instar 1.3.919 → 1.3.920
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/data/benchmarkDivergenceRegistry.d.ts.map +1 -1
- package/dist/data/benchmarkDivergenceRegistry.js +8 -0
- package/dist/data/benchmarkDivergenceRegistry.js.map +1 -1
- package/package.json +1 -1
- package/src/data/benchmarkDivergenceRegistry.ts +8 -0
- package/src/data/benchmarkPredictions.json +33 -0
- package/src/data/builtin-manifest.json +2 -2
- package/upgrades/1.3.920.md +74 -0
- package/upgrades/side-effects/benchmark-baseline-first-capture.md +89 -0
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"benchmarkDivergenceRegistry.d.ts","sourceRoot":"","sources":["../../src/data/benchmarkDivergenceRegistry.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAYH;;;;GAIG;AACH,eAAO,MAAM,cAAc,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAG3D,CAAC;AAIF,eAAO,MAAM,8BAA8B,IAAI,CAAC;AAEhD;;;;;;GAMG;AACH,eAAO,MAAM,sBAAsB,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,
|
|
1
|
+
{"version":3,"file":"benchmarkDivergenceRegistry.d.ts","sourceRoot":"","sources":["../../src/data/benchmarkDivergenceRegistry.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAYH;;;;GAIG;AACH,eAAO,MAAM,cAAc,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAG3D,CAAC;AAIF,eAAO,MAAM,8BAA8B,IAAI,CAAC;AAEhD;;;;;;GAMG;AACH,eAAO,MAAM,sBAAsB,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAgBnE,CAAC;AAEF,+EAA+E;AAC/E,wBAAgB,gBAAgB,CAAC,iBAAiB,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAGzE;AAED;iFACiF;AACjF,wBAAgB,oBAAoB,CAAC,KAAK,GAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAA0B,GAAG,MAAM,EAAE,CAS/G;AAID,MAAM,WAAW,mBAAmB;IAClC,2EAA2E;IAC3E,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B;;;;OAIG;IACH,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED,eAAO,MAAM,wBAAwB,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,mBAAmB,CAAC,CASlF,CAAC;AAIF,iFAAiF;AACjF,wBAAgB,oBAAoB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAE7D;AAED,wDAAwD;AACxD,wBAAgB,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAErD;AAED,mFAAmF;AACnF,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAU9D;AAID,MAAM,WAAW,eAAe;IAC9B,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,aAAa,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACtF,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;CAC3B;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,OAAO,CAAC;IACjB,6EAA6E;IAC7E,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;CACxC;AAED,qFAAqF;AACrF,eAAO,MAAM,mBAAmB,uCAAuC,CAAC;AAgBxE;;;;;;GAMG;AACH,wBAAgB,mBAAmB,CAAC,YAAY,EAAE,MAAM,EAAE,QAAQ,GAAE,CAAC,CAAC,EAAE,MAAM,KAAK,MAA2C,GAAG,eAAe,CAkD/I;AAED;mFACmF;AACnF,wBAAgB,iBAAiB,CAAC,WAAW,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,MAAM,GAAG,MAAM,CAGlF"}
|
|
@@ -52,6 +52,14 @@ export const MODEL_ID_NORMALIZATION = {
|
|
|
52
52
|
'claude-haiku-4-5-20251001': 'claude-haiku-4-5-20251001',
|
|
53
53
|
'gpt-5.5': 'gpt-5.5',
|
|
54
54
|
'gpt-5.5-codex': 'gpt-5.5-codex',
|
|
55
|
+
// The model actually serving `messaging-tone-gate` in production (verified
|
|
56
|
+
// 2026-07-23 on the live meter: attribution.models = ['gpt-5.4-mini'] via
|
|
57
|
+
// codex-cli). Its absence made the highest-volume enrolled pair report
|
|
58
|
+
// `no-benched-baseline (unmapped)` — i.e. the one pair with real traffic was
|
|
59
|
+
// structurally unmeasurable. An identity mapping, like the rest: the battery
|
|
60
|
+
// reports runs under the same immutable public id production records.
|
|
61
|
+
'gpt-5.4-mini': 'gpt-5.4-mini',
|
|
62
|
+
'gemini-2.5-flash': 'gemini-2.5-flash',
|
|
55
63
|
'gemini-2.5-pro': 'gemini-2.5-pro',
|
|
56
64
|
'gemini-3.1-pro': 'gemini-3.1-pro',
|
|
57
65
|
};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"benchmarkDivergenceRegistry.js","sourceRoot":"","sources":["../../src/data/benchmarkDivergenceRegistry.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,KAAK,MAAM,MAAM,aAAa,CAAC;AACtC,OAAO,EAAE,uCAAuC,EAAE,MAAM,8CAA8C,CAAC;AACvG,OAAO,EAAE,yBAAyB,EAAE,MAAM,8BAA8B,CAAC;AACzE,OAAO,EAAE,0BAA0B,EAAE,sBAAsB,EAAE,MAAM,yBAAyB,CAAC;AAC7F,OAAO,EAAE,SAAS,EAAE,MAAM,oCAAoC,CAAC;AAE/D,gFAAgF;AAEhF;;;;GAIG;AACH,MAAM,CAAC,MAAM,cAAc,GAAqC;IAC9D,CAAC,0BAA0B,CAAC,EAAE,iBAAiB;IAC/C,CAAC,sBAAsB,CAAC,EAAE,WAAW;CACtC,CAAC;AAEF,gFAAgF;AAEhF,MAAM,CAAC,MAAM,8BAA8B,GAAG,CAAC,CAAC;AAEhD;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAqC;IACtE,iBAAiB,EAAE,iBAAiB;IACpC,gBAAgB,EAAE,gBAAgB;IAClC,2BAA2B,EAAE,2BAA2B;IACxD,SAAS,EAAE,SAAS;IACpB,eAAe,EAAE,eAAe;IAChC,gBAAgB,EAAE,gBAAgB;IAClC,gBAAgB,EAAE,gBAAgB;CACnC,CAAC;AAEF,+EAA+E;AAC/E,MAAM,UAAU,gBAAgB,CAAC,iBAAyB;IACxD,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,cAAc,CAAC,IAAI,CAAC,sBAAsB,EAAE,iBAAiB,CAAC;QAAE,OAAO,IAAI,CAAC;IAClG,OAAO,sBAAsB,CAAC,iBAAiB,CAAC,CAAC;AACnD,CAAC;AAED;iFACiF;AACjF,MAAM,UAAU,oBAAoB,CAAC,QAA0C,sBAAsB;IACnG,MAAM,UAAU,GAAa,EAAE,CAAC;IAChC,KAAK,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACpD,KAAK,MAAM,EAAE,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,EAAE,CAAC;YACjC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC;gBAAE,UAAU,CAAC,IAAI,CAAC,iCAAiC,EAAE,EAAE,CAAC,CAAC;YAChF,IAAI,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,KAAK,QAAQ;gBAAE,UAAU,CAAC,IAAI,CAAC,gCAAgC,EAAE,EAAE,CAAC,CAAC;QACrG,CAAC;IACH,CAAC;IACD,OAAO,UAAU,CAAC;AACpB,CAAC;AAeD,MAAM,CAAC,MAAM,wBAAwB,GAAkD;IACrF,iBAAiB,EAAE;QACjB,QAAQ,EAAE,uCAAuC;QACjD,MAAM,EAAE,uFAAuF;KAChG;IACD,WAAW,EAAE;QACX,QAAQ,EAAE,yBAAyB;QACnC,MAAM,EAAE,yDAAyD;KAClE;CACF,CAAC;AAEF,gFAAgF;AAEhF,iFAAiF;AACjF,MAAM,UAAU,oBAAoB,CAAC,QAAgB;IACnD,OAAO,QAAQ,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;AAC9D,CAAC;AAED,wDAAwD;AACxD,MAAM,UAAU,YAAY,CAAC,QAAgB;IAC3C,OAAO,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,oBAAoB,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAClG,CAAC;AAED,mFAAmF;AACnF,MAAM,UAAU,gBAAgB,CAAC,MAAc;IAC7C,MAAM,KAAK,GAAG,wBAAwB,CAAC,MAAM,CAAC,CAAC;IAC/C,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,CAAC,QAAQ,KAAK,QAAQ,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAC7F,IAAI,CAAC;QACH,OAAO,YAAY,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IACtC,CAAC;IAAC,MAAM,CAAC;QACP,0EAA0E;QAC1E,oEAAoE;QACpE,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAkBD,qFAAqF;AACrF,MAAM,CAAC,MAAM,mBAAmB,GAAG,oCAAoC,CAAC;AAExE,SAAS,SAAS,CAAC,CAAU;IAC3B,OAAO,OAAO,CAAC,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AACpF,CAAC;AAED,SAAS,gBAAgB,CAAC,CAAU;IAClC,OAAO,OAAO,CAAC,KAAK,QAAQ,IAAI,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AAC7F,CAAC;AAED,SAAS,aAAa,CAAC,CAAU;IAC/B,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,CAAC,MAAM,GAAG,EAAE;QAAE,OAAO,IAAI,CAAC;IACxD,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACzB,OAAO,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AACxC,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,mBAAmB,CAAC,YAAoB,EAAE,WAAkC,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC,EAAE,OAAO,CAAC;IAC5H,IAAI,GAAY,CAAC;IACjB,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC,CAAC;IAC3C,CAAC;IAAC,MAAM,CAAC;QACP,oEAAoE;QACpE,sEAAsE;QACtE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;IACzD,CAAC;IACD,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC;QAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;IAClH,MAAM,IAAI,GAAG,GAA8B,CAAC;IAC5C,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,IAAI,OAAO,IAAI,CAAC,KAAK,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC;QACzF,CAAC,CAAE,IAAI,CAAC,KAAiC;QACzC,CAAC,CAAC,EAAE,CAAC;IACP,MAAM,KAAK,GAAoC,EAAE,CAAC;IAClD,IAAI,aAAa,GAAkB,aAAa,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAClE,KAAK,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC1D,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC;YAAE,SAAS;QACtC,IAAI,CAAC,QAAQ,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC;YAAE,SAAS;QACnF,MAAM,CAAC,GAAG,QAAmC,CAAC;QAC9C,MAAM,WAAW,GAAG,CAAC,CAAC,QAAQ,IAAI,OAAO,CAAC,CAAC,QAAQ,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC;YAC5F,CAAC,CAAE,CAAC,CAAC,QAAoC;YACzC,CAAC,CAAC,EAAE,CAAC;QACP,MAAM,QAAQ,GAAgC,EAAE,CAAC;QACjD,KAAK,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,CAAC;YAC9D,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC;gBAAE,SAAS;YACvC,IAAI,CAAC,QAAQ,IAAI,OAAO,QAAQ,KAAK,QAAQ;gBAAE,SAAS;YACxD,MAAM,CAAC,GAAG,QAAmC,CAAC;YAC9C,MAAM,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;YACvC,MAAM,MAAM,GAAG,gBAAgB,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;YAC1C,MAAM,aAAa,GAAG,gBAAgB,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC;YACxD,IAAI,QAAQ,KAAK,IAAI,IAAI,MAAM,KAAK,IAAI,IAAI,aAAa,KAAK,IAAI;gBAAE,SAAS;YAC7E,IAAI,aAAa,IAAI,CAAC,IAAI,MAAM,GAAG,aAAa;gBAAE,SAAS;YAC3D,uEAAuE;YACvE,IAAI,IAAI,CAAC,GAAG,CAAC,QAAQ,GAAG,MAAM,GAAG,aAAa,CAAC,GAAG,KAAK;gBAAE,SAAS;YAClE,QAAQ,CAAC,OAAO,CAAC,GAAG,EAAE,QAAQ,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC;QAC1D,CAAC;QACD,MAAM,UAAU,GAAG,aAAa,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;QAC/C,IAAI,UAAU,IAAI,CAAC,CAAC,aAAa,IAAI,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;YAAE,aAAa,GAAG,UAAU,CAAC;QACrH,KAAK,CAAC,MAAM,CAAC,GAAG;YACd,QAAQ;YACR,mBAAmB,EAAE,OAAO,CAAC,CAAC,mBAAmB,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,mBAAmB,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI;YAC3G,iBAAiB,EACf,OAAO,CAAC,CAAC,iBAAiB,KAAK,QAAQ,IAAI,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC,iBAAiB,CAAC;gBACpF,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAC,WAAW,EAAE;gBACnC,CAAC,CAAC,IAAI;YACV,UAAU;SACX,CAAC;IACJ,CAAC;IACD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,aAAa,EAAE,KAAK,EAAE,CAAC;AAC7D,CAAC;AAED;mFACmF;AACnF,MAAM,UAAU,iBAAiB,CAAC,WAAmB,EAAE,UAAmB;IACxE,MAAM,GAAG,GAAG,UAAU,IAAI,OAAO,UAAU,KAAK,QAAQ,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,mBAAmB,CAAC;IAC5F,OAAO,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC;AAClE,CAAC"}
|
|
1
|
+
{"version":3,"file":"benchmarkDivergenceRegistry.js","sourceRoot":"","sources":["../../src/data/benchmarkDivergenceRegistry.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,KAAK,MAAM,MAAM,aAAa,CAAC;AACtC,OAAO,EAAE,uCAAuC,EAAE,MAAM,8CAA8C,CAAC;AACvG,OAAO,EAAE,yBAAyB,EAAE,MAAM,8BAA8B,CAAC;AACzE,OAAO,EAAE,0BAA0B,EAAE,sBAAsB,EAAE,MAAM,yBAAyB,CAAC;AAC7F,OAAO,EAAE,SAAS,EAAE,MAAM,oCAAoC,CAAC;AAE/D,gFAAgF;AAEhF;;;;GAIG;AACH,MAAM,CAAC,MAAM,cAAc,GAAqC;IAC9D,CAAC,0BAA0B,CAAC,EAAE,iBAAiB;IAC/C,CAAC,sBAAsB,CAAC,EAAE,WAAW;CACtC,CAAC;AAEF,gFAAgF;AAEhF,MAAM,CAAC,MAAM,8BAA8B,GAAG,CAAC,CAAC;AAEhD;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAqC;IACtE,iBAAiB,EAAE,iBAAiB;IACpC,gBAAgB,EAAE,gBAAgB;IAClC,2BAA2B,EAAE,2BAA2B;IACxD,SAAS,EAAE,SAAS;IACpB,eAAe,EAAE,eAAe;IAChC,2EAA2E;IAC3E,0EAA0E;IAC1E,uEAAuE;IACvE,6EAA6E;IAC7E,6EAA6E;IAC7E,sEAAsE;IACtE,cAAc,EAAE,cAAc;IAC9B,kBAAkB,EAAE,kBAAkB;IACtC,gBAAgB,EAAE,gBAAgB;IAClC,gBAAgB,EAAE,gBAAgB;CACnC,CAAC;AAEF,+EAA+E;AAC/E,MAAM,UAAU,gBAAgB,CAAC,iBAAyB;IACxD,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,cAAc,CAAC,IAAI,CAAC,sBAAsB,EAAE,iBAAiB,CAAC;QAAE,OAAO,IAAI,CAAC;IAClG,OAAO,sBAAsB,CAAC,iBAAiB,CAAC,CAAC;AACnD,CAAC;AAED;iFACiF;AACjF,MAAM,UAAU,oBAAoB,CAAC,QAA0C,sBAAsB;IACnG,MAAM,UAAU,GAAa,EAAE,CAAC;IAChC,KAAK,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACpD,KAAK,MAAM,EAAE,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,EAAE,CAAC;YACjC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC;gBAAE,UAAU,CAAC,IAAI,CAAC,iCAAiC,EAAE,EAAE,CAAC,CAAC;YAChF,IAAI,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,KAAK,QAAQ;gBAAE,UAAU,CAAC,IAAI,CAAC,gCAAgC,EAAE,EAAE,CAAC,CAAC;QACrG,CAAC;IACH,CAAC;IACD,OAAO,UAAU,CAAC;AACpB,CAAC;AAeD,MAAM,CAAC,MAAM,wBAAwB,GAAkD;IACrF,iBAAiB,EAAE;QACjB,QAAQ,EAAE,uCAAuC;QACjD,MAAM,EAAE,uFAAuF;KAChG;IACD,WAAW,EAAE;QACX,QAAQ,EAAE,yBAAyB;QACnC,MAAM,EAAE,yDAAyD;KAClE;CACF,CAAC;AAEF,gFAAgF;AAEhF,iFAAiF;AACjF,MAAM,UAAU,oBAAoB,CAAC,QAAgB;IACnD,OAAO,QAAQ,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;AAC9D,CAAC;AAED,wDAAwD;AACxD,MAAM,UAAU,YAAY,CAAC,QAAgB;IAC3C,OAAO,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,oBAAoB,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAClG,CAAC;AAED,mFAAmF;AACnF,MAAM,UAAU,gBAAgB,CAAC,MAAc;IAC7C,MAAM,KAAK,GAAG,wBAAwB,CAAC,MAAM,CAAC,CAAC;IAC/C,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,CAAC,QAAQ,KAAK,QAAQ,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAC7F,IAAI,CAAC;QACH,OAAO,YAAY,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IACtC,CAAC;IAAC,MAAM,CAAC;QACP,0EAA0E;QAC1E,oEAAoE;QACpE,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAkBD,qFAAqF;AACrF,MAAM,CAAC,MAAM,mBAAmB,GAAG,oCAAoC,CAAC;AAExE,SAAS,SAAS,CAAC,CAAU;IAC3B,OAAO,OAAO,CAAC,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AACpF,CAAC;AAED,SAAS,gBAAgB,CAAC,CAAU;IAClC,OAAO,OAAO,CAAC,KAAK,QAAQ,IAAI,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AAC7F,CAAC;AAED,SAAS,aAAa,CAAC,CAAU;IAC/B,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,CAAC,MAAM,GAAG,EAAE;QAAE,OAAO,IAAI,CAAC;IACxD,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACzB,OAAO,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AACxC,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,mBAAmB,CAAC,YAAoB,EAAE,WAAkC,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC,EAAE,OAAO,CAAC;IAC5H,IAAI,GAAY,CAAC;IACjB,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC,CAAC;IAC3C,CAAC;IAAC,MAAM,CAAC;QACP,oEAAoE;QACpE,sEAAsE;QACtE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;IACzD,CAAC;IACD,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC;QAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;IAClH,MAAM,IAAI,GAAG,GAA8B,CAAC;IAC5C,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,IAAI,OAAO,IAAI,CAAC,KAAK,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC;QACzF,CAAC,CAAE,IAAI,CAAC,KAAiC;QACzC,CAAC,CAAC,EAAE,CAAC;IACP,MAAM,KAAK,GAAoC,EAAE,CAAC;IAClD,IAAI,aAAa,GAAkB,aAAa,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAClE,KAAK,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC1D,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC;YAAE,SAAS;QACtC,IAAI,CAAC,QAAQ,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC;YAAE,SAAS;QACnF,MAAM,CAAC,GAAG,QAAmC,CAAC;QAC9C,MAAM,WAAW,GAAG,CAAC,CAAC,QAAQ,IAAI,OAAO,CAAC,CAAC,QAAQ,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC;YAC5F,CAAC,CAAE,CAAC,CAAC,QAAoC;YACzC,CAAC,CAAC,EAAE,CAAC;QACP,MAAM,QAAQ,GAAgC,EAAE,CAAC;QACjD,KAAK,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,CAAC;YAC9D,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC;gBAAE,SAAS;YACvC,IAAI,CAAC,QAAQ,IAAI,OAAO,QAAQ,KAAK,QAAQ;gBAAE,SAAS;YACxD,MAAM,CAAC,GAAG,QAAmC,CAAC;YAC9C,MAAM,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;YACvC,MAAM,MAAM,GAAG,gBAAgB,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;YAC1C,MAAM,aAAa,GAAG,gBAAgB,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC;YACxD,IAAI,QAAQ,KAAK,IAAI,IAAI,MAAM,KAAK,IAAI,IAAI,aAAa,KAAK,IAAI;gBAAE,SAAS;YAC7E,IAAI,aAAa,IAAI,CAAC,IAAI,MAAM,GAAG,aAAa;gBAAE,SAAS;YAC3D,uEAAuE;YACvE,IAAI,IAAI,CAAC,GAAG,CAAC,QAAQ,GAAG,MAAM,GAAG,aAAa,CAAC,GAAG,KAAK;gBAAE,SAAS;YAClE,QAAQ,CAAC,OAAO,CAAC,GAAG,EAAE,QAAQ,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC;QAC1D,CAAC;QACD,MAAM,UAAU,GAAG,aAAa,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;QAC/C,IAAI,UAAU,IAAI,CAAC,CAAC,aAAa,IAAI,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;YAAE,aAAa,GAAG,UAAU,CAAC;QACrH,KAAK,CAAC,MAAM,CAAC,GAAG;YACd,QAAQ;YACR,mBAAmB,EAAE,OAAO,CAAC,CAAC,mBAAmB,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,mBAAmB,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI;YAC3G,iBAAiB,EACf,OAAO,CAAC,CAAC,iBAAiB,KAAK,QAAQ,IAAI,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC,iBAAiB,CAAC;gBACpF,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAC,WAAW,EAAE;gBACnC,CAAC,CAAC,IAAI;YACV,UAAU;SACX,CAAC;IACJ,CAAC;IACD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,aAAa,EAAE,KAAK,EAAE,CAAC;AAC7D,CAAC;AAED;mFACmF;AACnF,MAAM,UAAU,iBAAiB,CAAC,WAAmB,EAAE,UAAmB;IACxE,MAAM,GAAG,GAAG,UAAU,IAAI,OAAO,UAAU,KAAK,QAAQ,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,mBAAmB,CAAC;IAC5F,OAAO,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC;AAClE,CAAC"}
|
package/package.json
CHANGED
|
@@ -58,6 +58,14 @@ export const MODEL_ID_NORMALIZATION: Readonly<Record<string, string>> = {
|
|
|
58
58
|
'claude-haiku-4-5-20251001': 'claude-haiku-4-5-20251001',
|
|
59
59
|
'gpt-5.5': 'gpt-5.5',
|
|
60
60
|
'gpt-5.5-codex': 'gpt-5.5-codex',
|
|
61
|
+
// The model actually serving `messaging-tone-gate` in production (verified
|
|
62
|
+
// 2026-07-23 on the live meter: attribution.models = ['gpt-5.4-mini'] via
|
|
63
|
+
// codex-cli). Its absence made the highest-volume enrolled pair report
|
|
64
|
+
// `no-benched-baseline (unmapped)` — i.e. the one pair with real traffic was
|
|
65
|
+
// structurally unmeasurable. An identity mapping, like the rest: the battery
|
|
66
|
+
// reports runs under the same immutable public id production records.
|
|
67
|
+
'gpt-5.4-mini': 'gpt-5.4-mini',
|
|
68
|
+
'gemini-2.5-flash': 'gemini-2.5-flash',
|
|
61
69
|
'gemini-2.5-pro': 'gemini-2.5-pro',
|
|
62
70
|
'gemini-3.1-pro': 'gemini-3.1-pro',
|
|
63
71
|
};
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
{
|
|
2
|
+
"_doc": "Content-free INSTAR-Bench predictions mirror (benchmark-divergence-detector FD1). Counts, not just rates, so the prediction carries its own sampling uncertainty. Regenerate by re-running the battery against the CURRENT prompt templates and re-stamping benchedPromptHash.",
|
|
3
|
+
"capturedAt": "2026-07-23T22:30:00.000Z",
|
|
4
|
+
"tasks": {
|
|
5
|
+
"tone-gate": {
|
|
6
|
+
"perModel": {
|
|
7
|
+
"gpt-5.4-mini": {
|
|
8
|
+
"passRate": 1,
|
|
9
|
+
"passes": 14,
|
|
10
|
+
"deterministic": 14
|
|
11
|
+
},
|
|
12
|
+
"gpt-5.5": {
|
|
13
|
+
"passRate": 0.9643,
|
|
14
|
+
"passes": 27,
|
|
15
|
+
"deterministic": 28
|
|
16
|
+
},
|
|
17
|
+
"gemini-2.5-flash": {
|
|
18
|
+
"passRate": 1,
|
|
19
|
+
"passes": 14,
|
|
20
|
+
"deterministic": 14
|
|
21
|
+
},
|
|
22
|
+
"claude-haiku-4-5-20251001": {
|
|
23
|
+
"passRate": 0.9286,
|
|
24
|
+
"passes": 13,
|
|
25
|
+
"deterministic": 14
|
|
26
|
+
}
|
|
27
|
+
},
|
|
28
|
+
"benchedPromptSource": "src/core/MessagingToneGate.ts:TONE_GATE_PROMPT_TEMPLATE",
|
|
29
|
+
"benchedPromptHash": "4ae0cf6b02663c21a695393727bd05662334f0d8f1a9d3e45d8d9b2141208d54",
|
|
30
|
+
"capturedAt": "2026-07-23T22:30:00.000Z"
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
}
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
{
|
|
2
2
|
"$schema": "./builtin-manifest.schema.json",
|
|
3
3
|
"schemaVersion": 1,
|
|
4
|
-
"generatedAt": "2026-07-
|
|
5
|
-
"instarVersion": "1.3.
|
|
4
|
+
"generatedAt": "2026-07-23T22:58:25.584Z",
|
|
5
|
+
"instarVersion": "1.3.920",
|
|
6
6
|
"entryCount": 202,
|
|
7
7
|
"entries": {
|
|
8
8
|
"hook:session-start": {
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
# Upgrade Guide — vNEXT
|
|
2
|
+
|
|
3
|
+
<!-- assembled-by: assemble-next-md -->
|
|
4
|
+
<!-- bump: patch -->
|
|
5
|
+
|
|
6
|
+
## What Changed
|
|
7
|
+
|
|
8
|
+
The Benchmark-Divergence Detector now ships with its first captured predictions baseline (`src/data/benchmarkPredictions.json`), so `GET /benchmark-divergence` reports `mirror.present: true` instead of degrading every finding to `precondition-failed`.
|
|
9
|
+
|
|
10
|
+
Two model ids were added to the fail-closed normalization table, including `gpt-5.4-mini` — the model actually serving `messaging-tone-gate` in production. Its absence meant the one enrolled pair with real traffic resolved as unmapped indefinitely, so the comparator was quietly measuring nothing there.
|
|
11
|
+
|
|
12
|
+
The baseline was produced by re-running the battery against the CURRENT prompt templates rather than reusing July's numbers: the tone-gate prompt had grown from ~35,500 to ~39,900 characters since that run, so the old results would have been stale-on-arrival and correctly suppressed by the template-hash precondition.
|
|
13
|
+
|
|
14
|
+
## What to Tell Your User
|
|
15
|
+
|
|
16
|
+
Nothing to do. The model-quality benchmark now has a current baseline captured against the instructions actually running today, rather than a version from three weeks ago. One honest caveat: this is half of a two-part loop — the comparison also needs real-world outcome grades, which don't exist for this scenario yet, so the drift check stays quiet until that lands.
|
|
17
|
+
|
|
18
|
+
## Summary of New Capabilities
|
|
19
|
+
|
|
20
|
+
- `GET /benchmark-divergence` has a real baseline to compare against, with per-model counts (not just rates) so prediction-side sampling uncertainty is weighable.
|
|
21
|
+
|
|
22
|
+
## Evidence
|
|
23
|
+
|
|
24
|
+
**Before** — `GET /benchmark-divergence` on the serving machine (2026-07-23, v1.3.914):
|
|
25
|
+
|
|
26
|
+
```
|
|
27
|
+
{"mirror":{"present":false,"capturedAt":null,"staleDays":null,"stale":true},
|
|
28
|
+
"summary":{"byVerdict":{},"unmappedModels":[],"missingModelShare":0.02},"findings":[]}
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
Every comparison degraded to `precondition-failed`; nothing could be measured.
|
|
32
|
+
|
|
33
|
+
**The unmapped-model defect** — the live meter reports the enrolled pair's
|
|
34
|
+
production model:
|
|
35
|
+
|
|
36
|
+
```
|
|
37
|
+
messaging-tone-gate → attribution.models: ["gpt-5.4-mini"], frameworks: ["codex-cli"]
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
`gpt-5.4-mini` was absent from `MODEL_ID_NORMALIZATION`, and FD5 is fail-closed
|
|
41
|
+
exact-match, so that pair resolved `no-benched-baseline (unmapped)` indefinitely.
|
|
42
|
+
|
|
43
|
+
**The staleness that made a back-fill useless** — benched vs live template size for
|
|
44
|
+
the two enrolled tasks:
|
|
45
|
+
|
|
46
|
+
| task | benched | live | match? |
|
|
47
|
+
|---|---|---|---|
|
|
48
|
+
| tone-gate | 35,499 chars | 39,886 chars | no |
|
|
49
|
+
| zombie-classify | 2,029 chars | 1,373 chars | no |
|
|
50
|
+
|
|
51
|
+
Building a mirror from July's numbers would have carried a benched-prompt hash that
|
|
52
|
+
never matches the live template, so the Q0 precondition would have suppressed every
|
|
53
|
+
finding — a captured-looking baseline that measures nothing.
|
|
54
|
+
|
|
55
|
+
**After** — the battery re-run against the regenerated template (14 deterministic
|
|
56
|
+
cases per route, subscription doors), loaded back through the real
|
|
57
|
+
`loadBenchmarkMirror`:
|
|
58
|
+
|
|
59
|
+
```
|
|
60
|
+
present=true capturedAt=2026-07-23T22:30:00.000Z
|
|
61
|
+
tasks=tone-gate
|
|
62
|
+
perModel=gpt-5.4-mini,gpt-5.5,gemini-2.5-flash,claude-haiku-4-5-20251001
|
|
63
|
+
benchedPromptHash=4ae0cf6b02663c21… (= sha256 of the LIVE template)
|
|
64
|
+
normalize(gpt-5.4-mini)=gpt-5.4-mini
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
Pass rates against the current prompt: `gpt-5.4-mini` 14/14, `gemini-2.5-flash`
|
|
68
|
+
14/14, `gpt-5.5` 27/28, `claude-haiku-4-5` 13/14. Notably `gpt-5.4-mini` scored
|
|
69
|
+
6/14 against the OLD template in July — the prompt improved substantially, which is
|
|
70
|
+
the opposite of the decay the staleness concern assumed.
|
|
71
|
+
|
|
72
|
+
**Honest limit:** the drift comparison still cannot fire, because the real-world
|
|
73
|
+
side of it (`messaging-tone-gate` outcome grades) is all-`unknown`. This change
|
|
74
|
+
makes the prediction side current and measurable; it does not close the loop.
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
# Side-effects review — first benchmark predictions baseline
|
|
2
|
+
|
|
3
|
+
**Change:** ships `src/data/benchmarkPredictions.json` (the first captured
|
|
4
|
+
INSTAR-Bench predictions mirror) and adds two rows to `MODEL_ID_NORMALIZATION`.
|
|
5
|
+
|
|
6
|
+
## What actually changed
|
|
7
|
+
|
|
8
|
+
1. `src/data/benchmarkPredictions.json` — new, content-free: per (task, model)
|
|
9
|
+
`passRate`/`passes`/`deterministic`, the benched prompt source, and the benched
|
|
10
|
+
prompt hash. No prompts, no case text, no outputs.
|
|
11
|
+
2. `MODEL_ID_NORMALIZATION` — adds `gpt-5.4-mini` and `gemini-2.5-flash`, both
|
|
12
|
+
identity mappings.
|
|
13
|
+
3. `tests/unit/benchmarkDivergenceRegistry.test.ts` — the pin that asserted the
|
|
14
|
+
mirror is ABSENT now asserts it is PRESENT and well-formed.
|
|
15
|
+
|
|
16
|
+
## Why (two real defects, not just "capture a baseline")
|
|
17
|
+
|
|
18
|
+
**(a) The comparator had no baseline at all.** `GET /benchmark-divergence`
|
|
19
|
+
reported `mirror.present: false`, so every finding degraded to
|
|
20
|
+
`precondition-failed`. Measured on the serving machine 2026-07-23.
|
|
21
|
+
|
|
22
|
+
**(b) The production model was unmapped — the one pair with traffic was
|
|
23
|
+
structurally unmeasurable.** The live meter reports `messaging-tone-gate` running
|
|
24
|
+
on `gpt-5.4-mini` via codex-cli. That id was absent from the normalization table,
|
|
25
|
+
and FD5 is (correctly) fail-closed exact-match, so the pair resolved
|
|
26
|
+
`no-benched-baseline (unmapped)` — indefinitely, and quietly. A comparator that
|
|
27
|
+
reports "nothing to compare" forever looks identical to one reporting health.
|
|
28
|
+
|
|
29
|
+
**(c) The pre-existing bench results were stale against a drifted prompt.** The
|
|
30
|
+
July battery scored a tone-gate prompt of 35,499 chars; the live template is
|
|
31
|
+
39,886. Building a mirror from those numbers would have been stale-on-arrival —
|
|
32
|
+
the Q0 template-hash precondition would have (correctly) suppressed every finding.
|
|
33
|
+
So the battery was **re-run against the current template**, not back-filled.
|
|
34
|
+
|
|
35
|
+
## How the numbers were produced
|
|
36
|
+
|
|
37
|
+
The bench task's template file was regenerated from the live
|
|
38
|
+
`TONE_GATE_PROMPT_TEMPLATE` export: verbatim rules body + the harness's single
|
|
39
|
+
per-case slot, which is exactly the declared `promptFidelity` for this task. The
|
|
40
|
+
battery then ran 14 deterministic cases per route.
|
|
41
|
+
|
|
42
|
+
`benchedPromptHash` is the sha256 of the LIVE exported template, so the detector's
|
|
43
|
+
Q0 precondition passes only while the prompt genuinely matches — and starts failing
|
|
44
|
+
the moment it drifts again, which is the intended tripwire.
|
|
45
|
+
|
|
46
|
+
## Blast radius
|
|
47
|
+
|
|
48
|
+
- **Surface:** one read-only observability route (`GET /benchmark-divergence`).
|
|
49
|
+
- **Authority:** none. Findings are `advisory: true` by construction; nothing gates.
|
|
50
|
+
- **Data:** a static JSON file read through existing FD9 clamps. A malformed mirror
|
|
51
|
+
is already a first-class `present: false` state, never a throw.
|
|
52
|
+
- **Fail direction unchanged:** an unmapped model still resolves fail-closed.
|
|
53
|
+
|
|
54
|
+
## Risk analysis
|
|
55
|
+
|
|
56
|
+
**Could this manufacture a false finding?** Only if the real-world side had a rate
|
|
57
|
+
to compare — it does not. `messaging-tone-gate` currently settles every decision
|
|
58
|
+
`unknown`, so the detector has no real grade-rate and every comparison stays
|
|
59
|
+
non-actionable. **This ships one half of a two-halves loop, and that is stated
|
|
60
|
+
rather than implied**: the mirror makes the benchmark current; it does not make the
|
|
61
|
+
drift comparison useful until outcome grading lands.
|
|
62
|
+
|
|
63
|
+
**Sample size honesty.** 14 deterministic cases per model is a small battery. FD3's
|
|
64
|
+
Wilson half-width term already accounts for that on the prediction side (a 14-case
|
|
65
|
+
battery carries roughly ±0.25 at p=0.5), so a small battery *widens* the flag
|
|
66
|
+
threshold rather than manufacturing divergence. The counts — not just the rates —
|
|
67
|
+
are recorded precisely so that math can run.
|
|
68
|
+
|
|
69
|
+
**Cost.** The run was intended to use subscription doors only. One metered
|
|
70
|
+
OpenRouter route matched the inclusion filter by substring (`claude-haiku` matched
|
|
71
|
+
`or-claude-haiku-45`) and billed **$0.87** against a key with a $25 daily / $60
|
|
72
|
+
lifetime cap. Disclosed rather than absorbed; that route's result is EXCLUDED from
|
|
73
|
+
the mirror, which carries subscription-door results only.
|
|
74
|
+
|
|
75
|
+
## Migration parity
|
|
76
|
+
|
|
77
|
+
None required. A shipped data file and a source constant; existing agents pick both
|
|
78
|
+
up with the normal version bump. No installed-file, config, hook, or template
|
|
79
|
+
surface.
|
|
80
|
+
|
|
81
|
+
## Testing
|
|
82
|
+
|
|
83
|
+
Registry suite (14), core (unchanged), analyzer, routes, E2E alive, migration and
|
|
84
|
+
job-template suites — 97 green. `tsc --noEmit` clean.
|
|
85
|
+
|
|
86
|
+
## Rollback
|
|
87
|
+
|
|
88
|
+
Delete the JSON (the loader returns to `present: false`, today's behaviour) or
|
|
89
|
+
revert the commit. Nothing persists; no state to unwind.
|