@miller-tech/uap 1.179.12 → 1.183.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.tsbuildinfo +1 -1
- package/dist/benchmarks/harness-card.d.ts +10 -0
- package/dist/benchmarks/harness-card.d.ts.map +1 -1
- package/dist/benchmarks/harness-card.js +1 -0
- package/dist/benchmarks/harness-card.js.map +1 -1
- package/dist/benchmarks/paired/adapter.d.ts +25 -0
- package/dist/benchmarks/paired/adapter.d.ts.map +1 -1
- package/dist/benchmarks/paired/adapter.js +57 -6
- package/dist/benchmarks/paired/adapter.js.map +1 -1
- package/dist/benchmarks/paired/report.d.ts +86 -0
- package/dist/benchmarks/paired/report.d.ts.map +1 -1
- package/dist/benchmarks/paired/report.js +145 -1
- package/dist/benchmarks/paired/report.js.map +1 -1
- package/dist/benchmarks/paired/runner.d.ts.map +1 -1
- package/dist/benchmarks/paired/runner.js +6 -1
- package/dist/benchmarks/paired/runner.js.map +1 -1
- package/dist/benchmarks/paired/stats.d.ts +13 -1
- package/dist/benchmarks/paired/stats.d.ts.map +1 -1
- package/dist/benchmarks/paired/stats.js +6 -0
- package/dist/benchmarks/paired/stats.js.map +1 -1
- package/dist/bin/cli.js +2 -0
- package/dist/bin/cli.js.map +1 -1
- package/dist/cli/bench.d.ts +4 -0
- package/dist/cli/bench.d.ts.map +1 -1
- package/dist/cli/bench.js +112 -2
- package/dist/cli/bench.js.map +1 -1
- package/dist/cli/harness.d.ts.map +1 -1
- package/dist/cli/harness.js +2 -0
- package/dist/cli/harness.js.map +1 -1
- package/dist/self-harness/decide.d.ts +5 -0
- package/dist/self-harness/decide.d.ts.map +1 -1
- package/dist/self-harness/decide.js +19 -0
- package/dist/self-harness/decide.js.map +1 -1
- package/package.json +1 -1
- package/src/policies/enforcers/__pycache__/_common.cpython-312.pyc +0 -0
- package/templates/hooks/__pycache__/deliver_autoroute.cpython-312.pyc +0 -0
- package/tools/agents/scripts/__pycache__/toolcall_path_normalizer.cpython-312.pyc +0 -0
|
@@ -43,6 +43,16 @@ export interface HarnessCardInput {
|
|
|
43
43
|
sandboxed?: boolean;
|
|
44
44
|
/** Estimated-token ceiling for one agentic session, if set. */
|
|
45
45
|
contextTokenBudget?: number;
|
|
46
|
+
/**
|
|
47
|
+
* Per-completion `max_tokens` ceiling. Disclosed because a reasoning model
|
|
48
|
+
* spends this budget on a hidden thinking channel BEFORE emitting any answer
|
|
49
|
+
* token, so a budget set for the answer alone silently yields EMPTY
|
|
50
|
+
* completions (finish_reason=length) that look like format non-compliance.
|
|
51
|
+
* The raw bench adapter hardcoded 4096 while this repo's profile for the same
|
|
52
|
+
* local model allocates 81920 — a 20x starvation that pinned both arms toward
|
|
53
|
+
* the floor and appeared nowhere in any report.
|
|
54
|
+
*/
|
|
55
|
+
completionTokenBudget?: number;
|
|
46
56
|
/** Max tool-call rounds before a final answer is forced. */
|
|
47
57
|
maxToolRounds?: number;
|
|
48
58
|
/** Per-command bash timeout, ms. */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"harness-card.d.ts","sourceRoot":"","sources":["../../src/benchmarks/harness-card.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,4EAA4E;AAC5E,MAAM,WAAW,YAAY;IAC3B,kCAAkC;IAClC,IAAI,EAAE,GAAG,GAAG,GAAG,GAAG,GAAG,GAAG,GAAG,GAAG,GAAG,GAAG,GAAG,GAAG,GAAG,CAAC;IAC9C,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,KAAK,CAAC;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CAC/C;AAED,MAAM,WAAW,WAAW;IAC1B,gFAAgF;IAChF,OAAO,EAAE,MAAM,CAAC;IAChB,gCAAgC;IAChC,UAAU,EAAE,MAAM,CAAC;IACnB,kDAAkD;IAClD,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,YAAY,EAAE,CAAC;CACxB;AAED,MAAM,WAAW,gBAAgB;IAC/B,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,gDAAgD;IAChD,KAAK,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAC1B,wCAAwC;IACxC,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,6CAA6C;IAC7C,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,+DAA+D;IAC/D,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,4DAA4D;IAC5D,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,oCAAoC;IACpC,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,sCAAsC;IACtC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,gDAAgD;IAChD,YAAY,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACjC,6DAA6D;IAC7D,UAAU,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAC/B;;;;OAIG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,qDAAqD;IACrD,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,+DAA+D;IAC/D,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,6DAA6D;IAC7D,YAAY,CAAC,EAAE,OAAO,CAAC;CACxB;AAYD;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,gBAAgB,GAAG,WAAW,
|
|
1
|
+
{"version":3,"file":"harness-card.d.ts","sourceRoot":"","sources":["../../src/benchmarks/harness-card.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,4EAA4E;AAC5E,MAAM,WAAW,YAAY;IAC3B,kCAAkC;IAClC,IAAI,EAAE,GAAG,GAAG,GAAG,GAAG,GAAG,GAAG,GAAG,GAAG,GAAG,GAAG,GAAG,GAAG,GAAG,CAAC;IAC9C,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,KAAK,CAAC;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CAC/C;AAED,MAAM,WAAW,WAAW;IAC1B,gFAAgF;IAChF,OAAO,EAAE,MAAM,CAAC;IAChB,gCAAgC;IAChC,UAAU,EAAE,MAAM,CAAC;IACnB,kDAAkD;IAClD,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,YAAY,EAAE,CAAC;CACxB;AAED,MAAM,WAAW,gBAAgB;IAC/B,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,gDAAgD;IAChD,KAAK,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAC1B,wCAAwC;IACxC,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,6CAA6C;IAC7C,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,+DAA+D;IAC/D,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B;;;;;;;;OAQG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,4DAA4D;IAC5D,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,oCAAoC;IACpC,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,sCAAsC;IACtC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,gDAAgD;IAChD,YAAY,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACjC,6DAA6D;IAC7D,UAAU,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAC/B;;;;OAIG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,qDAAqD;IACrD,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,+DAA+D;IAC/D,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,6DAA6D;IAC7D,YAAY,CAAC,EAAE,OAAO,CAAC;CACxB;AAYD;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,gBAAgB,GAAG,WAAW,CAqFrE;AAED,gFAAgF;AAChF,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,WAAW,GAAG,MAAM,CAoB3D;AAED;;;;;GAKG;AACH,eAAO,MAAM,mBAAmB,wIAWtB,CAAC"}
|
|
@@ -60,6 +60,7 @@ export function buildHarnessCard(input) {
|
|
|
60
60
|
name: 'Context',
|
|
61
61
|
fields: [
|
|
62
62
|
{ key: 'token_budget', value: num(input.contextTokenBudget) },
|
|
63
|
+
{ key: 'completion_budget', value: num(input.completionTokenBudget) },
|
|
63
64
|
{ key: 'read_window_bytes', value: num(input.readWindowBytes) },
|
|
64
65
|
{ key: 'memory_mode', value: input.memoryMode ?? 'unknown' },
|
|
65
66
|
{ key: 'compression', value: 'semantic units + dynamic compressor' },
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"harness-card.js","sourceRoot":"","sources":["../../src/benchmarks/harness-card.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;
|
|
1
|
+
{"version":3,"file":"harness-card.js","sourceRoot":"","sources":["../../src/benchmarks/harness-card.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAiEH,MAAM,YAAY,GAAG,GAAG,CAAC;AAEzB,SAAS,GAAG,CAAC,CAAsB;IACjC,OAAO,CAAC,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;AACxD,CAAC;AAED,SAAS,GAAG,CAAC,CAAqB;IAChC,OAAO,CAAC,KAAK,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;AAC/C,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,gBAAgB,CAAC,KAAuB;IACtD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,IAAI,EAAE,CAAC;IAChC,OAAO;QACL,OAAO,EAAE,YAAY;QACrB,UAAU,EAAE,KAAK,CAAC,UAAU;QAC5B,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,MAAM,EAAE;YACN;gBACE,IAAI,EAAE,GAAG;gBACT,IAAI,EAAE,WAAW;gBACjB,MAAM,EAAE;oBACN,EAAE,GAAG,EAAE,WAAW,EAAE,KAAK,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,gCAAgC,CAAC,CAAC,CAAC,cAAc,EAAE;oBAChG,EAAE,GAAG,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE;oBAC7C,EAAE,GAAG,EAAE,iBAAiB,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,CAAC,aAAa,CAAC,EAAE;oBAC3D,EAAE,GAAG,EAAE,eAAe,EAAE,KAAK,EAAE,oCAAoC,EAAE;iBACtE;aACF;YACD;gBACE,IAAI,EAAE,GAAG;gBACT,IAAI,EAAE,MAAM;gBACZ,MAAM,EAAE;oBACN,EAAE,GAAG,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE;oBACxE,EAAE,GAAG,EAAE,cAAc,EAAE,KAAK,EAAE,uCAAuC,EAAE;oBACvE,EAAE,GAAG,EAAE,eAAe,EAAE,KAAK,EAAE,KAAK,CAAC,YAAY,IAAI,SAAS,EAAE;oBAChE,EAAE,GAAG,EAAE,aAAa,EAAE,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,SAAS,EAAE;oBACxF,EAAE,GAAG,EAAE,cAAc,EAAE,KAAK,EAAE,2CAA2C,EAAE;iBAC5E;aACF;YACD;gBACE,IAAI,EAAE,GAAG;gBACT,IAAI,EAAE,SAAS;gBACf,MAAM,EAAE;oBACN,EAAE,GAAG,EAAE,cAAc,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,CAAC,kBAAkB,CAAC,EAAE;oBAC7D,EAAE,GAAG,EAAE,mBAAmB,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,CAAC,qBAAqB,CAAC,EAAE;oBACrE,EAAE,GAAG,EAAE,mBAAmB,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,CAAC,eAAe,CAAC,EAAE;oBAC/D,EAAE,GAAG,EAAE,aAAa,EAAE,KAAK,EAAE,KAAK,CAAC,UAAU,IAAI,SAAS,EAAE;oBAC5D,EAAE,GAAG,EAAE,aAAa,EAAE,KAAK,EAAE,qCAAqC,EAAE;iBACrE;aACF;YACD;gBACE,IAAI,EAAE,GAAG;gBACT,IAAI,EAAE,YAAY;gBAClB,MAAM,EAAE;oBACN,EAAE,GAAG,EAAE,iBAAiB,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,CAAC,aAAa,CAAC,EAAE;oBAC3D,EAAE,GAAG,EAAE,eAAe,EAAE,KAAK,EAAE,2CAA2C,EAAE;oBAC5E,EAAE,GAAG,EAAE,cAAc,EAAE,KAAK,EAAE,8CAA8C,EAAE;oBAC9E,EAAE,GAAG,EAAE,OAAO,EAAE,KAAK,EAAE,8CAA8C,EAAE;iBACxE;aACF;YACD;gBACE,IAAI,EAAE,GAAG;gBACT,IAAI,EAAE,eAAe;gBACrB,MAAM,EAAE;oBACN,EAAE,GAAG,EAAE,kBAAkB,EAAE,KAAK,EAAE,gDAAgD,EAAE;oBACpF,EAAE,GAAG,EAAE,QAAQ,EAAE,KAAK,EAAE,mCAAmC,EAAE;oBAC7D,EAAE,GAAG,EAAE,aAAa,EAAE,KAAK,EAAE,oCAAoC,EAAE;iBACpE;aACF;YACD;gBACE,IAAI,EAAE,GAAG;gBACT,IAAI,EAAE,cAAc;gBACpB,MAAM,EAAE;oBACN;wBACE,GAAG,EAAE,OAAO;wBACZ,KAAK,EACH,KAAK,CAAC,YAAY,IAAI,KAAK,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC;4BACjD,CAAC,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC;4BAC/B,CAAC,CAAC,SAAS;qBAChB;oBACD,EAAE,GAAG,EAAE,cAAc,EAAE,KAAK,EAAE,iDAAiD,EAAE;iBAClF;aACF;YACD;gBACE,IAAI,EAAE,GAAG;gBACT,IAAI,EAAE,YAAY;gBAClB,MAAM,EAAE;oBACN,EAAE,GAAG,EAAE,iBAAiB,EAAE,KAAK,EAAE,+DAA+D,EAAE;oBAClG,EAAE,GAAG,EAAE,YAAY,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE;oBAClD,EAAE,GAAG,EAAE,eAAe,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,CAAC,YAAY,CAAC,EAAE;oBACxD,EAAE,GAAG,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC,KAAK,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,MAAM,EAAE;oBAC3E,EAAE,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,4CAA4C,EAAE;iBACxE;aACF;SACF;KACF,CAAC;AACJ,CAAC;AAED,gFAAgF;AAChF,MAAM,UAAU,iBAAiB,CAAC,IAAiB;IACjD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,CAAC,IAAI,CAAC,wCAAwC,IAAI,CAAC,OAAO,GAAG,CAAC,CAAC;IACpE,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,YAAY,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC;IAC5C,IAAI,IAAI,CAAC,KAAK;QAAE,KAAK,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC;IACzD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;IACxC,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;IAC5B,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QAChC,KAAK,MAAM,CAAC,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;YAC7B,KAAK,CAAC,IAAI,CAAC,KAAK,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,IAAI,MAAM,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;QACxE,CAAC;IACH,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CACR,yFAAyF;QACvF,iEAAiE,CACpE,CAAC;IACF,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG;IACjC,iBAAiB;IACjB,SAAS;IACT,qBAAqB;IACrB,gBAAgB;IAChB,+EAA+E;IAC/E,8EAA8E;IAC9E,qEAAqE;IACrE,eAAe;IACf,mBAAmB;IACnB,cAAc;CACN,CAAC"}
|
|
@@ -107,6 +107,31 @@ export declare function parseFileBlocks(text: string): {
|
|
|
107
107
|
path: string;
|
|
108
108
|
content: string;
|
|
109
109
|
}[];
|
|
110
|
+
/**
|
|
111
|
+
* Completion budget. Reasoning models spend this budget on a hidden thinking
|
|
112
|
+
* channel BEFORE emitting any answer tokens, so a budget that merely fits the
|
|
113
|
+
* answer silently produces an EMPTY completion with finish_reason=length.
|
|
114
|
+
*
|
|
115
|
+
* Measured on real-gate-power (15 tasks, this model, first turn):
|
|
116
|
+
*
|
|
117
|
+
* max_tokens=4096 8/15 zero-block, and ALL EIGHT were finish_reason=length
|
|
118
|
+
* with 0 content and 11k-15k chars of reasoning. The seven
|
|
119
|
+
* successes landed at 1982-3964 tokens — pressed right up
|
|
120
|
+
* against the cap. Zero of the fifteen were malformed.
|
|
121
|
+
* max_tokens=8192 2/15 zero-block. Successes ranged to 7631 tokens.
|
|
122
|
+
*
|
|
123
|
+
* Note the shape of what is left: re-probing those last two succeeded using
|
|
124
|
+
* 6196 and 5035 tokens — under 8192 — so they were not deterministically too
|
|
125
|
+
* expensive, they were the long tail of a stochastic thinking length. The
|
|
126
|
+
* budget therefore has to clear the TAIL, not the mean; 16384 is a bit over 2x
|
|
127
|
+
* the largest observed success. This costs nothing on cells that finish early,
|
|
128
|
+
* since generation stops at finish_reason=stop either way.
|
|
129
|
+
*
|
|
130
|
+
* For scale, this repo's own profile for the same local model (qwen36.json)
|
|
131
|
+
* allocates 81920. The bench was starving the model ~20x relative to how the
|
|
132
|
+
* product runs it, which pinned BOTH arms toward the floor.
|
|
133
|
+
*/
|
|
134
|
+
export declare function rawMaxTokens(): number;
|
|
110
135
|
export declare class RawCompletionAdapter implements AgentAdapter {
|
|
111
136
|
readonly id = "raw";
|
|
112
137
|
private readonly endpoint;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"adapter.d.ts","sourceRoot":"","sources":["../../../src/benchmarks/paired/adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAQH,OAAO,EAAE,YAAY,EAAE,eAAe,EAAE,cAAc,EAAc,MAAM,YAAY,CAAC;AAMvF,MAAM,MAAM,WAAW,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,cAAc,CAAC,CAAC;AAEtF,MAAM,WAAW,uBAAuB;IACtC,EAAE,EAAE,MAAM,CAAC;IACX,+CAA+C;IAC/C,GAAG,EAAE,MAAM,CAAC;IACZ;;;OAGG;IACH,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,sDAAsD;IACtD,UAAU,EAAE,WAAW,CAAC;IACxB,uEAAuE;IACvE,kBAAkB,CAAC,EAAE,OAAO,CAAC;CAC9B;AAOD,qBAAa,iBAAkB,YAAW,YAAY;IAExC,OAAO,CAAC,QAAQ,CAAC,GAAG;IADhC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;gBACS,GAAG,EAAE,uBAAuB;IAInD,GAAG,CAAC,GAAG,EAAE,eAAe,GAAG,OAAO,CAAC,cAAc,CAAC;CA8CzD;AA6GD,uFAAuF;AACvF,eAAO,MAAM,gBAAgB,EAAE,WAiB9B,CAAC;AAEF;;;;;;;;;GASG;AACH,eAAO,MAAM,kBAAkB,EAAE,WAsChC,CAAC;AAMF;;;;;GAKG;AACH,eAAO,MAAM,iBAAiB,EAAE,WAiB/B,CAAC;AAEF,2EAA2E;AAC3E,wBAAgB,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,iBAAiB,CAoB9D;AAED,sDAAsD;AACtD,wBAAgB,eAAe,CAAC,KAAK,EAAE,MAAM,GAAG,iBAAiB,CAShE;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,iBAAiB,CAM/D;AAMD,MAAM,WAAW,YAAY;IAC3B,0EAA0E;IAC1E,WAAW,EAAE,MAAM,CAAC,MAAM,GAAG,QAAQ,GAAG,MAAM,EAAE,MAAM,CAAC,CAAC;IACxD,2EAA2E;IAC3E,gBAAgB,EAAE,MAAM,CAAC;IACzB,uEAAuE;IACvE,UAAU,EAAE,MAAM,CAAC;IACnB,0BAA0B,EAAE,MAAM,CAAC;CACpC;AASD;;;;;GAKG;AACH,qBAAa,WAAY,YAAW,YAAY;IAElC,OAAO,CAAC,QAAQ,CAAC,QAAQ;IADrC,QAAQ,CAAC,EAAE,UAAU;gBACQ,QAAQ,GAAE,YAA2B;IAE5D,GAAG,CAAC,GAAG,EAAE,eAAe,GAAG,OAAO,CAAC,cAAc,CAAC;CA6BzD;AAED,wCAAwC;AACxC,wBAAgB,MAAM,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,CAOxC;AAYD,MAAM,WAAW,gBAAgB;IAC/B,+DAA+D;IAC/D,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,kEAAkE;IAClE,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AASD,8EAA8E;AAC9E,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,EAAE,CASjF;
|
|
1
|
+
{"version":3,"file":"adapter.d.ts","sourceRoot":"","sources":["../../../src/benchmarks/paired/adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAQH,OAAO,EAAE,YAAY,EAAE,eAAe,EAAE,cAAc,EAAc,MAAM,YAAY,CAAC;AAMvF,MAAM,MAAM,WAAW,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,cAAc,CAAC,CAAC;AAEtF,MAAM,WAAW,uBAAuB;IACtC,EAAE,EAAE,MAAM,CAAC;IACX,+CAA+C;IAC/C,GAAG,EAAE,MAAM,CAAC;IACZ;;;OAGG;IACH,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,sDAAsD;IACtD,UAAU,EAAE,WAAW,CAAC;IACxB,uEAAuE;IACvE,kBAAkB,CAAC,EAAE,OAAO,CAAC;CAC9B;AAOD,qBAAa,iBAAkB,YAAW,YAAY;IAExC,OAAO,CAAC,QAAQ,CAAC,GAAG;IADhC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;gBACS,GAAG,EAAE,uBAAuB;IAInD,GAAG,CAAC,GAAG,EAAE,eAAe,GAAG,OAAO,CAAC,cAAc,CAAC;CA8CzD;AA6GD,uFAAuF;AACvF,eAAO,MAAM,gBAAgB,EAAE,WAiB9B,CAAC;AAEF;;;;;;;;;GASG;AACH,eAAO,MAAM,kBAAkB,EAAE,WAsChC,CAAC;AAMF;;;;;GAKG;AACH,eAAO,MAAM,iBAAiB,EAAE,WAiB/B,CAAC;AAEF,2EAA2E;AAC3E,wBAAgB,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,iBAAiB,CAoB9D;AAED,sDAAsD;AACtD,wBAAgB,eAAe,CAAC,KAAK,EAAE,MAAM,GAAG,iBAAiB,CAShE;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,iBAAiB,CAM/D;AAMD,MAAM,WAAW,YAAY;IAC3B,0EAA0E;IAC1E,WAAW,EAAE,MAAM,CAAC,MAAM,GAAG,QAAQ,GAAG,MAAM,EAAE,MAAM,CAAC,CAAC;IACxD,2EAA2E;IAC3E,gBAAgB,EAAE,MAAM,CAAC;IACzB,uEAAuE;IACvE,UAAU,EAAE,MAAM,CAAC;IACnB,0BAA0B,EAAE,MAAM,CAAC;CACpC;AASD;;;;;GAKG;AACH,qBAAa,WAAY,YAAW,YAAY;IAElC,OAAO,CAAC,QAAQ,CAAC,QAAQ;IADrC,QAAQ,CAAC,EAAE,UAAU;gBACQ,QAAQ,GAAE,YAA2B;IAE5D,GAAG,CAAC,GAAG,EAAE,eAAe,GAAG,OAAO,CAAC,cAAc,CAAC;CA6BzD;AAED,wCAAwC;AACxC,wBAAgB,MAAM,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,CAOxC;AAYD,MAAM,WAAW,gBAAgB;IAC/B,+DAA+D;IAC/D,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,kEAAkE;IAClE,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AASD,8EAA8E;AAC9E,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,EAAE,CASjF;AAiDD;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAgB,YAAY,IAAI,MAAM,CAGrC;AA+CD,qBAAa,oBAAqB,YAAW,YAAY;IACvD,QAAQ,CAAC,EAAE,SAAS;IACpB,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IACtC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;gBACzB,GAAG,GAAE,gBAAqB;IAOhC,GAAG,CAAC,GAAG,EAAE,eAAe,GAAG,OAAO,CAAC,cAAc,CAAC;CA+FzD;AAED;;;;;;;;GAQG;AACH,qBAAa,iBAAkB,YAAW,YAAY;IACpD,QAAQ,CAAC,EAAE,aAAa;IACxB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;gBACrB,OAAO,CAAC,EAAE,MAAM;IAGtB,GAAG,CAAC,GAAG,EAAE,eAAe,GAAG,OAAO,CAAC,cAAc,CAAC;CAiCzD"}
|
|
@@ -414,27 +414,67 @@ function applyFileBlocks(workdir, blocks) {
|
|
|
414
414
|
}
|
|
415
415
|
return written;
|
|
416
416
|
}
|
|
417
|
+
/**
|
|
418
|
+
* Completion budget. Reasoning models spend this budget on a hidden thinking
|
|
419
|
+
* channel BEFORE emitting any answer tokens, so a budget that merely fits the
|
|
420
|
+
* answer silently produces an EMPTY completion with finish_reason=length.
|
|
421
|
+
*
|
|
422
|
+
* Measured on real-gate-power (15 tasks, this model, first turn):
|
|
423
|
+
*
|
|
424
|
+
* max_tokens=4096 8/15 zero-block, and ALL EIGHT were finish_reason=length
|
|
425
|
+
* with 0 content and 11k-15k chars of reasoning. The seven
|
|
426
|
+
* successes landed at 1982-3964 tokens — pressed right up
|
|
427
|
+
* against the cap. Zero of the fifteen were malformed.
|
|
428
|
+
* max_tokens=8192 2/15 zero-block. Successes ranged to 7631 tokens.
|
|
429
|
+
*
|
|
430
|
+
* Note the shape of what is left: re-probing those last two succeeded using
|
|
431
|
+
* 6196 and 5035 tokens — under 8192 — so they were not deterministically too
|
|
432
|
+
* expensive, they were the long tail of a stochastic thinking length. The
|
|
433
|
+
* budget therefore has to clear the TAIL, not the mean; 16384 is a bit over 2x
|
|
434
|
+
* the largest observed success. This costs nothing on cells that finish early,
|
|
435
|
+
* since generation stops at finish_reason=stop either way.
|
|
436
|
+
*
|
|
437
|
+
* For scale, this repo's own profile for the same local model (qwen36.json)
|
|
438
|
+
* allocates 81920. The bench was starving the model ~20x relative to how the
|
|
439
|
+
* product runs it, which pinned BOTH arms toward the floor.
|
|
440
|
+
*/
|
|
441
|
+
export function rawMaxTokens() {
|
|
442
|
+
const n = Number(process.env.UAP_RAW_MAX_TOKENS ?? 16384);
|
|
443
|
+
return Number.isFinite(n) && n > 0 ? n : 16384;
|
|
444
|
+
}
|
|
417
445
|
async function chatCompletion(endpoint, model, messages, temperature, timeoutMs) {
|
|
418
446
|
const ctrl = new AbortController();
|
|
419
447
|
const timer = setTimeout(() => ctrl.abort(), timeoutMs);
|
|
448
|
+
const fail = (error) => ({
|
|
449
|
+
content: '',
|
|
450
|
+
tokens: 0,
|
|
451
|
+
error,
|
|
452
|
+
finishReason: null,
|
|
453
|
+
truncated: false,
|
|
454
|
+
});
|
|
420
455
|
try {
|
|
421
456
|
const res = await fetch(endpoint, {
|
|
422
457
|
method: 'POST',
|
|
423
458
|
headers: { 'Content-Type': 'application/json' },
|
|
424
|
-
body: JSON.stringify({ model, messages, temperature, max_tokens:
|
|
459
|
+
body: JSON.stringify({ model, messages, temperature, max_tokens: rawMaxTokens() }),
|
|
425
460
|
signal: ctrl.signal,
|
|
426
461
|
});
|
|
427
462
|
if (!res.ok)
|
|
428
|
-
return
|
|
463
|
+
return fail(`http ${res.status}`);
|
|
429
464
|
const data = (await res.json());
|
|
465
|
+
const choice = data.choices?.[0];
|
|
466
|
+
const content = choice?.message?.content ?? '';
|
|
467
|
+
const finishReason = choice?.finish_reason ?? null;
|
|
430
468
|
return {
|
|
431
|
-
content
|
|
469
|
+
content,
|
|
432
470
|
tokens: data.usage?.total_tokens ?? 0,
|
|
433
471
|
error: null,
|
|
472
|
+
finishReason,
|
|
473
|
+
truncated: finishReason === 'length' && content.trim() === '',
|
|
434
474
|
};
|
|
435
475
|
}
|
|
436
476
|
catch (e) {
|
|
437
|
-
return
|
|
477
|
+
return fail(e instanceof Error ? e.message : String(e));
|
|
438
478
|
}
|
|
439
479
|
finally {
|
|
440
480
|
clearTimeout(timer);
|
|
@@ -477,6 +517,7 @@ export class RawCompletionAdapter {
|
|
|
477
517
|
let totalTokens = 0;
|
|
478
518
|
let turns = 0;
|
|
479
519
|
let gateRuns = 0;
|
|
520
|
+
let truncatedTurns = 0;
|
|
480
521
|
let error = null;
|
|
481
522
|
const logParts = [];
|
|
482
523
|
// Budget parity: the lazy bare attempt counts INSIDE the same iteration
|
|
@@ -497,7 +538,16 @@ export class RawCompletionAdapter {
|
|
|
497
538
|
}
|
|
498
539
|
const blocks = parseFileBlocks(chat.content);
|
|
499
540
|
applyFileBlocks(ctx.workdir, blocks);
|
|
500
|
-
|
|
541
|
+
if (chat.truncated)
|
|
542
|
+
truncatedTurns++;
|
|
543
|
+
// A turn that wrote nothing has two very different causes, and the record
|
|
544
|
+
// has to say which: the model answered in the wrong format, or it never
|
|
545
|
+
// answered at all because the budget ran out mid-reasoning. Conflating
|
|
546
|
+
// them sent the first investigation after a prompt-format fix for what
|
|
547
|
+
// was actually a max_tokens ceiling.
|
|
548
|
+
logParts.push(`--- turn ${turns}: ${blocks.length} file(s)` +
|
|
549
|
+
(chat.truncated ? ' [TRUNCATED: budget exhausted before any answer]' : '') +
|
|
550
|
+
` finish=${chat.finishReason ?? 'n/a'} ---`);
|
|
501
551
|
if (!useGate)
|
|
502
552
|
break;
|
|
503
553
|
const gate = spawnSync('bash', ['-lc', ctx.task.gateCmd], {
|
|
@@ -520,7 +570,8 @@ export class RawCompletionAdapter {
|
|
|
520
570
|
toolCalls: useGate ? gateRuns : 0,
|
|
521
571
|
wellFormed: null,
|
|
522
572
|
error,
|
|
523
|
-
rawLog: logParts.join('\n')
|
|
573
|
+
rawLog: logParts.join('\n') +
|
|
574
|
+
(truncatedTurns > 0 ? `\n[${truncatedTurns}/${turns} turn(s) truncated mid-reasoning]` : ''),
|
|
524
575
|
};
|
|
525
576
|
}
|
|
526
577
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"adapter.js","sourceRoot":"","sources":["../../../src/benchmarks/paired/adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AACjD,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,YAAY,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AACnF,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AAExD,OAAO,EAAE,gBAAgB,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAC9D,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,EAAiD,UAAU,EAAE,MAAM,YAAY,CAAC;AAuBvF,mFAAmF;AACnF,MAAM,gBAAgB,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC;AAC1C,yEAAyE;AACzE,MAAM,kBAAkB,GAAG,MAAM,CAAC;AAElC,MAAM,OAAO,iBAAiB;IAEC;IADpB,EAAE,CAAS;IACpB,YAA6B,GAA4B;QAA5B,QAAG,GAAH,GAAG,CAAyB;QACvD,IAAI,CAAC,EAAE,GAAG,GAAG,CAAC,EAAE,CAAC;IACnB,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,GAAoB;QAC5B,iEAAiE;QACjE,IAAI,QAAQ,GAAa,EAAE,CAAC;QAC5B,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;YAC/B,QAAQ,GAAG,gBAAgB,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,SAAS,CAAC,CAAC,aAAa,CAAC;QACxE,CAAC;QAED,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACnC,CAAC,CAAC,OAAO,CAAC,eAAe,EAAE,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,OAAO,CAAC,WAAW,EAAE,GAAG,CAAC,OAAO,CAAC,CACnF,CAAC;QAEF,MAAM,GAAG,GAAsB;YAC7B,GAAG,YAAY,EAAE;YACjB,GAAG,WAAW,CAAC,GAAG,CAAC,SAAS,CAAC;YAC7B,eAAe,EAAE,GAAG,CAAC,KAAK;SAC3B,CAAC;QAEF,MAAM,GAAG,GAAG,GAAG,CAAC,IAAI,CAAC,eAAe,CAAC;QACrC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,UAAU,CACvE,IAAI,CAAC,GAAG,CAAC,GAAG,EACZ,IAAI,EACJ;YACE,GAAG,EAAE,GAAG,CAAC,OAAO;YAChB,GAAG;YACH,SAAS,EAAE,GAAG,GAAG,IAAI;YACrB,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS;SACtE,CACF,CAAC;QAEF,IAAI,KAAK,GAAkB,IAAI,CAAC;QAChC,IAAI,QAAQ;YAAE,KAAK,GAAG,yBAAyB,GAAG,GAAG,CAAC;aACjD,IAAI,UAAU;YAAE,KAAK,GAAG,UAAU,CAAC;aACnC,IAAI,MAAM,KAAK,CAAC;YAAE,KAAK,GAAG,gBAAgB,MAAM,EAAE,CAAC;QAExD,MAAM,MAAM,GAAG,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAEnE,OAAO;YACL,MAAM,EAAE,MAAM,CAAC,MAAM,IAAI,IAAI;YAC7B,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,IAAI;YAC/B,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,IAAI;YAC3B,SAAS,EAAE,MAAM,CAAC,SAAS,IAAI,IAAI;YACnC,UAAU,EAAE,MAAM,CAAC,UAAU,IAAI,IAAI;YACrC,KAAK;YACL,MAAM,EAAE,eAAe,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,QAAQ,KAAK,MAAM,mBAAmB,MAAM,EAAE;SAC7F,CAAC;IACJ,CAAC;CACF;AAiBD;;;;;;;;GAQG;AACH,SAAS,UAAU,CACjB,GAAW,EACX,IAAc,EACd,IAAuB;IAEvB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QAC7B,IAAI,KAAK,CAAC;QACV,IAAI,CAAC;YACH,KAAK,GAAG,KAAK,CAAC,GAAG,EAAE,IAAI,EAAE;gBACvB,GAAG,EAAE,IAAI,CAAC,GAAG;gBACb,GAAG,EAAE,IAAI,CAAC,GAAG;gBACb,QAAQ,EAAE,OAAO,CAAC,QAAQ,KAAK,OAAO,EAAE,6BAA6B;gBACrE,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC;aAChC,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO,CAAC;gBACN,MAAM,EAAE,EAAE;gBACV,MAAM,EAAE,EAAE;gBACV,MAAM,EAAE,IAAI;gBACZ,QAAQ,EAAE,KAAK;gBACf,UAAU,EAAE,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;aACvD,CAAC,CAAC;YACH,OAAO;QACT,CAAC;QAED,IAAI,MAAM,GAAG,EAAE,CAAC;QAChB,IAAI,MAAM,GAAG,EAAE,CAAC;QAChB,IAAI,QAAQ,GAAG,KAAK,CAAC;QACrB,IAAI,UAAU,GAAkB,IAAI,CAAC;QACrC,IAAI,OAAO,GAAG,KAAK,CAAC;QAEpB,MAAM,MAAM,GAAG,CAAC,GAAW,EAAE,KAAa,EAAU,EAAE,CACpD,GAAG,CAAC,MAAM,IAAI,gBAAgB,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QACvE,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACtE,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAEtE,MAAM,SAAS,GAAG,CAAC,MAAsB,EAAE,EAAE;YAC3C,IAAI,CAAC;gBACH,IAAI,KAAK,CAAC,GAAG,IAAI,IAAI,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO;oBAAE,OAAO,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;;oBACnF,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC1B,CAAC;YAAC,MAAM,CAAC;gBACP,IAAI,CAAC;oBACH,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;gBACxB,CAAC;gBAAC,MAAM,CAAC;oBACP,kBAAkB;gBACpB,CAAC;YACH,CAAC;QACH,CAAC,CAAC;QAEF,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,EAAE;YAChC,QAAQ,GAAG,IAAI,CAAC;YAChB,SAAS,CAAC,SAAS,CAAC,CAAC;QACvB,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;QACnB,yEAAyE;QACzE,wEAAwE;QACxE,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,EAAE;YAChC,SAAS,CAAC,SAAS,CAAC,CAAC;YACrB,MAAM,CAAC,IAAI,CAAC,CAAC;QACf,CAAC,EAAE,IAAI,CAAC,SAAS,GAAG,kBAAkB,CAAC,CAAC;QAExC,MAAM,MAAM,GAAG,CAAC,MAAqB,EAAE,EAAE;YACvC,IAAI,OAAO;gBAAE,OAAO;YACpB,OAAO,GAAG,IAAI,CAAC;YACf,YAAY,CAAC,SAAS,CAAC,CAAC;YACxB,YAAY,CAAC,SAAS,CAAC,CAAC;YACxB,OAAO,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,CAAC,CAAC;QAC5D,CAAC,CAAC;QAEF,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,CAAQ,EAAE,EAAE;YAC7B,UAAU,GAAG,CAAC,CAAC,OAAO,CAAC;YACvB,MAAM,CAAC,IAAI,CAAC,CAAC;QACf,CAAC,CAAC,CAAC;QACH,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAmB,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;QAEzD,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI;YAAE,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACvD,KAAK,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC;IACrB,CAAC,CAAC,CAAC;AACL,CAAC;AAED,8EAA8E;AAC9E,yBAAyB;AACzB,8EAA8E;AAE9E,uFAAuF;AACvF,MAAM,CAAC,MAAM,gBAAgB,GAAgB,CAAC,MAAM,EAAE,EAAE;IACtD,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;QACtC,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,IAAI,GAAG,CAAC,OAAO,EAAE,KAAK,IAAI,EAAE,CAAC;QACpD,MAAM,MAAM,GACV,CAAC,KAAK,CAAC,YAAY,IAAI,CAAC,CAAC;YACzB,CAAC,KAAK,CAAC,aAAa,IAAI,CAAC,CAAC;YAC1B,CAAC,KAAK,CAAC,uBAAuB,IAAI,CAAC,CAAC;YACpC,CAAC,KAAK,CAAC,2BAA2B,IAAI,CAAC,CAAC,CAAC;QAC3C,OAAO;YACL,MAAM,EAAE,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI;YAClC,OAAO,EAAE,OAAO,GAAG,CAAC,cAAc,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC,CAAC,IAAI;YAC3E,KAAK,EAAE,OAAO,GAAG,CAAC,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI;SAChE,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC,CAAC;AAEF;;;;;;;;;GASG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAgB,CAAC,MAAM,EAAE,EAAE;IACxD,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,IAAI,SAAS,GAAG,KAAK,CAAC;IACtB,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QACtC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QAC5B,IAAI,CAAC,OAAO;YAAE,SAAS;QACvB,IAAI,EAA2B,CAAC;QAChC,IAAI,CAAC;YACH,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAA4B,CAAC;QACtD,CAAC;QAAC,MAAM,CAAC;YACP,SAAS;QACX,CAAC;QACD,MAAM,IAAI,GAAG,EAAE,CAAC,IAAI,CAAC;QACrB,IAAI,IAAI,KAAK,UAAU;YAAE,SAAS,EAAE,CAAC;QACrC,IAAI,IAAI,KAAK,aAAa,EAAE,CAAC;YAC3B,KAAK,EAAE,CAAC;YACR,MAAM,IAAI,GAAG,CAAC,EAAE,CAAC,IAAI,IAAI,EAAE,CAA4B,CAAC;YACxD,MAAM,EAAE,GAAG,CAAC,IAAI,CAAC,MAAM,IAAI,EAAE,CAA4B,CAAC;YAC1D,MAAM,KAAK,GAAG,CAAC,EAAE,CAAC,KAAK,IAAI,EAAE,CAA4B,CAAC;YAC1D,MAAM,KAAK,GAAG,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC;YAC5B,MAAM,MAAM,GAAG,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC;YAC9B,MAAM,SAAS,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAClC,IAAI,EAAE,CAAC,KAAK,IAAI,IAAI,IAAI,EAAE,CAAC,MAAM,IAAI,IAAI,EAAE,CAAC;gBAC1C,MAAM,IAAI,KAAK,GAAG,MAAM,GAAG,SAAS,CAAC;gBACrC,SAAS,GAAG,IAAI,CAAC;YACnB,CAAC;YACD,IAAI,IAAI,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACzB,CAAC;IACH,CAAC;IACD,OAAO;QACL,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI;QACjC,OAAO,EAAE,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI;QAC/B,KAAK,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI;QAC/B,SAAS,EAAE,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI;KAC5C,CAAC;AACJ,CAAC,CAAC;AAEF,SAAS,GAAG,CAAC,CAAU;IACrB,OAAO,OAAO,CAAC,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAC7D,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAgB,CAAC,MAAM,EAAE,EAAE;IACvD,MAAM,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;IAC1C,IAAI,CAAC,CAAC;QAAE,OAAO,EAAE,CAAC;IAClB,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAA4B,CAAC;QACxD,MAAM,KAAK,GAAG,CAAC,GAAG,CAAC,KAAK,IAAI,EAAE,CAA4B,CAAC;QAC3D,MAAM,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;QAC1D,MAAM,KAAK,GAAG,GAAG,CAAC,GAAG,CAAC,KAAK,IAAI,GAAG,CAAC,OAAO,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;QACzD,MAAM,IAAI,GAAG,GAAG,CAAC,GAAG,CAAC,IAAI,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC3C,OAAO;YACL,MAAM,EAAE,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI;YAClC,OAAO,EAAE,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI;YAC/B,KAAK,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI;SAChC,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC,CAAC;AAEF,2EAA2E;AAC3E,MAAM,UAAU,aAAa,CAAC,KAAa;IACzC,OAAO,IAAI,iBAAiB,CAAC;QAC3B,EAAE,EAAE,QAAQ;QACZ,GAAG,EAAE,QAAQ;QACb,0EAA0E;QAC1E,yEAAyE;QACzE,4EAA4E;QAC5E,6EAA6E;QAC7E,sEAAsE;QACtE,IAAI,EAAE;YACJ,IAAI;YACJ,eAAe;YACf,iBAAiB;YACjB,MAAM;YACN,gCAAgC;YAChC,SAAS;YACT,KAAK;SACN;QACD,UAAU,EAAE,gBAAgB;KAC7B,CAAC,CAAC;AACL,CAAC;AAED,sDAAsD;AACtD,MAAM,UAAU,eAAe,CAAC,KAAa;IAC3C,OAAO,IAAI,iBAAiB,CAAC;QAC3B,EAAE,EAAE,UAAU;QACd,GAAG,EAAE,UAAU;QACf,oEAAoE;QACpE,wDAAwD;QACxD,IAAI,EAAE,CAAC,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,eAAe,CAAC;QAC1F,UAAU,EAAE,kBAAkB;KAC/B,CAAC,CAAC;AACL,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,UAAU,cAAc,CAAC,KAAa;IAC1C,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,MAAM,CAAC;IAClD,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,gBAAgB;QACvC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC;QACzD,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,eAAe,CAAC,CAAC;IAC/C,OAAO,IAAI,iBAAiB,CAAC,EAAE,EAAE,EAAE,gBAAgB,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,EAAE,iBAAiB,EAAE,CAAC,CAAC;AACnG,CAAC;AAgBD,MAAM,YAAY,GAAiB;IACjC,WAAW,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE;IACnD,gBAAgB,EAAE,IAAI;IACtB,UAAU,EAAE,IAAI;IAChB,0BAA0B,EAAE,GAAG;CAChC,CAAC;AAEF;;;;;GAKG;AACH,MAAM,OAAO,WAAW;IAEO;IADpB,EAAE,GAAG,MAAM,CAAC;IACrB,YAA6B,WAAyB,YAAY;QAArC,aAAQ,GAAR,QAAQ,CAA6B;IAAG,CAAC;IAEtE,KAAK,CAAC,GAAG,CAAC,GAAoB;QAC5B,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC;YAAE,gBAAgB,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,SAAS,CAAC,CAAC;QAE7E,MAAM,CAAC,GAAG,GAAG,CAAC,SAAS,CAAC,UAAU,CAAC,IAAI,CAAC;QACxC,MAAM,cAAc,GAAG,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,0BAA0B,CAAC;QACpE,2EAA2E;QAC3E,sEAAsE;QACtE,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;QAClD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CACxB,IAAI,EACJ,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CACpF,CAAC;QACF,MAAM,QAAQ,GAAG,IAAI,GAAG,SAAS,CAAC;QAElC,4EAA4E;QAC5E,kFAAkF;QAClF,IAAI,QAAQ;YAAE,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,aAAa,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;QAE/E,OAAO;YACL,qEAAqE;YACrE,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,UAAU,GAAG,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC;YAC3E,OAAO,EAAE,IAAI;YACb,KAAK,EAAE,CAAC,GAAG,CAAC;YACZ,SAAS,EAAE,CAAC,GAAG,CAAC;YAChB,UAAU,EAAE,IAAI;YAChB,KAAK,EAAE,IAAI;YACX,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,iBAAiB;SACvD,CAAC;IACJ,CAAC;CACF;AAED,wCAAwC;AACxC,MAAM,UAAU,MAAM,CAAC,CAAS;IAC9B,IAAI,CAAC,GAAG,UAAU,KAAK,CAAC,CAAC;IACzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAClC,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QACrB,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;IAC7B,CAAC;IACD,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,UAAU,CAAC;AAChC,CAAC;AAoBD,MAAM,kBAAkB,GACtB,+EAA+E;IAC/E,+EAA+E;IAC/E,8DAA8D;IAC9D,gEAAgE;IAChE,6EAA6E,CAAC;AAEhF,8EAA8E;AAC9E,MAAM,UAAU,eAAe,CAAC,IAAY;IAC1C,MAAM,GAAG,GAAwC,EAAE,CAAC;IACpD,MAAM,EAAE,GAAG,mDAAmD,CAAC;IAC/D,IAAI,CAAyB,CAAC;IAC9B,OAAO,CAAC,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QACpC,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACzB,IAAI,IAAI;YAAE,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAC9C,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,kEAAkE;AAClE,SAAS,aAAa,CAAC,OAAe;IACpC,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,CAAC,cAAc,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC;IACzE,MAAM,MAAM,GAAG,qDAAqD,CAAC;IACrE,MAAM,KAAK,GAAwC,EAAE,CAAC;IACtD,MAAM,IAAI,GAAG,CAAC,GAAW,EAAQ,EAAE;QACjC,KAAK,MAAM,IAAI,IAAI,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC;YACpC,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC;gBAAE,SAAS;YAC7B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;YAC5B,MAAM,EAAE,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC;YACzB,IAAI,EAAE,CAAC,WAAW,EAAE;gBAAE,IAAI,CAAC,GAAG,CAAC,CAAC;iBAC3B,IAAI,EAAE,CAAC,IAAI,IAAI,EAAE,GAAG,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,KAAK,cAAc,EAAE,CAAC;gBAC9E,IAAI,CAAC;oBACH,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,CAAC,OAAO,EAAE,GAAG,CAAC,EAAE,OAAO,EAAE,YAAY,CAAC,GAAG,EAAE,OAAO,CAAC,EAAE,CAAC,CAAC;gBACpF,CAAC;gBAAC,MAAM,CAAC;oBACP,qBAAqB;gBACvB,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC,CAAC;IACF,IAAI,CAAC,OAAO,CAAC,CAAC;IACd,OAAO,KAAK,CAAC;AACf,CAAC;AAED,mEAAmE;AACnE,SAAS,eAAe,CAAC,OAAe,EAAE,MAA2C;IACnF,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;QACtC,IAAI,IAAI,KAAK,OAAO,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,OAAO,GAAG,GAAG,CAAC;YAAE,SAAS,CAAC,YAAY;QAC/E,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC9C,aAAa,CAAC,IAAI,EAAE,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QACxC,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAQD,KAAK,UAAU,cAAc,CAC3B,QAAgB,EAChB,KAAa,EACb,QAA6C,EAC7C,WAAmB,EACnB,SAAiB;IAEjB,MAAM,IAAI,GAAG,IAAI,eAAe,EAAE,CAAC;IACnC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,SAAS,CAAC,CAAC;IACxD,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,QAAQ,EAAE;YAChC,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;YAC/C,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,WAAW,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;YACxE,MAAM,EAAE,IAAI,CAAC,MAAM;SACpB,CAAC,CAAC;QACH,IAAI,CAAC,GAAG,CAAC,EAAE;YAAE,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,QAAQ,GAAG,CAAC,MAAM,EAAE,EAAE,CAAC;QAC5E,MAAM,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAG7B,CAAC;QACF,OAAO;YACL,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,IAAI,EAAE;YAClD,MAAM,EAAE,IAAI,CAAC,KAAK,EAAE,YAAY,IAAI,CAAC;YACrC,KAAK,EAAE,IAAI;SACZ,CAAC;IACJ,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;IACvF,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,KAAK,CAAC,CAAC;IACtB,CAAC;AACH,CAAC;AAED,MAAM,OAAO,oBAAoB;IACtB,EAAE,GAAG,KAAK,CAAC;IACH,QAAQ,CAAS;IACjB,YAAY,CAAS;IACrB,WAAW,CAAS;IACrC,YAAY,MAAwB,EAAE;QACpC,IAAI,CAAC,QAAQ;YACX,GAAG,CAAC,QAAQ,IAAI,OAAO,CAAC,GAAG,CAAC,gBAAgB,IAAI,2CAA2C,CAAC;QAC9F,IAAI,CAAC,YAAY,GAAG,GAAG,CAAC,YAAY,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,kBAAkB,IAAI,CAAC,CAAC,CAAC;QACpF,IAAI,CAAC,WAAW,GAAG,GAAG,CAAC,WAAW,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,GAAG,CAAC,CAAC;IACvF,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,GAAoB;QAC5B,MAAM,OAAO,GAAG,GAAG,CAAC,SAAS,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACnF,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QAEzC,yEAAyE;QACzE,2EAA2E;QAC3E,uEAAuE;QACvE,0EAA0E;QAC1E,MAAM,aAAa,GAAG,CAAC,IAAa,EAAE,OAAsB,EAAuC,EAAE;YACnG,MAAM,KAAK,GAAG,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI,KAAK,WAAW,CAAC,CAAC,CAAC;YAC1F,MAAM,QAAQ,GAAG,KAAK;iBACnB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,CAAC,IAAI,QAAQ,CAAC,CAAC,OAAO,aAAa,CAAC;iBAC3D,IAAI,CAAC,MAAM,CAAC,CAAC;YAChB,OAAO;gBACL,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,kBAAkB,EAAE;gBAC/C;oBACE,IAAI,EAAE,MAAM;oBACZ,OAAO,EACL,SAAS,GAAG,CAAC,IAAI,CAAC,WAAW,yBAAyB,QAAQ,MAAM;wBACpE,CAAC,OAAO;4BACN,CAAC,CAAC,kDAAkD,OAAO,uGAAuG;4BAClK,CAAC,CAAC,kFAAkF,CAAC;iBAC1F;aACF,CAAC;QACJ,CAAC,CAAC;QAEF,IAAI,WAAW,GAAG,CAAC,CAAC;QACpB,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,IAAI,KAAK,GAAkB,IAAI,CAAC;QAChC,MAAM,QAAQ,GAAa,EAAE,CAAC;QAC9B,wEAAwE;QACxE,sEAAsE;QACtE,yEAAyE;QACzE,0DAA0D;QAC1D,MAAM,QAAQ,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC;QACjD,IAAI,WAAW,GAAkB,IAAI,CAAC;QAEtC,KAAK,IAAI,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,QAAQ,EAAE,IAAI,EAAE,EAAE,CAAC;YAC3C,MAAM,IAAI,GAAG,IAAI,IAAI,IAAI,KAAK,CAAC,CAAC;YAChC,MAAM,QAAQ,GAAG,aAAa,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;YAClD,MAAM,IAAI,GAAG,MAAM,cAAc,CAC/B,IAAI,CAAC,QAAQ,EACb,GAAG,CAAC,KAAK,EACT,QAAQ,EACR,IAAI,CAAC,WAAW,EAChB,GAAG,CAAC,IAAI,CAAC,eAAe,GAAG,IAAI,CAChC,CAAC;YACF,KAAK,EAAE,CAAC;YACR,WAAW,IAAI,IAAI,CAAC,MAAM,CAAC;YAC3B,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;gBACf,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;gBACnB,MAAM;YACR,CAAC;YACD,MAAM,MAAM,GAAG,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAC7C,eAAe,CAAC,GAAG,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YACrC,QAAQ,CAAC,IAAI,CAAC,YAAY,KAAK,KAAK,MAAM,CAAC,MAAM,cAAc,CAAC,CAAC;YAEjE,IAAI,CAAC,OAAO;gBAAE,MAAM;YAEpB,MAAM,IAAI,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,GAAG,CAAC,IAAI,CAAC,OAAiB,CAAC,EAAE;gBAClE,GAAG,EAAE,GAAG,CAAC,OAAO;gBAChB,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,GAAG,CAAC,IAAI,CAAC,gBAAgB,GAAG,IAAI;gBACzC,GAAG,EAAE,YAAY,EAAE;aACpB,CAAC,CAAC;YACH,QAAQ,EAAE,CAAC;YACX,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;gBAAE,MAAM;YAC7B,IAAI,IAAI,KAAK,QAAQ,GAAG,CAAC;gBAAE,MAAM,CAAC,gBAAgB;YAClD,WAAW,GAAG,GAAG,IAAI,CAAC,MAAM,IAAI,EAAE,KAAK,IAAI,CAAC,MAAM,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC;QAC1E,CAAC;QAED,OAAO;YACL,MAAM,EAAE,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI;YAC5C,OAAO,EAAE,IAAI;YACb,KAAK;YACL,SAAS,EAAE,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;YACjC,UAAU,EAAE,IAAI;YAChB,KAAK;YACL,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC;SAC5B,CAAC;IACJ,CAAC;CACF;AAED;;;;;;;;GAQG;AACH,MAAM,OAAO,iBAAiB;IACnB,EAAE,GAAG,SAAS,CAAC;IACP,OAAO,CAAS;IACjC,YAAY,OAAgB;QAC1B,IAAI,CAAC,OAAO,GAAG,OAAO,IAAI,OAAO,CAAC,GAAG,CAAC,qBAAqB,IAAI,iBAAiB,CAAC;IACnF,CAAC;IACD,KAAK,CAAC,GAAG,CAAC,GAAoB;QAC5B,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,eAAe,CAAC,CAAC;QACnD,MAAM,QAAQ,GAAG,GAAG,CAAC,SAAS,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC,CAAC;QACrD,MAAM,IAAI,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,gBAAgB,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;QAChF,IAAI,QAAQ,EAAE,CAAC;YACb,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,sBAAsB,EAAE,aAAa,EAAE,GAAG,EAAE,WAAW,EAAE,gBAAgB,CAAC,CAAC;QACpG,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACtC,MAAM,GAAG,GAAsB,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,iBAAiB,EAAE,GAAG,CAAC,KAAK,EAAE,CAAC;QAChF,IAAI,GAAG,CAAC,SAAS,CAAC,IAAI;YAAE,GAAG,CAAC,gBAAgB,GAAG,GAAG,CAAC;QACnD,MAAM,SAAS,GAAG,GAAG,CAAC,IAAI,CAAC,eAAe,GAAG,IAAI,GAAG,CAAC,CAAC;QACtD,OAAO,IAAI,OAAO,CAAC,CAAC,cAAc,EAAE,EAAE;YACpC,QAAQ,CAAC,OAAO,CAAC,QAAQ,EAAE,IAAI,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,OAAO,EAAE,GAAG,EAAE,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,EAAE,GAAG,IAAI,GAAG,IAAI,EAAE,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE;gBAC3H,IAAI,KAAK,GAAG,CAAC,CAAC;gBACd,IAAI,CAAC;oBACH,MAAM,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;oBAC1C,IAAI,CAAC,EAAE,CAAC;wBACN,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAuB,CAAC;wBACjD,IAAI,OAAO,CAAC,CAAC,KAAK,KAAK,QAAQ;4BAAE,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;oBACnD,CAAC;gBACH,CAAC;gBAAC,MAAM,CAAC,CAAC,yBAAyB,CAAC,CAAC;gBACrC,cAAc,CAAC;oBACb,MAAM,EAAE,IAAI;oBACZ,OAAO,EAAE,IAAI;oBACb,KAAK;oBACL,SAAS,EAAE,CAAC;oBACZ,UAAU,EAAE,IAAI;oBAChB,KAAK,EAAE,GAAG,IAAK,GAAoC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI;oBAC7E,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC;iBAC5B,CAAC,CAAC;YACL,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;CACF"}
|
|
1
|
+
{"version":3,"file":"adapter.js","sourceRoot":"","sources":["../../../src/benchmarks/paired/adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AACjD,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,YAAY,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AACnF,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AAExD,OAAO,EAAE,gBAAgB,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAC9D,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,EAAiD,UAAU,EAAE,MAAM,YAAY,CAAC;AAuBvF,mFAAmF;AACnF,MAAM,gBAAgB,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC;AAC1C,yEAAyE;AACzE,MAAM,kBAAkB,GAAG,MAAM,CAAC;AAElC,MAAM,OAAO,iBAAiB;IAEC;IADpB,EAAE,CAAS;IACpB,YAA6B,GAA4B;QAA5B,QAAG,GAAH,GAAG,CAAyB;QACvD,IAAI,CAAC,EAAE,GAAG,GAAG,CAAC,EAAE,CAAC;IACnB,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,GAAoB;QAC5B,iEAAiE;QACjE,IAAI,QAAQ,GAAa,EAAE,CAAC;QAC5B,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;YAC/B,QAAQ,GAAG,gBAAgB,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,SAAS,CAAC,CAAC,aAAa,CAAC;QACxE,CAAC;QAED,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACnC,CAAC,CAAC,OAAO,CAAC,eAAe,EAAE,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,OAAO,CAAC,WAAW,EAAE,GAAG,CAAC,OAAO,CAAC,CACnF,CAAC;QAEF,MAAM,GAAG,GAAsB;YAC7B,GAAG,YAAY,EAAE;YACjB,GAAG,WAAW,CAAC,GAAG,CAAC,SAAS,CAAC;YAC7B,eAAe,EAAE,GAAG,CAAC,KAAK;SAC3B,CAAC;QAEF,MAAM,GAAG,GAAG,GAAG,CAAC,IAAI,CAAC,eAAe,CAAC;QACrC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,UAAU,CACvE,IAAI,CAAC,GAAG,CAAC,GAAG,EACZ,IAAI,EACJ;YACE,GAAG,EAAE,GAAG,CAAC,OAAO;YAChB,GAAG;YACH,SAAS,EAAE,GAAG,GAAG,IAAI;YACrB,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS;SACtE,CACF,CAAC;QAEF,IAAI,KAAK,GAAkB,IAAI,CAAC;QAChC,IAAI,QAAQ;YAAE,KAAK,GAAG,yBAAyB,GAAG,GAAG,CAAC;aACjD,IAAI,UAAU;YAAE,KAAK,GAAG,UAAU,CAAC;aACnC,IAAI,MAAM,KAAK,CAAC;YAAE,KAAK,GAAG,gBAAgB,MAAM,EAAE,CAAC;QAExD,MAAM,MAAM,GAAG,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAEnE,OAAO;YACL,MAAM,EAAE,MAAM,CAAC,MAAM,IAAI,IAAI;YAC7B,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,IAAI;YAC/B,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,IAAI;YAC3B,SAAS,EAAE,MAAM,CAAC,SAAS,IAAI,IAAI;YACnC,UAAU,EAAE,MAAM,CAAC,UAAU,IAAI,IAAI;YACrC,KAAK;YACL,MAAM,EAAE,eAAe,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,QAAQ,KAAK,MAAM,mBAAmB,MAAM,EAAE;SAC7F,CAAC;IACJ,CAAC;CACF;AAiBD;;;;;;;;GAQG;AACH,SAAS,UAAU,CACjB,GAAW,EACX,IAAc,EACd,IAAuB;IAEvB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QAC7B,IAAI,KAAK,CAAC;QACV,IAAI,CAAC;YACH,KAAK,GAAG,KAAK,CAAC,GAAG,EAAE,IAAI,EAAE;gBACvB,GAAG,EAAE,IAAI,CAAC,GAAG;gBACb,GAAG,EAAE,IAAI,CAAC,GAAG;gBACb,QAAQ,EAAE,OAAO,CAAC,QAAQ,KAAK,OAAO,EAAE,6BAA6B;gBACrE,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC;aAChC,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO,CAAC;gBACN,MAAM,EAAE,EAAE;gBACV,MAAM,EAAE,EAAE;gBACV,MAAM,EAAE,IAAI;gBACZ,QAAQ,EAAE,KAAK;gBACf,UAAU,EAAE,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;aACvD,CAAC,CAAC;YACH,OAAO;QACT,CAAC;QAED,IAAI,MAAM,GAAG,EAAE,CAAC;QAChB,IAAI,MAAM,GAAG,EAAE,CAAC;QAChB,IAAI,QAAQ,GAAG,KAAK,CAAC;QACrB,IAAI,UAAU,GAAkB,IAAI,CAAC;QACrC,IAAI,OAAO,GAAG,KAAK,CAAC;QAEpB,MAAM,MAAM,GAAG,CAAC,GAAW,EAAE,KAAa,EAAU,EAAE,CACpD,GAAG,CAAC,MAAM,IAAI,gBAAgB,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QACvE,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACtE,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAEtE,MAAM,SAAS,GAAG,CAAC,MAAsB,EAAE,EAAE;YAC3C,IAAI,CAAC;gBACH,IAAI,KAAK,CAAC,GAAG,IAAI,IAAI,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO;oBAAE,OAAO,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;;oBACnF,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC1B,CAAC;YAAC,MAAM,CAAC;gBACP,IAAI,CAAC;oBACH,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;gBACxB,CAAC;gBAAC,MAAM,CAAC;oBACP,kBAAkB;gBACpB,CAAC;YACH,CAAC;QACH,CAAC,CAAC;QAEF,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,EAAE;YAChC,QAAQ,GAAG,IAAI,CAAC;YAChB,SAAS,CAAC,SAAS,CAAC,CAAC;QACvB,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;QACnB,yEAAyE;QACzE,wEAAwE;QACxE,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,EAAE;YAChC,SAAS,CAAC,SAAS,CAAC,CAAC;YACrB,MAAM,CAAC,IAAI,CAAC,CAAC;QACf,CAAC,EAAE,IAAI,CAAC,SAAS,GAAG,kBAAkB,CAAC,CAAC;QAExC,MAAM,MAAM,GAAG,CAAC,MAAqB,EAAE,EAAE;YACvC,IAAI,OAAO;gBAAE,OAAO;YACpB,OAAO,GAAG,IAAI,CAAC;YACf,YAAY,CAAC,SAAS,CAAC,CAAC;YACxB,YAAY,CAAC,SAAS,CAAC,CAAC;YACxB,OAAO,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,CAAC,CAAC;QAC5D,CAAC,CAAC;QAEF,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,CAAQ,EAAE,EAAE;YAC7B,UAAU,GAAG,CAAC,CAAC,OAAO,CAAC;YACvB,MAAM,CAAC,IAAI,CAAC,CAAC;QACf,CAAC,CAAC,CAAC;QACH,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAmB,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;QAEzD,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI;YAAE,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACvD,KAAK,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC;IACrB,CAAC,CAAC,CAAC;AACL,CAAC;AAED,8EAA8E;AAC9E,yBAAyB;AACzB,8EAA8E;AAE9E,uFAAuF;AACvF,MAAM,CAAC,MAAM,gBAAgB,GAAgB,CAAC,MAAM,EAAE,EAAE;IACtD,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;QACtC,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,IAAI,GAAG,CAAC,OAAO,EAAE,KAAK,IAAI,EAAE,CAAC;QACpD,MAAM,MAAM,GACV,CAAC,KAAK,CAAC,YAAY,IAAI,CAAC,CAAC;YACzB,CAAC,KAAK,CAAC,aAAa,IAAI,CAAC,CAAC;YAC1B,CAAC,KAAK,CAAC,uBAAuB,IAAI,CAAC,CAAC;YACpC,CAAC,KAAK,CAAC,2BAA2B,IAAI,CAAC,CAAC,CAAC;QAC3C,OAAO;YACL,MAAM,EAAE,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI;YAClC,OAAO,EAAE,OAAO,GAAG,CAAC,cAAc,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC,CAAC,IAAI;YAC3E,KAAK,EAAE,OAAO,GAAG,CAAC,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI;SAChE,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC,CAAC;AAEF;;;;;;;;;GASG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAgB,CAAC,MAAM,EAAE,EAAE;IACxD,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,IAAI,SAAS,GAAG,KAAK,CAAC;IACtB,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QACtC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QAC5B,IAAI,CAAC,OAAO;YAAE,SAAS;QACvB,IAAI,EAA2B,CAAC;QAChC,IAAI,CAAC;YACH,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAA4B,CAAC;QACtD,CAAC;QAAC,MAAM,CAAC;YACP,SAAS;QACX,CAAC;QACD,MAAM,IAAI,GAAG,EAAE,CAAC,IAAI,CAAC;QACrB,IAAI,IAAI,KAAK,UAAU;YAAE,SAAS,EAAE,CAAC;QACrC,IAAI,IAAI,KAAK,aAAa,EAAE,CAAC;YAC3B,KAAK,EAAE,CAAC;YACR,MAAM,IAAI,GAAG,CAAC,EAAE,CAAC,IAAI,IAAI,EAAE,CAA4B,CAAC;YACxD,MAAM,EAAE,GAAG,CAAC,IAAI,CAAC,MAAM,IAAI,EAAE,CAA4B,CAAC;YAC1D,MAAM,KAAK,GAAG,CAAC,EAAE,CAAC,KAAK,IAAI,EAAE,CAA4B,CAAC;YAC1D,MAAM,KAAK,GAAG,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC;YAC5B,MAAM,MAAM,GAAG,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC;YAC9B,MAAM,SAAS,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAClC,IAAI,EAAE,CAAC,KAAK,IAAI,IAAI,IAAI,EAAE,CAAC,MAAM,IAAI,IAAI,EAAE,CAAC;gBAC1C,MAAM,IAAI,KAAK,GAAG,MAAM,GAAG,SAAS,CAAC;gBACrC,SAAS,GAAG,IAAI,CAAC;YACnB,CAAC;YACD,IAAI,IAAI,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACzB,CAAC;IACH,CAAC;IACD,OAAO;QACL,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI;QACjC,OAAO,EAAE,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI;QAC/B,KAAK,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI;QAC/B,SAAS,EAAE,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI;KAC5C,CAAC;AACJ,CAAC,CAAC;AAEF,SAAS,GAAG,CAAC,CAAU;IACrB,OAAO,OAAO,CAAC,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAC7D,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAgB,CAAC,MAAM,EAAE,EAAE;IACvD,MAAM,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;IAC1C,IAAI,CAAC,CAAC;QAAE,OAAO,EAAE,CAAC;IAClB,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAA4B,CAAC;QACxD,MAAM,KAAK,GAAG,CAAC,GAAG,CAAC,KAAK,IAAI,EAAE,CAA4B,CAAC;QAC3D,MAAM,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;QAC1D,MAAM,KAAK,GAAG,GAAG,CAAC,GAAG,CAAC,KAAK,IAAI,GAAG,CAAC,OAAO,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;QACzD,MAAM,IAAI,GAAG,GAAG,CAAC,GAAG,CAAC,IAAI,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC3C,OAAO;YACL,MAAM,EAAE,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI;YAClC,OAAO,EAAE,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI;YAC/B,KAAK,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI;SAChC,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC,CAAC;AAEF,2EAA2E;AAC3E,MAAM,UAAU,aAAa,CAAC,KAAa;IACzC,OAAO,IAAI,iBAAiB,CAAC;QAC3B,EAAE,EAAE,QAAQ;QACZ,GAAG,EAAE,QAAQ;QACb,0EAA0E;QAC1E,yEAAyE;QACzE,4EAA4E;QAC5E,6EAA6E;QAC7E,sEAAsE;QACtE,IAAI,EAAE;YACJ,IAAI;YACJ,eAAe;YACf,iBAAiB;YACjB,MAAM;YACN,gCAAgC;YAChC,SAAS;YACT,KAAK;SACN;QACD,UAAU,EAAE,gBAAgB;KAC7B,CAAC,CAAC;AACL,CAAC;AAED,sDAAsD;AACtD,MAAM,UAAU,eAAe,CAAC,KAAa;IAC3C,OAAO,IAAI,iBAAiB,CAAC;QAC3B,EAAE,EAAE,UAAU;QACd,GAAG,EAAE,UAAU;QACf,oEAAoE;QACpE,wDAAwD;QACxD,IAAI,EAAE,CAAC,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,eAAe,CAAC;QAC1F,UAAU,EAAE,kBAAkB;KAC/B,CAAC,CAAC;AACL,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,UAAU,cAAc,CAAC,KAAa;IAC1C,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,MAAM,CAAC;IAClD,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,gBAAgB;QACvC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC;QACzD,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,eAAe,CAAC,CAAC;IAC/C,OAAO,IAAI,iBAAiB,CAAC,EAAE,EAAE,EAAE,gBAAgB,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,EAAE,iBAAiB,EAAE,CAAC,CAAC;AACnG,CAAC;AAgBD,MAAM,YAAY,GAAiB;IACjC,WAAW,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE;IACnD,gBAAgB,EAAE,IAAI;IACtB,UAAU,EAAE,IAAI;IAChB,0BAA0B,EAAE,GAAG;CAChC,CAAC;AAEF;;;;;GAKG;AACH,MAAM,OAAO,WAAW;IAEO;IADpB,EAAE,GAAG,MAAM,CAAC;IACrB,YAA6B,WAAyB,YAAY;QAArC,aAAQ,GAAR,QAAQ,CAA6B;IAAG,CAAC;IAEtE,KAAK,CAAC,GAAG,CAAC,GAAoB;QAC5B,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC;YAAE,gBAAgB,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,SAAS,CAAC,CAAC;QAE7E,MAAM,CAAC,GAAG,GAAG,CAAC,SAAS,CAAC,UAAU,CAAC,IAAI,CAAC;QACxC,MAAM,cAAc,GAAG,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,0BAA0B,CAAC;QACpE,2EAA2E;QAC3E,sEAAsE;QACtE,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;QAClD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CACxB,IAAI,EACJ,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CACpF,CAAC;QACF,MAAM,QAAQ,GAAG,IAAI,GAAG,SAAS,CAAC;QAElC,4EAA4E;QAC5E,kFAAkF;QAClF,IAAI,QAAQ;YAAE,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,aAAa,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;QAE/E,OAAO;YACL,qEAAqE;YACrE,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,UAAU,GAAG,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC;YAC3E,OAAO,EAAE,IAAI;YACb,KAAK,EAAE,CAAC,GAAG,CAAC;YACZ,SAAS,EAAE,CAAC,GAAG,CAAC;YAChB,UAAU,EAAE,IAAI;YAChB,KAAK,EAAE,IAAI;YACX,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,iBAAiB;SACvD,CAAC;IACJ,CAAC;CACF;AAED,wCAAwC;AACxC,MAAM,UAAU,MAAM,CAAC,CAAS;IAC9B,IAAI,CAAC,GAAG,UAAU,KAAK,CAAC,CAAC;IACzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAClC,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QACrB,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;IAC7B,CAAC;IACD,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,UAAU,CAAC;AAChC,CAAC;AAoBD,MAAM,kBAAkB,GACtB,+EAA+E;IAC/E,+EAA+E;IAC/E,8DAA8D;IAC9D,gEAAgE;IAChE,6EAA6E,CAAC;AAEhF,8EAA8E;AAC9E,MAAM,UAAU,eAAe,CAAC,IAAY;IAC1C,MAAM,GAAG,GAAwC,EAAE,CAAC;IACpD,MAAM,EAAE,GAAG,mDAAmD,CAAC;IAC/D,IAAI,CAAyB,CAAC;IAC9B,OAAO,CAAC,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QACpC,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACzB,IAAI,IAAI;YAAE,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAC9C,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,kEAAkE;AAClE,SAAS,aAAa,CAAC,OAAe;IACpC,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,CAAC,cAAc,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC;IACzE,MAAM,MAAM,GAAG,qDAAqD,CAAC;IACrE,MAAM,KAAK,GAAwC,EAAE,CAAC;IACtD,MAAM,IAAI,GAAG,CAAC,GAAW,EAAQ,EAAE;QACjC,KAAK,MAAM,IAAI,IAAI,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC;YACpC,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC;gBAAE,SAAS;YAC7B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;YAC5B,MAAM,EAAE,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC;YACzB,IAAI,EAAE,CAAC,WAAW,EAAE;gBAAE,IAAI,CAAC,GAAG,CAAC,CAAC;iBAC3B,IAAI,EAAE,CAAC,IAAI,IAAI,EAAE,GAAG,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,KAAK,cAAc,EAAE,CAAC;gBAC9E,IAAI,CAAC;oBACH,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,CAAC,OAAO,EAAE,GAAG,CAAC,EAAE,OAAO,EAAE,YAAY,CAAC,GAAG,EAAE,OAAO,CAAC,EAAE,CAAC,CAAC;gBACpF,CAAC;gBAAC,MAAM,CAAC;oBACP,qBAAqB;gBACvB,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC,CAAC;IACF,IAAI,CAAC,OAAO,CAAC,CAAC;IACd,OAAO,KAAK,CAAC;AACf,CAAC;AAED,mEAAmE;AACnE,SAAS,eAAe,CAAC,OAAe,EAAE,MAA2C;IACnF,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;QACtC,IAAI,IAAI,KAAK,OAAO,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,OAAO,GAAG,GAAG,CAAC;YAAE,SAAS,CAAC,YAAY;QAC/E,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC9C,aAAa,CAAC,IAAI,EAAE,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QACxC,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAYD;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,MAAM,UAAU,YAAY;IAC1B,MAAM,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,kBAAkB,IAAI,KAAK,CAAC,CAAC;IAC1D,OAAO,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;AACjD,CAAC;AAED,KAAK,UAAU,cAAc,CAC3B,QAAgB,EAChB,KAAa,EACb,QAA6C,EAC7C,WAAmB,EACnB,SAAiB;IAEjB,MAAM,IAAI,GAAG,IAAI,eAAe,EAAE,CAAC;IACnC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,SAAS,CAAC,CAAC;IACxD,MAAM,IAAI,GAAG,CAAC,KAAa,EAAc,EAAE,CAAC,CAAC;QAC3C,OAAO,EAAE,EAAE;QACX,MAAM,EAAE,CAAC;QACT,KAAK;QACL,YAAY,EAAE,IAAI;QAClB,SAAS,EAAE,KAAK;KACjB,CAAC,CAAC;IACH,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,QAAQ,EAAE;YAChC,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;YAC/C,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,WAAW,EAAE,UAAU,EAAE,YAAY,EAAE,EAAE,CAAC;YAClF,MAAM,EAAE,IAAI,CAAC,MAAM;SACpB,CAAC,CAAC;QACH,IAAI,CAAC,GAAG,CAAC,EAAE;YAAE,OAAO,IAAI,CAAC,QAAQ,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;QAC/C,MAAM,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAG7B,CAAC;QACF,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC;QACjC,MAAM,OAAO,GAAG,MAAM,EAAE,OAAO,EAAE,OAAO,IAAI,EAAE,CAAC;QAC/C,MAAM,YAAY,GAAG,MAAM,EAAE,aAAa,IAAI,IAAI,CAAC;QACnD,OAAO;YACL,OAAO;YACP,MAAM,EAAE,IAAI,CAAC,KAAK,EAAE,YAAY,IAAI,CAAC;YACrC,KAAK,EAAE,IAAI;YACX,YAAY;YACZ,SAAS,EAAE,YAAY,KAAK,QAAQ,IAAI,OAAO,CAAC,IAAI,EAAE,KAAK,EAAE;SAC9D,CAAC;IACJ,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO,IAAI,CAAC,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1D,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,KAAK,CAAC,CAAC;IACtB,CAAC;AACH,CAAC;AAED,MAAM,OAAO,oBAAoB;IACtB,EAAE,GAAG,KAAK,CAAC;IACH,QAAQ,CAAS;IACjB,YAAY,CAAS;IACrB,WAAW,CAAS;IACrC,YAAY,MAAwB,EAAE;QACpC,IAAI,CAAC,QAAQ;YACX,GAAG,CAAC,QAAQ,IAAI,OAAO,CAAC,GAAG,CAAC,gBAAgB,IAAI,2CAA2C,CAAC;QAC9F,IAAI,CAAC,YAAY,GAAG,GAAG,CAAC,YAAY,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,kBAAkB,IAAI,CAAC,CAAC,CAAC;QACpF,IAAI,CAAC,WAAW,GAAG,GAAG,CAAC,WAAW,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,GAAG,CAAC,CAAC;IACvF,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,GAAoB;QAC5B,MAAM,OAAO,GAAG,GAAG,CAAC,SAAS,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACnF,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QAEzC,yEAAyE;QACzE,2EAA2E;QAC3E,uEAAuE;QACvE,0EAA0E;QAC1E,MAAM,aAAa,GAAG,CAAC,IAAa,EAAE,OAAsB,EAAuC,EAAE;YACnG,MAAM,KAAK,GAAG,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI,KAAK,WAAW,CAAC,CAAC,CAAC;YAC1F,MAAM,QAAQ,GAAG,KAAK;iBACnB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,CAAC,IAAI,QAAQ,CAAC,CAAC,OAAO,aAAa,CAAC;iBAC3D,IAAI,CAAC,MAAM,CAAC,CAAC;YAChB,OAAO;gBACL,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,kBAAkB,EAAE;gBAC/C;oBACE,IAAI,EAAE,MAAM;oBACZ,OAAO,EACL,SAAS,GAAG,CAAC,IAAI,CAAC,WAAW,yBAAyB,QAAQ,MAAM;wBACpE,CAAC,OAAO;4BACN,CAAC,CAAC,kDAAkD,OAAO,uGAAuG;4BAClK,CAAC,CAAC,kFAAkF,CAAC;iBAC1F;aACF,CAAC;QACJ,CAAC,CAAC;QAEF,IAAI,WAAW,GAAG,CAAC,CAAC;QACpB,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,IAAI,cAAc,GAAG,CAAC,CAAC;QACvB,IAAI,KAAK,GAAkB,IAAI,CAAC;QAChC,MAAM,QAAQ,GAAa,EAAE,CAAC;QAC9B,wEAAwE;QACxE,sEAAsE;QACtE,yEAAyE;QACzE,0DAA0D;QAC1D,MAAM,QAAQ,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC;QACjD,IAAI,WAAW,GAAkB,IAAI,CAAC;QAEtC,KAAK,IAAI,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,QAAQ,EAAE,IAAI,EAAE,EAAE,CAAC;YAC3C,MAAM,IAAI,GAAG,IAAI,IAAI,IAAI,KAAK,CAAC,CAAC;YAChC,MAAM,QAAQ,GAAG,aAAa,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;YAClD,MAAM,IAAI,GAAG,MAAM,cAAc,CAC/B,IAAI,CAAC,QAAQ,EACb,GAAG,CAAC,KAAK,EACT,QAAQ,EACR,IAAI,CAAC,WAAW,EAChB,GAAG,CAAC,IAAI,CAAC,eAAe,GAAG,IAAI,CAChC,CAAC;YACF,KAAK,EAAE,CAAC;YACR,WAAW,IAAI,IAAI,CAAC,MAAM,CAAC;YAC3B,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;gBACf,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;gBACnB,MAAM;YACR,CAAC;YACD,MAAM,MAAM,GAAG,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAC7C,eAAe,CAAC,GAAG,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YACrC,IAAI,IAAI,CAAC,SAAS;gBAAE,cAAc,EAAE,CAAC;YACrC,0EAA0E;YAC1E,wEAAwE;YACxE,uEAAuE;YACvE,uEAAuE;YACvE,qCAAqC;YACrC,QAAQ,CAAC,IAAI,CACX,YAAY,KAAK,KAAK,MAAM,CAAC,MAAM,UAAU;gBAC3C,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,kDAAkD,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC1E,WAAW,IAAI,CAAC,YAAY,IAAI,KAAK,MAAM,CAC9C,CAAC;YAEF,IAAI,CAAC,OAAO;gBAAE,MAAM;YAEpB,MAAM,IAAI,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,GAAG,CAAC,IAAI,CAAC,OAAiB,CAAC,EAAE;gBAClE,GAAG,EAAE,GAAG,CAAC,OAAO;gBAChB,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,GAAG,CAAC,IAAI,CAAC,gBAAgB,GAAG,IAAI;gBACzC,GAAG,EAAE,YAAY,EAAE;aACpB,CAAC,CAAC;YACH,QAAQ,EAAE,CAAC;YACX,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;gBAAE,MAAM;YAC7B,IAAI,IAAI,KAAK,QAAQ,GAAG,CAAC;gBAAE,MAAM,CAAC,gBAAgB;YAClD,WAAW,GAAG,GAAG,IAAI,CAAC,MAAM,IAAI,EAAE,KAAK,IAAI,CAAC,MAAM,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC;QAC1E,CAAC;QAED,OAAO;YACL,MAAM,EAAE,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI;YAC5C,OAAO,EAAE,IAAI;YACb,KAAK;YACL,SAAS,EAAE,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;YACjC,UAAU,EAAE,IAAI;YAChB,KAAK;YACL,MAAM,EACJ,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC;gBACnB,CAAC,cAAc,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,cAAc,IAAI,KAAK,mCAAmC,CAAC,CAAC,CAAC,EAAE,CAAC;SAC/F,CAAC;IACJ,CAAC;CACF;AAED;;;;;;;;GAQG;AACH,MAAM,OAAO,iBAAiB;IACnB,EAAE,GAAG,SAAS,CAAC;IACP,OAAO,CAAS;IACjC,YAAY,OAAgB;QAC1B,IAAI,CAAC,OAAO,GAAG,OAAO,IAAI,OAAO,CAAC,GAAG,CAAC,qBAAqB,IAAI,iBAAiB,CAAC;IACnF,CAAC;IACD,KAAK,CAAC,GAAG,CAAC,GAAoB;QAC5B,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,eAAe,CAAC,CAAC;QACnD,MAAM,QAAQ,GAAG,GAAG,CAAC,SAAS,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC,CAAC;QACrD,MAAM,IAAI,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,gBAAgB,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;QAChF,IAAI,QAAQ,EAAE,CAAC;YACb,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,sBAAsB,EAAE,aAAa,EAAE,GAAG,EAAE,WAAW,EAAE,gBAAgB,CAAC,CAAC;QACpG,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACtC,MAAM,GAAG,GAAsB,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,iBAAiB,EAAE,GAAG,CAAC,KAAK,EAAE,CAAC;QAChF,IAAI,GAAG,CAAC,SAAS,CAAC,IAAI;YAAE,GAAG,CAAC,gBAAgB,GAAG,GAAG,CAAC;QACnD,MAAM,SAAS,GAAG,GAAG,CAAC,IAAI,CAAC,eAAe,GAAG,IAAI,GAAG,CAAC,CAAC;QACtD,OAAO,IAAI,OAAO,CAAC,CAAC,cAAc,EAAE,EAAE;YACpC,QAAQ,CAAC,OAAO,CAAC,QAAQ,EAAE,IAAI,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,OAAO,EAAE,GAAG,EAAE,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,EAAE,GAAG,IAAI,GAAG,IAAI,EAAE,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE;gBAC3H,IAAI,KAAK,GAAG,CAAC,CAAC;gBACd,IAAI,CAAC;oBACH,MAAM,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;oBAC1C,IAAI,CAAC,EAAE,CAAC;wBACN,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAuB,CAAC;wBACjD,IAAI,OAAO,CAAC,CAAC,KAAK,KAAK,QAAQ;4BAAE,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;oBACnD,CAAC;gBACH,CAAC;gBAAC,MAAM,CAAC,CAAC,yBAAyB,CAAC,CAAC;gBACrC,cAAc,CAAC;oBACb,MAAM,EAAE,IAAI;oBACZ,OAAO,EAAE,IAAI;oBACb,KAAK;oBACL,SAAS,EAAE,CAAC;oBACZ,UAAU,EAAE,IAAI;oBAChB,KAAK,EAAE,GAAG,IAAK,GAAoC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI;oBAC7E,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC;iBAC5B,CAAC,CAAC;YACL,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;CACF"}
|
|
@@ -67,6 +67,13 @@ export interface AnalysisReport {
|
|
|
67
67
|
successRate: number;
|
|
68
68
|
meanTokens: number | null;
|
|
69
69
|
}[];
|
|
70
|
+
/**
|
|
71
|
+
* Whether this run could have detected a difference at all (see
|
|
72
|
+
* `assessDiscrimination`). A suite where every arm scores 100% — or 0% —
|
|
73
|
+
* yields `delta=+0.000, significant=false`, which reads like "no effect" but
|
|
74
|
+
* actually means "no measurement".
|
|
75
|
+
*/
|
|
76
|
+
discrimination: DiscriminationVerdict;
|
|
70
77
|
/**
|
|
71
78
|
* ETCSOVG disclosure card for the harness these numbers were produced by
|
|
72
79
|
* (harness plan F). Harness variance dominates model variance 7.8x
|
|
@@ -76,6 +83,83 @@ export interface AnalysisReport {
|
|
|
76
83
|
*/
|
|
77
84
|
harnessCard?: HarnessCard;
|
|
78
85
|
}
|
|
86
|
+
/**
|
|
87
|
+
* Can this run tell the arms apart at all?
|
|
88
|
+
*
|
|
89
|
+
* A paired benchmark reports `delta=+0.000, significant=false` in two very
|
|
90
|
+
* different situations: the arms genuinely perform the same, or the SUITE
|
|
91
|
+
* cannot separate them because every arm sits at the ceiling (or the floor).
|
|
92
|
+
* The numbers are identical; the conclusions are opposite. Reporting only the
|
|
93
|
+
* delta invites reading "no measurement" as "no effect" — three consecutive
|
|
94
|
+
* runs against `real-gate` (100%/100%) and `smoke` (0%/0%, wrong adapter) were
|
|
95
|
+
* each read as a null result before anyone noticed nothing had been measured.
|
|
96
|
+
*
|
|
97
|
+
* So the verdict is computed and printed alongside the delta, always.
|
|
98
|
+
*/
|
|
99
|
+
export type DiscriminationStatus = 'ok'
|
|
100
|
+
/** Every arm solved (nearly) everything — the suite cannot separate them. */
|
|
101
|
+
| 'ceiling'
|
|
102
|
+
/** Nothing was solved — suite too hard, or adapter/harness misconfigured. */
|
|
103
|
+
| 'floor'
|
|
104
|
+
/**
|
|
105
|
+
* Arms differed nowhere on CORRECTNESS: zero discordant pairs. This is the
|
|
106
|
+
* criterion McNemar tests on, and it is the honest name for the failure — a
|
|
107
|
+
* bootstrap over an all-zero delta vector returns CI [0.000, 0.000], which
|
|
108
|
+
* reads as "conclusively no effect" when it means "no information".
|
|
109
|
+
*/
|
|
110
|
+
| 'no-discordant-pairs'
|
|
111
|
+
/** The interval spans both zero and the effect worth having. */
|
|
112
|
+
| 'underpowered'
|
|
113
|
+
/** Nothing to compare — a report with no treatment arm. */
|
|
114
|
+
| 'no-comparisons';
|
|
115
|
+
export interface DiscriminationVerdict {
|
|
116
|
+
status: DiscriminationStatus;
|
|
117
|
+
/**
|
|
118
|
+
* True when a CORRECTNESS difference could have shown up. Efficiency results
|
|
119
|
+
* are judged separately — see `efficiencyUsable`.
|
|
120
|
+
*/
|
|
121
|
+
usable: boolean;
|
|
122
|
+
/**
|
|
123
|
+
* True when at least one continuous metric (tokens/turns/cost/latency) or the
|
|
124
|
+
* quality score carries a non-degenerate interval.
|
|
125
|
+
*
|
|
126
|
+
* This exists because "same accuracy, 40% fewer tokens" is a real, citable
|
|
127
|
+
* result — and the first cut of this check refused it, printing "not evidence
|
|
128
|
+
* of anything" directly above a token table reading 🟢 WIN. A saturated suite
|
|
129
|
+
* measures correctness badly and efficiency perfectly well.
|
|
130
|
+
*/
|
|
131
|
+
efficiencyUsable: boolean;
|
|
132
|
+
reason: string;
|
|
133
|
+
minSuccess: number;
|
|
134
|
+
maxSuccess: number;
|
|
135
|
+
/** Paired cells in the PRIMARY comparison. */
|
|
136
|
+
pairedCells: number;
|
|
137
|
+
/** Discordant correctness pairs in the PRIMARY comparison (McNemar b+c). */
|
|
138
|
+
discordantPairs: number;
|
|
139
|
+
}
|
|
140
|
+
export interface DiscriminationOptions {
|
|
141
|
+
/** Every arm at or above this is a ceiling. Default 0.98. */
|
|
142
|
+
ceilingAt?: number;
|
|
143
|
+
/** Every arm at or below this is a floor. Default 0.02. */
|
|
144
|
+
floorAt?: number;
|
|
145
|
+
/** Fewer paired cells than this cannot support a verdict. Default 8. */
|
|
146
|
+
minPairedCells?: number;
|
|
147
|
+
/**
|
|
148
|
+
* Smallest correctness delta worth detecting, in success-rate units. A run is
|
|
149
|
+
* inconclusive when its interval contains BOTH zero and this effect.
|
|
150
|
+
* Default 0.25.
|
|
151
|
+
*/
|
|
152
|
+
minDetectableEffect?: number;
|
|
153
|
+
}
|
|
154
|
+
/**
|
|
155
|
+
* Can this run tell the arms apart?
|
|
156
|
+
*
|
|
157
|
+
* Judged against the PRIMARY comparison (the first treatment arm), not folded
|
|
158
|
+
* across every arm: an `--ablation` run has 6 comparisons, and summing their
|
|
159
|
+
* discordance let one noisy leave-one-out arm mask a null primary, while taking
|
|
160
|
+
* the widest CI let it refuse a tight primary.
|
|
161
|
+
*/
|
|
162
|
+
export declare function assessDiscrimination(perCondition: ConditionSummary[], comparisons: Comparison[], opts?: DiscriminationOptions): DiscriminationVerdict;
|
|
79
163
|
export interface AnalyzeOptions extends PairedOptions {
|
|
80
164
|
baselineLabel?: string;
|
|
81
165
|
/** ROPE half-width for correctness (success-rate units, 0..1). A success-rate
|
|
@@ -94,6 +178,8 @@ export interface AnalyzeOptions extends PairedOptions {
|
|
|
94
178
|
* a guessed one.
|
|
95
179
|
*/
|
|
96
180
|
harness?: HarnessCardInput;
|
|
181
|
+
/** Thresholds for the ceiling/floor/underpowered verdict. */
|
|
182
|
+
discrimination?: DiscriminationOptions;
|
|
97
183
|
}
|
|
98
184
|
export declare function analyze(output: RunnerOutput, opts?: AnalyzeOptions): AnalysisReport;
|
|
99
185
|
export declare function renderMarkdown(r: AnalysisReport): string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"report.d.ts","sourceRoot":"","sources":["../../../src/benchmarks/paired/report.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAGL,aAAa,EAGb,iBAAiB,EACjB,aAAa,EAEb,OAAO,EACR,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAsB,gBAAgB,EAA2B,MAAM,YAAY,CAAC;AAC3F,OAAO,EAGL,KAAK,WAAW,EAChB,KAAK,gBAAgB,EACtB,MAAM,oBAAoB,CAAC;AAE5B,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,CAAC,EAAE,MAAM,CAAC;IACV,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,sFAAsF;IACtF,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;CAC5B;AAED,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,2EAA2E;IAC3E,WAAW,EAAE;QACX,YAAY,EAAE,MAAM,CAAC;QACrB,aAAa,EAAE,MAAM,CAAC;QACtB,KAAK,EAAE,iBAAiB,CAAC;QACzB,OAAO,EAAE,aAAa,CAAC;QACvB,yEAAyE;QACzE,OAAO,EAAE,OAAO,CAAC;KAClB,CAAC;IACF,iFAAiF;IACjF,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC,gBAAgB,EAAE,iBAAiB,CAAC,CAAC,CAAC;IAC9D,+EAA+E;IAC/E,cAAc,EAAE,OAAO,CAAC,MAAM,CAAC,gBAAgB,EAAE,OAAO,CAAC,CAAC,CAAC;IAC3D;;;;OAIG;IACH,OAAO,CAAC,EAAE,iBAAiB,CAAC;IAC5B,4DAA4D;IAC5D,cAAc,CAAC,EAAE,OAAO,CAAC;CAC1B;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE;QACJ,KAAK,EAAE,MAAM,CAAC;QACd,OAAO,EAAE,MAAM,CAAC;QAChB,MAAM,EAAE,MAAM,CAAC;QACf,SAAS,EAAE,MAAM,CAAC;QAClB,UAAU,EAAE,MAAM,EAAE,CAAC;QACrB,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,EAAE,MAAM,CAAC;QAClB,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC;IACF,YAAY,EAAE,gBAAgB,EAAE,CAAC;IACjC,WAAW,EAAE,UAAU,EAAE,CAAC;IAC1B,uDAAuD;IACvD,MAAM,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE,EAAE,CAAC;IAC5E;;;;;;OAMG;IACH,WAAW,CAAC,EAAE,WAAW,CAAC;CAC3B;AA6DD,MAAM,WAAW,cAAe,SAAQ,aAAa;IACnD,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;2EAEuE;IACvE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;gEAC4D;IAC5D,aAAa,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,gBAAgB,EAAE,MAAM,CAAC,CAAC,CAAC;IAC1D;uEACmE;IACnE,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;;;OAIG;IACH,OAAO,CAAC,EAAE,gBAAgB,CAAC;
|
|
1
|
+
{"version":3,"file":"report.d.ts","sourceRoot":"","sources":["../../../src/benchmarks/paired/report.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAGL,aAAa,EAGb,iBAAiB,EACjB,aAAa,EAEb,OAAO,EACR,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAsB,gBAAgB,EAA2B,MAAM,YAAY,CAAC;AAC3F,OAAO,EAGL,KAAK,WAAW,EAChB,KAAK,gBAAgB,EACtB,MAAM,oBAAoB,CAAC;AAE5B,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,CAAC,EAAE,MAAM,CAAC;IACV,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,sFAAsF;IACtF,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;CAC5B;AAED,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,2EAA2E;IAC3E,WAAW,EAAE;QACX,YAAY,EAAE,MAAM,CAAC;QACrB,aAAa,EAAE,MAAM,CAAC;QACtB,KAAK,EAAE,iBAAiB,CAAC;QACzB,OAAO,EAAE,aAAa,CAAC;QACvB,yEAAyE;QACzE,OAAO,EAAE,OAAO,CAAC;KAClB,CAAC;IACF,iFAAiF;IACjF,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC,gBAAgB,EAAE,iBAAiB,CAAC,CAAC,CAAC;IAC9D,+EAA+E;IAC/E,cAAc,EAAE,OAAO,CAAC,MAAM,CAAC,gBAAgB,EAAE,OAAO,CAAC,CAAC,CAAC;IAC3D;;;;OAIG;IACH,OAAO,CAAC,EAAE,iBAAiB,CAAC;IAC5B,4DAA4D;IAC5D,cAAc,CAAC,EAAE,OAAO,CAAC;CAC1B;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE;QACJ,KAAK,EAAE,MAAM,CAAC;QACd,OAAO,EAAE,MAAM,CAAC;QAChB,MAAM,EAAE,MAAM,CAAC;QACf,SAAS,EAAE,MAAM,CAAC;QAClB,UAAU,EAAE,MAAM,EAAE,CAAC;QACrB,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,EAAE,MAAM,CAAC;QAClB,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC;IACF,YAAY,EAAE,gBAAgB,EAAE,CAAC;IACjC,WAAW,EAAE,UAAU,EAAE,CAAC;IAC1B,uDAAuD;IACvD,MAAM,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE,EAAE,CAAC;IAC5E;;;;;OAKG;IACH,cAAc,EAAE,qBAAqB,CAAC;IACtC;;;;;;OAMG;IACH,WAAW,CAAC,EAAE,WAAW,CAAC;CAC3B;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,MAAM,oBAAoB,GAC5B,IAAI;AACN,6EAA6E;GAC3E,SAAS;AACX,6EAA6E;GAC3E,OAAO;AACT;;;;;GAKG;GACD,qBAAqB;AACvB,gEAAgE;GAC9D,cAAc;AAChB,2DAA2D;GACzD,gBAAgB,CAAC;AAErB,MAAM,WAAW,qBAAqB;IACpC,MAAM,EAAE,oBAAoB,CAAC;IAC7B;;;OAGG;IACH,MAAM,EAAE,OAAO,CAAC;IAChB;;;;;;;;OAQG;IACH,gBAAgB,EAAE,OAAO,CAAC;IAC1B,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,8CAA8C;IAC9C,WAAW,EAAE,MAAM,CAAC;IACpB,4EAA4E;IAC5E,eAAe,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,qBAAqB;IACpC,6DAA6D;IAC7D,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,2DAA2D;IAC3D,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,wEAAwE;IACxE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;;;OAIG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;CAC9B;AAyBD;;;;;;;GAOG;AACH,wBAAgB,oBAAoB,CAClC,YAAY,EAAE,gBAAgB,EAAE,EAChC,WAAW,EAAE,UAAU,EAAE,EACzB,IAAI,GAAE,qBAA0B,GAC/B,qBAAqB,CAmGvB;AA6DD,MAAM,WAAW,cAAe,SAAQ,aAAa;IACnD,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;2EAEuE;IACvE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;gEAC4D;IAC5D,aAAa,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,gBAAgB,EAAE,MAAM,CAAC,CAAC,CAAC;IAC1D;uEACmE;IACnE,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;;;OAIG;IACH,OAAO,CAAC,EAAE,gBAAgB,CAAC;IAC3B,6DAA6D;IAC7D,cAAc,CAAC,EAAE,qBAAqB,CAAC;CACxC;AAED,wBAAgB,OAAO,CAAC,MAAM,EAAE,YAAY,EAAE,IAAI,GAAE,cAAmB,GAAG,cAAc,CA+GvF;AA+BD,wBAAgB,cAAc,CAAC,CAAC,EAAE,cAAc,GAAG,MAAM,CA+GxD"}
|
|
@@ -10,6 +10,127 @@
|
|
|
10
10
|
import { mcnemar, mean, pairedDelta, verdict, } from './stats.js';
|
|
11
11
|
import { CONTINUOUS_METRICS } from './types.js';
|
|
12
12
|
import { buildHarnessCard, renderHarnessCard, } from '../harness-card.js';
|
|
13
|
+
function finite(n) {
|
|
14
|
+
return typeof n === 'number' && Number.isFinite(n);
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* Does any continuous metric or quality score carry a real difference?
|
|
18
|
+
*
|
|
19
|
+
* Keyed on a NON-ZERO paired mean, not on interval width. Zero width means
|
|
20
|
+
* opposite things for the two kinds of measure: an all-zero correctness delta is
|
|
21
|
+
* no information, while a constant -1200-token delta across every cell is the
|
|
22
|
+
* strongest efficiency result there is — perfectly consistent, zero variance.
|
|
23
|
+
* Testing width alone refused exactly that.
|
|
24
|
+
*/
|
|
25
|
+
function hasEfficiencySignal(c) {
|
|
26
|
+
if (!c)
|
|
27
|
+
return false;
|
|
28
|
+
const carries = (pd) => Boolean(pd && pd.n > 0 && finite(pd.meanDelta) && pd.meanDelta !== 0);
|
|
29
|
+
for (const m of CONTINUOUS_METRICS) {
|
|
30
|
+
if (carries(c.metrics[m]))
|
|
31
|
+
return true;
|
|
32
|
+
}
|
|
33
|
+
return carries(c.quality);
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* Can this run tell the arms apart?
|
|
37
|
+
*
|
|
38
|
+
* Judged against the PRIMARY comparison (the first treatment arm), not folded
|
|
39
|
+
* across every arm: an `--ablation` run has 6 comparisons, and summing their
|
|
40
|
+
* discordance let one noisy leave-one-out arm mask a null primary, while taking
|
|
41
|
+
* the widest CI let it refuse a tight primary.
|
|
42
|
+
*/
|
|
43
|
+
export function assessDiscrimination(perCondition, comparisons, opts = {}) {
|
|
44
|
+
const ceilingAt = opts.ceilingAt ?? 0.98;
|
|
45
|
+
const floorAt = opts.floorAt ?? 0.02;
|
|
46
|
+
const minCells = opts.minPairedCells ?? 8;
|
|
47
|
+
const mde = opts.minDetectableEffect ?? 0.25;
|
|
48
|
+
const rates = perCondition.map((c) => c.successRate).filter(finite);
|
|
49
|
+
const minSuccess = rates.length ? Math.min(...rates) : 0;
|
|
50
|
+
const maxSuccess = rates.length ? Math.max(...rates) : 0;
|
|
51
|
+
const primary = comparisons[0];
|
|
52
|
+
const efficiencyUsable = hasEfficiencySignal(primary);
|
|
53
|
+
const d = primary?.correctness.delta;
|
|
54
|
+
const pairedCells = finite(d?.n) ? d.n : 0;
|
|
55
|
+
const discordantPairs = primary
|
|
56
|
+
? primary.correctness.mcnemar.onlyTreatment + primary.correctness.mcnemar.onlyBaseline
|
|
57
|
+
: 0;
|
|
58
|
+
const base = { minSuccess, maxSuccess, pairedCells, discordantPairs, efficiencyUsable };
|
|
59
|
+
const alsoEfficiency = efficiencyUsable
|
|
60
|
+
? ' Efficiency deltas (tokens/turns/latency) below ARE valid and may be cited.'
|
|
61
|
+
: '';
|
|
62
|
+
if (!primary) {
|
|
63
|
+
return {
|
|
64
|
+
status: 'no-comparisons',
|
|
65
|
+
usable: false,
|
|
66
|
+
reason: 'no treatment arm to compare against the baseline — nothing was measured.',
|
|
67
|
+
...base,
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
if (rates.length > 0 && minSuccess >= ceilingAt) {
|
|
71
|
+
return {
|
|
72
|
+
status: 'ceiling',
|
|
73
|
+
usable: false,
|
|
74
|
+
reason: `every condition solved ${(minSuccess * 100).toFixed(0)}%+ of cells — the suite cannot separate ` +
|
|
75
|
+
`them on correctness. Either it is too easy, or the verify command always passes (a mock adapter ` +
|
|
76
|
+
`against a mock suite looks identical). A zero correctness delta here means NO MEASUREMENT.` +
|
|
77
|
+
alsoEfficiency,
|
|
78
|
+
...base,
|
|
79
|
+
};
|
|
80
|
+
}
|
|
81
|
+
if (rates.length > 0 && maxSuccess <= floorAt) {
|
|
82
|
+
return {
|
|
83
|
+
status: 'floor',
|
|
84
|
+
usable: false,
|
|
85
|
+
reason: `no condition solved anything (max ${(maxSuccess * 100).toFixed(0)}%) — the suite is too hard, ` +
|
|
86
|
+
`or the adapter/suite are mismatched (a mock-only suite scores 0% against every real model, ` +
|
|
87
|
+
`which looks identical to "too hard"). A zero delta here means NO MEASUREMENT.` + alsoEfficiency,
|
|
88
|
+
...base,
|
|
89
|
+
};
|
|
90
|
+
}
|
|
91
|
+
if (discordantPairs === 0) {
|
|
92
|
+
return {
|
|
93
|
+
status: 'no-discordant-pairs',
|
|
94
|
+
usable: false,
|
|
95
|
+
reason: `the arms produced identical correctness on all ${pairedCells} paired cells (0 discordant), so ` +
|
|
96
|
+
`the run carries no information about a correctness difference — the CI collapses to [0, 0], ` +
|
|
97
|
+
`which LOOKS conclusive and is not.` + alsoEfficiency,
|
|
98
|
+
...base,
|
|
99
|
+
};
|
|
100
|
+
}
|
|
101
|
+
if (pairedCells > 0 && pairedCells < minCells) {
|
|
102
|
+
return {
|
|
103
|
+
status: 'underpowered',
|
|
104
|
+
usable: false,
|
|
105
|
+
reason: `only ${pairedCells} paired cells — too few to distinguish a real effect from noise. ` +
|
|
106
|
+
`Raise --epochs or add tasks before reading the delta.` + alsoEfficiency,
|
|
107
|
+
...base,
|
|
108
|
+
};
|
|
109
|
+
}
|
|
110
|
+
// Inconclusive = the interval admits both "no effect" and an effect worth
|
|
111
|
+
// having. Half-width alone got this wrong in both directions: it passed
|
|
112
|
+
// [-0.24, +0.24] (cannot tell a 24pp regression from a 24pp win) and refused
|
|
113
|
+
// [0.02, 0.55] (a proven win with a long tail).
|
|
114
|
+
const ci = d?.ci;
|
|
115
|
+
const significant = Boolean(d?.significant);
|
|
116
|
+
if (!significant && ci && finite(ci.lower) && finite(ci.upper) && ci.lower < mde && ci.upper > -mde) {
|
|
117
|
+
return {
|
|
118
|
+
status: 'underpowered',
|
|
119
|
+
usable: false,
|
|
120
|
+
reason: `the correctness CI [${ci.lower.toFixed(2)}, ${ci.upper.toFixed(2)}] contains both zero and the ` +
|
|
121
|
+
`±${mde} effect worth detecting — this run could not have told a real difference from none. ` +
|
|
122
|
+
`Raise --epochs or add tasks.` + alsoEfficiency,
|
|
123
|
+
...base,
|
|
124
|
+
};
|
|
125
|
+
}
|
|
126
|
+
return {
|
|
127
|
+
status: 'ok',
|
|
128
|
+
usable: true,
|
|
129
|
+
reason: `conditions span ${(minSuccess * 100).toFixed(0)}%-${(maxSuccess * 100).toFixed(0)}% over ` +
|
|
130
|
+
`${pairedCells} paired cells with ${discordantPairs} discordant — a difference could show up here.`,
|
|
131
|
+
...base,
|
|
132
|
+
};
|
|
133
|
+
}
|
|
13
134
|
function cellKey(r) {
|
|
14
135
|
return `${r.taskId}#${r.seed}`;
|
|
15
136
|
}
|
|
@@ -155,6 +276,7 @@ export function analyze(output, opts = {}) {
|
|
|
155
276
|
successRate: c.successRate,
|
|
156
277
|
meanTokens: c.meanTokens,
|
|
157
278
|
})),
|
|
279
|
+
discrimination: assessDiscrimination(perCondition, comparisons, opts.discrimination),
|
|
158
280
|
// Default the card's model to the one the run actually used, so the caller
|
|
159
281
|
// cannot accidentally disclose a different model than it benchmarked.
|
|
160
282
|
...(opts.harness
|
|
@@ -182,7 +304,13 @@ function sig(p) {
|
|
|
182
304
|
}
|
|
183
305
|
/** Win/tie/loss badge under the ROPE norm — ties within noise are NOT wins. */
|
|
184
306
|
function vbadge(v) {
|
|
185
|
-
|
|
307
|
+
if (v === 'win')
|
|
308
|
+
return '🟢 WIN';
|
|
309
|
+
if (v === 'loss')
|
|
310
|
+
return '🔴 LOSS';
|
|
311
|
+
if (v === 'borderline')
|
|
312
|
+
return '🟡 BORDERLINE';
|
|
313
|
+
return '⚪ TIE';
|
|
186
314
|
}
|
|
187
315
|
export function renderMarkdown(r) {
|
|
188
316
|
const L = [];
|
|
@@ -193,6 +321,22 @@ export function renderMarkdown(r) {
|
|
|
193
321
|
`**Baseline:** \`${r.meta.baseline}\``);
|
|
194
322
|
L.push(`*${r.meta.startedAt} → ${r.meta.finishedAt}*`);
|
|
195
323
|
L.push('');
|
|
324
|
+
// Ahead of every number, because a reader who quotes a delta from a run that
|
|
325
|
+
// could not measure anything has been misled by the report, not by the data.
|
|
326
|
+
const d = r.discrimination;
|
|
327
|
+
if (!d.usable) {
|
|
328
|
+
const what = d.efficiencyUsable ? 'NO CORRECTNESS SIGNAL' : 'NO USABLE SIGNAL';
|
|
329
|
+
L.push(`> ⚠️ **${what} (${d.status})** — ${d.reason}`);
|
|
330
|
+
L.push('>');
|
|
331
|
+
L.push(d.efficiencyUsable
|
|
332
|
+
? '> The CORRECTNESS delta below is not evidence of anything. The efficiency deltas are.'
|
|
333
|
+
: '> The deltas below are not evidence of anything. Do not cite them.');
|
|
334
|
+
L.push('');
|
|
335
|
+
}
|
|
336
|
+
else {
|
|
337
|
+
L.push(`> Signal check: ${d.reason}`);
|
|
338
|
+
L.push('');
|
|
339
|
+
}
|
|
196
340
|
L.push(`## Per-condition summary`);
|
|
197
341
|
L.push('');
|
|
198
342
|
const anyQuality = r.perCondition.some((c) => c.meanQuality != null);
|