@crowandraven/evalkeeper 0.1.1 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/attention.js +2 -1
- package/dist/attention.js.map +1 -1
- package/dist/board.d.ts +2 -0
- package/dist/glossary.d.ts +1 -1
- package/dist/glossary.js +1 -1
- package/dist/glossary.js.map +1 -1
- package/dist/health.js +1 -1
- package/dist/health.js.map +1 -1
- package/dist/importCases.d.ts +94 -0
- package/dist/importCases.js +185 -0
- package/dist/importCases.js.map +1 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/labels.d.ts +9 -3
- package/dist/labels.js +2 -0
- package/dist/labels.js.map +1 -1
- package/dist/memoryStore.js +3 -2
- package/dist/memoryStore.js.map +1 -1
- package/dist/models.d.ts +44 -7
- package/dist/models.js +48 -7
- package/dist/models.js.map +1 -1
- package/dist/runner.d.ts +25 -1
- package/dist/runner.js +64 -7
- package/dist/runner.js.map +1 -1
- package/dist/settings.d.ts +42 -3
- package/dist/settings.js +101 -11
- package/dist/settings.js.map +1 -1
- package/dist/store.d.ts +18 -2
- package/dist/suggest.js +4 -2
- package/dist/suggest.js.map +1 -1
- package/dist/testing.js +48 -0
- package/dist/testing.js.map +1 -1
- package/dist/types.d.ts +11 -0
- package/dist/types.js.map +1 -1
- package/dist/weekly.js +5 -2
- package/dist/weekly.js.map +1 -1
- package/package.json +2 -2
- package/src/attention.ts +2 -1
- package/src/board.ts +2 -0
- package/src/glossary.ts +1 -1
- package/src/health.ts +3 -1
- package/src/importCases.ts +252 -0
- package/src/index.ts +1 -0
- package/src/labels.ts +9 -4
- package/src/memoryStore.ts +6 -2
- package/src/models.ts +83 -10
- package/src/runner.ts +102 -15
- package/src/settings.ts +125 -14
- package/src/store.ts +19 -2
- package/src/suggest.ts +3 -1
- package/src/testing.ts +56 -0
- package/src/types.ts +11 -0
- package/src/weekly.ts +5 -2
- package/templates/app/evals/[suite]/page.tsx +1 -2
- package/templates/app/evals/feedback/page.tsx +11 -6
- package/templates/app/evals/page.tsx +40 -0
package/dist/settings.js
CHANGED
|
@@ -5,12 +5,27 @@ export class SettingsConflict extends Error {
|
|
|
5
5
|
this.name = "SettingsConflict";
|
|
6
6
|
}
|
|
7
7
|
}
|
|
8
|
+
/**
|
|
9
|
+
* A feature has no settings: no version yet, and no default in code. Apps
|
|
10
|
+
* whose models are chosen in an admin page, not in code, see this until
|
|
11
|
+
* the feature's first version is recorded (`seed` or `change`).
|
|
12
|
+
*/
|
|
13
|
+
export class NoSettingsError extends Error {
|
|
14
|
+
/** The feature with no settings. */
|
|
15
|
+
operation;
|
|
16
|
+
constructor(operation) {
|
|
17
|
+
super(`${operation} has no settings yet: no version, and no default in code.`);
|
|
18
|
+
this.name = "NoSettingsError";
|
|
19
|
+
this.operation = operation;
|
|
20
|
+
}
|
|
21
|
+
}
|
|
8
22
|
/** Whether two settings are the same in every field. */
|
|
9
23
|
export function sameSettings(a, b) {
|
|
10
24
|
return (a.model === b.model &&
|
|
11
25
|
a.fallbackModel === b.fallbackModel &&
|
|
12
26
|
a.reasoningEffort === b.reasoningEffort &&
|
|
13
|
-
a.maxOutputTokens === b.maxOutputTokens
|
|
27
|
+
a.maxOutputTokens === b.maxOutputTokens &&
|
|
28
|
+
(a.temperature ?? null) === (b.temperature ?? null));
|
|
14
29
|
}
|
|
15
30
|
/** Just the settings of a version. */
|
|
16
31
|
export function settingsOf(v) {
|
|
@@ -19,6 +34,7 @@ export function settingsOf(v) {
|
|
|
19
34
|
fallbackModel: v.fallbackModel,
|
|
20
35
|
reasoningEffort: v.reasoningEffort,
|
|
21
36
|
maxOutputTokens: v.maxOutputTokens,
|
|
37
|
+
temperature: v.temperature ?? null,
|
|
22
38
|
};
|
|
23
39
|
}
|
|
24
40
|
/** What changed between two settings, in plain words. */
|
|
@@ -34,6 +50,8 @@ export function describeChange(from, to, label = (m) => m) {
|
|
|
34
50
|
out.push(`Reasoning: ${from.reasoningEffort ?? "default"} → ${to.reasoningEffort ?? "default"}`);
|
|
35
51
|
if (from.maxOutputTokens !== to.maxOutputTokens)
|
|
36
52
|
out.push(`Output limit: ${from.maxOutputTokens ?? "default"} → ${to.maxOutputTokens ?? "default"}`);
|
|
53
|
+
if ((from.temperature ?? null) !== (to.temperature ?? null))
|
|
54
|
+
out.push(`Temperature: ${from.temperature ?? "default"} → ${to.temperature ?? "default"}`);
|
|
37
55
|
return out;
|
|
38
56
|
}
|
|
39
57
|
/**
|
|
@@ -54,38 +72,68 @@ export function describeChange(from, to, label = (m) => m) {
|
|
|
54
72
|
* });
|
|
55
73
|
* await settings.rollBack("ticket_triage", versionId, { by: admin.name, reason: "Worse in production" });
|
|
56
74
|
* ```
|
|
75
|
+
*
|
|
76
|
+
* When models are chosen in an admin page rather than in code, return null
|
|
77
|
+
* from `defaults` and record each feature's existing config as version 1
|
|
78
|
+
* with `seed`:
|
|
79
|
+
*
|
|
80
|
+
* ```ts
|
|
81
|
+
* const settings = createSettings({ store, defaults: () => null });
|
|
82
|
+
* for (const row of await db.aiConfig.findMany())
|
|
83
|
+
* await settings.seed(row.operation, settingsFromRow(row), {
|
|
84
|
+
* by: null,
|
|
85
|
+
* reason: "Imported from the admin config",
|
|
86
|
+
* });
|
|
87
|
+
* ```
|
|
57
88
|
*/
|
|
58
89
|
export function createSettings(opts) {
|
|
59
90
|
const { store } = opts;
|
|
60
91
|
const prompt = async (op) => opts.promptVersion ? await opts.promptVersion(op) : null;
|
|
61
|
-
/**
|
|
92
|
+
/**
|
|
93
|
+
* The live settings: the live version's, else the default from code.
|
|
94
|
+
* Throws `NoSettingsError` when the feature has neither.
|
|
95
|
+
*/
|
|
62
96
|
async function current(operation) {
|
|
63
97
|
const version = await store.liveVersion(operation);
|
|
64
|
-
|
|
65
|
-
settings:
|
|
66
|
-
|
|
67
|
-
|
|
98
|
+
if (version)
|
|
99
|
+
return { settings: settingsOf(version), version };
|
|
100
|
+
const fallback = opts.defaults(operation);
|
|
101
|
+
if (!fallback)
|
|
102
|
+
throw new NoSettingsError(operation);
|
|
103
|
+
return { settings: fallback, version };
|
|
68
104
|
}
|
|
69
|
-
/** The runner's route for a feature, from its live settings. */
|
|
105
|
+
/** The runner's route for a feature, from its live settings; throws `NoSettingsError` when it has none. */
|
|
70
106
|
async function route(operation) {
|
|
71
107
|
const { settings } = await current(operation);
|
|
72
108
|
return {
|
|
73
109
|
model: settings.model,
|
|
74
110
|
fallbackModel: settings.fallbackModel,
|
|
75
111
|
reasoningEffort: settings.reasoningEffort,
|
|
112
|
+
temperature: settings.temperature ?? null,
|
|
113
|
+
maxOutputTokens: settings.maxOutputTokens,
|
|
76
114
|
};
|
|
77
115
|
}
|
|
78
116
|
/**
|
|
79
117
|
* Makes new settings live as a new version. The first change to a
|
|
80
118
|
* feature first records its default as version 1, so there's always
|
|
81
|
-
* something to roll back to
|
|
82
|
-
*
|
|
119
|
+
* something to roll back to; a feature with no default gets `next` as
|
|
120
|
+
* version 1. With `evidence`, it's a promotion. Returns null when
|
|
121
|
+
* nothing changed.
|
|
83
122
|
*/
|
|
84
123
|
async function change(operation, next, by) {
|
|
85
124
|
const promptVersion = await prompt(operation);
|
|
86
125
|
let live = await store.liveVersion(operation);
|
|
87
126
|
if (!live) {
|
|
88
127
|
const initial = opts.defaults(operation);
|
|
128
|
+
if (!initial)
|
|
129
|
+
return store.switchTo(operation, {
|
|
130
|
+
create: { settings: settingsOf(next), promptVersion },
|
|
131
|
+
kind: by.evidence === undefined ? "CHANGE" : "PROMOTE",
|
|
132
|
+
reason: by.reason,
|
|
133
|
+
by: by.by,
|
|
134
|
+
evidence: by.evidence,
|
|
135
|
+
expectLive: null,
|
|
136
|
+
});
|
|
89
137
|
if (sameSettings(initial, next))
|
|
90
138
|
return null;
|
|
91
139
|
live = await store.switchTo(operation, {
|
|
@@ -107,7 +155,11 @@ export function createSettings(opts) {
|
|
|
107
155
|
expectLive: live.id,
|
|
108
156
|
});
|
|
109
157
|
}
|
|
110
|
-
/**
|
|
158
|
+
/**
|
|
159
|
+
* Changes only some settings (e.g. the fallback), keeping the rest.
|
|
160
|
+
* Throws `NoSettingsError` for a feature with no settings to keep: use
|
|
161
|
+
* `change` or `seed` with full settings first.
|
|
162
|
+
*/
|
|
111
163
|
async function update(operation, patch, by) {
|
|
112
164
|
const { settings } = await current(operation);
|
|
113
165
|
return change(operation, { ...settings, ...patch }, by);
|
|
@@ -158,6 +210,35 @@ export function createSettings(opts) {
|
|
|
158
210
|
throw e;
|
|
159
211
|
}
|
|
160
212
|
}
|
|
213
|
+
/**
|
|
214
|
+
* Records existing settings as a feature's version 1, for an app moving
|
|
215
|
+
* its model choices into versioned settings (an import from its own
|
|
216
|
+
* config table). Only when the feature has no version yet: otherwise it
|
|
217
|
+
* changes nothing and returns null, so an import can run again safely.
|
|
218
|
+
* `promptVersion` defaults to the instructions' current hash.
|
|
219
|
+
*/
|
|
220
|
+
async function seed(operation, initial, by) {
|
|
221
|
+
if (await store.liveVersion(operation))
|
|
222
|
+
return null;
|
|
223
|
+
const promptVersion = by.promptVersion !== undefined
|
|
224
|
+
? by.promptVersion
|
|
225
|
+
: await prompt(operation);
|
|
226
|
+
try {
|
|
227
|
+
return await store.switchTo(operation, {
|
|
228
|
+
create: { settings: settingsOf(initial), promptVersion },
|
|
229
|
+
kind: "CHANGE",
|
|
230
|
+
reason: by.reason,
|
|
231
|
+
by: by.by,
|
|
232
|
+
expectLive: null,
|
|
233
|
+
});
|
|
234
|
+
}
|
|
235
|
+
catch (e) {
|
|
236
|
+
// Another request seeded it first: that version stands.
|
|
237
|
+
if (e instanceof SettingsConflict)
|
|
238
|
+
return null;
|
|
239
|
+
throw e;
|
|
240
|
+
}
|
|
241
|
+
}
|
|
161
242
|
/** Every switch, newest first, with the versions it went from and to. */
|
|
162
243
|
async function history(operation, limit = 50) {
|
|
163
244
|
const [changes, versions] = await Promise.all([
|
|
@@ -178,6 +259,15 @@ export function createSettings(opts) {
|
|
|
178
259
|
];
|
|
179
260
|
});
|
|
180
261
|
}
|
|
181
|
-
return {
|
|
262
|
+
return {
|
|
263
|
+
current,
|
|
264
|
+
route,
|
|
265
|
+
change,
|
|
266
|
+
update,
|
|
267
|
+
rollBack,
|
|
268
|
+
syncPrompt,
|
|
269
|
+
seed,
|
|
270
|
+
history,
|
|
271
|
+
};
|
|
182
272
|
}
|
|
183
273
|
//# sourceMappingURL=settings.js.map
|
package/dist/settings.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"settings.js","sourceRoot":"","sources":["../src/settings.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"settings.js","sourceRoot":"","sources":["../src/settings.ts"],"names":[],"mappings":"AAyGA,4DAA4D;AAC5D,MAAM,OAAO,gBAAiB,SAAQ,KAAK;IACzC,YAAY,SAAiB;QAC3B,KAAK,CACH,oBAAoB,SAAS,2CAA2C,CACzE,CAAC;QACF,IAAI,CAAC,IAAI,GAAG,kBAAkB,CAAC;IACjC,CAAC;CACF;AAED;;;;GAIG;AACH,MAAM,OAAO,eAAgB,SAAQ,KAAK;IACxC,oCAAoC;IAC3B,SAAS,CAAS;IAC3B,YAAY,SAAiB;QAC3B,KAAK,CACH,GAAG,SAAS,2DAA2D,CACxE,CAAC;QACF,IAAI,CAAC,IAAI,GAAG,iBAAiB,CAAC;QAC9B,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;IAC7B,CAAC;CACF;AAED,wDAAwD;AACxD,MAAM,UAAU,YAAY,CAAC,CAAW,EAAE,CAAW;IACnD,OAAO,CACL,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,KAAK;QACnB,CAAC,CAAC,aAAa,KAAK,CAAC,CAAC,aAAa;QACnC,CAAC,CAAC,eAAe,KAAK,CAAC,CAAC,eAAe;QACvC,CAAC,CAAC,eAAe,KAAK,CAAC,CAAC,eAAe;QACvC,CAAC,CAAC,CAAC,WAAW,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,WAAW,IAAI,IAAI,CAAC,CACpD,CAAC;AACJ,CAAC;AAED,sCAAsC;AACtC,MAAM,UAAU,UAAU,CAAC,CAAW;IACpC,OAAO;QACL,KAAK,EAAE,CAAC,CAAC,KAAK;QACd,aAAa,EAAE,CAAC,CAAC,aAAa;QAC9B,eAAe,EAAE,CAAC,CAAC,eAAe;QAClC,eAAe,EAAE,CAAC,CAAC,eAAe;QAClC,WAAW,EAAE,CAAC,CAAC,WAAW,IAAI,IAAI;KACnC,CAAC;AACJ,CAAC;AAED,yDAAyD;AACzD,MAAM,UAAU,cAAc,CAC5B,IAAqB,EACrB,EAAY,EACZ,QAAmC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAE3C,IAAI,CAAC,IAAI;QAAE,OAAO,CAAC,SAAS,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAC/C,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,IAAI,IAAI,CAAC,KAAK,KAAK,EAAE,CAAC,KAAK;QACzB,GAAG,CAAC,IAAI,CAAC,SAAS,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAC9D,IAAI,IAAI,CAAC,aAAa,KAAK,EAAE,CAAC,aAAa;QACzC,GAAG,CAAC,IAAI,CACN,aAAa,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,MAAM,MAAM,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAChI,CAAC;IACJ,IAAI,IAAI,CAAC,eAAe,KAAK,EAAE,CAAC,eAAe;QAC7C,GAAG,CAAC,IAAI,CACN,cAAc,IAAI,CAAC,eAAe,IAAI,SAAS,MAAM,EAAE,CAAC,eAAe,IAAI,SAAS,EAAE,CACvF,CAAC;IACJ,IAAI,IAAI,CAAC,eAAe,KAAK,EAAE,CAAC,eAAe;QAC7C,GAAG,CAAC,IAAI,CACN,iBAAiB,IAAI,CAAC,eAAe,IAAI,SAAS,MAAM,EAAE,CAAC,eAAe,IAAI,SAAS,EAAE,CAC1F,CAAC;IACJ,IAAI,CAAC,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,WAAW,IAAI,IAAI,CAAC;QACzD,GAAG,CAAC,IAAI,CACN,gBAAgB,IAAI,CAAC,WAAW,IAAI,SAAS,MAAM,EAAE,CAAC,WAAW,IAAI,SAAS,EAAE,CACjF,CAAC;IACJ,OAAO,GAAG,CAAC;AACb,CAAC;AAQD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AACH,MAAM,UAAU,cAAc,CAAC,IAU9B;IACC,MAAM,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC;IACvB,MAAM,MAAM,GAAG,KAAK,EAAE,EAAU,EAAE,EAAE,CAClC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAE3D;;;OAGG;IACH,KAAK,UAAU,OAAO,CACpB,SAAiB;QAEjB,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC;QACnD,IAAI,OAAO;YAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,CAAC;QAC/D,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;QAC1C,IAAI,CAAC,QAAQ;YAAE,MAAM,IAAI,eAAe,CAAC,SAAS,CAAC,CAAC;QACpD,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC;IACzC,CAAC;IAED,2GAA2G;IAC3G,KAAK,UAAU,KAAK,CAAC,SAAiB;QACpC,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,CAAC;QAC9C,OAAO;YACL,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,aAAa,EAAE,QAAQ,CAAC,aAAa;YACrC,eAAe,EAAE,QAAQ,CAAC,eAAe;YACzC,WAAW,EAAE,QAAQ,CAAC,WAAW,IAAI,IAAI;YACzC,eAAe,EAAE,QAAQ,CAAC,eAAe;SAC1C,CAAC;IACJ,CAAC;IAED;;;;;;OAMG;IACH,KAAK,UAAU,MAAM,CACnB,SAAiB,EACjB,IAAc,EACd,EAAoE;QAEpE,MAAM,aAAa,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC,CAAC;QAC9C,IAAI,IAAI,GAAG,MAAM,KAAK,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC;QAC9C,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;YACzC,IAAI,CAAC,OAAO;gBACV,OAAO,KAAK,CAAC,QAAQ,CAAC,SAAS,EAAE;oBAC/B,MAAM,EAAE,EAAE,QAAQ,EAAE,UAAU,CAAC,IAAI,CAAC,EAAE,aAAa,EAAE;oBACrD,IAAI,EAAE,EAAE,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS;oBACtD,MAAM,EAAE,EAAE,CAAC,MAAM;oBACjB,EAAE,EAAE,EAAE,CAAC,EAAE;oBACT,QAAQ,EAAE,EAAE,CAAC,QAAQ;oBACrB,UAAU,EAAE,IAAI;iBACjB,CAAC,CAAC;YACL,IAAI,YAAY,CAAC,OAAO,EAAE,IAAI,CAAC;gBAAE,OAAO,IAAI,CAAC;YAC7C,IAAI,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,SAAS,EAAE;gBACrC,MAAM,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,aAAa,EAAE;gBAC5C,IAAI,EAAE,QAAQ;gBACd,MAAM,EAAE,gDAAgD;gBACxD,EAAE,EAAE,IAAI;gBACR,UAAU,EAAE,IAAI;aACjB,CAAC,CAAC;QACL,CAAC;aAAM,IAAI,YAAY,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC;YAAE,OAAO,IAAI,CAAC;QAC7D,OAAO,KAAK,CAAC,QAAQ,CAAC,SAAS,EAAE;YAC/B,MAAM,EAAE,EAAE,QAAQ,EAAE,UAAU,CAAC,IAAI,CAAC,EAAE,aAAa,EAAE;YACrD,IAAI,EAAE,EAAE,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS;YACtD,MAAM,EAAE,EAAE,CAAC,MAAM;YACjB,EAAE,EAAE,EAAE,CAAC,EAAE;YACT,QAAQ,EAAE,EAAE,CAAC,QAAQ;YACrB,UAAU,EAAE,IAAI,CAAC,EAAE;SACpB,CAAC,CAAC;IACL,CAAC;IAED;;;;OAIG;IACH,KAAK,UAAU,MAAM,CACnB,SAAiB,EACjB,KAAwB,EACxB,EAAoE;QAEpE,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,CAAC;QAC9C,OAAO,MAAM,CAAC,SAAS,EAAE,EAAE,GAAG,QAAQ,EAAE,GAAG,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC;IAC1D,CAAC;IAED,2CAA2C;IAC3C,KAAK,UAAU,QAAQ,CACrB,SAAiB,EACjB,SAAiB,EACjB,EAAyC;QAEzC,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;QACrD,MAAM,EAAE,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,SAAS,CAAC,CAAC;QACpD,IAAI,CAAC,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;QAC7D,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC;QAChD,IAAI,IAAI,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE;YAAE,OAAO,EAAE,CAAC;QAClC,OAAO,KAAK,CAAC,QAAQ,CAAC,SAAS,EAAE;YAC/B,SAAS,EAAE,EAAE,CAAC,EAAE;YAChB,IAAI,EAAE,UAAU;YAChB,MAAM,EAAE,EAAE,CAAC,MAAM;YACjB,EAAE,EAAE,EAAE,CAAC,EAAE;YACT,UAAU,EAAE,IAAI,EAAE,EAAE,IAAI,IAAI;SAC7B,CAAC,CAAC;IACL,CAAC;IAED;;;;;OAKG;IACH,KAAK,UAAU,UAAU,CACvB,SAAiB;QAEjB,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC;QAChD,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC,CAAC;QACpC,IAAI,CAAC,IAAI,IAAI,CAAC,GAAG,IAAI,GAAG,KAAK,IAAI,CAAC,aAAa;YAAE,OAAO,IAAI,CAAC;QAC7D,IAAI,CAAC;YACH,OAAO,MAAM,KAAK,CAAC,QAAQ,CAAC,SAAS,EAAE;gBACrC,MAAM,EAAE,EAAE,QAAQ,EAAE,UAAU,CAAC,IAAI,CAAC,EAAE,aAAa,EAAE,GAAG,EAAE;gBAC1D,IAAI,EAAE,QAAQ;gBACd,MAAM,EAAE,IAAI,CAAC,aAAa;oBACxB,CAAC,CAAC,yBAAyB,IAAI,CAAC,aAAa,MAAM,GAAG,GAAG;oBACzD,CAAC,CAAC,0BAA0B,GAAG,GAAG;gBACpC,EAAE,EAAE,IAAI;gBACR,UAAU,EAAE,IAAI,CAAC,EAAE;aACpB,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,kDAAkD;YAClD,IAAI,CAAC,YAAY,gBAAgB;gBAAE,OAAO,KAAK,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC;YACvE,MAAM,CAAC,CAAC;QACV,CAAC;IACH,CAAC;IAED;;;;;;OAMG;IACH,KAAK,UAAU,IAAI,CACjB,SAAiB,EACjB,OAAiB,EACjB,EAIC;QAED,IAAI,MAAM,KAAK,CAAC,WAAW,CAAC,SAAS,CAAC;YAAE,OAAO,IAAI,CAAC;QACpD,MAAM,aAAa,GACjB,EAAE,CAAC,aAAa,KAAK,SAAS;YAC5B,CAAC,CAAC,EAAE,CAAC,aAAa;YAClB,CAAC,CAAC,MAAM,MAAM,CAAC,SAAS,CAAC,CAAC;QAC9B,IAAI,CAAC;YACH,OAAO,MAAM,KAAK,CAAC,QAAQ,CAAC,SAAS,EAAE;gBACrC,MAAM,EAAE,EAAE,QAAQ,EAAE,UAAU,CAAC,OAAO,CAAC,EAAE,aAAa,EAAE;gBACxD,IAAI,EAAE,QAAQ;gBACd,MAAM,EAAE,EAAE,CAAC,MAAM;gBACjB,EAAE,EAAE,EAAE,CAAC,EAAE;gBACT,UAAU,EAAE,IAAI;aACjB,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,wDAAwD;YACxD,IAAI,CAAC,YAAY,gBAAgB;gBAAE,OAAO,IAAI,CAAC;YAC/C,MAAM,CAAC,CAAC;QACV,CAAC;IACH,CAAC;IAED,yEAAyE;IACzE,KAAK,UAAU,OAAO,CACpB,SAAiB,EACjB,KAAK,GAAG,EAAE;QAEV,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAC5C,KAAK,CAAC,WAAW,CAAC,SAAS,EAAE,EAAE,KAAK,EAAE,CAAC;YACvC,KAAK,CAAC,YAAY,CAAC,SAAS,CAAC;SAC9B,CAAC,CAAC;QACH,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACrD,OAAO,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE;YAC3B,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC;YACnC,IAAI,CAAC,EAAE;gBAAE,OAAO,EAAE,CAAC;YACnB,OAAO;gBACL;oBACE,GAAG,CAAC;oBACJ,EAAE;oBACF,IAAI,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,aAAa,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI;iBACnE;aACF,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC;IAED,OAAO;QACL,OAAO;QACP,KAAK;QACL,MAAM;QACN,MAAM;QACN,QAAQ;QACR,UAAU;QACV,IAAI;QACJ,OAAO;KACR,CAAC;AACJ,CAAC"}
|
package/dist/store.d.ts
CHANGED
|
@@ -69,6 +69,16 @@ export type RunRecord = {
|
|
|
69
69
|
promptVersion: string;
|
|
70
70
|
/** The reasoning setting tested; null: the model's own default. */
|
|
71
71
|
reasoningEffort: string | null;
|
|
72
|
+
/**
|
|
73
|
+
* The temperature tested; null or unset: none sent. Stores keep it, so a
|
|
74
|
+
* test at another temperature than the live one shows as out of date.
|
|
75
|
+
*/
|
|
76
|
+
temperature?: number | null;
|
|
77
|
+
/**
|
|
78
|
+
* The output-token cap tested; null or unset: the call's own default.
|
|
79
|
+
* Kept like temperature: a test at another cap is out of date.
|
|
80
|
+
*/
|
|
81
|
+
maxOutputTokens?: number | null;
|
|
72
82
|
set: EvalSet;
|
|
73
83
|
repeats: number;
|
|
74
84
|
/** Changes when the cases or their answers do, so tests compare. */
|
|
@@ -117,8 +127,14 @@ export type NewCase = Omit<CaseRecord, "id" | "createdAt" | "updatedAt" | "label
|
|
|
117
127
|
* database, and run describeStoreContract against it.
|
|
118
128
|
*/
|
|
119
129
|
export interface EvalStore {
|
|
120
|
-
/**
|
|
121
|
-
|
|
130
|
+
/**
|
|
131
|
+
* A suite's cases that aren't retired, with their answers, oldest first.
|
|
132
|
+
* With `includeRetired`, retired ones too (an import checks them, so a
|
|
133
|
+
* case someone retired isn't added again).
|
|
134
|
+
*/
|
|
135
|
+
listCases(suite: string, opts?: {
|
|
136
|
+
includeRetired?: boolean;
|
|
137
|
+
}): Promise<CaseRecord[]>;
|
|
122
138
|
createCase(data: NewCase): Promise<CaseRecord>;
|
|
123
139
|
/** Adds an answer to a case (answers are never overwritten). */
|
|
124
140
|
addLabel(caseId: string, label: NewLabel): Promise<LabelRecord>;
|
package/dist/suggest.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { retiresLabel } from "./models.js";
|
|
1
|
+
import { isBaseline, retiresLabel } from "./models.js";
|
|
2
2
|
import { at } from "./util.js";
|
|
3
3
|
/**
|
|
4
4
|
* How many serious checks a model's test failed; Infinity without a test.
|
|
@@ -43,7 +43,9 @@ models = {}) {
|
|
|
43
43
|
const retiring = (r) => retiresLabel(models, r.model);
|
|
44
44
|
const limitMs = priority.maxSeconds ? priority.maxSeconds * 1000 : Infinity;
|
|
45
45
|
const tooSlow = (r) => ms(r) !== Infinity && ms(r) > limitMs;
|
|
46
|
-
|
|
46
|
+
// Baselines are code, compared on the board but never suggested.
|
|
47
|
+
const tested = rows.filter((r) => !isBaseline(models, r.model) &&
|
|
48
|
+
r.test &&
|
|
47
49
|
!r.test.stale &&
|
|
48
50
|
!r.test.failedAnswers &&
|
|
49
51
|
!retiring(r) &&
|
package/dist/suggest.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"suggest.js","sourceRoot":"","sources":["../src/suggest.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAqB,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"suggest.js","sourceRoot":"","sources":["../src/suggest.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,YAAY,EAAqB,MAAM,aAAa,CAAC;AAI1E,OAAO,EAAE,EAAE,EAAE,MAAM,WAAW,CAAC;AA0D/B;;GAEG;AACH,MAAM,UAAU,eAAe,CAAC,GAAe,EAAE,MAAkB;IACjE,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;IACtB,IAAI,CAAC,IAAI;QAAE,OAAO,QAAQ,CAAC;IAC3B,OAAO,MAAM;SACV,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC;SACvC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACf,MAAM,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAC9B,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACtC,CAAC,EAAE,CAAC,CAAC,CAAC;AACV,CAAC;AAED,MAAM,QAAQ,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,GAAG,CAAC,mBAAmB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;AAC5E,MAAM,IAAI,GAAG,CAAC,CAAa,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAC3E,MAAM,GAAG,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC;AACrD,MAAM,IAAI,GAAG,CAAC,CAAa,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,aAAa,IAAI,QAAQ,CAAC;AAClE,MAAM,EAAE,GAAG,CAAC,CAAa,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,IAAI,QAAQ,CAAC;AACxD,wEAAwE;AACxE,MAAM,cAAc,GAAG,GAAG,CAAC;AAC3B,6DAA6D;AAC7D,MAAM,IAAI,GAAG,GAAG,CAAC;AACjB,0BAA0B;AAC1B,MAAM,WAAW,GAAG,CAAC,CAAS,EAAE,CAAS,EAAE,EAAE,CAC3C,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC;AAE3D;;;;;;;;;;GAUG;AACH,MAAM,UAAU,OAAO,CACrB,IAAS,EACT,MAAkB,EAClB,WAA0B,EAAE,KAAK,EAAE,SAAS,EAAE;AAC9C,wEAAwE;AACxE,SAAuB,EAAE;IAEzB,MAAM,QAAQ,GAAG,CAAC,KAAa,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,QAAQ,IAAI,KAAK,CAAC;IACrE,MAAM,QAAQ,GAAG,CAAC,CAAa,EAAE,EAAE,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC;IAClE,MAAM,OAAO,GAAG,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC;IAC5E,MAAM,OAAO,GAAG,CAAC,CAAa,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,QAAQ,IAAI,EAAE,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC;IACzE,iEAAiE;IACjE,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CACxB,CAAC,CAAC,EAAE,EAAE,CACJ,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC;QAC5B,CAAC,CAAC,IAAI;QACN,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK;QACb,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa;QACrB,CAAC,QAAQ,CAAC,CAAC,CAAC;QACZ,CAAC,OAAO,CAAC,CAAC,CAAC,CACd,CAAC;IACF,IAAI,CAAC,MAAM,CAAC,MAAM;QAAE,OAAO,IAAI,CAAC;IAChC,MAAM,OAAO,GAAG,CAAC,CAAI,EAAE,EAAE,CAAC,eAAe,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;IACrD,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC;IAChD,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACzC,MAAM,WAAW,GAAG,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IACvD,MAAM,QAAQ,GACZ,OAAO,EAAE,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,IAAI,OAAO,CAAC,OAAO,CAAC;QACtD,CAAC,CAAC,GAAG,OAAO,CAAC,KAAK,UAAU,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,wBAAwB,QAAQ,CAAC,UAAU,UAAU;QAChH,CAAC,CAAC,IAAI,CAAC;IAEX,kDAAkD;IAClD,IAAI,IAAO,CAAC;IACZ,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;IACvD,IAAI,CAAC,IAAI,EAAE,CAAC;QACV,IAAI,CAAC,QAAQ,CAAC,MAAM;YAAE,OAAO,IAAI,CAAC;QAClC,IAAI,GAAG,EAAE,CACP,CAAC,GAAG,QAAQ,CAAC,CAAC,IAAI,CAChB,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACP,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CACpE,EACD,CAAC,CACF,CAAC;QACF,OAAO,CAAC,IAAI,CACV,WAAW,IAAI,QAAQ;YACrB,CAAC,CAAC,GAAG,WAAW,IAAI,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,KAAK,eAAe,WAAW,GAAG,CAAC,CAAC,CAAC,QAAQ,IAAI,IAAI,CAAC,KAAK,gCAAgC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,UAAU,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG;YACvL,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,oBAAoB,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,UAAU,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,0DAA0D,CAChJ,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,MAAM,GAAG,GAAG,CAAC,CAAI,EAAiB,EAAE;YAClC,IAAI,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC;gBAAE,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC;YAC1C,IAAI,CAAC,CAAC,MAAM,KAAK,QAAQ;gBACvB,OAAO,GAAG,CAAC,CAAC,KAAK,kCAAkC,IAAI,CAAC,KAAK,KAAK,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC;YAC/G,IAAI,OAAO,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC;gBACjC,OAAO,GAAG,CAAC,CAAC,KAAK,sCAAsC,IAAI,CAAC,KAAK,KAAK,OAAO,CAAC,CAAC,CAAC,YAAY,OAAO,CAAC,IAAI,CAAC,mCAAmC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC;YAC1L,IAAI,QAAQ,CAAC,KAAK,KAAK,MAAM,IAAI,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;gBAC/D,OAAO,GAAG,CAAC,CAAC,KAAK,+BAA+B,IAAI,CAAC,KAAK,KAAK,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,QAAQ,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC,oBAAoB,CAAC;YACjL,IAAI,QAAQ,CAAC,KAAK,KAAK,OAAO,IAAI,WAAW,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC;gBAC5D,OAAO,GAAG,CAAC,CAAC,KAAK,+BAA+B,IAAI,CAAC,KAAK,KAAK,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,yBAAyB,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,aAAa,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC;YACzM,OAAO,IAAI,CAAC;QACd,CAAC,CAAC;QACF,sEAAsE;QACtE,uCAAuC;QACvC,MAAM,KAAK,GAAG,CAAC,CAAI,EAAiB,EAAE;YACpC,IACE,QAAQ,CAAC,KAAK,KAAK,SAAS;gBAC5B,CAAC,CAAC,MAAM,KAAK,MAAM;gBACnB,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,GAAG,cAAc;gBACrC,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;gBAE1B,OAAO,IAAI,CAAC;YACd,MAAM,KAAK,GAAG,GAAG,CAAC,CAAC,KAAK,4BAA4B,IAAI,CAAC,KAAK,KAAK,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,0EAA0E,CAAC;YACrL,IAAI,QAAQ,CAAC,KAAK,KAAK,MAAM,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI;gBAC3D,OAAO,GAAG,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC,4CAA4C,CAAC;YAClH,IAAI,QAAQ,CAAC,KAAK,KAAK,OAAO,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,GAAG,IAAI;gBACxD,OAAO,GAAG,KAAK,oBAAoB,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,aAAa,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,6BAA6B,CAAC;YACrI,OAAO,IAAI,CAAC;QACd,CAAC,CAAC;QACF,MAAM,UAAU,GAAG,QAAQ;aACxB,MAAM,CACL,CAAC,CAAC,EAAE,EAAE,CACJ,CAAC,CAAC,CAAC,IAAI;YACP,CAAC,CAAC,MAAM,KAAK,OAAO;YACpB,OAAO,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC;YAC3B,GAAG,CAAC,CAAC,CAAC,CACT;aACA,IAAI,CACH,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACP,MAAM,CAAC,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC;YAC7D,gDAAgD;YAChD,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC;YAC7D,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC;YACjB,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;YACvB,CAAC,QAAQ,CAAC,KAAK,KAAK,OAAO;gBACzB,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC;gBACpC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAC1C,CAAC;QACJ,IAAI,UAAU,CAAC,MAAM,EAAE,CAAC;YACtB,IAAI,GAAG,EAAE,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;YACzB,MAAM,MAAM,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC;YACzB,IAAI,MAAM;gBAAE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACnC,CAAC;aAAM,CAAC;YACN,IAAI,GAAG,IAAI,CAAC;YACZ,OAAO,CAAC,IAAI,CACV,QAAQ,IAAI,CAAC,KAAK,mDAAmD,CACtE,CAAC;YACF,8DAA8D;YAC9D,MAAM,KAAK,GAAG,MAAM;iBACjB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC;iBAC9C,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACxC,IAAI,KAAK;gBACP,OAAO,CAAC,IAAI,CACV,GAAG,KAAK,CAAC,KAAK,oBAAoB,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,YAAY,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,+DAA+D,OAAO,CAAC,KAAK,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,yCAAyC,OAAO,CAAC,KAAK,CAAC,YAAY,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,+BAA+B,CACrS,CAAC;QACN,CAAC;IACH,CAAC;IAED,wEAAwE;IACxE,kEAAkE;IAClE,MAAM,QAAQ,GACZ,MAAM;SACH,MAAM,CACL,CAAC,CAAC,EAAE,EAAE,CACJ,CAAC,CAAC,CAAC,OAAO;QACV,CAAC,CAAC,CAAC,QAAQ;QACX,CAAC,CAAC,GAAG,KAAK,IAAI,CAAC,GAAG;QAClB,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAC7C;SACA,IAAI,CACH,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACP,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CACpE,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC;IACjB,IAAI,QAAQ;QACV,OAAO,CAAC,IAAI,CACV,aAAa,QAAQ,CAAC,KAAK,gDAAgD,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,KAAK,QAAQ,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,IAAI,CACnI,CAAC;IAEJ,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC;IAC3B,OAAO;QACL,IAAI,EAAE,IAAI;QACV,QAAQ;QACR,QAAQ;QACR,IAAI,EAAE,QAAQ,IAAI,CAAC,CAAC,QAAQ,IAAI,QAAQ,CAAC,QAAQ,CAAC;QAClD,OAAO;KACR,CAAC;AACJ,CAAC"}
|
package/dist/testing.js
CHANGED
|
@@ -77,6 +77,8 @@ export function describeStoreContract(name, make) {
|
|
|
77
77
|
expect((await store.listCases("s")).map((c) => c.id)).toEqual([
|
|
78
78
|
second.id,
|
|
79
79
|
]);
|
|
80
|
+
// On request, retired ones too, still oldest first.
|
|
81
|
+
expect((await store.listCases("s", { includeRetired: true })).map((c) => c.id)).toEqual([first.id, second.id]);
|
|
80
82
|
});
|
|
81
83
|
it("adds answers without overwriting, and changes their status", async () => {
|
|
82
84
|
const store = await make();
|
|
@@ -109,6 +111,26 @@ export function describeStoreContract(name, make) {
|
|
|
109
111
|
expect((await store.listRuns("s", { status: "SUCCEEDED" })).map((r) => r.id)).toEqual([a.id]);
|
|
110
112
|
expect(await store.getRun("missing")).toBeNull();
|
|
111
113
|
});
|
|
114
|
+
it("keeps the temperature a run was tested at", async () => {
|
|
115
|
+
const store = await make();
|
|
116
|
+
const warm = await store.createRun({ ...newRun("s"), temperature: 0.7 });
|
|
117
|
+
const none = await store.createRun({ ...newRun("s"), temperature: null });
|
|
118
|
+
expect((await store.getRun(warm.id))?.temperature).toBe(0.7);
|
|
119
|
+
expect((await store.getRun(none.id))?.temperature ?? null).toBeNull();
|
|
120
|
+
});
|
|
121
|
+
it("keeps the output cap a run was tested at", async () => {
|
|
122
|
+
const store = await make();
|
|
123
|
+
const capped = await store.createRun({
|
|
124
|
+
...newRun("s"),
|
|
125
|
+
maxOutputTokens: 4000,
|
|
126
|
+
});
|
|
127
|
+
const none = await store.createRun({
|
|
128
|
+
...newRun("s"),
|
|
129
|
+
maxOutputTokens: null,
|
|
130
|
+
});
|
|
131
|
+
expect((await store.getRun(capped.id))?.maxOutputTokens).toBe(4000);
|
|
132
|
+
expect((await store.getRun(none.id))?.maxOutputTokens ?? null).toBeNull();
|
|
133
|
+
});
|
|
112
134
|
it("updates a run only while it has the status it's guarded by", async () => {
|
|
113
135
|
const store = await make();
|
|
114
136
|
const r = await store.createRun(newRun("s"));
|
|
@@ -329,6 +351,32 @@ export function describeSettingsStoreContract(name, make) {
|
|
|
329
351
|
expect((await store.liveVersion("triage"))?.model).toBe("b");
|
|
330
352
|
expect(await store.listVersions("triage")).toHaveLength(2);
|
|
331
353
|
});
|
|
354
|
+
it("keeps a version's reasoning setting as named, even a newer one", async () => {
|
|
355
|
+
const store = await make();
|
|
356
|
+
const v1 = await store.switchTo("triage", {
|
|
357
|
+
...create("a", null),
|
|
358
|
+
create: {
|
|
359
|
+
settings: { ...settings("a"), reasoningEffort: "xhigh" },
|
|
360
|
+
promptVersion: "p1",
|
|
361
|
+
},
|
|
362
|
+
});
|
|
363
|
+
expect(v1.reasoningEffort).toBe("xhigh");
|
|
364
|
+
expect((await store.liveVersion("triage"))?.reasoningEffort).toBe("xhigh");
|
|
365
|
+
});
|
|
366
|
+
it("keeps a version's temperature", async () => {
|
|
367
|
+
const store = await make();
|
|
368
|
+
const v1 = await store.switchTo("triage", {
|
|
369
|
+
...create("a", null),
|
|
370
|
+
create: {
|
|
371
|
+
settings: { ...settings("a"), temperature: 0.4 },
|
|
372
|
+
promptVersion: "p1",
|
|
373
|
+
},
|
|
374
|
+
});
|
|
375
|
+
expect(v1.temperature).toBe(0.4);
|
|
376
|
+
expect((await store.liveVersion("triage"))?.temperature).toBe(0.4);
|
|
377
|
+
const v2 = await store.switchTo("triage", create("b", v1.id));
|
|
378
|
+
expect(v2.temperature ?? null).toBeNull();
|
|
379
|
+
});
|
|
332
380
|
it("keeps each feature's versions and live one apart", async () => {
|
|
333
381
|
const store = await make();
|
|
334
382
|
await store.switchTo("triage", create("a", null));
|
package/dist/testing.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"testing.js","sourceRoot":"","sources":["../src/testing.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EACL,gBAAgB,GAGjB,MAAM,eAAe,CAAC;AAIvB;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,qBAAqB,CACnC,IAAY,EACZ,IAA0C;IAE1C,MAAM,OAAO,GAAG,CAAC,KAAa,EAAE,OAAyB,EAAE,EAAW,EAAE,CAAC,CAAC;QACxE,KAAK;QACL,KAAK,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE;QAC1C,IAAI,EAAE,CAAC,GAAG,EAAE,GAAG,CAAC;QAChB,KAAK,EAAE,KAAK;QACZ,MAAM,EAAE,QAAQ;QAChB,MAAM,EAAE,OAAO;QACf,SAAS,EAAE,IAAI;QACf,IAAI,EAAE,IAAI;QACV,MAAM,EAAE;YACN;gBACE,IAAI,EAAE,UAAU;gBAChB,MAAM,EAAE,WAAW;gBACnB,KAAK,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE;gBACxB,SAAS,EAAE,OAAO;gBAClB,OAAO,EAAE,IAAI;gBACb,IAAI,EAAE,IAAI;aACX;SACF;QACD,GAAG,IAAI;KACR,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,CAAC,KAAa,EAAE,EAAE,CAAC,CAAC;QACjC,KAAK;QACL,KAAK,EAAE,SAAS;QAChB,aAAa,EAAE,IAAI;QACnB,eAAe,EAAE,IAAI;QACrB,GAAG,EAAE,KAAc;QACnB,OAAO,EAAE,CAAC;QACV,cAAc,EAAE,IAAI;QACpB,aAAa,EAAE,GAAG;QAClB,gBAAgB,EAAE,IAAI;QACtB,SAAS,EAAE,IAAI;KAChB,CAAC,CAAC;IAEH,QAAQ,CAAC,uBAAuB,IAAI,EAAE,EAAE,GAAG,EAAE;QAC3C,EAAE,CAAC,kDAAkD,EAAE,KAAK,IAAI,EAAE;YAChE,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC;YAC/C,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;YAC5C,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAC9B,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;YACnE,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;YACzC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YACvC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC;gBACtC,IAAI,EAAE,UAAU;gBAChB,MAAM,EAAE,WAAW;gBACnB,KAAK,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE;aACzB,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8DAA8D,EAAE,KAAK,IAAI,EAAE;YAC5E,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC;YACnD,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC;YACpD,MAAM,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;YACzC,MAAM,KAAK,CAAC,aAAa,CAAC,KAAK,CAAC,EAAE,EAAE,SAAS,CAAC,CAAC;YAC/C,MAAM,CAAC,CAAC,MAAM,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC;gBAC5D,MAAM,CAAC,EAAE;aACV,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4DAA4D,EAAE,KAAK,IAAI,EAAE;YAC1E,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC;YAC/D,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,EAAE;gBACnC,IAAI,EAAE,UAAU;gBAChB,MAAM,EAAE,SAAS;gBACjB,KAAK,EAAE,EAAE,IAAI,EAAE,GAAG,EAAE;gBACpB,SAAS,EAAE,OAAO;gBAClB,OAAO,EAAE,IAAI;gBACb,IAAI,EAAE,IAAI;aACX,CAAC,CAAC;YACH,MAAM,KAAK,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,EAAE,WAAW,CAAC,CAAC;YAC9C,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;YAC5C,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YACvC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACtD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8DAA8D,EAAE,KAAK,IAAI,EAAE;YAC5E,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YAC7C,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;YAC3C,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YAC7C,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACjC,MAAM,CAAC,CAAC,MAAM,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC;gBAC3D,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,EAAE;aACL,CAAC,CAAC;YACH,MAAM,CAAC,MAAM,KAAK,CAAC,QAAQ,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAChE,MAAM,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,CAAC;YACrD,MAAM,CACJ,CAAC,MAAM,KAAK,CAAC,QAAQ,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CACtE,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YAClB,MAAM,CAAC,MAAM,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;QACnD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4DAA4D,EAAE,KAAK,IAAI,EAAE;YAC1E,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YAC7C,MAAM,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;YACzE,MAAM,KAAK,CAAC,SAAS,CACnB,CAAC,CAAC,EAAE,EACJ,EAAE,MAAM,EAAE,WAAW,EAAE,EACvB,EAAE,MAAM,EAAE,SAAS,EAAE,CACtB,CAAC;YACF,MAAM,CAAC,CAAC,MAAM,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC5D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8DAA8D,EAAE,KAAK,IAAI,EAAE;YAC5E,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC;YAC/C,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YAC7C,MAAM,IAAI,GAAG;gBACX,KAAK,EAAE,CAAC,CAAC,EAAE;gBACX,MAAM,EAAE,CAAC,CAAC,EAAE;gBACZ,MAAM,EAAE,CAAC;gBACT,WAAW,EAAE,GAAG;gBAChB,MAAM,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE;gBACzB,KAAK,EAAE,IAAI;gBACX,MAAM,EAAE,IAAI;gBACZ,OAAO,EAAE,IAAI;gBACb,EAAE,EAAE,EAAE;gBACN,MAAM,EAAE,KAAK;aACd,CAAC;YACF,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;YACzC,MAAM,KAAK,CAAC,YAAY,CAAC,EAAE,GAAG,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;YACpD,MAAM,KAAK,CAAC,YAAY,CAAC,EAAE,GAAG,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;YACnE,MAAM,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;YAClE,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAC3C,MAAM,CAAC,IAAI,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAC7B,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC,OAAO,CAAC;gBACtD,EAAE,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE;aAC7B,CAAC,CAAC;YACH,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,kBAAkB,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;YAC/D,MAAM,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAC7B,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;YACjD,MAAM,CACJ,MAAM,KAAK,CAAC,kBAAkB,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,CAAC,CACnE,CAAC,QAAQ,EAAE,CAAC;QACf,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,sCAAsC,EAAE,KAAK,IAAI,EAAE;YACpD,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,MAAM,IAAI,GACR;gBACE,GAAG,EAAE,UAAU;gBACf,SAAS,EAAE,UAAU;gBACrB,QAAQ,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE;gBAC9B,KAAK,EAAE,UAAU;gBACjB,WAAW,EAAE,kBAAkB;gBAC/B,aAAa,EAAE,GAAG;gBAClB,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC;gBACrC,UAAU,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC;gBAC1C,MAAM,EAAE;oBACN;wBACE,GAAG,EAAE,OAAO;wBACZ,KAAK,EAAE,SAAS;wBAChB,WAAW,EAAE,EAAE;wBACf,IAAI,EAAE,OAAO;wBACb,QAAQ,EAAE,SAAS;qBACpB;iBACF;gBACD,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;gBAClD,KAAK,EAAE,KAAK,EAAE,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC;oBAC9B,EAAE,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,KAAK,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE;iBAC7D;gBACD,aAAa,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC;gBACnC,KAAK,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI;aACrB,CAAC;YACJ,MAAM,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;YAC5C,MAAM,MAAM,GAAG,YAAY,CAAC;gBAC1B,KAAK;gBACL,MAAM,EAAE,CAAC,IAAI,CAAC;gBACd,MAAM,EAAE,EAAE,SAAS,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE;gBAC7D,aAAa,EAAE,GAAG,EAAE,CAAC,IAAI;aAC1B,CAAC,CAAC;YACH,MAAM,EAAE,GAAG,EAAE,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC;gBACrC,KAAK,EAAE,UAAU;gBACjB,KAAK,EAAE,SAAS;aACjB,CAAC,CAAC;YACH,MAAM,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAC7B,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACxC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YACvC,MAAM,CAAC,IAAI,EAAE,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAChD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,6BAA6B,CAC3C,IAAY,EACZ,IAAkD;IAElD,MAAM,MAAM,GAAG,CAAC,OAA2B,EAAE,EAAa,EAAE,CAAC,CAAC;QAC5D,SAAS,EAAE,QAAQ;QACnB,QAAQ,EAAE,UAAU;QACpB,OAAO,EAAE,UAAU;QACnB,MAAM,EAAE,MAAM;QACd,MAAM,EAAE,aAAa;QACrB,OAAO,EAAE,SAAS;QAClB,KAAK,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,mBAAmB,EAAE;QACvD,MAAM,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE;QAC3B,GAAG,IAAI;KACR,CAAC,CAAC;IAEH,QAAQ,CAAC,2BAA2B,IAAI,EAAE,EAAE,GAAG,EAAE;QAC/C,EAAE,CAAC,yDAAyD,EAAE,KAAK,IAAI,EAAE;YACvE,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;YACrC,MAAM,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC;gBACtB,SAAS,EAAE,QAAQ;gBACnB,MAAM,EAAE,MAAM;gBACd,MAAM,EAAE,aAAa;gBACrB,OAAO,EAAE,SAAS;gBAClB,KAAK,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,mBAAmB,EAAE;gBACvD,MAAM,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE;gBAC3B,MAAM,EAAE,MAAM;gBACd,MAAM,EAAE,IAAI;aACb,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,mEAAmE,EAAE,KAAK,IAAI,EAAE;YACjF,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;YACzC,MAAM,KAAK,CAAC,iBAAiB,CAAC,KAAK,CAAC,EAAE,EAAE,WAAW,CAAC,CAAC;YACrD,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;YAC3D,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAClC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACpC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,CAAC;YAClC,MAAM,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC;YAClD,MAAM,CAAC,MAAM,KAAK,CAAC,YAAY,EAAE,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACrD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,qBAAqB,EAAE,KAAK,IAAI,EAAE;YACnC,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,MAAM,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;YAC3B,MAAM,KAAK,CAAC,WAAW,CAAC,UAAU,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC;YAC1D,MAAM,CAAC,MAAM,KAAK,CAAC,YAAY,EAAE,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACrD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4DAA4D,EAAE,KAAK,IAAI,EAAE;YAC1E,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;YACtD,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;YAC3C,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;YACpE,MAAM,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,GAAG,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC;YAClE,MAAM,CACJ,CAAC,MAAM,KAAK,CAAC,YAAY,CAAC,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CACrE,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YACxB,MAAM,CAAC,MAAM,KAAK,CAAC,YAAY,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YACnE,MAAM,KAAK,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YACvD,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,YAAY,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC;YAC5D,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC;YACvE,MAAM,CACJ,MAAM,KAAK,CAAC,YAAY,CAAC,EAAE,KAAK,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,EAAE,CAAC,CACnE,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAClB,MAAM,CAAC,MAAM,KAAK,CAAC,YAAY,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACjE,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,6BAA6B,CAC3C,IAAY,EACZ,IAAkD;IAElD,MAAM,QAAQ,GAAG,CAAC,KAAa,EAAY,EAAE,CAAC,CAAC;QAC7C,KAAK;QACL,aAAa,EAAE,IAAI;QACnB,eAAe,EAAE,IAAI;QACrB,eAAe,EAAE,IAAI;KACtB,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,CAAC,KAAa,EAAE,UAAyB,EAAE,EAAE,CAAC,CAAC;QAC5D,MAAM,EAAE,EAAE,QAAQ,EAAE,QAAQ,CAAC,KAAK,CAAC,EAAE,aAAa,EAAE,IAAI,EAAE;QAC1D,IAAI,EAAE,QAAiB;QACvB,MAAM,EAAE,MAAM,KAAK,EAAE;QACrB,EAAE,EAAE,UAAU;QACd,UAAU;KACX,CAAC,CAAC;IAEH,QAAQ,CAAC,2BAA2B,IAAI,EAAE,EAAE,GAAG,EAAE;QAC/C,EAAE,CAAC,gEAAgE,EAAE,KAAK,IAAI,EAAE;YAC9E,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,MAAM,CAAC,MAAM,KAAK,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;YACrD,MAAM,EAAE,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC;YAC7D,MAAM,EAAE,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;YAC9D,MAAM,CAAC,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;YAC/C,MAAM,CAAC,EAAE,CAAC,CAAC,aAAa,CAAC;gBACvB,SAAS,EAAE,QAAQ;gBACnB,KAAK,EAAE,GAAG;gBACV,aAAa,EAAE,IAAI;aACpB,CAAC,CAAC;YACH,MAAM,CAAC,CAAC,MAAM,KAAK,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;YAC5D,MAAM,CAAC,CAAC,MAAM,KAAK,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CACvE,CAAC,CAAC,EAAE,CAAC,CAAC,CACP,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8DAA8D,EAAE,KAAK,IAAI,EAAE;YAC5E,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,MAAM,EAAE,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC;YAC7D,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE;gBAC7B,GAAG,MAAM,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,CAAC;gBACrB,IAAI,EAAE,SAAS;gBACf,QAAQ,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE;aAC7C,CAAC,CAAC;YACH,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,GAAG,MAAM,KAAK,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;YAC1D,MAAM,CAAC,MAAM,CAAC,CAAC,aAAa,CAAC;gBAC3B,IAAI,EAAE,SAAS;gBACf,aAAa,EAAE,EAAE,CAAC,EAAE;gBACpB,MAAM,EAAE,MAAM;gBACd,EAAE,EAAE,UAAU;gBACd,QAAQ,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE;aAC7C,CAAC,CAAC;YACH,MAAM,CAAC,KAAK,CAAC,CAAC,aAAa,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;YACrE,MAAM,CAAC,MAAM,KAAK,CAAC,WAAW,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC1E,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uDAAuD,EAAE,KAAK,IAAI,EAAE;YACrE,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,MAAM,EAAE,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC;YAC7D,MAAM,EAAE,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;YAC9D,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE;gBAC7B,SAAS,EAAE,EAAE,CAAC,EAAE;gBAChB,IAAI,EAAE,UAAU;gBAChB,MAAM,EAAE,aAAa;gBACrB,EAAE,EAAE,UAAU;gBACd,UAAU,EAAE,EAAE,CAAC,EAAE;aAClB,CAAC,CAAC;YACH,MAAM,CAAC,CAAC,MAAM,KAAK,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;YAC5D,MAAM,CAAC,MAAM,KAAK,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC7D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8DAA8D,EAAE,KAAK,IAAI,EAAE;YAC5E,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,MAAM,EAAE,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC;YAC7D,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;YACnD,MAAM,MAAM,CACV,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAC7C,CAAC,OAAO,CAAC,cAAc,CAAC,gBAAgB,CAAC,CAAC;YAC3C,MAAM,CAAC,CAAC,MAAM,KAAK,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC7D,MAAM,CAAC,MAAM,KAAK,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC7D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kDAAkD,EAAE,KAAK,IAAI,EAAE;YAChE,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC;YAClD,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC;YACjE,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC7B,MAAM,CAAC,CAAC,MAAM,KAAK,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC/D,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC"}
|
|
1
|
+
{"version":3,"file":"testing.js","sourceRoot":"","sources":["../src/testing.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EACL,gBAAgB,GAGjB,MAAM,eAAe,CAAC;AAIvB;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,qBAAqB,CACnC,IAAY,EACZ,IAA0C;IAE1C,MAAM,OAAO,GAAG,CAAC,KAAa,EAAE,OAAyB,EAAE,EAAW,EAAE,CAAC,CAAC;QACxE,KAAK;QACL,KAAK,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE;QAC1C,IAAI,EAAE,CAAC,GAAG,EAAE,GAAG,CAAC;QAChB,KAAK,EAAE,KAAK;QACZ,MAAM,EAAE,QAAQ;QAChB,MAAM,EAAE,OAAO;QACf,SAAS,EAAE,IAAI;QACf,IAAI,EAAE,IAAI;QACV,MAAM,EAAE;YACN;gBACE,IAAI,EAAE,UAAU;gBAChB,MAAM,EAAE,WAAW;gBACnB,KAAK,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE;gBACxB,SAAS,EAAE,OAAO;gBAClB,OAAO,EAAE,IAAI;gBACb,IAAI,EAAE,IAAI;aACX;SACF;QACD,GAAG,IAAI;KACR,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,CAAC,KAAa,EAAE,EAAE,CAAC,CAAC;QACjC,KAAK;QACL,KAAK,EAAE,SAAS;QAChB,aAAa,EAAE,IAAI;QACnB,eAAe,EAAE,IAAI;QACrB,GAAG,EAAE,KAAc;QACnB,OAAO,EAAE,CAAC;QACV,cAAc,EAAE,IAAI;QACpB,aAAa,EAAE,GAAG;QAClB,gBAAgB,EAAE,IAAI;QACtB,SAAS,EAAE,IAAI;KAChB,CAAC,CAAC;IAEH,QAAQ,CAAC,uBAAuB,IAAI,EAAE,EAAE,GAAG,EAAE;QAC3C,EAAE,CAAC,kDAAkD,EAAE,KAAK,IAAI,EAAE;YAChE,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC;YAC/C,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;YAC5C,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAC9B,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;YACnE,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;YACzC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YACvC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC;gBACtC,IAAI,EAAE,UAAU;gBAChB,MAAM,EAAE,WAAW;gBACnB,KAAK,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE;aACzB,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8DAA8D,EAAE,KAAK,IAAI,EAAE;YAC5E,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC;YACnD,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC;YACpD,MAAM,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;YACzC,MAAM,KAAK,CAAC,aAAa,CAAC,KAAK,CAAC,EAAE,EAAE,SAAS,CAAC,CAAC;YAC/C,MAAM,CAAC,CAAC,MAAM,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC;gBAC5D,MAAM,CAAC,EAAE;aACV,CAAC,CAAC;YACH,oDAAoD;YACpD,MAAM,CACJ,CAAC,MAAM,KAAK,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CACxE,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;QACnC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4DAA4D,EAAE,KAAK,IAAI,EAAE;YAC1E,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC;YAC/D,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,EAAE;gBACnC,IAAI,EAAE,UAAU;gBAChB,MAAM,EAAE,SAAS;gBACjB,KAAK,EAAE,EAAE,IAAI,EAAE,GAAG,EAAE;gBACpB,SAAS,EAAE,OAAO;gBAClB,OAAO,EAAE,IAAI;gBACb,IAAI,EAAE,IAAI;aACX,CAAC,CAAC;YACH,MAAM,KAAK,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,EAAE,WAAW,CAAC,CAAC;YAC9C,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;YAC5C,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YACvC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACtD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8DAA8D,EAAE,KAAK,IAAI,EAAE;YAC5E,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YAC7C,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;YAC3C,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YAC7C,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACjC,MAAM,CAAC,CAAC,MAAM,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC;gBAC3D,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,EAAE;aACL,CAAC,CAAC;YACH,MAAM,CAAC,MAAM,KAAK,CAAC,QAAQ,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAChE,MAAM,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,CAAC;YACrD,MAAM,CACJ,CAAC,MAAM,KAAK,CAAC,QAAQ,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CACtE,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YAClB,MAAM,CAAC,MAAM,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;QACnD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;YACzD,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC,EAAE,GAAG,MAAM,CAAC,GAAG,CAAC,EAAE,WAAW,EAAE,GAAG,EAAE,CAAC,CAAC;YACzE,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC,EAAE,GAAG,MAAM,CAAC,GAAG,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC;YAC1E,MAAM,CAAC,CAAC,MAAM,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC7D,MAAM,CAAC,CAAC,MAAM,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,EAAE,WAAW,IAAI,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC;QACxE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0CAA0C,EAAE,KAAK,IAAI,EAAE;YACxD,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC;gBACnC,GAAG,MAAM,CAAC,GAAG,CAAC;gBACd,eAAe,EAAE,IAAI;aACtB,CAAC,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC;gBACjC,GAAG,MAAM,CAAC,GAAG,CAAC;gBACd,eAAe,EAAE,IAAI;aACtB,CAAC,CAAC;YACH,MAAM,CAAC,CAAC,MAAM,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,EAAE,eAAe,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACpE,MAAM,CAAC,CAAC,MAAM,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,EAAE,eAAe,IAAI,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC;QAC5E,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4DAA4D,EAAE,KAAK,IAAI,EAAE;YAC1E,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YAC7C,MAAM,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;YACzE,MAAM,KAAK,CAAC,SAAS,CACnB,CAAC,CAAC,EAAE,EACJ,EAAE,MAAM,EAAE,WAAW,EAAE,EACvB,EAAE,MAAM,EAAE,SAAS,EAAE,CACtB,CAAC;YACF,MAAM,CAAC,CAAC,MAAM,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC5D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8DAA8D,EAAE,KAAK,IAAI,EAAE;YAC5E,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC;YAC/C,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YAC7C,MAAM,IAAI,GAAG;gBACX,KAAK,EAAE,CAAC,CAAC,EAAE;gBACX,MAAM,EAAE,CAAC,CAAC,EAAE;gBACZ,MAAM,EAAE,CAAC;gBACT,WAAW,EAAE,GAAG;gBAChB,MAAM,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE;gBACzB,KAAK,EAAE,IAAI;gBACX,MAAM,EAAE,IAAI;gBACZ,OAAO,EAAE,IAAI;gBACb,EAAE,EAAE,EAAE;gBACN,MAAM,EAAE,KAAK;aACd,CAAC;YACF,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;YACzC,MAAM,KAAK,CAAC,YAAY,CAAC,EAAE,GAAG,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;YACpD,MAAM,KAAK,CAAC,YAAY,CAAC,EAAE,GAAG,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;YACnE,MAAM,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;YAClE,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAC3C,MAAM,CAAC,IAAI,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAC7B,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC,OAAO,CAAC;gBACtD,EAAE,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE;aAC7B,CAAC,CAAC;YACH,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,kBAAkB,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;YAC/D,MAAM,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAC7B,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;YACjD,MAAM,CACJ,MAAM,KAAK,CAAC,kBAAkB,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,CAAC,CACnE,CAAC,QAAQ,EAAE,CAAC;QACf,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,sCAAsC,EAAE,KAAK,IAAI,EAAE;YACpD,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,MAAM,IAAI,GACR;gBACE,GAAG,EAAE,UAAU;gBACf,SAAS,EAAE,UAAU;gBACrB,QAAQ,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE;gBAC9B,KAAK,EAAE,UAAU;gBACjB,WAAW,EAAE,kBAAkB;gBAC/B,aAAa,EAAE,GAAG;gBAClB,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC;gBACrC,UAAU,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC;gBAC1C,MAAM,EAAE;oBACN;wBACE,GAAG,EAAE,OAAO;wBACZ,KAAK,EAAE,SAAS;wBAChB,WAAW,EAAE,EAAE;wBACf,IAAI,EAAE,OAAO;wBACb,QAAQ,EAAE,SAAS;qBACpB;iBACF;gBACD,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;gBAClD,KAAK,EAAE,KAAK,EAAE,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC;oBAC9B,EAAE,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,KAAK,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE;iBAC7D;gBACD,aAAa,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC;gBACnC,KAAK,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI;aACrB,CAAC;YACJ,MAAM,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;YAC5C,MAAM,MAAM,GAAG,YAAY,CAAC;gBAC1B,KAAK;gBACL,MAAM,EAAE,CAAC,IAAI,CAAC;gBACd,MAAM,EAAE,EAAE,SAAS,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE;gBAC7D,aAAa,EAAE,GAAG,EAAE,CAAC,IAAI;aAC1B,CAAC,CAAC;YACH,MAAM,EAAE,GAAG,EAAE,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC;gBACrC,KAAK,EAAE,UAAU;gBACjB,KAAK,EAAE,SAAS;aACjB,CAAC,CAAC;YACH,MAAM,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAC7B,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACxC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YACvC,MAAM,CAAC,IAAI,EAAE,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAChD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,6BAA6B,CAC3C,IAAY,EACZ,IAAkD;IAElD,MAAM,MAAM,GAAG,CAAC,OAA2B,EAAE,EAAa,EAAE,CAAC,CAAC;QAC5D,SAAS,EAAE,QAAQ;QACnB,QAAQ,EAAE,UAAU;QACpB,OAAO,EAAE,UAAU;QACnB,MAAM,EAAE,MAAM;QACd,MAAM,EAAE,aAAa;QACrB,OAAO,EAAE,SAAS;QAClB,KAAK,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,mBAAmB,EAAE;QACvD,MAAM,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE;QAC3B,GAAG,IAAI;KACR,CAAC,CAAC;IAEH,QAAQ,CAAC,2BAA2B,IAAI,EAAE,EAAE,GAAG,EAAE;QAC/C,EAAE,CAAC,yDAAyD,EAAE,KAAK,IAAI,EAAE;YACvE,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;YACrC,MAAM,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC;gBACtB,SAAS,EAAE,QAAQ;gBACnB,MAAM,EAAE,MAAM;gBACd,MAAM,EAAE,aAAa;gBACrB,OAAO,EAAE,SAAS;gBAClB,KAAK,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,mBAAmB,EAAE;gBACvD,MAAM,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE;gBAC3B,MAAM,EAAE,MAAM;gBACd,MAAM,EAAE,IAAI;aACb,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,mEAAmE,EAAE,KAAK,IAAI,EAAE;YACjF,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;YACzC,MAAM,KAAK,CAAC,iBAAiB,CAAC,KAAK,CAAC,EAAE,EAAE,WAAW,CAAC,CAAC;YACrD,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;YAC3D,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAClC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACpC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,CAAC;YAClC,MAAM,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC;YAClD,MAAM,CAAC,MAAM,KAAK,CAAC,YAAY,EAAE,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACrD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,qBAAqB,EAAE,KAAK,IAAI,EAAE;YACnC,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,MAAM,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;YAC3B,MAAM,KAAK,CAAC,WAAW,CAAC,UAAU,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC;YAC1D,MAAM,CAAC,MAAM,KAAK,CAAC,YAAY,EAAE,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACrD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4DAA4D,EAAE,KAAK,IAAI,EAAE;YAC1E,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;YACtD,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;YAC3C,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;YACpE,MAAM,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,GAAG,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC;YAClE,MAAM,CACJ,CAAC,MAAM,KAAK,CAAC,YAAY,CAAC,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CACrE,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YACxB,MAAM,CAAC,MAAM,KAAK,CAAC,YAAY,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YACnE,MAAM,KAAK,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YACvD,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,YAAY,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC;YAC5D,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC;YACvE,MAAM,CACJ,MAAM,KAAK,CAAC,YAAY,CAAC,EAAE,KAAK,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,EAAE,CAAC,CACnE,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAClB,MAAM,CAAC,MAAM,KAAK,CAAC,YAAY,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACjE,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,6BAA6B,CAC3C,IAAY,EACZ,IAAkD;IAElD,MAAM,QAAQ,GAAG,CAAC,KAAa,EAAY,EAAE,CAAC,CAAC;QAC7C,KAAK;QACL,aAAa,EAAE,IAAI;QACnB,eAAe,EAAE,IAAI;QACrB,eAAe,EAAE,IAAI;KACtB,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,CAAC,KAAa,EAAE,UAAyB,EAAE,EAAE,CAAC,CAAC;QAC5D,MAAM,EAAE,EAAE,QAAQ,EAAE,QAAQ,CAAC,KAAK,CAAC,EAAE,aAAa,EAAE,IAAI,EAAE;QAC1D,IAAI,EAAE,QAAiB;QACvB,MAAM,EAAE,MAAM,KAAK,EAAE;QACrB,EAAE,EAAE,UAAU;QACd,UAAU;KACX,CAAC,CAAC;IAEH,QAAQ,CAAC,2BAA2B,IAAI,EAAE,EAAE,GAAG,EAAE;QAC/C,EAAE,CAAC,gEAAgE,EAAE,KAAK,IAAI,EAAE;YAC9E,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,MAAM,CAAC,MAAM,KAAK,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;YACrD,MAAM,EAAE,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC;YAC7D,MAAM,EAAE,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;YAC9D,MAAM,CAAC,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;YAC/C,MAAM,CAAC,EAAE,CAAC,CAAC,aAAa,CAAC;gBACvB,SAAS,EAAE,QAAQ;gBACnB,KAAK,EAAE,GAAG;gBACV,aAAa,EAAE,IAAI;aACpB,CAAC,CAAC;YACH,MAAM,CAAC,CAAC,MAAM,KAAK,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;YAC5D,MAAM,CAAC,CAAC,MAAM,KAAK,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CACvE,CAAC,CAAC,EAAE,CAAC,CAAC,CACP,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8DAA8D,EAAE,KAAK,IAAI,EAAE;YAC5E,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,MAAM,EAAE,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC;YAC7D,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE;gBAC7B,GAAG,MAAM,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,CAAC;gBACrB,IAAI,EAAE,SAAS;gBACf,QAAQ,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE;aAC7C,CAAC,CAAC;YACH,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,GAAG,MAAM,KAAK,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;YAC1D,MAAM,CAAC,MAAM,CAAC,CAAC,aAAa,CAAC;gBAC3B,IAAI,EAAE,SAAS;gBACf,aAAa,EAAE,EAAE,CAAC,EAAE;gBACpB,MAAM,EAAE,MAAM;gBACd,EAAE,EAAE,UAAU;gBACd,QAAQ,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE;aAC7C,CAAC,CAAC;YACH,MAAM,CAAC,KAAK,CAAC,CAAC,aAAa,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;YACrE,MAAM,CAAC,MAAM,KAAK,CAAC,WAAW,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC1E,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uDAAuD,EAAE,KAAK,IAAI,EAAE;YACrE,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,MAAM,EAAE,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC;YAC7D,MAAM,EAAE,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;YAC9D,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE;gBAC7B,SAAS,EAAE,EAAE,CAAC,EAAE;gBAChB,IAAI,EAAE,UAAU;gBAChB,MAAM,EAAE,aAAa;gBACrB,EAAE,EAAE,UAAU;gBACd,UAAU,EAAE,EAAE,CAAC,EAAE;aAClB,CAAC,CAAC;YACH,MAAM,CAAC,CAAC,MAAM,KAAK,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;YAC5D,MAAM,CAAC,MAAM,KAAK,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC7D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8DAA8D,EAAE,KAAK,IAAI,EAAE;YAC5E,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,MAAM,EAAE,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC;YAC7D,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;YACnD,MAAM,MAAM,CACV,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAC7C,CAAC,OAAO,CAAC,cAAc,CAAC,gBAAgB,CAAC,CAAC;YAC3C,MAAM,CAAC,CAAC,MAAM,KAAK,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC7D,MAAM,CAAC,MAAM,KAAK,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC7D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gEAAgE,EAAE,KAAK,IAAI,EAAE;YAC9E,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,MAAM,EAAE,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE;gBACxC,GAAG,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC;gBACpB,MAAM,EAAE;oBACN,QAAQ,EAAE,EAAE,GAAG,QAAQ,CAAC,GAAG,CAAC,EAAE,eAAe,EAAE,OAAO,EAAE;oBACxD,aAAa,EAAE,IAAI;iBACpB;aACF,CAAC,CAAC;YACH,MAAM,CAAC,EAAE,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACzC,MAAM,CAAC,CAAC,MAAM,KAAK,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,EAAE,eAAe,CAAC,CAAC,IAAI,CAC/D,OAAO,CACR,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+BAA+B,EAAE,KAAK,IAAI,EAAE;YAC7C,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,MAAM,EAAE,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE;gBACxC,GAAG,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC;gBACpB,MAAM,EAAE;oBACN,QAAQ,EAAE,EAAE,GAAG,QAAQ,CAAC,GAAG,CAAC,EAAE,WAAW,EAAE,GAAG,EAAE;oBAChD,aAAa,EAAE,IAAI;iBACpB;aACF,CAAC,CAAC;YACH,MAAM,CAAC,EAAE,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACjC,MAAM,CAAC,CAAC,MAAM,KAAK,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACnE,MAAM,EAAE,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;YAC9D,MAAM,CAAC,EAAE,CAAC,WAAW,IAAI,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC;QAC5C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kDAAkD,EAAE,KAAK,IAAI,EAAE;YAChE,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC;YAClD,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC;YACjE,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC7B,MAAM,CAAC,CAAC,MAAM,KAAK,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC/D,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC"}
|
package/dist/types.d.ts
CHANGED
|
@@ -92,6 +92,17 @@ export type Suite<Input, Output, Label> = {
|
|
|
92
92
|
model: string;
|
|
93
93
|
/** A reasoning setting to test; undefined: the operation's own. */
|
|
94
94
|
reasoningEffort?: ReasoningEffort | null;
|
|
95
|
+
/**
|
|
96
|
+
* The temperature the test runs at: the operation's live one when the
|
|
97
|
+
* test was started. Null: send none. Undefined from runners before
|
|
98
|
+
* temperature was versioned.
|
|
99
|
+
*/
|
|
100
|
+
temperature?: number | null;
|
|
101
|
+
/**
|
|
102
|
+
* The output-token cap the test runs at, like temperature. Null: send
|
|
103
|
+
* none (the call's own default).
|
|
104
|
+
*/
|
|
105
|
+
maxOutputTokens?: number | null;
|
|
95
106
|
onCost: (usd: number) => void;
|
|
96
107
|
}): Promise<Output>;
|
|
97
108
|
/**
|
package/dist/types.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAoDA,gFAAgF;AAChF,MAAM,UAAU,QAAQ,CAAC,CAAW;IAClC,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;AAC3D,CAAC;AAED,qEAAqE;AACrE,MAAM,UAAU,SAAS,CACvB,IAAY,EACZ,GAAW;IAEX,OAAO,IAAI,IAAI,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,IAAI,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC;AACvE,CAAC;AAcD,mDAAmD;AACnD,MAAM,CAAC,MAAM,UAAU,GAA2C;IAChE,KAAK,EACH,oLAAoL;IACtL,IAAI,EAAE,4LAA4L;IAClM,OAAO,EACL,wHAAwH;CAC3H,CAAC;
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAoDA,gFAAgF;AAChF,MAAM,UAAU,QAAQ,CAAC,CAAW;IAClC,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;AAC3D,CAAC;AAED,qEAAqE;AACrE,MAAM,UAAU,SAAS,CACvB,IAAY,EACZ,GAAW;IAEX,OAAO,IAAI,IAAI,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,IAAI,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC;AACvE,CAAC;AAcD,mDAAmD;AACnD,MAAM,CAAC,MAAM,UAAU,GAA2C;IAChE,KAAK,EACH,oLAAoL;IACtL,IAAI,EAAE,4LAA4L;IAClM,OAAO,EACL,wHAAwH;CAC3H,CAAC;AA8IF,wEAAwE;AACxE,MAAM,UAAU,MAAM,CAAC,CAAU;IAC/B,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,IAAI;QAAE,OAAO,EAAE,CAAC;IAC7C,IAAI,OAAO,CAAC,KAAK,QAAQ;QAAE,OAAO,CAAC,CAAC;IACpC,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,OAAO,CAAC,KAAK,SAAS;QAAE,OAAO,MAAM,CAAC,CAAC,CAAC,CAAC;IACtE,OAAO,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;AAC3B,CAAC;AAED,uDAAuD;AACvD,MAAM,UAAU,QAAQ,CAAC,IAAc,EAAE,IAAY;IACnD,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACzB,OAAO,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;AAC/C,CAAC;AAED,2EAA2E;AAC3E,MAAM,UAAU,UAAU,CAAC,KAAgB,EAAE,KAAc;IACzD,MAAM,CAAC,GACL,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ;QAChC,CAAC,CAAE,KAAiC,CAAC,KAAK,CAAC,IAAI,CAAC;QAChD,CAAC,CAAC,SAAS,CAAC;IAChB,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;QAAE,OAAO,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACtD,OAAO,MAAM,CAAC,CAAC,CAAC,CAAC;AACnB,CAAC;AAED,sEAAsE;AACtE,MAAM,UAAU,OAAO,CAAC,IAAc,EAAE,IAAY;IAClD,OAAO,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC;SACxB,KAAK,CAAC,IAAI,CAAC;SACX,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;SACpB,MAAM,CAAC,OAAO,CAAC,CAAC;AACrB,CAAC"}
|
package/dist/weekly.js
CHANGED
|
@@ -52,11 +52,14 @@ export function weeklySummary(args) {
|
|
|
52
52
|
}
|
|
53
53
|
if (r.status !== "SUCCEEDED")
|
|
54
54
|
continue;
|
|
55
|
-
// The last check of the same model,
|
|
56
|
-
// more earlier: anything else changed for a known
|
|
55
|
+
// The last check of the same model, temperature, output cap,
|
|
56
|
+
// instructions, and grading, a day or more earlier: anything else changed for a known
|
|
57
|
+
// reason.
|
|
57
58
|
const prior = scheduled.find((p) => p.suite === r.suite &&
|
|
58
59
|
p.status === "SUCCEEDED" &&
|
|
59
60
|
p.model === r.model &&
|
|
61
|
+
(p.temperature ?? null) === (r.temperature ?? null) &&
|
|
62
|
+
(p.maxOutputTokens ?? null) === (r.maxOutputTokens ?? null) &&
|
|
60
63
|
p.promptVersion === r.promptVersion &&
|
|
61
64
|
p.graderVersion === r.graderVersion &&
|
|
62
65
|
+r.startedAt - +p.startedAt >= DAY);
|
package/dist/weekly.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"weekly.js","sourceRoot":"","sources":["../src/weekly.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAqB,MAAM,aAAa,CAAC;AAE9D,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAEnE;;;;;;GAMG;AAEH,2EAA2E;AAC3E,MAAM,CAAC,MAAM,SAAS,GAAG,UAAU,CAAC;AAEpC,MAAM,GAAG,GAAG,UAAU,CAAC;AAqCvB,0EAA0E;AAC1E,MAAM,UAAU,QAAQ,CAAC,CAAoB;IAC3C,IAAI,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IACpB,OAAO,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,KAAK,CAAC,KAAK;QAC7B,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,KAAK;QAC/B,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC;AAC5B,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,aAAa,CAAC,IAU7B;IACC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,IAAI,IAAI,EAAE,CAAC;IACvC,MAAM,KAAK,GAAG,CAAC,GAAW,EAAE,EAAE,CAC5B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,GAAG,CAAC,EAAE,KAAK,IAAI,GAAG,CAAC;IACvD,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI;SACxB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,SAAS,IAAI,CAAC,CAAC,GAAG,KAAK,OAAO,CAAC;SAC7D,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IAC/C,MAAM,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,IAAI,IAAI,CAAC;IAC/C,wEAAwE;IACxE,MAAM,KAAK,GAAG,MAAM;QAClB,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,SAAS,GAAG,GAAG,CAAC;QACvD,CAAC,CAAC,EAAE,CAAC;IAEP,MAAM,KAAK,GAA2B,EAAE,CAAC;IACzC,MAAM,MAAM,GAA4B,EAAE,CAAC;IAC3C,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,IAAI,CAAC,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YAC1B,MAAM,CAAC,IAAI,CAAC;gBACV,KAAK,EAAE,CAAC,CAAC,KAAK;gBACd,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC;gBACrB,KAAK,EAAE,CAAC,CAAC,KAAK,IAAI,eAAe;aAClC,CAAC,CAAC;YACH,SAAS;QACX,CAAC;QACD,IAAI,CAAC,CAAC,MAAM,KAAK,WAAW;YAAE,SAAS;QACvC,
|
|
1
|
+
{"version":3,"file":"weekly.js","sourceRoot":"","sources":["../src/weekly.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAqB,MAAM,aAAa,CAAC;AAE9D,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAEnE;;;;;;GAMG;AAEH,2EAA2E;AAC3E,MAAM,CAAC,MAAM,SAAS,GAAG,UAAU,CAAC;AAEpC,MAAM,GAAG,GAAG,UAAU,CAAC;AAqCvB,0EAA0E;AAC1E,MAAM,UAAU,QAAQ,CAAC,CAAoB;IAC3C,IAAI,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IACpB,OAAO,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,KAAK,CAAC,KAAK;QAC7B,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,KAAK;QAC/B,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC;AAC5B,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,aAAa,CAAC,IAU7B;IACC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,IAAI,IAAI,EAAE,CAAC;IACvC,MAAM,KAAK,GAAG,CAAC,GAAW,EAAE,EAAE,CAC5B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,GAAG,CAAC,EAAE,KAAK,IAAI,GAAG,CAAC;IACvD,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI;SACxB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,SAAS,IAAI,CAAC,CAAC,GAAG,KAAK,OAAO,CAAC;SAC7D,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IAC/C,MAAM,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,IAAI,IAAI,CAAC;IAC/C,wEAAwE;IACxE,MAAM,KAAK,GAAG,MAAM;QAClB,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,SAAS,GAAG,GAAG,CAAC;QACvD,CAAC,CAAC,EAAE,CAAC;IAEP,MAAM,KAAK,GAA2B,EAAE,CAAC;IACzC,MAAM,MAAM,GAA4B,EAAE,CAAC;IAC3C,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,IAAI,CAAC,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YAC1B,MAAM,CAAC,IAAI,CAAC;gBACV,KAAK,EAAE,CAAC,CAAC,KAAK;gBACd,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC;gBACrB,KAAK,EAAE,CAAC,CAAC,KAAK,IAAI,eAAe;aAClC,CAAC,CAAC;YACH,SAAS;QACX,CAAC;QACD,IAAI,CAAC,CAAC,MAAM,KAAK,WAAW;YAAE,SAAS;QACvC,6DAA6D;QAC7D,sFAAsF;QACtF,UAAU;QACV,MAAM,KAAK,GAAG,SAAS,CAAC,IAAI,CAC1B,CAAC,CAAC,EAAE,EAAE,CACJ,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,KAAK;YACnB,CAAC,CAAC,MAAM,KAAK,WAAW;YACxB,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,KAAK;YACnB,CAAC,CAAC,CAAC,WAAW,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,WAAW,IAAI,IAAI,CAAC;YACnD,CAAC,CAAC,CAAC,eAAe,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,eAAe,IAAI,IAAI,CAAC;YAC3D,CAAC,CAAC,aAAa,KAAK,CAAC,CAAC,aAAa;YACnC,CAAC,CAAC,aAAa,KAAK,CAAC,CAAC,aAAa;YACnC,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,SAAS,IAAI,GAAG,CACrC,CAAC;QACF,MAAM,MAAM,GAAG,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QACnC,MAAM,QAAQ,GAAG,QAAQ,CAAC,KAAK,EAAE,OAAO,IAAI,IAAI,CAAC,CAAC;QAClD,IAAI,MAAM,KAAK,IAAI,IAAI,QAAQ,KAAK,IAAI,IAAI,OAAO,CAAC,QAAQ,EAAE,MAAM,CAAC;YACnE,KAAK,CAAC,IAAI,CAAC;gBACT,KAAK,EAAE,CAAC,CAAC,KAAK;gBACd,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC;gBACrB,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC,KAAK;gBAC7C,QAAQ;gBACR,MAAM;aACP,CAAC,CAAC;IACP,CAAC;IAED,MAAM,QAAQ,GAA8B,EAAE,CAAC;IAC/C,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QAC5B,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QACvC,IAAI,CAAC,KAAK;YAAE,SAAS;QACrB,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI;YAC1B,CAAC,KAAK,CAAC,KAAK,EAAE,MAAM,CAAC;YACrB,CAAC,KAAK,CAAC,aAAa,EAAE,UAAU,CAAC;SACzB,EAAE,CAAC;YACX,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC;YAC9D,IAAI,KAAK,IAAI,IAAI,IAAI,WAAW,CAAC,IAAI,EAAE,KAAK,CAAC;gBAC3C,QAAQ,CAAC,IAAI,CAAC;oBACZ,KAAK,EAAE,CAAC,CAAC,GAAG;oBACZ,KAAK,EAAE,CAAC,CAAC,KAAK;oBACd,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,KAAK,IAAI,KAAK;oBACzC,IAAI;oBACJ,IAAI,EAAE,YAAY,CAAC,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,IAAI;oBAC9C,IAAI,EAAE,SAAS,CAAC,IAAI,EAAE,KAAK,CAAC;iBAC7B,CAAC,CAAC;QACP,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAG,CAAC,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IAC1D,OAAO;QACL,MAAM;QACN,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE,OAAO,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC;QACjE,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,MAAM;QAC3D,KAAK;QACL,MAAM;QACN,QAAQ;QACR,MAAM;QACN,KAAK,EACH,MAAM,KAAK,IAAI;YACf,CAAC,KAAK,CAAC,MAAM;YACb,CAAC,MAAM,CAAC,MAAM;YACd,CAAC,QAAQ,CAAC,MAAM;YAChB,CAAC,MAAM,CAAC,MAAM;KACjB,CAAC;AACJ,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crowandraven/evalkeeper",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"description": "
|
|
3
|
+
"version": "0.3.0",
|
|
4
|
+
"description": "An open-source eval harness for TypeScript apps: test each AI feature against answers people confirm, compare models on accuracy, cost, and speed, and choose the live model on evidence.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
7
7
|
"main": "./dist/index.js",
|
package/src/attention.ts
CHANGED
|
@@ -43,7 +43,8 @@ export function attention(boards: SuiteBoard[]): Attention {
|
|
|
43
43
|
let slowest: Attention["slowest"] = null;
|
|
44
44
|
let costliest: Attention["costliest"] = null;
|
|
45
45
|
for (const b of boards) {
|
|
46
|
-
|
|
46
|
+
// The best model's score: a baseline is code, not a model to switch to.
|
|
47
|
+
const tested = b.rows.filter((r) => !r.baseline && current(r));
|
|
47
48
|
if (tested.length) {
|
|
48
49
|
const rateOf = (r: BoardRow) => {
|
|
49
50
|
const t = current(r);
|
package/src/board.ts
CHANGED
|
@@ -18,6 +18,8 @@ export type BoardRow = {
|
|
|
18
18
|
label: string;
|
|
19
19
|
live: boolean;
|
|
20
20
|
fallback: boolean;
|
|
21
|
+
/** Code, not a model (`ModelInfo.baseline`): compared, never made live or the fallback. */
|
|
22
|
+
baseline: boolean;
|
|
21
23
|
testing: boolean;
|
|
22
24
|
/** The newest test failed: why, in a line. Older results still show. */
|
|
23
25
|
failed: string | null;
|
package/src/glossary.ts
CHANGED
|
@@ -54,7 +54,7 @@ export const GLOSSARY = {
|
|
|
54
54
|
weeklyCheck:
|
|
55
55
|
"Each suite's quick set, answered again on its live model every week. A drop of 15 points or more against last week's check of the same model, instructions, and grading is drift.",
|
|
56
56
|
versions:
|
|
57
|
-
"Each numbered version of this feature's settings (model, fallback, reasoning), with who changed it, when, why, and
|
|
57
|
+
"Each numbered version of this feature's settings (model, fallback, reasoning, output limit, temperature), with who changed it, when, why, and its evidence. Any version can be made live again.",
|
|
58
58
|
suites:
|
|
59
59
|
"One suite per AI feature: its cases, the answers people confirmed, and the checks that grade any model against them. Open one to compare models and choose the live one.",
|
|
60
60
|
answerKey:
|
package/src/health.ts
CHANGED
|
@@ -50,7 +50,9 @@ export function suiteHealth(
|
|
|
50
50
|
models: ModelCatalog = {},
|
|
51
51
|
): Health {
|
|
52
52
|
const liveRow = rows.find((r) => r.live) ?? null;
|
|
53
|
-
const testedModels = rows.filter(
|
|
53
|
+
const testedModels = rows.filter(
|
|
54
|
+
(r) => !r.baseline && r.test && !r.test.stale,
|
|
55
|
+
).length;
|
|
54
56
|
const t = liveRow?.test;
|
|
55
57
|
const live =
|
|
56
58
|
liveRow && t ? { label: liveRow.label, rate: t.right / t.total } : null;
|