@crowandraven/evalkeeper 0.2.0 → 0.4.0

package/dist/testing.js

@@ -71,12 +71,17 @@ export function describeStoreContract(name, make) {
         it("lists a suite's cases oldest first, leaving out retired ones", async () => {
             const store = await make();
             const first = await store.createCase(newCase("s"));
+            // Apart in time: two cases made in the same millisecond have no
+            // oldest one.
+            await new Promise((r) => setTimeout(r, 5));
             const second = await store.createCase(newCase("s"));
             await store.createCase(newCase("other"));
             await store.setCaseStatus(first.id, "RETIRED");
             expect((await store.listCases("s")).map((c) => c.id)).toEqual([
                 second.id,
             ]);
+            // On request, retired ones too, still oldest first.
+            expect((await store.listCases("s", { includeRetired: true })).map((c) => c.id)).toEqual([first.id, second.id]);
         });
         it("adds answers without overwriting, and changes their status", async () => {
             const store = await make();
@@ -116,6 +121,19 @@ export function describeStoreContract(name, make) {
             expect((await store.getRun(warm.id))?.temperature).toBe(0.7);
             expect((await store.getRun(none.id))?.temperature ?? null).toBeNull();
         });
+        it("keeps the output cap a run was tested at", async () => {
+            const store = await make();
+            const capped = await store.createRun({
+                ...newRun("s"),
+                maxOutputTokens: 4000,
+            });
+            const none = await store.createRun({
+                ...newRun("s"),
+                maxOutputTokens: null,
+            });
+            expect((await store.getRun(capped.id))?.maxOutputTokens).toBe(4000);
+            expect((await store.getRun(none.id))?.maxOutputTokens ?? null).toBeNull();
+        });
         it("updates a run only while it has the status it's guarded by", async () => {
             const store = await make();
             const r = await store.createRun(newRun("s"));
@@ -153,6 +171,53 @@ export function describeStoreContract(name, make) {
             expect(found?.output).toEqual({ text: "HELLO" });
             expect(await store.findReusableOutput("h", new Date(Date.now() + 60_000))).toBeNull();
         });
+        it("lists several tests' outputs and reusable hashes in one call, where it can", async () => {
+            const store = await make();
+            const c = await store.createCase(newCase("s"));
+            const [r1, r2, r3] = await Promise.all([
+                store.createRun(newRun("s")),
+                store.createRun(newRun("s")),
+                store.createRun(newRun("s")),
+            ]);
+            const base = {
+                caseId: c.id,
+                repeat: 1,
+                output: { text: "HELLO" },
+                error: null,
+                scores: null,
+                costUsd: 0.01,
+                ms: 12,
+                reused: false,
+            };
+            await store.createOutput({ ...base, runId: r1.id, requestHash: "a" });
+            await store.createOutput({ ...base, runId: r2.id, requestHash: "b" });
+            await store.createOutput({
+                ...base,
+                runId: r2.id,
+                requestHash: "c",
+                reused: true,
+            });
+            await store.createOutput({
+                ...base,
+                runId: r3.id,
+                requestHash: "d",
+                error: "boom",
+                output: null,
+            });
+            if (store.listOutputsFor) {
+                const outs = await store.listOutputsFor([r1.id, r2.id]);
+                expect(outs.map((o) => o.requestHash).sort()).toEqual(["a", "b", "c"]);
+                expect(await store.listOutputsFor([])).toEqual([]);
+            }
+            if (store.findReusableHashes) {
+                expect([
+                    ...(await store.findReusableHashes(["a", "b", "c", "d", "e"], new Date(0))),
+                ].sort()).toEqual(["a", "b"]);
+                expect((await store.findReusableHashes(["a"], new Date(Date.now() + 60_000)))
+                    .size).toBe(0);
+                expect((await store.findReusableHashes([], new Date(0))).size).toBe(0);
+            }
+        });
         it("runs a whole test through the runner", async () => {
             const store = await make();
             const echo = {
@@ -336,6 +401,18 @@ export function describeSettingsStoreContract(name, make) {
             expect((await store.liveVersion("triage"))?.model).toBe("b");
             expect(await store.listVersions("triage")).toHaveLength(2);
         });
+        it("keeps a version's reasoning setting as named, even a newer one", async () => {
+            const store = await make();
+            const v1 = await store.switchTo("triage", {
+                ...create("a", null),
+                create: {
+                    settings: { ...settings("a"), reasoningEffort: "xhigh" },
+                    promptVersion: "p1",
+                },
+            });
+            expect(v1.reasoningEffort).toBe("xhigh");
+            expect((await store.liveVersion("triage"))?.reasoningEffort).toBe("xhigh");
+        });
         it("keeps a version's temperature", async () => {
             const store = await make();
             const v1 = await store.switchTo("triage", {

package/dist/testing.js.map

@@ -1 +1 @@
-{"version":3,"file":"testing.js","sourceRoot":"","sources":["../src/testing.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EACL,gBAAgB,GAGjB,MAAM,eAAe,CAAC;AAIvB;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,qBAAqB,CACnC,IAAY,EACZ,IAA0C;IAE1C,MAAM,OAAO,GAAG,CAAC,KAAa,EAAE,OAAyB,EAAE,EAAW,EAAE,CAAC,CAAC;QACxE,KAAK;QACL,KAAK,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE;QAC1C,IAAI,EAAE,CAAC,GAAG,EAAE,GAAG,CAAC;QAChB,KAAK,EAAE,KAAK;QACZ,MAAM,EAAE,QAAQ;QAChB,MAAM,EAAE,OAAO;QACf,SAAS,EAAE,IAAI;QACf,IAAI,EAAE,IAAI;QACV,MAAM,EAAE;YACN;gBACE,IAAI,EAAE,UAAU;gBAChB,MAAM,EAAE,WAAW;gBACnB,KAAK,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE;gBACxB,SAAS,EAAE,OAAO;gBAClB,OAAO,EAAE,IAAI;gBACb,IAAI,EAAE,IAAI;aACX;SACF;QACD,GAAG,IAAI;KACR,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,CAAC,KAAa,EAAE,EAAE,CAAC,CAAC;QACjC,KAAK;QACL,KAAK,EAAE,SAAS;QAChB,aAAa,EAAE,IAAI;QACnB,eAAe,EAAE,IAAI;QACrB,GAAG,EAAE,KAAc;QACnB,OAAO,EAAE,CAAC;QACV,cAAc,EAAE,IAAI;QACpB,aAAa,EAAE,GAAG;QAClB,gBAAgB,EAAE,IAAI;QACtB,SAAS,EAAE,IAAI;KAChB,CAAC,CAAC;IAEH,QAAQ,CAAC,uBAAuB,IAAI,EAAE,EAAE,GAAG,EAAE;QAC3C,EAAE,CAAC,kDAAkD,EAAE,KAAK,IAAI,EAAE;YAChE,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC;YAC/C,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;YAC5C,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAC9B,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;YACnE,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;YACzC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YACvC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC;gBACtC,IAAI,EAAE,UAAU;gBAChB,MAAM,EAAE,WAAW;gBACnB,KAAK,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE;aACzB,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8DAA8D,EAAE,KAAK,IAAI,EAAE;YAC5E,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC;YACnD,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC;YACpD,MAAM,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;YACzC,MAAM,KAAK,CAAC,aAAa,CAAC,KAAK,CAAC,EAAE,EAAE,SAAS,CAAC,CAAC;YAC/C,MAAM,CAAC,CAAC,MAAM,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC;gBAC5D,MAAM,CAAC,EAAE;aACV,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4DAA4D,EAAE,KAAK,IAAI,EAAE;YAC1E,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC;YAC/D,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,EAAE;gBACnC,IAAI,EAAE,UAAU;gBAChB,MAAM,EAAE,SAAS;gBACjB,KAAK,EAAE,EAAE,IAAI,EAAE,GAAG,EAAE;gBACpB,SAAS,EAAE,OAAO;gBAClB,OAAO,EAAE,IAAI;gBACb,IAAI,EAAE,IAAI;aACX,CAAC,CAAC;YACH,MAAM,KAAK,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,EAAE,WAAW,CAAC,CAAC;YAC9C,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;YAC5C,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YACvC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACtD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8DAA8D,EAAE,KAAK,IAAI,EAAE;YAC5E,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YAC7C,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;YAC3C,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YAC7C,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACjC,MAAM,CAAC,CAAC,MAAM,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC;gBAC3D,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,EAAE;aACL,CAAC,CAAC;YACH,MAAM,CAAC,MAAM,KAAK,CAAC,QAAQ,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAChE,MAAM,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,CAAC;YACrD,MAAM,CACJ,CAAC,MAAM,KAAK,CAAC,QAAQ,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CACtE,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YAClB,MAAM,CAAC,MAAM,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;QACnD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;YACzD,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC,EAAE,GAAG,MAAM,CAAC,GAAG,CAAC,EAAE,WAAW,EAAE,GAAG,EAAE,CAAC,CAAC;YACzE,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC,EAAE,GAAG,MAAM,CAAC,GAAG,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC;YAC1E,MAAM,CAAC,CAAC,MAAM,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC7D,MAAM,CAAC,CAAC,MAAM,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,EAAE,WAAW,IAAI,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC;QACxE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4DAA4D,EAAE,KAAK,IAAI,EAAE;YAC1E,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YAC7C,MAAM,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;YACzE,MAAM,KAAK,CAAC,SAAS,CACnB,CAAC,CAAC,EAAE,EACJ,EAAE,MAAM,EAAE,WAAW,EAAE,EACvB,EAAE,MAAM,EAAE,SAAS,EAAE,CACtB,CAAC;YACF,MAAM,CAAC,CAAC,MAAM,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC5D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8DAA8D,EAAE,KAAK,IAAI,EAAE;YAC5E,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC;YAC/C,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YAC7C,MAAM,IAAI,GAAG;gBACX,KAAK,EAAE,CAAC,CAAC,EAAE;gBACX,MAAM,EAAE,CAAC,CAAC,EAAE;gBACZ,MAAM,EAAE,CAAC;gBACT,WAAW,EAAE,GAAG;gBAChB,MAAM,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE;gBACzB,KAAK,EAAE,IAAI;gBACX,MAAM,EAAE,IAAI;gBACZ,OAAO,EAAE,IAAI;gBACb,EAAE,EAAE,EAAE;gBACN,MAAM,EAAE,KAAK;aACd,CAAC;YACF,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;YACzC,MAAM,KAAK,CAAC,YAAY,CAAC,EAAE,GAAG,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;YACpD,MAAM,KAAK,CAAC,YAAY,CAAC,EAAE,GAAG,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;YACnE,MAAM,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;YAClE,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAC3C,MAAM,CAAC,IAAI,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAC7B,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC,OAAO,CAAC;gBACtD,EAAE,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE;aAC7B,CAAC,CAAC;YACH,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,kBAAkB,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;YAC/D,MAAM,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAC7B,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;YACjD,MAAM,CACJ,MAAM,KAAK,CAAC,kBAAkB,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,CAAC,CACnE,CAAC,QAAQ,EAAE,CAAC;QACf,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,sCAAsC,EAAE,KAAK,IAAI,EAAE;YACpD,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,MAAM,IAAI,GACR;gBACE,GAAG,EAAE,UAAU;gBACf,SAAS,EAAE,UAAU;gBACrB,QAAQ,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE;gBAC9B,KAAK,EAAE,UAAU;gBACjB,WAAW,EAAE,kBAAkB;gBAC/B,aAAa,EAAE,GAAG;gBAClB,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC;gBACrC,UAAU,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC;gBAC1C,MAAM,EAAE;oBACN;wBACE,GAAG,EAAE,OAAO;wBACZ,KAAK,EAAE,SAAS;wBAChB,WAAW,EAAE,EAAE;wBACf,IAAI,EAAE,OAAO;wBACb,QAAQ,EAAE,SAAS;qBACpB;iBACF;gBACD,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;gBAClD,KAAK,EAAE,KAAK,EAAE,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC;oBAC9B,EAAE,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,KAAK,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE;iBAC7D;gBACD,aAAa,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC;gBACnC,KAAK,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI;aACrB,CAAC;YACJ,MAAM,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;YAC5C,MAAM,MAAM,GAAG,YAAY,CAAC;gBAC1B,KAAK;gBACL,MAAM,EAAE,CAAC,IAAI,CAAC;gBACd,MAAM,EAAE,EAAE,SAAS,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE;gBAC7D,aAAa,EAAE,GAAG,EAAE,CAAC,IAAI;aAC1B,CAAC,CAAC;YACH,MAAM,EAAE,GAAG,EAAE,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC;gBACrC,KAAK,EAAE,UAAU;gBACjB,KAAK,EAAE,SAAS;aACjB,CAAC,CAAC;YACH,MAAM,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAC7B,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACxC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YACvC,MAAM,CAAC,IAAI,EAAE,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAChD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,6BAA6B,CAC3C,IAAY,EACZ,IAAkD;IAElD,MAAM,MAAM,GAAG,CAAC,OAA2B,EAAE,EAAa,EAAE,CAAC,CAAC;QAC5D,SAAS,EAAE,QAAQ;QACnB,QAAQ,EAAE,UAAU;QACpB,OAAO,EAAE,UAAU;QACnB,MAAM,EAAE,MAAM;QACd,MAAM,EAAE,aAAa;QACrB,OAAO,EAAE,SAAS;QAClB,KAAK,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,mBAAmB,EAAE;QACvD,MAAM,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE;QAC3B,GAAG,IAAI;KACR,CAAC,CAAC;IAEH,QAAQ,CAAC,2BAA2B,IAAI,EAAE,EAAE,GAAG,EAAE;QAC/C,EAAE,CAAC,yDAAyD,EAAE,KAAK,IAAI,EAAE;YACvE,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;YACrC,MAAM,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC;gBACtB,SAAS,EAAE,QAAQ;gBACnB,MAAM,EAAE,MAAM;gBACd,MAAM,EAAE,aAAa;gBACrB,OAAO,EAAE,SAAS;gBAClB,KAAK,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,mBAAmB,EAAE;gBACvD,MAAM,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE;gBAC3B,MAAM,EAAE,MAAM;gBACd,MAAM,EAAE,IAAI;aACb,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,mEAAmE,EAAE,KAAK,IAAI,EAAE;YACjF,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;YACzC,MAAM,KAAK,CAAC,iBAAiB,CAAC,KAAK,CAAC,EAAE,EAAE,WAAW,CAAC,CAAC;YACrD,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;YAC3D,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAClC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACpC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,CAAC;YAClC,MAAM,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC;YAClD,MAAM,CAAC,MAAM,KAAK,CAAC,YAAY,EAAE,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACrD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,qBAAqB,EAAE,KAAK,IAAI,EAAE;YACnC,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,MAAM,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;YAC3B,MAAM,KAAK,CAAC,WAAW,CAAC,UAAU,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC;YAC1D,MAAM,CAAC,MAAM,KAAK,CAAC,YAAY,EAAE,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACrD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4DAA4D,EAAE,KAAK,IAAI,EAAE;YAC1E,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;YACtD,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;YAC3C,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;YACpE,MAAM,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,GAAG,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC;YAClE,MAAM,CACJ,CAAC,MAAM,KAAK,CAAC,YAAY,CAAC,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CACrE,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YACxB,MAAM,CAAC,MAAM,KAAK,CAAC,YAAY,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YACnE,MAAM,KAAK,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YACvD,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,YAAY,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC;YAC5D,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC;YACvE,MAAM,CACJ,MAAM,KAAK,CAAC,YAAY,CAAC,EAAE,KAAK,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,EAAE,CAAC,CACnE,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAClB,MAAM,CAAC,MAAM,KAAK,CAAC,YAAY,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACjE,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,6BAA6B,CAC3C,IAAY,EACZ,IAAkD;IAElD,MAAM,QAAQ,GAAG,CAAC,KAAa,EAAY,EAAE,CAAC,CAAC;QAC7C,KAAK;QACL,aAAa,EAAE,IAAI;QACnB,eAAe,EAAE,IAAI;QACrB,eAAe,EAAE,IAAI;KACtB,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,CAAC,KAAa,EAAE,UAAyB,EAAE,EAAE,CAAC,CAAC;QAC5D,MAAM,EAAE,EAAE,QAAQ,EAAE,QAAQ,CAAC,KAAK,CAAC,EAAE,aAAa,EAAE,IAAI,EAAE;QAC1D,IAAI,EAAE,QAAiB;QACvB,MAAM,EAAE,MAAM,KAAK,EAAE;QACrB,EAAE,EAAE,UAAU;QACd,UAAU;KACX,CAAC,CAAC;IAEH,QAAQ,CAAC,2BAA2B,IAAI,EAAE,EAAE,GAAG,EAAE;QAC/C,EAAE,CAAC,gEAAgE,EAAE,KAAK,IAAI,EAAE;YAC9E,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,MAAM,CAAC,MAAM,KAAK,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;YACrD,MAAM,EAAE,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC;YAC7D,MAAM,EAAE,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;YAC9D,MAAM,CAAC,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;YAC/C,MAAM,CAAC,EAAE,CAAC,CAAC,aAAa,CAAC;gBACvB,SAAS,EAAE,QAAQ;gBACnB,KAAK,EAAE,GAAG;gBACV,aAAa,EAAE,IAAI;aACpB,CAAC,CAAC;YACH,MAAM,CAAC,CAAC,MAAM,KAAK,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;YAC5D,MAAM,CAAC,CAAC,MAAM,KAAK,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CACvE,CAAC,CAAC,EAAE,CAAC,CAAC,CACP,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8DAA8D,EAAE,KAAK,IAAI,EAAE;YAC5E,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,MAAM,EAAE,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC;YAC7D,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE;gBAC7B,GAAG,MAAM,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,CAAC;gBACrB,IAAI,EAAE,SAAS;gBACf,QAAQ,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE;aAC7C,CAAC,CAAC;YACH,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,GAAG,MAAM,KAAK,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;YAC1D,MAAM,CAAC,MAAM,CAAC,CAAC,aAAa,CAAC;gBAC3B,IAAI,EAAE,SAAS;gBACf,aAAa,EAAE,EAAE,CAAC,EAAE;gBACpB,MAAM,EAAE,MAAM;gBACd,EAAE,EAAE,UAAU;gBACd,QAAQ,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE;aAC7C,CAAC,CAAC;YACH,MAAM,CAAC,KAAK,CAAC,CAAC,aAAa,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;YACrE,MAAM,CAAC,MAAM,KAAK,CAAC,WAAW,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC1E,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uDAAuD,EAAE,KAAK,IAAI,EAAE;YACrE,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,MAAM,EAAE,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC;YAC7D,MAAM,EAAE,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;YAC9D,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE;gBAC7B,SAAS,EAAE,EAAE,CAAC,EAAE;gBAChB,IAAI,EAAE,UAAU;gBAChB,MAAM,EAAE,aAAa;gBACrB,EAAE,EAAE,UAAU;gBACd,UAAU,EAAE,EAAE,CAAC,EAAE;aAClB,CAAC,CAAC;YACH,MAAM,CAAC,CAAC,MAAM,KAAK,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;YAC5D,MAAM,CAAC,MAAM,KAAK,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC7D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8DAA8D,EAAE,KAAK,IAAI,EAAE;YAC5E,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,MAAM,EAAE,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC;YAC7D,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;YACnD,MAAM,MAAM,CACV,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAC7C,CAAC,OAAO,CAAC,cAAc,CAAC,gBAAgB,CAAC,CAAC;YAC3C,MAAM,CAAC,CAAC,MAAM,KAAK,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC7D,MAAM,CAAC,MAAM,KAAK,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC7D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+BAA+B,EAAE,KAAK,IAAI,EAAE;YAC7C,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,MAAM,EAAE,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE;gBACxC,GAAG,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC;gBACpB,MAAM,EAAE;oBACN,QAAQ,EAAE,EAAE,GAAG,QAAQ,CAAC,GAAG,CAAC,EAAE,WAAW,EAAE,GAAG,EAAE;oBAChD,aAAa,EAAE,IAAI;iBACpB;aACF,CAAC,CAAC;YACH,MAAM,CAAC,EAAE,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACjC,MAAM,CAAC,CAAC,MAAM,KAAK,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACnE,MAAM,EAAE,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;YAC9D,MAAM,CAAC,EAAE,CAAC,WAAW,IAAI,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC;QAC5C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kDAAkD,EAAE,KAAK,IAAI,EAAE;YAChE,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC;YAClD,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC;YACjE,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC7B,MAAM,CAAC,CAAC,MAAM,KAAK,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC/D,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC"}
+{"version":3,"file":"testing.js","sourceRoot":"","sources":["../src/testing.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EACL,gBAAgB,GAGjB,MAAM,eAAe,CAAC;AAIvB;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,qBAAqB,CACnC,IAAY,EACZ,IAA0C;IAE1C,MAAM,OAAO,GAAG,CAAC,KAAa,EAAE,OAAyB,EAAE,EAAW,EAAE,CAAC,CAAC;QACxE,KAAK;QACL,KAAK,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE;QAC1C,IAAI,EAAE,CAAC,GAAG,EAAE,GAAG,CAAC;QAChB,KAAK,EAAE,KAAK;QACZ,MAAM,EAAE,QAAQ;QAChB,MAAM,EAAE,OAAO;QACf,SAAS,EAAE,IAAI;QACf,IAAI,EAAE,IAAI;QACV,MAAM,EAAE;YACN;gBACE,IAAI,EAAE,UAAU;gBAChB,MAAM,EAAE,WAAW;gBACnB,KAAK,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE;gBACxB,SAAS,EAAE,OAAO;gBAClB,OAAO,EAAE,IAAI;gBACb,IAAI,EAAE,IAAI;aACX;SACF;QACD,GAAG,IAAI;KACR,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,CAAC,KAAa,EAAE,EAAE,CAAC,CAAC;QACjC,KAAK;QACL,KAAK,EAAE,SAAS;QAChB,aAAa,EAAE,IAAI;QACnB,eAAe,EAAE,IAAI;QACrB,GAAG,EAAE,KAAc;QACnB,OAAO,EAAE,CAAC;QACV,cAAc,EAAE,IAAI;QACpB,aAAa,EAAE,GAAG;QAClB,gBAAgB,EAAE,IAAI;QACtB,SAAS,EAAE,IAAI;KAChB,CAAC,CAAC;IAEH,QAAQ,CAAC,uBAAuB,IAAI,EAAE,EAAE,GAAG,EAAE;QAC3C,EAAE,CAAC,kDAAkD,EAAE,KAAK,IAAI,EAAE;YAChE,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC;YAC/C,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;YAC5C,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAC9B,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;YACnE,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;YACzC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YACvC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC;gBACtC,IAAI,EAAE,UAAU;gBAChB,MAAM,EAAE,WAAW;gBACnB,KAAK,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE;aACzB,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8DAA8D,EAAE,KAAK,IAAI,EAAE;YAC5E,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC;YACnD,gEAAgE;YAChE,cAAc;YACd,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;YAC3C,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC;YACpD,MAAM,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;YACzC,MAAM,KAAK,CAAC,aAAa,CAAC,KAAK,CAAC,EAAE,EAAE,SAAS,CAAC,CAAC;YAC/C,MAAM,CAAC,CAAC,MAAM,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC;gBAC5D,MAAM,CAAC,EAAE;aACV,CAAC,CAAC;YACH,oDAAoD;YACpD,MAAM,CACJ,CAAC,MAAM,KAAK,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CACxE,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;QACnC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4DAA4D,EAAE,KAAK,IAAI,EAAE;YAC1E,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC;YAC/D,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,EAAE;gBACnC,IAAI,EAAE,UAAU;gBAChB,MAAM,EAAE,SAAS;gBACjB,KAAK,EAAE,EAAE,IAAI,EAAE,GAAG,EAAE;gBACpB,SAAS,EAAE,OAAO;gBAClB,OAAO,EAAE,IAAI;gBACb,IAAI,EAAE,IAAI;aACX,CAAC,CAAC;YACH,MAAM,KAAK,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,EAAE,WAAW,CAAC,CAAC;YAC9C,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;YAC5C,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YACvC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACtD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8DAA8D,EAAE,KAAK,IAAI,EAAE;YAC5E,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YAC7C,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;YAC3C,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YAC7C,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACjC,MAAM,CAAC,CAAC,MAAM,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC;gBAC3D,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,EAAE;aACL,CAAC,CAAC;YACH,MAAM,CAAC,MAAM,KAAK,CAAC,QAAQ,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAChE,MAAM,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,CAAC;YACrD,MAAM,CACJ,CAAC,MAAM,KAAK,CAAC,QAAQ,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CACtE,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YAClB,MAAM,CAAC,MAAM,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;QACnD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;YACzD,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC,EAAE,GAAG,MAAM,CAAC,GAAG,CAAC,EAAE,WAAW,EAAE,GAAG,EAAE,CAAC,CAAC;YACzE,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC,EAAE,GAAG,MAAM,CAAC,GAAG,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC;YAC1E,MAAM,CAAC,CAAC,MAAM,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC7D,MAAM,CAAC,CAAC,MAAM,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,EAAE,WAAW,IAAI,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC;QACxE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0CAA0C,EAAE,KAAK,IAAI,EAAE;YACxD,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC;gBACnC,GAAG,MAAM,CAAC,GAAG,CAAC;gBACd,eAAe,EAAE,IAAI;aACtB,CAAC,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC;gBACjC,GAAG,MAAM,CAAC,GAAG,CAAC;gBACd,eAAe,EAAE,IAAI;aACtB,CAAC,CAAC;YACH,MAAM,CAAC,CAAC,MAAM,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,EAAE,eAAe,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACpE,MAAM,CAAC,CAAC,MAAM,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,EAAE,eAAe,IAAI,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC;QAC5E,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4DAA4D,EAAE,KAAK,IAAI,EAAE;YAC1E,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YAC7C,MAAM,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;YACzE,MAAM,KAAK,CAAC,SAAS,CACnB,CAAC,CAAC,EAAE,EACJ,EAAE,MAAM,EAAE,WAAW,EAAE,EACvB,EAAE,MAAM,EAAE,SAAS,EAAE,CACtB,CAAC;YACF,MAAM,CAAC,CAAC,MAAM,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC5D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8DAA8D,EAAE,KAAK,IAAI,EAAE;YAC5E,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC;YAC/C,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YAC7C,MAAM,IAAI,GAAG;gBACX,KAAK,EAAE,CAAC,CAAC,EAAE;gBACX,MAAM,EAAE,CAAC,CAAC,EAAE;gBACZ,MAAM,EAAE,CAAC;gBACT,WAAW,EAAE,GAAG;gBAChB,MAAM,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE;gBACzB,KAAK,EAAE,IAAI;gBACX,MAAM,EAAE,IAAI;gBACZ,OAAO,EAAE,IAAI;gBACb,EAAE,EAAE,EAAE;gBACN,MAAM,EAAE,KAAK;aACd,CAAC;YACF,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;YACzC,MAAM,KAAK,CAAC,YAAY,CAAC,EAAE,GAAG,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;YACpD,MAAM,KAAK,CAAC,YAAY,CAAC,EAAE,GAAG,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;YACnE,MAAM,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;YAClE,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAC3C,MAAM,CAAC,IAAI,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAC7B,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC,OAAO,CAAC;gBACtD,EAAE,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE;aAC7B,CAAC,CAAC;YACH,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,kBAAkB,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;YAC/D,MAAM,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAC7B,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;YACjD,MAAM,CACJ,MAAM,KAAK,CAAC,kBAAkB,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,CAAC,CACnE,CAAC,QAAQ,EAAE,CAAC;QACf,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4EAA4E,EAAE,KAAK,IAAI,EAAE;YAC1F,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC;YAC/C,MAAM,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;gBACrC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBAC5B,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBAC5B,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;aAC7B,CAAC,CAAC;YACH,MAAM,IAAI,GAAG;gBACX,MAAM,EAAE,CAAC,CAAC,EAAE;gBACZ,MAAM,EAAE,CAAC;gBACT,MAAM,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE;gBACzB,KAAK,EAAE,IAAI;gBACX,MAAM,EAAE,IAAI;gBACZ,OAAO,EAAE,IAAI;gBACb,EAAE,EAAE,EAAE;gBACN,MAAM,EAAE,KAAK;aACd,CAAC;YACF,MAAM,KAAK,CAAC,YAAY,CAAC,EAAE,GAAG,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,EAAE,EAAE,WAAW,EAAE,GAAG,EAAE,CAAC,CAAC;YACtE,MAAM,KAAK,CAAC,YAAY,CAAC,EAAE,GAAG,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,EAAE,EAAE,WAAW,EAAE,GAAG,EAAE,CAAC,CAAC;YACtE,MAAM,KAAK,CAAC,YAAY,CAAC;gBACvB,GAAG,IAAI;gBACP,KAAK,EAAE,EAAE,CAAC,EAAE;gBACZ,WAAW,EAAE,GAAG;gBAChB,MAAM,EAAE,IAAI;aACb,CAAC,CAAC;YACH,MAAM,KAAK,CAAC,YAAY,CAAC;gBACvB,GAAG,IAAI;gBACP,KAAK,EAAE,EAAE,CAAC,EAAE;gBACZ,WAAW,EAAE,GAAG;gBAChB,KAAK,EAAE,MAAM;gBACb,MAAM,EAAE,IAAI;aACb,CAAC,CAAC;YACH,IAAI,KAAK,CAAC,cAAc,EAAE,CAAC;gBACzB,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,cAAc,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;gBACxD,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;gBACvE,MAAM,CAAC,MAAM,KAAK,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;YACrD,CAAC;YACD,IAAI,KAAK,CAAC,kBAAkB,EAAE,CAAC;gBAC7B,MAAM,CACJ;oBACE,GAAG,CAAC,MAAM,KAAK,CAAC,kBAAkB,CAChC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,EACzB,IAAI,IAAI,CAAC,CAAC,CAAC,CACZ,CAAC;iBACH,CAAC,IAAI,EAAE,CACT,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;gBACtB,MAAM,CACJ,CAAC,MAAM,KAAK,CAAC,kBAAkB,CAAC,CAAC,GAAG,CAAC,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,CAAC,CAAC;qBACnE,IAAI,CACR,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBACV,MAAM,CAAC,CAAC,MAAM,KAAK,CAAC,kBAAkB,CAAC,EAAE,EAAE,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACzE,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,sCAAsC,EAAE,KAAK,IAAI,EAAE;YACpD,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,MAAM,IAAI,GACR;gBACE,GAAG,EAAE,UAAU;gBACf,SAAS,EAAE,UAAU;gBACrB,QAAQ,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE;gBAC9B,KAAK,EAAE,UAAU;gBACjB,WAAW,EAAE,kBAAkB;gBAC/B,aAAa,EAAE,GAAG;gBAClB,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC;gBACrC,UAAU,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC;gBAC1C,MAAM,EAAE;oBACN;wBACE,GAAG,EAAE,OAAO;wBACZ,KAAK,EAAE,SAAS;wBAChB,WAAW,EAAE,EAAE;wBACf,IAAI,EAAE,OAAO;wBACb,QAAQ,EAAE,SAAS;qBACpB;iBACF;gBACD,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;gBAClD,KAAK,EAAE,KAAK,EAAE,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC;oBAC9B,EAAE,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,KAAK,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE;iBAC7D;gBACD,aAAa,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC;gBACnC,KAAK,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI;aACrB,CAAC;YACJ,MAAM,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;YAC5C,MAAM,MAAM,GAAG,YAAY,CAAC;gBAC1B,KAAK;gBACL,MAAM,EAAE,CAAC,IAAI,CAAC;gBACd,MAAM,EAAE,EAAE,SAAS,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE;gBAC7D,aAAa,EAAE,GAAG,EAAE,CAAC,IAAI;aAC1B,CAAC,CAAC;YACH,MAAM,EAAE,GAAG,EAAE,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC;gBACrC,KAAK,EAAE,UAAU;gBACjB,KAAK,EAAE,SAAS;aACjB,CAAC,CAAC;YACH,MAAM,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAC7B,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACxC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YACvC,MAAM,CAAC,IAAI,EAAE,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAChD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,6BAA6B,CAC3C,IAAY,EACZ,IAAkD;IAElD,MAAM,MAAM,GAAG,CAAC,OAA2B,EAAE,EAAa,EAAE,CAAC,CAAC;QAC5D,SAAS,EAAE,QAAQ;QACnB,QAAQ,EAAE,UAAU;QACpB,OAAO,EAAE,UAAU;QACnB,MAAM,EAAE,MAAM;QACd,MAAM,EAAE,aAAa;QACrB,OAAO,EAAE,SAAS;QAClB,KAAK,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,mBAAmB,EAAE;QACvD,MAAM,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE;QAC3B,GAAG,IAAI;KACR,CAAC,CAAC;IAEH,QAAQ,CAAC,2BAA2B,IAAI,EAAE,EAAE,GAAG,EAAE;QAC/C,EAAE,CAAC,yDAAyD,EAAE,KAAK,IAAI,EAAE;YACvE,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;YACrC,MAAM,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC;gBACtB,SAAS,EAAE,QAAQ;gBACnB,MAAM,EAAE,MAAM;gBACd,MAAM,EAAE,aAAa;gBACrB,OAAO,EAAE,SAAS;gBAClB,KAAK,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,mBAAmB,EAAE;gBACvD,MAAM,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE;gBAC3B,MAAM,EAAE,MAAM;gBACd,MAAM,EAAE,IAAI;aACb,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,mEAAmE,EAAE,KAAK,IAAI,EAAE;YACjF,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;YACzC,MAAM,KAAK,CAAC,iBAAiB,CAAC,KAAK,CAAC,EAAE,EAAE,WAAW,CAAC,CAAC;YACrD,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;YAC3D,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAClC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACpC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,CAAC;YAClC,MAAM,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC;YAClD,MAAM,CAAC,MAAM,KAAK,CAAC,YAAY,EAAE,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACrD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,qBAAqB,EAAE,KAAK,IAAI,EAAE;YACnC,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,MAAM,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;YAC3B,MAAM,KAAK,CAAC,WAAW,CAAC,UAAU,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC;YAC1D,MAAM,CAAC,MAAM,KAAK,CAAC,YAAY,EAAE,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACrD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4DAA4D,EAAE,KAAK,IAAI,EAAE;YAC1E,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;YACtD,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;YAC3C,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;YACpE,MAAM,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,GAAG,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC;YAClE,MAAM,CACJ,CAAC,MAAM,KAAK,CAAC,YAAY,CAAC,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CACrE,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YACxB,MAAM,CAAC,MAAM,KAAK,CAAC,YAAY,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YACnE,MAAM,KAAK,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YACvD,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,YAAY,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC;YAC5D,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC;YACvE,MAAM,CACJ,MAAM,KAAK,CAAC,YAAY,CAAC,EAAE,KAAK,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,EAAE,CAAC,CACnE,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAClB,MAAM,CAAC,MAAM,KAAK,CAAC,YAAY,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACjE,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,6BAA6B,CAC3C,IAAY,EACZ,IAAkD;IAElD,MAAM,QAAQ,GAAG,CAAC,KAAa,EAAY,EAAE,CAAC,CAAC;QAC7C,KAAK;QACL,aAAa,EAAE,IAAI;QACnB,eAAe,EAAE,IAAI;QACrB,eAAe,EAAE,IAAI;KACtB,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,CAAC,KAAa,EAAE,UAAyB,EAAE,EAAE,CAAC,CAAC;QAC5D,MAAM,EAAE,EAAE,QAAQ,EAAE,QAAQ,CAAC,KAAK,CAAC,EAAE,aAAa,EAAE,IAAI,EAAE;QAC1D,IAAI,EAAE,QAAiB;QACvB,MAAM,EAAE,MAAM,KAAK,EAAE;QACrB,EAAE,EAAE,UAAU;QACd,UAAU;KACX,CAAC,CAAC;IAEH,QAAQ,CAAC,2BAA2B,IAAI,EAAE,EAAE,GAAG,EAAE;QAC/C,EAAE,CAAC,gEAAgE,EAAE,KAAK,IAAI,EAAE;YAC9E,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,MAAM,CAAC,MAAM,KAAK,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;YACrD,MAAM,EAAE,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC;YAC7D,MAAM,EAAE,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;YAC9D,MAAM,CAAC,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;YAC/C,MAAM,CAAC,EAAE,CAAC,CAAC,aAAa,CAAC;gBACvB,SAAS,EAAE,QAAQ;gBACnB,KAAK,EAAE,GAAG;gBACV,aAAa,EAAE,IAAI;aACpB,CAAC,CAAC;YACH,MAAM,CAAC,CAAC,MAAM,KAAK,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;YAC5D,MAAM,CAAC,CAAC,MAAM,KAAK,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CACvE,CAAC,CAAC,EAAE,CAAC,CAAC,CACP,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8DAA8D,EAAE,KAAK,IAAI,EAAE;YAC5E,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,MAAM,EAAE,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC;YAC7D,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE;gBAC7B,GAAG,MAAM,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,CAAC;gBACrB,IAAI,EAAE,SAAS;gBACf,QAAQ,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE;aAC7C,CAAC,CAAC;YACH,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,GAAG,MAAM,KAAK,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;YAC1D,MAAM,CAAC,MAAM,CAAC,CAAC,aAAa,CAAC;gBAC3B,IAAI,EAAE,SAAS;gBACf,aAAa,EAAE,EAAE,CAAC,EAAE;gBACpB,MAAM,EAAE,MAAM;gBACd,EAAE,EAAE,UAAU;gBACd,QAAQ,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE;aAC7C,CAAC,CAAC;YACH,MAAM,CAAC,KAAK,CAAC,CAAC,aAAa,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;YACrE,MAAM,CAAC,MAAM,KAAK,CAAC,WAAW,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC1E,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uDAAuD,EAAE,KAAK,IAAI,EAAE;YACrE,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,MAAM,EAAE,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC;YAC7D,MAAM,EAAE,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;YAC9D,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE;gBAC7B,SAAS,EAAE,EAAE,CAAC,EAAE;gBAChB,IAAI,EAAE,UAAU;gBAChB,MAAM,EAAE,aAAa;gBACrB,EAAE,EAAE,UAAU;gBACd,UAAU,EAAE,EAAE,CAAC,EAAE;aAClB,CAAC,CAAC;YACH,MAAM,CAAC,CAAC,MAAM,KAAK,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;YAC5D,MAAM,CAAC,MAAM,KAAK,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC7D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8DAA8D,EAAE,KAAK,IAAI,EAAE;YAC5E,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,MAAM,EAAE,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC;YAC7D,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;YACnD,MAAM,MAAM,CACV,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAC7C,CAAC,OAAO,CAAC,cAAc,CAAC,gBAAgB,CAAC,CAAC;YAC3C,MAAM,CAAC,CAAC,MAAM,KAAK,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC7D,MAAM,CAAC,MAAM,KAAK,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC7D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gEAAgE,EAAE,KAAK,IAAI,EAAE;YAC9E,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,MAAM,EAAE,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE;gBACxC,GAAG,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC;gBACpB,MAAM,EAAE;oBACN,QAAQ,EAAE,EAAE,GAAG,QAAQ,CAAC,GAAG,CAAC,EAAE,eAAe,EAAE,OAAO,EAAE;oBACxD,aAAa,EAAE,IAAI;iBACpB;aACF,CAAC,CAAC;YACH,MAAM,CAAC,EAAE,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACzC,MAAM,CAAC,CAAC,MAAM,KAAK,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,EAAE,eAAe,CAAC,CAAC,IAAI,CAC/D,OAAO,CACR,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+BAA+B,EAAE,KAAK,IAAI,EAAE;YAC7C,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,MAAM,EAAE,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE;gBACxC,GAAG,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC;gBACpB,MAAM,EAAE;oBACN,QAAQ,EAAE,EAAE,GAAG,QAAQ,CAAC,GAAG,CAAC,EAAE,WAAW,EAAE,GAAG,EAAE;oBAChD,aAAa,EAAE,IAAI;iBACpB;aACF,CAAC,CAAC;YACH,MAAM,CAAC,EAAE,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACjC,MAAM,CAAC,CAAC,MAAM,KAAK,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACnE,MAAM,EAAE,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;YAC9D,MAAM,CAAC,EAAE,CAAC,WAAW,IAAI,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC;QAC5C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kDAAkD,EAAE,KAAK,IAAI,EAAE;YAChE,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC;YAClD,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC;YACjE,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC7B,MAAM,CAAC,CAAC,MAAM,KAAK,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC/D,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC"}

package/dist/types.d.ts

@@ -98,6 +98,11 @@ export type Suite<Input, Output, Label> = {
          * temperature was versioned.
          */
         temperature?: number | null;
+        /**
+         * The output-token cap the test runs at, like temperature. Null: send
+         * none (the call's own default).
+         */
+        maxOutputTokens?: number | null;
         onCost: (usd: number) => void;
     }): Promise<Output>;
     /**

package/dist/types.js.map

@@ -1 +1 @@
-{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAoDA,gFAAgF;AAChF,MAAM,UAAU,QAAQ,CAAC,CAAW;IAClC,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;AAC3D,CAAC;AAED,qEAAqE;AACrE,MAAM,UAAU,SAAS,CACvB,IAAY,EACZ,GAAW;IAEX,OAAO,IAAI,IAAI,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,IAAI,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC;AACvE,CAAC;AAcD,mDAAmD;AACnD,MAAM,CAAC,MAAM,UAAU,GAA2C;IAChE,KAAK,EACH,oLAAoL;IACtL,IAAI,EAAE,4LAA4L;IAClM,OAAO,EACL,wHAAwH;CAC3H,CAAC;AAyIF,wEAAwE;AACxE,MAAM,UAAU,MAAM,CAAC,CAAU;IAC/B,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,IAAI;QAAE,OAAO,EAAE,CAAC;IAC7C,IAAI,OAAO,CAAC,KAAK,QAAQ;QAAE,OAAO,CAAC,CAAC;IACpC,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,OAAO,CAAC,KAAK,SAAS;QAAE,OAAO,MAAM,CAAC,CAAC,CAAC,CAAC;IACtE,OAAO,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;AAC3B,CAAC;AAED,uDAAuD;AACvD,MAAM,UAAU,QAAQ,CAAC,IAAc,EAAE,IAAY;IACnD,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACzB,OAAO,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;AAC/C,CAAC;AAED,2EAA2E;AAC3E,MAAM,UAAU,UAAU,CAAC,KAAgB,EAAE,KAAc;IACzD,MAAM,CAAC,GACL,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ;QAChC,CAAC,CAAE,KAAiC,CAAC,KAAK,CAAC,IAAI,CAAC;QAChD,CAAC,CAAC,SAAS,CAAC;IAChB,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;QAAE,OAAO,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACtD,OAAO,MAAM,CAAC,CAAC,CAAC,CAAC;AACnB,CAAC;AAED,sEAAsE;AACtE,MAAM,UAAU,OAAO,CAAC,IAAc,EAAE,IAAY;IAClD,OAAO,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC;SACxB,KAAK,CAAC,IAAI,CAAC;SACX,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;SACpB,MAAM,CAAC,OAAO,CAAC,CAAC;AACrB,CAAC"}
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAoDA,gFAAgF;AAChF,MAAM,UAAU,QAAQ,CAAC,CAAW;IAClC,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;AAC3D,CAAC;AAED,qEAAqE;AACrE,MAAM,UAAU,SAAS,CACvB,IAAY,EACZ,GAAW;IAEX,OAAO,IAAI,IAAI,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,IAAI,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC;AACvE,CAAC;AAcD,mDAAmD;AACnD,MAAM,CAAC,MAAM,UAAU,GAA2C;IAChE,KAAK,EACH,oLAAoL;IACtL,IAAI,EAAE,4LAA4L;IAClM,OAAO,EACL,wHAAwH;CAC3H,CAAC;AA8IF,wEAAwE;AACxE,MAAM,UAAU,MAAM,CAAC,CAAU;IAC/B,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,IAAI;QAAE,OAAO,EAAE,CAAC;IAC7C,IAAI,OAAO,CAAC,KAAK,QAAQ;QAAE,OAAO,CAAC,CAAC;IACpC,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,OAAO,CAAC,KAAK,SAAS;QAAE,OAAO,MAAM,CAAC,CAAC,CAAC,CAAC;IACtE,OAAO,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;AAC3B,CAAC;AAED,uDAAuD;AACvD,MAAM,UAAU,QAAQ,CAAC,IAAc,EAAE,IAAY;IACnD,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACzB,OAAO,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;AAC/C,CAAC;AAED,2EAA2E;AAC3E,MAAM,UAAU,UAAU,CAAC,KAAgB,EAAE,KAAc;IACzD,MAAM,CAAC,GACL,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ;QAChC,CAAC,CAAE,KAAiC,CAAC,KAAK,CAAC,IAAI,CAAC;QAChD,CAAC,CAAC,SAAS,CAAC;IAChB,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;QAAE,OAAO,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACtD,OAAO,MAAM,CAAC,CAAC,CAAC,CAAC;AACnB,CAAC;AAED,sEAAsE;AACtE,MAAM,UAAU,OAAO,CAAC,IAAc,EAAE,IAAY;IAClD,OAAO,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC;SACxB,KAAK,CAAC,IAAI,CAAC;SACX,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;SACpB,MAAM,CAAC,OAAO,CAAC,CAAC;AACrB,CAAC"}

package/dist/weekly.js

@@ -52,13 +52,14 @@ export function weeklySummary(args) {
         }
         if (r.status !== "SUCCEEDED")
             continue;
-        // The last check of the same model, temperature, instructions, and
-        // grading, a day or more earlier: anything else changed for a known
+        // The last check of the same model, temperature, output cap,
+        // instructions, and grading, a day or more earlier: anything else changed for a known
         // reason.
         const prior = scheduled.find((p) => p.suite === r.suite &&
             p.status === "SUCCEEDED" &&
             p.model === r.model &&
             (p.temperature ?? null) === (r.temperature ?? null) &&
+            (p.maxOutputTokens ?? null) === (r.maxOutputTokens ?? null) &&
             p.promptVersion === r.promptVersion &&
             p.graderVersion === r.graderVersion &&
             +r.startedAt - +p.startedAt >= DAY);

package/dist/weekly.js.map

@@ -1 +1 @@
-{"version":3,"file":"weekly.js","sourceRoot":"","sources":["../src/weekly.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAqB,MAAM,aAAa,CAAC;AAE9D,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAEnE;;;;;;GAMG;AAEH,2EAA2E;AAC3E,MAAM,CAAC,MAAM,SAAS,GAAG,UAAU,CAAC;AAEpC,MAAM,GAAG,GAAG,UAAU,CAAC;AAqCvB,0EAA0E;AAC1E,MAAM,UAAU,QAAQ,CAAC,CAAoB;IAC3C,IAAI,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IACpB,OAAO,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,KAAK,CAAC,KAAK;QAC7B,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,KAAK;QAC/B,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC;AAC5B,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,aAAa,CAAC,IAU7B;IACC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,IAAI,IAAI,EAAE,CAAC;IACvC,MAAM,KAAK,GAAG,CAAC,GAAW,EAAE,EAAE,CAC5B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,GAAG,CAAC,EAAE,KAAK,IAAI,GAAG,CAAC;IACvD,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI;SACxB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,SAAS,IAAI,CAAC,CAAC,GAAG,KAAK,OAAO,CAAC;SAC7D,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IAC/C,MAAM,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,IAAI,IAAI,CAAC;IAC/C,wEAAwE;IACxE,MAAM,KAAK,GAAG,MAAM;QAClB,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,SAAS,GAAG,GAAG,CAAC;QACvD,CAAC,CAAC,EAAE,CAAC;IAEP,MAAM,KAAK,GAA2B,EAAE,CAAC;IACzC,MAAM,MAAM,GAA4B,EAAE,CAAC;IAC3C,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,IAAI,CAAC,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YAC1B,MAAM,CAAC,IAAI,CAAC;gBACV,KAAK,EAAE,CAAC,CAAC,KAAK;gBACd,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC;gBACrB,KAAK,EAAE,CAAC,CAAC,KAAK,IAAI,eAAe;aAClC,CAAC,CAAC;YACH,SAAS;QACX,CAAC;QACD,IAAI,CAAC,CAAC,MAAM,KAAK,WAAW;YAAE,SAAS;QACvC,mEAAmE;QACnE,oEAAoE;QACpE,UAAU;QACV,MAAM,KAAK,GAAG,SAAS,CAAC,IAAI,CAC1B,CAAC,CAAC,EAAE,EAAE,CACJ,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,KAAK;YACnB,CAAC,CAAC,MAAM,KAAK,WAAW;YACxB,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,KAAK;YACnB,CAAC,CAAC,CAAC,WAAW,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,WAAW,IAAI,IAAI,CAAC;YACnD,CAAC,CAAC,aAAa,KAAK,CAAC,CAAC,aAAa;YACnC,CAAC,CAAC,aAAa,KAAK,CAAC,CAAC,aAAa;YACnC,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,SAAS,IAAI,GAAG,CACrC,CAAC;QACF,MAAM,MAAM,GAAG,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QACnC,MAAM,QAAQ,GAAG,QAAQ,CAAC,KAAK,EAAE,OAAO,IAAI,IAAI,CAAC,CAAC;QAClD,IAAI,MAAM,KAAK,IAAI,IAAI,QAAQ,KAAK,IAAI,IAAI,OAAO,CAAC,QAAQ,EAAE,MAAM,CAAC;YACnE,KAAK,CAAC,IAAI,CAAC;gBACT,KAAK,EAAE,CAAC,CAAC,KAAK;gBACd,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC;gBACrB,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC,KAAK;gBAC7C,QAAQ;gBACR,MAAM;aACP,CAAC,CAAC;IACP,CAAC;IAED,MAAM,QAAQ,GAA8B,EAAE,CAAC;IAC/C,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QAC5B,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QACvC,IAAI,CAAC,KAAK;YAAE,SAAS;QACrB,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI;YAC1B,CAAC,KAAK,CAAC,KAAK,EAAE,MAAM,CAAC;YACrB,CAAC,KAAK,CAAC,aAAa,EAAE,UAAU,CAAC;SACzB,EAAE,CAAC;YACX,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC;YAC9D,IAAI,KAAK,IAAI,IAAI,IAAI,WAAW,CAAC,IAAI,EAAE,KAAK,CAAC;gBAC3C,QAAQ,CAAC,IAAI,CAAC;oBACZ,KAAK,EAAE,CAAC,CAAC,GAAG;oBACZ,KAAK,EAAE,CAAC,CAAC,KAAK;oBACd,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,KAAK,IAAI,KAAK;oBACzC,IAAI;oBACJ,IAAI,EAAE,YAAY,CAAC,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,IAAI;oBAC9C,IAAI,EAAE,SAAS,CAAC,IAAI,EAAE,KAAK,CAAC;iBAC7B,CAAC,CAAC;QACP,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAG,CAAC,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IAC1D,OAAO;QACL,MAAM;QACN,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE,OAAO,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC;QACjE,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,MAAM;QAC3D,KAAK;QACL,MAAM;QACN,QAAQ;QACR,MAAM;QACN,KAAK,EACH,MAAM,KAAK,IAAI;YACf,CAAC,KAAK,CAAC,MAAM;YACb,CAAC,MAAM,CAAC,MAAM;YACd,CAAC,QAAQ,CAAC,MAAM;YAChB,CAAC,MAAM,CAAC,MAAM;KACjB,CAAC;AACJ,CAAC"}
+{"version":3,"file":"weekly.js","sourceRoot":"","sources":["../src/weekly.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAqB,MAAM,aAAa,CAAC;AAE9D,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAEnE;;;;;;GAMG;AAEH,2EAA2E;AAC3E,MAAM,CAAC,MAAM,SAAS,GAAG,UAAU,CAAC;AAEpC,MAAM,GAAG,GAAG,UAAU,CAAC;AAqCvB,0EAA0E;AAC1E,MAAM,UAAU,QAAQ,CAAC,CAAoB;IAC3C,IAAI,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IACpB,OAAO,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,KAAK,CAAC,KAAK;QAC7B,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,KAAK;QAC/B,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC;AAC5B,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,aAAa,CAAC,IAU7B;IACC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,IAAI,IAAI,EAAE,CAAC;IACvC,MAAM,KAAK,GAAG,CAAC,GAAW,EAAE,EAAE,CAC5B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,GAAG,CAAC,EAAE,KAAK,IAAI,GAAG,CAAC;IACvD,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI;SACxB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,SAAS,IAAI,CAAC,CAAC,GAAG,KAAK,OAAO,CAAC;SAC7D,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IAC/C,MAAM,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,IAAI,IAAI,CAAC;IAC/C,wEAAwE;IACxE,MAAM,KAAK,GAAG,MAAM;QAClB,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,SAAS,GAAG,GAAG,CAAC;QACvD,CAAC,CAAC,EAAE,CAAC;IAEP,MAAM,KAAK,GAA2B,EAAE,CAAC;IACzC,MAAM,MAAM,GAA4B,EAAE,CAAC;IAC3C,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,IAAI,CAAC,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YAC1B,MAAM,CAAC,IAAI,CAAC;gBACV,KAAK,EAAE,CAAC,CAAC,KAAK;gBACd,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC;gBACrB,KAAK,EAAE,CAAC,CAAC,KAAK,IAAI,eAAe;aAClC,CAAC,CAAC;YACH,SAAS;QACX,CAAC;QACD,IAAI,CAAC,CAAC,MAAM,KAAK,WAAW;YAAE,SAAS;QACvC,6DAA6D;QAC7D,sFAAsF;QACtF,UAAU;QACV,MAAM,KAAK,GAAG,SAAS,CAAC,IAAI,CAC1B,CAAC,CAAC,EAAE,EAAE,CACJ,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,KAAK;YACnB,CAAC,CAAC,MAAM,KAAK,WAAW;YACxB,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,KAAK;YACnB,CAAC,CAAC,CAAC,WAAW,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,WAAW,IAAI,IAAI,CAAC;YACnD,CAAC,CAAC,CAAC,eAAe,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,eAAe,IAAI,IAAI,CAAC;YAC3D,CAAC,CAAC,aAAa,KAAK,CAAC,CAAC,aAAa;YACnC,CAAC,CAAC,aAAa,KAAK,CAAC,CAAC,aAAa;YACnC,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,SAAS,IAAI,GAAG,CACrC,CAAC;QACF,MAAM,MAAM,GAAG,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QACnC,MAAM,QAAQ,GAAG,QAAQ,CAAC,KAAK,EAAE,OAAO,IAAI,IAAI,CAAC,CAAC;QAClD,IAAI,MAAM,KAAK,IAAI,IAAI,QAAQ,KAAK,IAAI,IAAI,OAAO,CAAC,QAAQ,EAAE,MAAM,CAAC;YACnE,KAAK,CAAC,IAAI,CAAC;gBACT,KAAK,EAAE,CAAC,CAAC,KAAK;gBACd,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC;gBACrB,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC,KAAK;gBAC7C,QAAQ;gBACR,MAAM;aACP,CAAC,CAAC;IACP,CAAC;IAED,MAAM,QAAQ,GAA8B,EAAE,CAAC;IAC/C,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QAC5B,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QACvC,IAAI,CAAC,KAAK;YAAE,SAAS;QACrB,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI;YAC1B,CAAC,KAAK,CAAC,KAAK,EAAE,MAAM,CAAC;YACrB,CAAC,KAAK,CAAC,aAAa,EAAE,UAAU,CAAC;SACzB,EAAE,CAAC;YACX,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC;YAC9D,IAAI,KAAK,IAAI,IAAI,IAAI,WAAW,CAAC,IAAI,EAAE,KAAK,CAAC;gBAC3C,QAAQ,CAAC,IAAI,CAAC;oBACZ,KAAK,EAAE,CAAC,CAAC,GAAG;oBACZ,KAAK,EAAE,CAAC,CAAC,KAAK;oBACd,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,KAAK,IAAI,KAAK;oBACzC,IAAI;oBACJ,IAAI,EAAE,YAAY,CAAC,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,IAAI;oBAC9C,IAAI,EAAE,SAAS,CAAC,IAAI,EAAE,KAAK,CAAC;iBAC7B,CAAC,CAAC;QACP,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAG,CAAC,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IAC1D,OAAO;QACL,MAAM;QACN,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE,OAAO,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC;QACjE,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,MAAM;QAC3D,KAAK;QACL,MAAM;QACN,QAAQ;QACR,MAAM;QACN,KAAK,EACH,MAAM,KAAK,IAAI;YACf,CAAC,KAAK,CAAC,MAAM;YACb,CAAC,MAAM,CAAC,MAAM;YACd,CAAC,QAAQ,CAAC,MAAM;YAChB,CAAC,MAAM,CAAC,MAAM;KACjB,CAAC;AACJ,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@crowandraven/evalkeeper",
-  "version": "0.2.0",
+  "version": "0.4.0",
   "description": "An open-source eval harness for TypeScript apps: test each AI feature against answers people confirm, compare models on accuracy, cost, and speed, and choose the live model on evidence.",
   "license": "MIT",
   "type": "module",

package/src/common.ts

@@ -77,9 +77,12 @@ export function withCountedWords(text: string): string {
  * A number with an optional "k" or "m" ("150k", "1.5m"). It starts only
  * where a number starts and reads it once (the lookahead and backreference
  * keep the digits from being retried), so long digit runs in model output
- * stay linear. A number glued to a word ("3rd", "10am") doesn't count.
+ * stay linear. A number glued to a word ("3rd", "10am") doesn't count. A
+ * comma belongs to a number only as a thousands separator ("150,000"), so
+ * "In 2020, I joined" reads 2020, not "2020,".
  */
-const NUMBER = /(?<![\d,.])(?=(\d[\d,]*(?:\.\d+)?))\1\s*([km])?\b/gi;
+const NUMBER =
+  /(?<![\d,.])(?=((?:\d{1,3}(?:,\d{3})+|\d+)(?:\.\d+)?))\1\s*([km])?\b/gi;
 
 /**
  * Numbers the text states that appear in none of the sources: likely invented.

package/src/importCases.ts

@@ -0,0 +1,252 @@
+import { nextSplit } from "./quickSet.js";
+import { stableJson } from "./stableJson.js";
+import type { EvalStore, NewCase, NewLabel } from "./store.js";
+import type { AnySuite } from "./types.js";
+
+/** One row of an import, mapped to the suite's shapes. */
+export type ImportedCase = {
+  /** Stable within its source (a row id in the file), so a rerun skips it. */
+  id: string;
+  /** The suite's input; checked against `suite.input`. */
+  input: unknown;
+  /** The answer, checked against `suite.labelValue`; null: none yet. */
+  label: unknown;
+  tags?: string[];
+  note?: string | null;
+  /** This row's set, instead of the import's `split`. */
+  split?: "DEV" | "TEST";
+};
+
+/**
+ * How imported answers arrive. Pending (the default): a person confirms
+ * each one on the pages before tests use it. Provisional: tests use them
+ * now, every score says it rests on unreviewed answers, and the rows carry
+ * `tag` so the review queue is findable. Use provisional only when the
+ * owner said so; `allowedBy` and `at` record who and when. Nothing is
+ * imported confirmed: only a person confirms an answer.
+ */
+export type ImportAnswers =
+  | { status: "PENDING" }
+  | {
+      status: "PROVISIONAL";
+      /** Who drafted the answers: an agent's name, recorded as the rater. */
+      rater: string;
+      /** The person who allowed using them before review. */
+      allowedBy: string;
+      /** When they allowed it; default now. */
+      at?: Date;
+      /** Added to each row's tags; default "needs-review". */
+      tag?: string;
+    };
+
+/** A row that couldn't be imported, and why. */
+export type ImportProblem = { id: string; problem: string };
+
+/** Rows that don't fit the suite. Nothing was written. */
+export class CaseImportError extends Error {
+  readonly problems: ImportProblem[];
+  constructor(source: string, problems: ImportProblem[]) {
+    super(
+      `${problems.length} row${problems.length === 1 ? "" : "s"} in ${source} can't be imported, so none were: ${problems
+        .slice(0, 5)
+        .map((p) => `${p.id} (${p.problem})`)
+        .join("; ")}${problems.length > 5 ? "; …" : ""}`,
+    );
+    this.name = "CaseImportError";
+    this.problems = problems;
+  }
+}
+
+/** What an import did. */
+export type ImportResult = {
+  /** Cases created. */
+  added: number;
+  /** Rows imported before (by source and id), left alone. */
+  skipped: number;
+};
+
+const sourceRefOf = (source: string, id: string) => `${source}#${id}`;
+
+/**
+ * Imports existing test data as cases for one suite. Every row is mapped
+ * and checked against the suite's input and answer schemas first; if any
+ * row fails, nothing is written and `CaseImportError` lists them. Rows
+ * imported before (same source and id, retired ones included) are skipped,
+ * so it can run again after the file grows.
+ *
+ * Splits: "DEV" for cases you tune against, "TEST" for a held-out file,
+ * "auto" (the default) for new cases, split like cases added by hand.
+ *
+ * Run it against your development database; an app's own guard should
+ * refuse a production one.
+ *
+ * @example
+ * ```ts
+ * const { added, skipped } = await importCases({
+ *   store,
+ *   suite: ticketTriage,
+ *   source: "tickets-heldout.json",
+ *   rows: JSON.parse(await readFile("tickets-heldout.json", "utf8")),
+ *   map: (r) => ({ id: r.id, input: { body: r.text }, label: { queue: r.queue } }),
+ *   split: "TEST",
+ * });
+ * ```
+ */
+export async function importCases<Row>(opts: {
+  store: EvalStore;
+  suite: AnySuite;
+  /** Where the rows came from, such as a file name. */
+  source: string;
+  rows: readonly Row[];
+  map: (row: Row, index: number) => ImportedCase;
+  split?: "DEV" | "TEST" | "auto";
+  answers?: ImportAnswers;
+}): Promise<ImportResult> {
+  const { store, suite, source } = opts;
+  const answers = opts.answers ?? { status: "PENDING" as const };
+
+  // 1. Map and check every row before writing anything.
+  const problems: ImportProblem[] = [];
+  const seen = new Set<string>();
+  const mapped: (ImportedCase & { input: unknown; label: unknown })[] = [];
+  opts.rows.forEach((row, i) => {
+    let c: ImportedCase;
+    try {
+      c = opts.map(row, i);
+    } catch (e) {
+      problems.push({
+        id: `row ${i + 1}`,
+        problem: e instanceof Error ? e.message : String(e),
+      });
+      return;
+    }
+    const id = c.id || `row ${i + 1}`;
+    if (!c.id) problems.push({ id, problem: "no id" });
+    else if (seen.has(c.id))
+      problems.push({ id, problem: "the same id as an earlier row" });
+    seen.add(c.id);
+    const input = suite.input.safeParse(c.input);
+    if (!input.success)
+      problems.push({
+        id,
+        problem: `input: ${input.error.issues[0]?.message ?? "invalid"}`,
+      });
+    let label: unknown = null;
+    if (c.label !== null && c.label !== undefined) {
+      const l = suite.labelValue.safeParse(c.label);
+      if (!l.success)
+        problems.push({
+          id,
+          problem: `answer: ${l.error.issues[0]?.message ?? "invalid"}`,
+        });
+      else label = l.data;
+    }
+    if (input.success) mapped.push({ ...c, input: input.data, label });
+  });
+  if (problems.length) throw new CaseImportError(source, problems);
+
+  // 2. Skip what's already there, retired cases included.
+  const existing = await store.listCases(suite.key, { includeRetired: true });
+  const refs = new Set(existing.map((c) => c.sourceRef));
+  const counts = { DEV: 0, TEST: 0 };
+  for (const c of existing) if (c.status !== "RETIRED") counts[c.split] += 1;
+
+  const at =
+    answers.status === "PROVISIONAL" ? (answers.at ?? new Date()) : null;
+  let added = 0;
+  let skipped = 0;
+  for (const c of mapped) {
+    const sourceRef = sourceRefOf(source, c.id);
+    if (refs.has(sourceRef)) {
+      skipped += 1;
+      continue;
+    }
+    const split =
+      c.split ??
+      (opts.split === "DEV" || opts.split === "TEST"
+        ? opts.split
+        : nextSplit(counts));
+    counts[split] += 1;
+    const labels: NewLabel[] = [];
+    if (c.label !== null)
+      labels.push(
+        answers.status === "PROVISIONAL"
+          ? {
+              kind: "EXPECTED",
+              status: "PROVISIONAL",
+              value: c.label,
+              raterRole: "AGENT",
+              raterId: answers.rater,
+              note: `Drafted by ${answers.rater}; ${answers.allowedBy} allowed testing on it before review (${at?.toISOString().slice(0, 10)}).`,
+            }
+          : {
+              kind: "EXPECTED",
+              status: "PENDING",
+              value: c.label,
+              raterRole: "IMPORT",
+              raterId: null,
+              note: null,
+            },
+      );
+    const tags = [...(c.tags ?? [])];
+    if (answers.status === "PROVISIONAL" && c.label !== null) {
+      const tag = answers.tag ?? "needs-review";
+      if (!tags.includes(tag)) tags.push(tag);
+    }
+    const data: NewCase = {
+      suite: suite.key,
+      input: c.input,
+      tags,
+      split,
+      // In use only with an answer tests may grade against.
+      status:
+        c.label !== null && answers.status === "PROVISIONAL"
+          ? "ACTIVE"
+          : "DRAFT",
+      source: "IMPORT",
+      sourceRef,
+      note: c.note ?? null,
+      labels,
+    };
+    await store.createCase(data);
+    refs.add(sourceRef);
+    added += 1;
+  }
+  return { added, skipped };
+}
+
+/** A short, stable id for a starter case: a hash of its input. */
+function starterId(input: unknown): string {
+  // FNV-1a: the core stays free of node:crypto; this only names rows.
+  let h = 0x811c9dc5;
+  for (const ch of stableJson(input)) {
+    h ^= ch.charCodeAt(0);
+    h = Math.imul(h, 0x01000193) >>> 0;
+  }
+  return h.toString(16).padStart(8, "0");
+}
+
+/**
+ * Adds a suite's `starter` cases as drafts with pending answers, for a
+ * person to confirm on the pages. Safe to run again: starters already
+ * added (by their input) are skipped.
+ */
+export function importStarters(
+  store: EvalStore,
+  suite: AnySuite,
+): Promise<ImportResult> {
+  return importCases({
+    store,
+    suite,
+    source: `starter:${suite.key}`,
+    rows: suite.starter ?? [],
+    map: (s) => ({
+      id: starterId(s.input),
+      input: s.input,
+      label: s.label,
+      tags: s.tags,
+      note: s.note,
+      split: s.split,
+    }),
+  });
+}

package/src/index.ts

@@ -31,3 +31,4 @@ export * from "./store.js";
 export * from "./labels.js";
 // Server-only (node:crypto): import from "@crowandraven/evalkeeper/runner".
 // In-memory store for tests and demos: "@crowandraven/evalkeeper/memory".
+export * from "./importCases.js";

package/src/labels.ts

@@ -1,11 +1,16 @@
-import type { LabelRecord } from "./store.js";
+/** What the answer-in-use rule needs from a label: the store's, or an app's own row. */
+export type LabelLike = { status: string; updatedAt: Date };
 
 /**
  * The EXPECTED answer tests use: the newest confirmed one, or failing that a
  * provisional one (put to use before review). Pending answers never count.
+ * Generic, so pages that read their own label rows (with extra columns)
+ * share the rule and keep their row type.
  */
-export function confirmedLabel(labels: LabelRecord[]): LabelRecord | null {
-  const newest = (status: LabelRecord["status"]) =>
+export function confirmedLabel<L extends LabelLike>(
+  labels: readonly L[],
+): L | null {
+  const newest = (status: string) =>
     labels
       .filter((l) => l.status === status)
       .sort((a, b) => +b.updatedAt - +a.updatedAt)[0] ?? null;
@@ -13,6 +18,6 @@ export function confirmedLabel(labels: LabelRecord[]): LabelRecord | null {
 }
 
 /** In use on an answer nobody has reviewed yet. */
-export function isProvisional(labels: LabelRecord[]): boolean {
+export function isProvisional(labels: readonly LabelLike[]): boolean {
   return confirmedLabel(labels)?.status === "PROVISIONAL";
 }

package/src/memoryStore.ts

@@ -163,9 +163,13 @@ export function memoryStore(): EvalStore &
         f.updatedAt = now();
       }
     },
-    async listCases(suite) {
+    async listCases(suite, opts) {
       return cases
-        .filter((c) => c.suite === suite && c.status !== "RETIRED")
+        .filter(
+          (c) =>
+            c.suite === suite &&
+            (opts?.includeRetired || c.status !== "RETIRED"),
+        )
         .sort((a, b) => +a.createdAt - +b.createdAt);
     },
     async createCase({ labels = [], ...data }) {
@@ -242,6 +246,10 @@ export function memoryStore(): EvalStore &
     async listOutputs(runId) {
       return outputs.filter((o) => o.runId === runId);
     },
+    async listOutputsFor(runIds) {
+      const ids = new Set(runIds);
+      return outputs.filter((o) => ids.has(o.runId));
+    },
     async findReusableOutput(requestHash, since) {
       return (
         outputs
@@ -255,6 +263,20 @@ export function memoryStore(): EvalStore &
           .sort((a, b) => +b.createdAt - +a.createdAt)[0] ?? null
       );
     },
+    async findReusableHashes(requestHashes, since) {
+      const wanted = new Set(requestHashes);
+      return new Set(
+        outputs
+          .filter(
+            (o) =>
+              wanted.has(o.requestHash) &&
+              !o.reused &&
+              !o.error &&
+              o.createdAt >= since,
+          )
+          .map((o) => o.requestHash),
+      );
+    },
     async createOutput(data) {
       const o: OutputRecord = { ...data, id: id("out"), createdAt: now() };
       outputs.push(o);