@crowandraven/evalkeeper 0.0.0-stage → 0.2.0

package/dist/stats.d.ts

@@ -0,0 +1,98 @@
+import type { CheckResult } from "./types.js";
+/**
+ * Summary numbers for a run: pass rates per check with a 95% range, so a
+ * small dataset doesn't look more certain than it is. Pure; no database.
+ */
+/** A pass rate with its 95% range; null when nothing was measured. */
+export type Rate = {
+    passed: number;
+    /** Outputs the check applied to. */
+    n: number;
+    rate: number | null;
+    /** 95% Wilson interval. */
+    low: number | null;
+    high: number | null;
+};
+/** Wilson score interval: honest for small n and rates near 0 or 1. */
+export declare function wilson(passed: number, n: number, z?: number): Rate;
+/** One graded answer, as the metrics read it. */
+export type ScoredOutput = {
+    caseId: string;
+    repeat: number;
+    /** null when the call failed. */
+    checks: CheckResult[] | null;
+    costUsd: number;
+    ms: number;
+    reused: boolean;
+};
+/**
+ * A test's numbers: answers right, partial credit, each check's pass rate, consistency,
+ * cost, time, and reuse.
+ */
+export type SuiteMetrics = {
+    cases: number;
+    outputs: number;
+    failed: number;
+    /** Every applicable check passed. Failed calls count as not passing. */
+    allPass: Rate;
+    /**
+     * Partial credit summed over answers (e.g. items right of items
+     * expected), where the suite gives it; the headline when present.
+     */
+    units: {
+        right: number;
+        total: number;
+    } | null;
+    byCheck: Record<string, Rate>;
+    /**
+     * Of cases run more than once, the share whose all-pass result was the
+     * same every time. Low: the model is inconsistent on these cases.
+     */
+    consistency: Rate | null;
+    /** What grading's judge spent, apart from the model under test (runner). */
+    judgeCostUsd?: number;
+    /** The error most failed answers share (runner). */
+    failureReason?: string | null;
+    /**
+     * Per check, the mean of each measure its answers carried (brier,
+     * logLoss, …), and, where answers carried a confidence, the calibration
+     * error. Only checks with measures appear.
+     */
+    measures?: Record<string, CheckMeasures>;
+    /** What this run spent: reused outputs cost nothing. */
+    costUsd: number;
+    /** What one answer costs on this model, reused or not. */
+    costPerAnswer: number | null;
+    /** How long one answer takes on this model. */
+    avgMs: number;
+    reused: number;
+};
+/**
+ * Whether an answer passed every check that applied. A failed answer (null) didn't.
+ */
+export declare function allPassed(checks: CheckResult[] | null): boolean;
+/** Sums each answer's partial credit; null when the suite gives none. */
+export declare function sumUnits(outputs: {
+    checks: CheckResult[] | null;
+}[]): {
+    right: number;
+    total: number;
+} | null;
+/** One answer's score from 0 to 1: its partial credit, else all-or-nothing. */
+export declare function answerScore(checks: CheckResult[] | null): number;
+/** A test's numbers from its graded answers. */
+/** One check's measures across a test. */
+export type CheckMeasures = {
+    /** Answers that carried measures. */
+    n: number;
+    /** The mean of each measure, over the answers that carried it. */
+    mean: Record<string, number>;
+    /** Expected calibration error, from confidence and correctness; null without both. */
+    calibrationError: number | null;
+};
+/** One check's measures across a test's answers; null when none carried any. */
+export declare function measuresFor(outputs: ScoredOutput[], key: string): CheckMeasures | null;
+/**
+ *
+ */
+export declare function suiteMetrics(outputs: ScoredOutput[], checkKeys: string[]): SuiteMetrics;

package/dist/stats.js

@@ -0,0 +1,119 @@
+import { calibrationError } from "./decisions.js";
+/** Wilson score interval: honest for small n and rates near 0 or 1. */
+export function wilson(passed, n, z = 1.96) {
+    if (n === 0)
+        return { passed, n, rate: null, low: null, high: null };
+    const p = passed / n;
+    const denom = 1 + (z * z) / n;
+    const center = (p + (z * z) / (2 * n)) / denom;
+    const half = (z * Math.sqrt((p * (1 - p)) / n + (z * z) / (4 * n * n))) / denom;
+    return {
+        passed,
+        n,
+        rate: p,
+        low: Math.max(0, center - half),
+        high: Math.min(1, center + half),
+    };
+}
+/**
+ * Whether an answer passed every check that applied. A failed answer (null) didn't.
+ */
+export function allPassed(checks) {
+    return checks !== null && checks.every((c) => c.pass !== false);
+}
+/** Sums each answer's partial credit; null when the suite gives none. */
+export function sumUnits(outputs) {
+    let right = 0;
+    let total = 0;
+    let any = false;
+    for (const o of outputs)
+        for (const c of o.checks ?? [])
+            if (c.units) {
+                any = true;
+                right += c.units.right;
+                total += c.units.total;
+            }
+    return any ? { right, total } : null;
+}
+/** One answer's score from 0 to 1: its partial credit, else all-or-nothing. */
+export function answerScore(checks) {
+    if (!checks)
+        return 0;
+    const u = checks.find((c) => c.units)?.units;
+    if (u)
+        return u.total ? u.right / u.total : 1;
+    return allPassed(checks) ? 1 : 0;
+}
+/** One check's measures across a test's answers; null when none carried any. */
+export function measuresFor(outputs, key) {
+    const all = outputs
+        .map((o) => o.checks?.find((c) => c.key === key)?.measures)
+        .filter((m) => m !== undefined);
+    if (!all.length)
+        return null;
+    const sums = new Map();
+    for (const m of all)
+        for (const [name, v] of Object.entries(m)) {
+            const s = sums.get(name) ?? { total: 0, n: 0 };
+            s.total += v;
+            s.n++;
+            sums.set(name, s);
+        }
+    const pairs = all.flatMap((m) => m.confidence !== undefined && m.correct !== undefined
+        ? [{ confidence: m.confidence, correct: m.correct === 1 }]
+        : []);
+    return {
+        n: all.length,
+        mean: Object.fromEntries([...sums].map(([k, s]) => [k, s.total / s.n])),
+        calibrationError: calibrationError(pairs),
+    };
+}
+/**
+ *
+ */
+export function suiteMetrics(outputs, checkKeys) {
+    const byCheck = {};
+    for (const key of checkKeys) {
+        const applied = outputs
+            .map((o) => o.checks?.find((c) => c.key === key))
+            .filter((c) => c !== undefined && c.pass !== null);
+        byCheck[key] = wilson(applied.filter((c) => c.pass).length, applied.length);
+    }
+    const measures = {};
+    for (const key of checkKeys) {
+        const m = measuresFor(outputs, key);
+        if (m)
+            measures[key] = m;
+    }
+    const byCase = new Map();
+    for (const o of outputs)
+        byCase.set(o.caseId, [
+            ...(byCase.get(o.caseId) ?? []),
+            allPassed(o.checks),
+        ]);
+    const repeated = [...byCase.values()].filter((r) => r.length > 1);
+    const called = outputs.filter((o) => !o.reused);
+    // Answers with a recorded time (older reused rows have none).
+    const timed = outputs.filter((o) => o.checks && o.ms > 0);
+    return {
+        cases: byCase.size,
+        outputs: outputs.length,
+        failed: outputs.filter((o) => !o.checks).length,
+        allPass: wilson(outputs.filter((o) => allPassed(o.checks)).length, outputs.length),
+        units: sumUnits(outputs),
+        byCheck,
+        ...(Object.keys(measures).length ? { measures } : {}),
+        consistency: repeated.length
+            ? wilson(repeated.filter((r) => r.every((x) => x === r[0])).length, repeated.length)
+            : null,
+        costUsd: called.reduce((t, o) => t + o.costUsd, 0),
+        costPerAnswer: timed.length
+            ? timed.reduce((t, o) => t + o.costUsd, 0) / timed.length
+            : null,
+        avgMs: timed.length
+            ? Math.round(timed.reduce((t, o) => t + o.ms, 0) / timed.length)
+            : 0,
+        reused: outputs.length - called.length,
+    };
+}
+//# sourceMappingURL=stats.js.map

package/dist/stats.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"stats.js","sourceRoot":"","sources":["../src/stats.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAkBlD,uEAAuE;AACvE,MAAM,UAAU,MAAM,CAAC,MAAc,EAAE,CAAS,EAAE,CAAC,GAAG,IAAI;IACxD,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IACrE,MAAM,CAAC,GAAG,MAAM,GAAG,CAAC,CAAC;IACrB,MAAM,KAAK,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;IAC9B,MAAM,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC;IAC/C,MAAM,IAAI,GACR,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC;IACrE,OAAO;QACL,MAAM;QACN,CAAC;QACD,IAAI,EAAE,CAAC;QACP,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAC/B,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;KACjC,CAAC;AACJ,CAAC;AAqDD;;GAEG;AACH,MAAM,UAAU,SAAS,CAAC,MAA4B;IACpD,OAAO,MAAM,KAAK,IAAI,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC;AAClE,CAAC;AAED,yEAAyE;AACzE,MAAM,UAAU,QAAQ,CACtB,OAA2C;IAE3C,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,GAAG,GAAG,KAAK,CAAC;IAChB,KAAK,MAAM,CAAC,IAAI,OAAO;QACrB,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,IAAI,EAAE;YAC5B,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC;gBACZ,GAAG,GAAG,IAAI,CAAC;gBACX,KAAK,IAAI,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC;gBACvB,KAAK,IAAI,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC;YACzB,CAAC;IACL,OAAO,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;AACvC,CAAC;AAED,+EAA+E;AAC/E,MAAM,UAAU,WAAW,CAAC,MAA4B;IACtD,IAAI,CAAC,MAAM;QAAE,OAAO,CAAC,CAAC;IACtB,MAAM,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC;IAC7C,IAAI,CAAC;QAAE,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9C,OAAO,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACnC,CAAC;AAaD,gFAAgF;AAChF,MAAM,UAAU,WAAW,CACzB,OAAuB,EACvB,GAAW;IAEX,MAAM,GAAG,GAAG,OAAO;SAChB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,GAAG,CAAC,EAAE,QAAQ,CAAC;SAC1D,MAAM,CAAC,CAAC,CAAC,EAA+B,EAAE,CAAC,CAAC,KAAK,SAAS,CAAC,CAAC;IAC/D,IAAI,CAAC,GAAG,CAAC,MAAM;QAAE,OAAO,IAAI,CAAC;IAC7B,MAAM,IAAI,GAAG,IAAI,GAAG,EAAwC,CAAC;IAC7D,KAAK,MAAM,CAAC,IAAI,GAAG;QACjB,KAAK,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;YAC1C,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC;YAC/C,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;YACb,CAAC,CAAC,CAAC,EAAE,CAAC;YACN,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QACpB,CAAC;IACH,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAC9B,CAAC,CAAC,UAAU,KAAK,SAAS,IAAI,CAAC,CAAC,OAAO,KAAK,SAAS;QACnD,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,UAAU,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,KAAK,CAAC,EAAE,CAAC;QAC1D,CAAC,CAAC,EAAE,CACP,CAAC;IACF,OAAO;QACL,CAAC,EAAE,GAAG,CAAC,MAAM;QACb,IAAI,EAAE,MAAM,CAAC,WAAW,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACvE,gBAAgB,EAAE,gBAAgB,CAAC,KAAK,CAAC;KAC1C,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,YAAY,CAC1B,OAAuB,EACvB,SAAmB;IAEnB,MAAM,OAAO,GAAyB,EAAE,CAAC;IACzC,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;QAC5B,MAAM,OAAO,GAAG,OAAO;aACpB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,GAAG,CAAC,CAAC;aAChD,MAAM,CAAC,CAAC,CAAC,EAAoB,EAAE,CAAC,CAAC,KAAK,SAAS,IAAI,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC;QACvE,OAAO,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IAC9E,CAAC;IACD,MAAM,QAAQ,GAAkC,EAAE,CAAC;IACnD,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;QAC5B,MAAM,CAAC,GAAG,WAAW,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;QACpC,IAAI,CAAC;YAAE,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC3B,CAAC;IACD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAqB,CAAC;IAC5C,KAAK,MAAM,CAAC,IAAI,OAAO;QACrB,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE;YACnB,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YAC/B,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC;SACpB,CAAC,CAAC;IACL,MAAM,QAAQ,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAClE,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IAChD,8DAA8D;IAC9D,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;IAC1D,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,IAAI;QAClB,OAAO,EAAE,OAAO,CAAC,MAAM;QACvB,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM;QAC/C,OAAO,EAAE,MAAM,CACb,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EACjD,OAAO,CAAC,MAAM,CACf;QACD,KAAK,EAAE,QAAQ,CAAC,OAAO,CAAC;QACxB,OAAO;QACP,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACrD,WAAW,EAAE,QAAQ,CAAC,MAAM;YAC1B,CAAC,CAAC,MAAM,CACJ,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,EACzD,QAAQ,CAAC,MAAM,CAChB;YACH,CAAC,CAAC,IAAI;QACR,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;QAClD,aAAa,EAAE,KAAK,CAAC,MAAM;YACzB,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM;YACzD,CAAC,CAAC,IAAI;QACR,KAAK,EAAE,KAAK,CAAC,MAAM;YACjB,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC;YAChE,CAAC,CAAC,CAAC;QACL,MAAM,EAAE,OAAO,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM;KACvC,CAAC;AACJ,CAAC"}

package/dist/store.d.ts

@@ -0,0 +1,151 @@
+import type { CheckResult } from "./types.js";
+import type { SuiteMetrics } from "./stats.js";
+/**
+ * Where Evalkeeper keeps cases, answers, tests, and outputs. The runner
+ * talks only to this interface: `@crowandraven/evalkeeper-prisma` implements it on your
+ * database, and `memoryStore()` keeps everything in memory for tests and
+ * demos. Every method is a plain read or write; the rules live in the
+ * runner.
+ */
+/** ALL: every case in use. SMOKE: the quick set. DEV and TEST: by split. */
+export type EvalSet = "ALL" | "SMOKE" | "DEV" | "TEST";
+/** Every set of cases a test can run on. */
+export declare const EVAL_SETS: EvalSet[];
+/**
+ * Where an answer stands: pending review, confirmed, provisional, or rejected.
+ */
+export type LabelStatus = "PENDING" | "CONFIRMED" | "PROVISIONAL" | "REJECTED";
+/**
+ * An answer to a case. EXPECTED is what a good answer looks like; a case's
+ * newest CONFIRMED one is what tests grade against. PENDING answers (from a
+ * model, or not yet reviewed) are never graded against; PROVISIONAL ones are
+ * put to use before review and say so.
+ */
+export type LabelRecord = {
+    id: string;
+    kind: "EXPECTED";
+    status: LabelStatus;
+    value: unknown;
+    /** Who gave it: "OWNER", "MODEL", a reviewer's role. */
+    raterRole: string;
+    raterId: string | null;
+    note: string | null;
+    createdAt: Date;
+    updatedAt: Date;
+};
+/** A stored case: a frozen input, its tags and set, and its answers. */
+export type CaseRecord = {
+    id: string;
+    suite: string;
+    /** The suite's input, frozen. */
+    input: unknown;
+    tags: string[];
+    split: "DEV" | "TEST";
+    status: "DRAFT" | "ACTIVE" | "RETIRED";
+    /** Where it came from: "OWNER", "MODEL_DRAFT", "FEEDBACK", … */
+    source: string;
+    /** What it was made from (an id in your app), to avoid adding it twice. */
+    sourceRef: string | null;
+    note: string | null;
+    createdAt: Date;
+    updatedAt: Date;
+    labels: LabelRecord[];
+};
+/** Where a test stands. */
+export type RunStatus = "RUNNING" | "SUCCEEDED" | "FAILED";
+/** What the runner stores on a finished test. */
+export type RunMetrics = SuiteMetrics & {
+    judgeCostUsd: number;
+    failureReason: string | null;
+};
+/**
+ * A stored test: one suite on one model, with what it ran with and its metrics.
+ */
+export type RunRecord = {
+    id: string;
+    suite: string;
+    model: string;
+    /** Hash of the instructions it ran with (see `instructionsHash`). */
+    promptVersion: string;
+    /** The reasoning setting tested; null: the model's own default. */
+    reasoningEffort: string | null;
+    /**
+     * The temperature tested; null or unset: none sent. Stores keep it, so a
+     * test at another temperature than the live one shows as out of date.
+     */
+    temperature?: number | null;
+    set: EvalSet;
+    repeats: number;
+    /** Changes when the cases or their answers do, so tests compare. */
+    datasetVersion: string | null;
+    /** The suite's grader version when last graded. */
+    graderVersion: string;
+    status: RunStatus;
+    metrics: RunMetrics | null;
+    error: string | null;
+    estimatedCostUsd: number | null;
+    /** Who started it; null for a scheduled check. */
+    startedBy: string | null;
+    startedAt: Date;
+    finishedAt: Date | null;
+};
+/** One stored answer in a test, with its grades, cost, and time. */
+export type OutputRecord = {
+    id: string;
+    runId: string;
+    caseId: string;
+    repeat: number;
+    /** The request's hash, so an identical one is reused, not paid twice. */
+    requestHash: string;
+    output: unknown;
+    /** Why the answer failed; null when it came back. */
+    error: string | null;
+    scores: CheckResult[] | null;
+    costUsd: number;
+    ms: number;
+    /** Copied from an identical earlier request: cost nothing this time. */
+    reused: boolean;
+    createdAt: Date;
+};
+/** A test to record; the store fills in the id, times, and status. */
+export type NewRun = Omit<RunRecord, "id" | "startedAt" | "finishedAt" | "status" | "metrics" | "error">;
+/** An answer to record. */
+export type NewOutput = Omit<OutputRecord, "id" | "createdAt">;
+/** An answer to add to a case. */
+export type NewLabel = Omit<LabelRecord, "id" | "createdAt" | "updatedAt">;
+/** A case to create, with its first answers. */
+export type NewCase = Omit<CaseRecord, "id" | "createdAt" | "updatedAt" | "labels"> & {
+    labels?: NewLabel[];
+};
+/**
+ * Where Evalkeeper keeps cases, answers, tests, and outputs. Implement it for another
+ * database, and run describeStoreContract against it.
+ */
+export interface EvalStore {
+    /** A suite's cases that aren't retired, with their answers, oldest first. */
+    listCases(suite: string): Promise<CaseRecord[]>;
+    createCase(data: NewCase): Promise<CaseRecord>;
+    /** Adds an answer to a case (answers are never overwritten). */
+    addLabel(caseId: string, label: NewLabel): Promise<LabelRecord>;
+    /** Changes an answer's status, e.g. a person confirming a draft. */
+    setLabelStatus(labelId: string, status: LabelStatus): Promise<void>;
+    /** Changes a case's status, e.g. retiring it. */
+    setCaseStatus(caseId: string, status: CaseRecord["status"]): Promise<void>;
+    getRun(id: string): Promise<RunRecord | null>;
+    /** Newest first. */
+    listRuns(suite: string, opts?: {
+        status?: RunStatus;
+        limit?: number;
+    }): Promise<RunRecord[]>;
+    createRun(data: NewRun): Promise<RunRecord>;
+    updateRun(id: string, patch: Partial<Omit<RunRecord, "id">>, 
+    /** Only if the run still has this status (a guard against races). */
+    onlyIf?: {
+        status: RunStatus;
+    }): Promise<void>;
+    listOutputs(runId: string): Promise<OutputRecord[]>;
+    /** The newest original (not reused, no error) output with this hash. */
+    findReusableOutput(requestHash: string, since: Date): Promise<OutputRecord | null>;
+    createOutput(data: NewOutput): Promise<OutputRecord>;
+    setOutputScores(id: string, scores: CheckResult[]): Promise<void>;
+}

package/dist/store.js

@@ -0,0 +1,3 @@
+/** Every set of cases a test can run on. */
+export const EVAL_SETS = ["ALL", "SMOKE", "DEV", "TEST"];
+//# sourceMappingURL=store.js.map

package/dist/store.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"store.js","sourceRoot":"","sources":["../src/store.ts"],"names":[],"mappings":"AAaA,4CAA4C;AAC5C,MAAM,CAAC,MAAM,SAAS,GAAc,CAAC,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC"}

package/dist/suggest.d.ts

@@ -0,0 +1,75 @@
+import { type ModelCatalog } from "./models.js";
+import type { Rate } from "./stats.js";
+import type { CheckDef, SuitePriority } from "./types.js";
+import type { Verdict } from "./compare.js";
+/**
+ * Which model to make live and which the fallback, from a suite's current
+ * tests, by fixed rules so the reason is always the real one. Switching
+ * live is conservative: the candidate must score at least as high as live
+ * and make no more serious mistakes, and then needs a real reason: clearly
+ * more accurate, at least two fewer serious mistakes, or, by the suite's
+ * priority, at least 30% faster (speed) or cheaper (cost). One exception,
+ * for suites that favor cost or speed: a model that scores a little lower
+ * (within 10 points, and no clear difference) with no more serious
+ * mistakes qualifies when it costs or takes at most half as much. A suite's
+ * time limit rules out slower models, live included. Otherwise live stays, and the best-scoring other
+ * model is named as one to watch. The fallback is the most accurate model
+ * from another provider; it needn't beat live, it only runs in an outage.
+ * Models the provider is retiring are never suggested, and a retiring live
+ * model is replaced by the best-scoring one that isn't. Pure.
+ */
+/** One model's row, as the suggestion reads it. */
+export type SuggestRow = {
+    key: string;
+    model: string;
+    label: string;
+    variant: boolean;
+    live: boolean;
+    fallback: boolean;
+    vsLive: Verdict | null;
+    /** May only be live (e.g. a model that can't stand in for a text model). */
+    liveOnly?: boolean;
+    /** May only be the fallback. */
+    fallbackOnly?: boolean;
+    test: {
+        right: number;
+        total: number;
+        byCheck: Record<string, Rate>;
+        avgMs: number | null;
+        costPerAnswer: number | null;
+        stale: boolean;
+        /** Answers that failed; a test with any is incomplete. */
+        failedAnswers?: number;
+    } | null;
+};
+/**
+ * Which model to make live and which the fallback, with the reasons in plain sentences.
+ */
+export type Suggestion<R extends SuggestRow = SuggestRow> = {
+    live: R;
+    fallback: R | null;
+    /** Live stays as it is. */
+    keepLive: boolean;
+    /** Nothing to change at all: live stays and the fallback is already set. */
+    same: boolean;
+    /** Why, in plain sentences. */
+    reasons: string[];
+};
+/**
+ * How many serious checks a model's test failed; Infinity without a test.
+ */
+export declare function seriousMistakes(row: SuggestRow, checks: CheckDef[]): number;
+/**
+ * Suggests the live and fallback models from a suite's board, by fixed rules, so the
+ * reason given is always the real one. Null with nothing current to go on.
+ *
+ * @example
+ * ```ts
+ * const board = await runner.board("ticket_triage");
+ * const s = suggest(board.rows, suite.checks, suite.priority, MODELS);
+ * if (s && !s.same) console.log(s.live.label, s.fallback?.label, s.reasons);
+ * ```
+ */
+export declare function suggest<R extends SuggestRow>(rows: R[], checks: CheckDef[], priority?: SuitePriority, 
+/** Your model catalog: providers (for the fallback) and retirements. */
+models?: ModelCatalog): Suggestion<R> | null;

package/dist/suggest.js

@@ -0,0 +1,152 @@
+import { isBaseline, retiresLabel } from "./models.js";
+import { at } from "./util.js";
+/**
+ * How many serious checks a model's test failed; Infinity without a test.
+ */
+export function seriousMistakes(row, checks) {
+    const test = row.test;
+    if (!test)
+        return Infinity;
+    return checks
+        .filter((c) => c.severity === "serious")
+        .reduce((t, c) => {
+        const r = test.byCheck[c.key];
+        return t + (r ? r.n - r.passed : 0);
+    }, 0);
+}
+const mistakes = (n) => `${n} serious mistake${n === 1 ? "" : "s"}`;
+const rate = (r) => (r.test ? r.test.right / r.test.total : 0);
+const pct = (x) => `${Math.round(x * 100)}%`;
+const cost = (r) => r.test?.costPerAnswer ?? Infinity;
+const ms = (r) => r.test?.avgMs ?? Infinity;
+/** A cost or speed trade-off may give up at most this much accuracy. */
+const MAX_TRADE_DROP = 0.1;
+/** ...and must cost or take at most this share of live's. */
+const HALF = 0.5;
+/** At least 30% lower. */
+const clearlyLess = (a, b) => Number.isFinite(a) && Number.isFinite(b) && a <= b * 0.7;
+/**
+ * Suggests the live and fallback models from a suite's board, by fixed rules, so the
+ * reason given is always the real one. Null with nothing current to go on.
+ *
+ * @example
+ * ```ts
+ * const board = await runner.board("ticket_triage");
+ * const s = suggest(board.rows, suite.checks, suite.priority, MODELS);
+ * if (s && !s.same) console.log(s.live.label, s.fallback?.label, s.reasons);
+ * ```
+ */
+export function suggest(rows, checks, priority = { favor: "quality" }, 
+/** Your model catalog: providers (for the fallback) and retirements. */
+models = {}) {
+    const provider = (model) => models[model]?.provider ?? model;
+    const retiring = (r) => retiresLabel(models, r.model);
+    const limitMs = priority.maxSeconds ? priority.maxSeconds * 1000 : Infinity;
+    const tooSlow = (r) => ms(r) !== Infinity && ms(r) > limitMs;
+    // Baselines are code, compared on the board but never suggested.
+    const tested = rows.filter((r) => !isBaseline(models, r.model) &&
+        r.test &&
+        !r.test.stale &&
+        !r.test.failedAnswers &&
+        !retiring(r) &&
+        !tooSlow(r));
+    if (!tested.length)
+        return null;
+    const serious = (r) => seriousMistakes(r, checks);
+    const live = tested.find((r) => r.live) ?? null;
+    const reasons = [];
+    const liveRow = rows.find((r) => r.live);
+    const liveRetires = liveRow ? retiring(liveRow) : null;
+    const liveSlow = liveRow?.test && !liveRow.test.stale && tooSlow(liveRow)
+        ? `${liveRow.label} takes ${(ms(liveRow) / 1000).toFixed(1)}s, over this suite's ${priority.maxSeconds}s limit.`
+        : null;
+    // Live: switch only for a real, safe improvement.
+    let next;
+    const liveable = tested.filter((r) => !r.fallbackOnly);
+    if (!live) {
+        if (!liveable.length)
+            return null;
+        next = at([...liveable].sort((a, b) => rate(b) - rate(a) || serious(a) - serious(b) || cost(a) - cost(b)), 0);
+        reasons.push(liveRetires || liveSlow
+            ? `${liveRetires && liveRow ? `${liveRow.label} retires in ${liveRetires}.` : liveSlow} ${next.label} scores highest of the rest (${pct(rate(next))}) with ${mistakes(serious(next))}.`
+            : `${next.label} scores highest (${pct(rate(next))}) with ${mistakes(serious(next))}. The live model has no up-to-date test to compare with.`);
+    }
+    else {
+        const why = (r) => {
+            if (rate(r) < rate(live))
+                return trade(r);
+            if (r.vsLive === "BETTER")
+                return `${r.label} is clearly more accurate than ${live.label}: ${pct(rate(r))} against ${pct(rate(live))}.`;
+            if (serious(r) <= serious(live) - 2)
+                return `${r.label} makes fewer serious mistakes than ${live.label} (${serious(r)} against ${serious(live)}), and scores at least as well (${pct(rate(r))} against ${pct(rate(live))}).`;
+            if (priority.favor === "cost" && clearlyLess(cost(r), cost(live)))
+                return `${r.label} scores at least as well as ${live.label} (${pct(rate(r))} against ${pct(rate(live))}) at ${Math.round((1 - cost(r) / cost(live)) * 100)}% less per answer.`;
+            if (priority.favor === "speed" && clearlyLess(ms(r), ms(live)))
+                return `${r.label} scores at least as well as ${live.label} (${pct(rate(r))} against ${pct(rate(live))}) and answers faster (${(ms(r) / 1000).toFixed(1)}s against ${(ms(live) / 1000).toFixed(1)}s).`;
+            return null;
+        };
+        // Cost and speed suites: a little less accurate, no clear difference,
+        // and at most half the cost (or time).
+        const trade = (r) => {
+            if (priority.favor === "quality" ||
+                r.vsLive !== "SAME" ||
+                rate(live) - rate(r) > MAX_TRADE_DROP ||
+                serious(r) > serious(live))
+                return null;
+            const close = `${r.label} is about as accurate as ${live.label} (${pct(rate(r))} against ${pct(rate(live))}: closer than these cases can tell apart), with no more serious mistakes`;
+            if (priority.favor === "cost" && cost(r) <= cost(live) * HALF)
+                return `${close}, at ${Math.round((1 - cost(r) / cost(live)) * 100)}% less per answer. This suite favors cost.`;
+            if (priority.favor === "speed" && ms(r) <= ms(live) * HALF)
+                return `${close}, and answers in ${(ms(r) / 1000).toFixed(1)}s against ${(ms(live) / 1000).toFixed(1)}s. This suite favors speed.`;
+            return null;
+        };
+        const candidates = liveable
+            .filter((r) => !r.live &&
+            r.vsLive !== "WORSE" &&
+            serious(r) <= serious(live) &&
+            why(r))
+            .sort((a, b) => Number(b.vsLive === "BETTER") - Number(a.vsLive === "BETTER") ||
+            // A trade-off only when nothing scores as well.
+            Number(rate(b) >= rate(live)) - Number(rate(a) >= rate(live)) ||
+            rate(b) - rate(a) ||
+            serious(a) - serious(b) ||
+            (priority.favor === "speed"
+                ? ms(a) - ms(b) || cost(a) - cost(b)
+                : cost(a) - cost(b) || ms(a) - ms(b)));
+        if (candidates.length) {
+            next = at(candidates, 0);
+            const reason = why(next);
+            if (reason)
+                reasons.push(reason);
+        }
+        else {
+            next = live;
+            reasons.push(`Keep ${live.label}: no other model is safely better on these cases.`);
+            // The one to watch: scored higher, but not by enough to tell.
+            const watch = tested
+                .filter((r) => !r.live && rate(r) > rate(live))
+                .sort((a, b) => rate(b) - rate(a))[0];
+            if (watch)
+                reasons.push(`${watch.label} scored highest (${pct(rate(watch))} against ${pct(rate(live))}), but with this many cases the gap can't be told from noise${serious(watch) > serious(live) ? `, and it makes more serious mistakes (${serious(watch)} against ${serious(live)})` : ""}. Add cases, then test again.`);
+        }
+    }
+    // Fallback: the most accurate from another provider, at the operation's
+    // setting (the setting is shared, so setting rows don't qualify).
+    const fallback = tested
+        .filter((r) => !r.variant &&
+        !r.liveOnly &&
+        r.key !== next.key &&
+        provider(r.model) !== provider(next.model))
+        .sort((a, b) => rate(b) - rate(a) || serious(a) - serious(b) || cost(a) - cost(b))[0] ?? null;
+    if (fallback)
+        reasons.push(`Fallback: ${fallback.label} is the most accurate from another provider (${pct(rate(fallback))}, ${mistakes(serious(fallback))}).`);
+    const keepLive = next.live;
+    return {
+        live: next,
+        fallback,
+        keepLive,
+        same: keepLive && (!fallback || fallback.fallback),
+        reasons,
+    };
+}
+//# sourceMappingURL=suggest.js.map

package/dist/suggest.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"suggest.js","sourceRoot":"","sources":["../src/suggest.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,YAAY,EAAqB,MAAM,aAAa,CAAC;AAI1E,OAAO,EAAE,EAAE,EAAE,MAAM,WAAW,CAAC;AA0D/B;;GAEG;AACH,MAAM,UAAU,eAAe,CAAC,GAAe,EAAE,MAAkB;IACjE,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;IACtB,IAAI,CAAC,IAAI;QAAE,OAAO,QAAQ,CAAC;IAC3B,OAAO,MAAM;SACV,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC;SACvC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACf,MAAM,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAC9B,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACtC,CAAC,EAAE,CAAC,CAAC,CAAC;AACV,CAAC;AAED,MAAM,QAAQ,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,GAAG,CAAC,mBAAmB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;AAC5E,MAAM,IAAI,GAAG,CAAC,CAAa,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAC3E,MAAM,GAAG,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC;AACrD,MAAM,IAAI,GAAG,CAAC,CAAa,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,aAAa,IAAI,QAAQ,CAAC;AAClE,MAAM,EAAE,GAAG,CAAC,CAAa,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,IAAI,QAAQ,CAAC;AACxD,wEAAwE;AACxE,MAAM,cAAc,GAAG,GAAG,CAAC;AAC3B,6DAA6D;AAC7D,MAAM,IAAI,GAAG,GAAG,CAAC;AACjB,0BAA0B;AAC1B,MAAM,WAAW,GAAG,CAAC,CAAS,EAAE,CAAS,EAAE,EAAE,CAC3C,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC;AAE3D;;;;;;;;;;GAUG;AACH,MAAM,UAAU,OAAO,CACrB,IAAS,EACT,MAAkB,EAClB,WAA0B,EAAE,KAAK,EAAE,SAAS,EAAE;AAC9C,wEAAwE;AACxE,SAAuB,EAAE;IAEzB,MAAM,QAAQ,GAAG,CAAC,KAAa,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,QAAQ,IAAI,KAAK,CAAC;IACrE,MAAM,QAAQ,GAAG,CAAC,CAAa,EAAE,EAAE,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC;IAClE,MAAM,OAAO,GAAG,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC;IAC5E,MAAM,OAAO,GAAG,CAAC,CAAa,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,QAAQ,IAAI,EAAE,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC;IACzE,iEAAiE;IACjE,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CACxB,CAAC,CAAC,EAAE,EAAE,CACJ,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC;QAC5B,CAAC,CAAC,IAAI;QACN,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK;QACb,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa;QACrB,CAAC,QAAQ,CAAC,CAAC,CAAC;QACZ,CAAC,OAAO,CAAC,CAAC,CAAC,CACd,CAAC;IACF,IAAI,CAAC,MAAM,CAAC,MAAM;QAAE,OAAO,IAAI,CAAC;IAChC,MAAM,OAAO,GAAG,CAAC,CAAI,EAAE,EAAE,CAAC,eAAe,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;IACrD,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC;IAChD,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACzC,MAAM,WAAW,GAAG,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IACvD,MAAM,QAAQ,GACZ,OAAO,EAAE,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,IAAI,OAAO,CAAC,OAAO,CAAC;QACtD,CAAC,CAAC,GAAG,OAAO,CAAC,KAAK,UAAU,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,wBAAwB,QAAQ,CAAC,UAAU,UAAU;QAChH,CAAC,CAAC,IAAI,CAAC;IAEX,kDAAkD;IAClD,IAAI,IAAO,CAAC;IACZ,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;IACvD,IAAI,CAAC,IAAI,EAAE,CAAC;QACV,IAAI,CAAC,QAAQ,CAAC,MAAM;YAAE,OAAO,IAAI,CAAC;QAClC,IAAI,GAAG,EAAE,CACP,CAAC,GAAG,QAAQ,CAAC,CAAC,IAAI,CAChB,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACP,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CACpE,EACD,CAAC,CACF,CAAC;QACF,OAAO,CAAC,IAAI,CACV,WAAW,IAAI,QAAQ;YACrB,CAAC,CAAC,GAAG,WAAW,IAAI,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,KAAK,eAAe,WAAW,GAAG,CAAC,CAAC,CAAC,QAAQ,IAAI,IAAI,CAAC,KAAK,gCAAgC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,UAAU,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG;YACvL,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,oBAAoB,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,UAAU,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,0DAA0D,CAChJ,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,MAAM,GAAG,GAAG,CAAC,CAAI,EAAiB,EAAE;YAClC,IAAI,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC;gBAAE,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC;YAC1C,IAAI,CAAC,CAAC,MAAM,KAAK,QAAQ;gBACvB,OAAO,GAAG,CAAC,CAAC,KAAK,kCAAkC,IAAI,CAAC,KAAK,KAAK,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC;YAC/G,IAAI,OAAO,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC;gBACjC,OAAO,GAAG,CAAC,CAAC,KAAK,sCAAsC,IAAI,CAAC,KAAK,KAAK,OAAO,CAAC,CAAC,CAAC,YAAY,OAAO,CAAC,IAAI,CAAC,mCAAmC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC;YAC1L,IAAI,QAAQ,CAAC,KAAK,KAAK,MAAM,IAAI,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;gBAC/D,OAAO,GAAG,CAAC,CAAC,KAAK,+BAA+B,IAAI,CAAC,KAAK,KAAK,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,QAAQ,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC,oBAAoB,CAAC;YACjL,IAAI,QAAQ,CAAC,KAAK,KAAK,OAAO,IAAI,WAAW,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC;gBAC5D,OAAO,GAAG,CAAC,CAAC,KAAK,+BAA+B,IAAI,CAAC,KAAK,KAAK,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,yBAAyB,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,aAAa,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC;YACzM,OAAO,IAAI,CAAC;QACd,CAAC,CAAC;QACF,sEAAsE;QACtE,uCAAuC;QACvC,MAAM,KAAK,GAAG,CAAC,CAAI,EAAiB,EAAE;YACpC,IACE,QAAQ,CAAC,KAAK,KAAK,SAAS;gBAC5B,CAAC,CAAC,MAAM,KAAK,MAAM;gBACnB,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,GAAG,cAAc;gBACrC,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;gBAE1B,OAAO,IAAI,CAAC;YACd,MAAM,KAAK,GAAG,GAAG,CAAC,CAAC,KAAK,4BAA4B,IAAI,CAAC,KAAK,KAAK,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,0EAA0E,CAAC;YACrL,IAAI,QAAQ,CAAC,KAAK,KAAK,MAAM,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI;gBAC3D,OAAO,GAAG,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC,4CAA4C,CAAC;YAClH,IAAI,QAAQ,CAAC,KAAK,KAAK,OAAO,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,GAAG,IAAI;gBACxD,OAAO,GAAG,KAAK,oBAAoB,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,aAAa,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,6BAA6B,CAAC;YACrI,OAAO,IAAI,CAAC;QACd,CAAC,CAAC;QACF,MAAM,UAAU,GAAG,QAAQ;aACxB,MAAM,CACL,CAAC,CAAC,EAAE,EAAE,CACJ,CAAC,CAAC,CAAC,IAAI;YACP,CAAC,CAAC,MAAM,KAAK,OAAO;YACpB,OAAO,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC;YAC3B,GAAG,CAAC,CAAC,CAAC,CACT;aACA,IAAI,CACH,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACP,MAAM,CAAC,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC;YAC7D,gDAAgD;YAChD,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC;YAC7D,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC;YACjB,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;YACvB,CAAC,QAAQ,CAAC,KAAK,KAAK,OAAO;gBACzB,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC;gBACpC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAC1C,CAAC;QACJ,IAAI,UAAU,CAAC,MAAM,EAAE,CAAC;YACtB,IAAI,GAAG,EAAE,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;YACzB,MAAM,MAAM,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC;YACzB,IAAI,MAAM;gBAAE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACnC,CAAC;aAAM,CAAC;YACN,IAAI,GAAG,IAAI,CAAC;YACZ,OAAO,CAAC,IAAI,CACV,QAAQ,IAAI,CAAC,KAAK,mDAAmD,CACtE,CAAC;YACF,8DAA8D;YAC9D,MAAM,KAAK,GAAG,MAAM;iBACjB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC;iBAC9C,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACxC,IAAI,KAAK;gBACP,OAAO,CAAC,IAAI,CACV,GAAG,KAAK,CAAC,KAAK,oBAAoB,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,YAAY,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,+DAA+D,OAAO,CAAC,KAAK,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,yCAAyC,OAAO,CAAC,KAAK,CAAC,YAAY,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,+BAA+B,CACrS,CAAC;QACN,CAAC;IACH,CAAC;IAED,wEAAwE;IACxE,kEAAkE;IAClE,MAAM,QAAQ,GACZ,MAAM;SACH,MAAM,CACL,CAAC,CAAC,EAAE,EAAE,CACJ,CAAC,CAAC,CAAC,OAAO;QACV,CAAC,CAAC,CAAC,QAAQ;QACX,CAAC,CAAC,GAAG,KAAK,IAAI,CAAC,GAAG;QAClB,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAC7C;SACA,IAAI,CACH,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACP,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CACpE,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC;IACjB,IAAI,QAAQ;QACV,OAAO,CAAC,IAAI,CACV,aAAa,QAAQ,CAAC,KAAK,gDAAgD,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,KAAK,QAAQ,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,IAAI,CACnI,CAAC;IAEJ,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC;IAC3B,OAAO;QACL,IAAI,EAAE,IAAI;QACV,QAAQ;QACR,QAAQ;QACR,IAAI,EAAE,QAAQ,IAAI,CAAC,CAAC,QAAQ,IAAI,QAAQ,CAAC,QAAQ,CAAC;QAClD,OAAO;KACR,CAAC;AACJ,CAAC"}