@cef-ai/eval 0.0.0-stage → 0.2.0

package/dist/summary.d.ts

@@ -0,0 +1,105 @@
+import type { DatasetDiff, Experiment, Metrics, RunRecord, Summary, VersionEntry } from "./types.js";
+export declare function emptyMetrics(): Metrics;
+/** Nearest-rank percentile of a sorted list; null when empty. */
+export declare function percentile(sorted: number[], p: number): number | null;
+/**
+ * Roll a set of runs up. `passed`/`failed` count runs whose `pass` is
+ * true/false; `unscored` those with null (no expectations, or the harness
+ * errored); `errored` those with status `error` (they are also unscored).
+ * `passRate` is passed / (passed + failed), null when nothing was judged
+ * (ROC calls it Accuracy). `fields` is the same per expected field, over every
+ * run's `eval/field@1` score for it; n/a scores (`pass: null`) are counted
+ * apart and never move a rate.
+ */
+export declare function summarize(runs: RunRecord[]): Summary;
+export type Change = "improved" | "regressed" | "unchanged" | "only-in-base" | "only-in-candidate";
+export interface MetricDelta {
+    base: number | null;
+    candidate: number | null;
+    /** candidate − base; null when either side is missing. */
+    delta: number | null;
+    /** delta / base; null when base is 0 or missing. */
+    relative: number | null;
+}
+export interface RunComparison {
+    caseId: string;
+    repeat: number;
+    change: Change;
+    base?: Pick<RunRecord, "status" | "pass" | "output" | "metrics">;
+    candidate?: Pick<RunRecord, "status" | "pass" | "output" | "metrics">;
+    /** Whether the two outputs differ, pass or not. */
+    outputChanged: boolean;
+    durationMs: MetricDelta;
+    totalTokens: MetricDelta;
+    cost: MetricDelta;
+    steps: MetricDelta;
+}
+export interface CaseComparison {
+    caseId: string;
+    change: Change;
+    /** Pass rate over the case's judged repeats, each side. */
+    basePassRate: number | null;
+    candidatePassRate: number | null;
+    /** Mean duration over the case's repeats, each side. */
+    durationMs: MetricDelta;
+    totalTokens: MetricDelta;
+    cost: MetricDelta;
+}
+export interface ExperimentComparison {
+    base: {
+        id: string;
+        workflowVersion?: string;
+        datasetVersion?: number;
+    };
+    candidate: {
+        id: string;
+        workflowVersion?: string;
+        datasetVersion?: number;
+    };
+    /**
+     * How the dataset changed from base to candidate (case ids; edited = another
+     * revision). Absent unless BOTH sides gave their version's `cases`: derived
+     * from run records, a case one side never ran (a cancelled or errored
+     * experiment) would read as added or removed though the dataset is the same.
+     */
+    datasetDiff?: DatasetDiff;
+    /** Cases compared: present on both sides at the same revision. */
+    comparedOn: number;
+    counts: Record<Change, number>;
+    cases: CaseComparison[];
+    runs: RunComparison[];
+    metrics: {
+        passRate: MetricDelta;
+        durationP50: MetricDelta;
+        durationP95: MetricDelta;
+        tokensPerRun: MetricDelta;
+        costPerRun: MetricDelta;
+        stepsPerRun: MetricDelta;
+        limitBreaches: MetricDelta;
+    };
+}
+export interface ComparisonSide {
+    experiment?: Pick<Experiment, "id" | "workflowVersion" | "datasetVersion">;
+    runs: RunRecord[];
+    /**
+     * The cases the side's dataset version pins. Default: read off the runs'
+     * `caseRev` — enough to pick the like-for-like cases, not to say how the
+     * dataset changed: `datasetDiff` is left out unless both sides give these.
+     */
+    cases?: ReadonlyArray<Pick<VersionEntry, "id" | "rev">>;
+}
+type Side = ComparisonSide;
+/**
+ * How a candidate experiment differs from a baseline: per run (case × repeat),
+ * per case (over its repeats), and in the summary metrics.
+ *
+ * LIKE FOR LIKE ONLY. When the dataset changed between the two experiments,
+ * the comparison covers only the cases present on both sides at the same
+ * revision (`comparedOn`), and `datasetDiff` (when both sides name their
+ * cases) says what was left out: a case whose expectation was edited is not
+ * evidence the workflow improved or regressed. Metrics are computed over the compared runs too. A run present on
+ * one side only (a repeat that did not run) is still reported as such.
+ */
+export declare function compareExperiments(base: Side, candidate: Side): ExperimentComparison;
+export {};
+//# sourceMappingURL=summary.d.ts.map

package/dist/summary.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"summary.d.ts","sourceRoot":"","sources":["../src/summary.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,WAAW,EAAE,UAAU,EAAuB,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAE1H,wBAAgB,YAAY,IAAI,OAAO,CAYtC;AAED,iEAAiE;AACjE,wBAAgB,UAAU,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAIrE;AAID;;;;;;;;GAQG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,SAAS,EAAE,GAAG,OAAO,CAiCpD;AAiBD,MAAM,MAAM,MAAM,GAAG,UAAU,GAAG,WAAW,GAAG,WAAW,GAAG,cAAc,GAAG,mBAAmB,CAAC;AAqBnG,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,0DAA0D;IAC1D,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,oDAAoD;IACpD,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;CACzB;AASD,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,IAAI,CAAC,SAAS,EAAE,QAAQ,GAAG,MAAM,GAAG,QAAQ,GAAG,SAAS,CAAC,CAAC;IACjE,SAAS,CAAC,EAAE,IAAI,CAAC,SAAS,EAAE,QAAQ,GAAG,MAAM,GAAG,QAAQ,GAAG,SAAS,CAAC,CAAC;IACtE,mDAAmD;IACnD,aAAa,EAAE,OAAO,CAAC;IACvB,UAAU,EAAE,WAAW,CAAC;IACxB,WAAW,EAAE,WAAW,CAAC;IACzB,IAAI,EAAE,WAAW,CAAC;IAClB,KAAK,EAAE,WAAW,CAAC;CACpB;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,2DAA2D;IAC3D,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,wDAAwD;IACxD,UAAU,EAAE,WAAW,CAAC;IACxB,WAAW,EAAE,WAAW,CAAC;IACzB,IAAI,EAAE,WAAW,CAAC;CACnB;AAED,MAAM,WAAW,oBAAoB;IACnC,IAAI,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,eAAe,CAAC,EAAE,MAAM,CAAC;QAAC,cAAc,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IACxE,SAAS,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,eAAe,CAAC,EAAE,MAAM,CAAC;QAAC,cAAc,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAC7E;;;;;OAKG;IACH,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,kEAAkE;IAClE,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC/B,KAAK,EAAE,cAAc,EAAE,CAAC;IACxB,IAAI,EAAE,aAAa,EAAE,CAAC;IACtB,OAAO,EAAE;QACP,QAAQ,EAAE,WAAW,CAAC;QACtB,WAAW,EAAE,WAAW,CAAC;QACzB,WAAW,EAAE,WAAW,CAAC;QACzB,YAAY,EAAE,WAAW,CAAC;QAC1B,UAAU,EAAE,WAAW,CAAC;QACxB,WAAW,EAAE,WAAW,CAAC;QACzB,aAAa,EAAE,WAAW,CAAC;KAC5B,CAAC;CACH;AAED,MAAM,WAAW,cAAc;IAC7B,UAAU,CAAC,EAAE,IAAI,CAAC,UAAU,EAAE,IAAI,GAAG,iBAAiB,GAAG,gBAAgB,CAAC,CAAC;IAC3E,IAAI,EAAE,SAAS,EAAE,CAAC;IAClB;;;;OAIG;IACH,KAAK,CAAC,EAAE,aAAa,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC;CACzD;AACD,KAAK,IAAI,GAAG,cAAc,CAAC;AAuB3B;;;;;;;;;;GAUG;AACH,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,GAAG,oBAAoB,CAoFpF"}

package/dist/summary.js

@@ -0,0 +1,219 @@
+/**
+ * An experiment's numbers, and how two experiments differ.
+ */
+import { deepEqual, FIELD_SCORER, LIMIT_SCORER } from "./score.js";
+import { diffVersionEntries } from "./storage.js";
+export function emptyMetrics() {
+    return {
+        durationMs: null,
+        inputTokens: 0,
+        outputTokens: 0,
+        totalTokens: 0,
+        cost: 0,
+        steps: 0,
+        modelCalls: 0,
+        cpuUnits: 0,
+        gpuUnits: 0,
+    };
+}
+/** Nearest-rank percentile of a sorted list; null when empty. */
+export function percentile(sorted, p) {
+    if (sorted.length === 0)
+        return null;
+    const rank = Math.ceil((p / 100) * sorted.length);
+    return sorted[Math.min(sorted.length, Math.max(1, rank)) - 1];
+}
+const round = (n, places = 6) => Math.round(n * 10 ** places) / 10 ** places;
+/**
+ * Roll a set of runs up. `passed`/`failed` count runs whose `pass` is
+ * true/false; `unscored` those with null (no expectations, or the harness
+ * errored); `errored` those with status `error` (they are also unscored).
+ * `passRate` is passed / (passed + failed), null when nothing was judged
+ * (ROC calls it Accuracy). `fields` is the same per expected field, over every
+ * run's `eval/field@1` score for it; n/a scores (`pass: null`) are counted
+ * apart and never move a rate.
+ */
+export function summarize(runs) {
+    const passed = runs.filter((r) => r.pass === true).length;
+    const failed = runs.filter((r) => r.pass === false).length;
+    const durations = runs
+        .map((r) => r.metrics.durationMs)
+        .filter((d) => typeof d === "number" && Number.isFinite(d))
+        .sort((a, b) => a - b);
+    const tokens = runs.reduce((s, r) => s + (r.metrics.totalTokens || 0), 0);
+    const cost = runs.reduce((s, r) => s + (r.metrics.cost || 0), 0);
+    const steps = runs.reduce((s, r) => s + (r.metrics.steps || 0), 0);
+    const n = runs.length;
+    return {
+        cases: new Set(runs.map((r) => r.caseId)).size,
+        runs: n,
+        passed,
+        failed,
+        errored: runs.filter((r) => r.status === "error").length,
+        unscored: runs.filter((r) => r.pass === null).length,
+        passRate: passed + failed === 0 ? null : round(passed / (passed + failed)),
+        durationMs: {
+            p50: percentile(durations, 50),
+            p95: percentile(durations, 95),
+            mean: durations.length ? round(durations.reduce((s, d) => s + d, 0) / durations.length, 1) : null,
+        },
+        tokens: { total: tokens, meanPerRun: n ? round(tokens / n, 2) : 0 },
+        cost: { total: round(cost), meanPerRun: n ? round(cost / n) : 0 },
+        steps: { meanPerRun: n ? round(steps / n, 2) : 0 },
+        limitBreaches: runs.reduce((s, r) => s + r.scores.filter((x) => x.scorer === LIMIT_SCORER && x.pass === false).length, 0),
+        fields: fieldAccuracy(runs),
+    };
+}
+function fieldAccuracy(runs) {
+    const out = {};
+    for (const r of runs) {
+        for (const s of r.scores) {
+            if (s.scorer !== FIELD_SCORER)
+                continue;
+            const f = (out[s.name] ??= { passed: 0, failed: 0, notApplicable: 0, rate: null });
+            if (s.pass === true)
+                f.passed++;
+            else if (s.pass === false)
+                f.failed++;
+            else
+                f.notApplicable++;
+        }
+    }
+    for (const f of Object.values(out))
+        f.rate = f.passed + f.failed === 0 ? null : round(f.passed / (f.passed + f.failed));
+    return out;
+}
+/** true > null > false: a pass beats "not judged", which beats a failure. */
+const rank = (p) => (p === true ? 2 : p === null ? 1 : 0);
+const changeBy = (d) => (d > 0 ? "improved" : d < 0 ? "regressed" : "unchanged");
+function changeOf(base, cand) {
+    return changeBy(rank(cand) - rank(base));
+}
+/**
+ * A case's standing over its repeats on the SAME scale a single run is ranked
+ * on: the mean run rank. For one repeat it is exactly the run's rank, so the
+ * case and its run never disagree; when every repeat was judged it orders
+ * cases as their pass rates do. A case whose repeats all went unscored sits
+ * between a passing and a failing one — not below a failure, which is what a
+ * pass rate of "none" compared as -1 made it.
+ */
+const caseRank = (runs) => runs.reduce((s, r) => s + rank(r.pass), 0) / runs.length;
+function delta(base, cand) {
+    const b = base ?? null;
+    const c = cand ?? null;
+    const d = b === null || c === null ? null : round(c - b);
+    return { base: b, candidate: c, delta: d, relative: d === null || !b ? null : round(d / b, 4) };
+}
+/** A side's cases at their revisions: given, or from its runs. */
+function casesOf(s) {
+    if (s.cases)
+        return s.cases.map(({ id, rev }) => ({ id, rev }));
+    const revs = new Map();
+    for (const r of s.runs)
+        revs.set(r.caseId, r.caseRev);
+    return [...revs].map(([id, rev]) => ({ id, rev }));
+}
+const passRateOf = (runs) => {
+    const judged = runs.filter((r) => r.pass !== null);
+    return judged.length ? round(judged.filter((r) => r.pass).length / judged.length) : null;
+};
+const meanOf = (xs) => {
+    const ok = xs.filter((x) => typeof x === "number");
+    return ok.length ? round(ok.reduce((s, x) => s + x, 0) / ok.length, 2) : null;
+};
+/** Same output, as field scoring judges sameness: key order does not count. */
+const sameJson = (a, b) => deepEqual(a ?? null, b ?? null);
+/**
+ * How a candidate experiment differs from a baseline: per run (case × repeat),
+ * per case (over its repeats), and in the summary metrics.
+ *
+ * LIKE FOR LIKE ONLY. When the dataset changed between the two experiments,
+ * the comparison covers only the cases present on both sides at the same
+ * revision (`comparedOn`), and `datasetDiff` (when both sides name their
+ * cases) says what was left out: a case whose expectation was edited is not
+ * evidence the workflow improved or regressed. Metrics are computed over the compared runs too. A run present on
+ * one side only (a repeat that did not run) is still reported as such.
+ */
+export function compareExperiments(base, candidate) {
+    const baseCases = casesOf(base);
+    const candCases = casesOf(candidate);
+    const datasetDiff = base.cases && candidate.cases ? diffVersionEntries(baseCases, candCases) : undefined;
+    const candRev = new Map(candCases.map((e) => [e.id, e.rev]));
+    const shared = new Set(baseCases.filter((e) => candRev.get(e.id) === e.rev).map((e) => e.id));
+    const baseRuns = base.runs.filter((r) => shared.has(r.caseId));
+    const candRuns = candidate.runs.filter((r) => shared.has(r.caseId));
+    const key = (r) => `${r.caseId}\u0000${r.repeat}`;
+    const b = new Map(baseRuns.map((r) => [key(r), r]));
+    const c = new Map(candRuns.map((r) => [key(r), r]));
+    const keys = [...new Set([...b.keys(), ...c.keys()])].sort();
+    const pick = (r) => ({ status: r.status, pass: r.pass, output: r.output, metrics: r.metrics });
+    const runs = keys.map((k) => {
+        const x = b.get(k);
+        const y = c.get(k);
+        const [caseId, repeat] = k.split("\u0000");
+        return {
+            caseId,
+            repeat: Number(repeat),
+            change: !x ? "only-in-candidate" : !y ? "only-in-base" : changeOf(x.pass, y.pass),
+            ...(x ? { base: pick(x) } : {}),
+            ...(y ? { candidate: pick(y) } : {}),
+            outputChanged: !!x && !!y && !sameJson(x.output, y.output),
+            durationMs: delta(x?.metrics.durationMs, y?.metrics.durationMs),
+            totalTokens: delta(x?.metrics.totalTokens, y?.metrics.totalTokens),
+            cost: delta(x?.metrics.cost, y?.metrics.cost),
+            steps: delta(x?.metrics.steps, y?.metrics.steps),
+        };
+    });
+    const caseIds = [...new Set([...baseRuns, ...candRuns].map((r) => r.caseId))].sort();
+    const cases = caseIds.map((caseId) => {
+        const xs = baseRuns.filter((r) => r.caseId === caseId);
+        const ys = candRuns.filter((r) => r.caseId === caseId);
+        const bp = passRateOf(xs);
+        const cp = passRateOf(ys);
+        const change = !xs.length
+            ? "only-in-candidate"
+            : !ys.length
+                ? "only-in-base"
+                : changeBy(round(caseRank(ys) - caseRank(xs)));
+        return {
+            caseId,
+            change,
+            basePassRate: bp,
+            candidatePassRate: cp,
+            durationMs: delta(meanOf(xs.map((r) => r.metrics.durationMs)), meanOf(ys.map((r) => r.metrics.durationMs))),
+            totalTokens: delta(meanOf(xs.map((r) => r.metrics.totalTokens)), meanOf(ys.map((r) => r.metrics.totalTokens))),
+            cost: delta(meanOf(xs.map((r) => r.metrics.cost)), meanOf(ys.map((r) => r.metrics.cost))),
+        };
+    });
+    const sb = summarize(baseRuns);
+    const sc = summarize(candRuns);
+    const counts = {
+        improved: 0, regressed: 0, unchanged: 0, "only-in-base": 0, "only-in-candidate": 0,
+    };
+    for (const r of runs)
+        counts[r.change]++;
+    const label = (s, fallback) => ({
+        id: s.experiment?.id ?? fallback,
+        ...(s.experiment?.workflowVersion !== undefined ? { workflowVersion: s.experiment.workflowVersion } : {}),
+        ...(s.experiment?.datasetVersion !== undefined ? { datasetVersion: s.experiment.datasetVersion } : {}),
+    });
+    return {
+        base: label(base, "base"),
+        candidate: label(candidate, "candidate"),
+        ...(datasetDiff ? { datasetDiff } : {}),
+        comparedOn: shared.size,
+        counts,
+        cases,
+        runs,
+        metrics: {
+            passRate: delta(sb.passRate, sc.passRate),
+            durationP50: delta(sb.durationMs.p50, sc.durationMs.p50),
+            durationP95: delta(sb.durationMs.p95, sc.durationMs.p95),
+            tokensPerRun: delta(sb.tokens.meanPerRun, sc.tokens.meanPerRun),
+            costPerRun: delta(sb.cost.meanPerRun, sc.cost.meanPerRun),
+            stepsPerRun: delta(sb.steps.meanPerRun, sc.steps.meanPerRun),
+            limitBreaches: delta(sb.limitBreaches, sc.limitBreaches),
+        },
+    };
+}
+//# sourceMappingURL=summary.js.map

package/dist/summary.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"summary.js","sourceRoot":"","sources":["../src/summary.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AACnE,OAAO,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;AAGlD,MAAM,UAAU,YAAY;IAC1B,OAAO;QACL,UAAU,EAAE,IAAI;QAChB,WAAW,EAAE,CAAC;QACd,YAAY,EAAE,CAAC;QACf,WAAW,EAAE,CAAC;QACd,IAAI,EAAE,CAAC;QACP,KAAK,EAAE,CAAC;QACR,UAAU,EAAE,CAAC;QACb,QAAQ,EAAE,CAAC;QACX,QAAQ,EAAE,CAAC;KACZ,CAAC;AACJ,CAAC;AAED,iEAAiE;AACjE,MAAM,UAAU,UAAU,CAAC,MAAgB,EAAE,CAAS;IACpD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACrC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;IAClD,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,GAAG,CAAC,CAAE,CAAC;AACjE,CAAC;AAED,MAAM,KAAK,GAAG,CAAC,CAAS,EAAE,MAAM,GAAG,CAAC,EAAU,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,IAAI,MAAM,CAAC,GAAG,EAAE,IAAI,MAAM,CAAC;AAE7F;;;;;;;;GAQG;AACH,MAAM,UAAU,SAAS,CAAC,IAAiB;IACzC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,MAAM,CAAC;IAC1D,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC,MAAM,CAAC;IAC3D,MAAM,SAAS,GAAG,IAAI;SACnB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC;SAChC,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;SACvE,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACzB,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC1E,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACjE,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACnE,MAAM,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC;IACtB,OAAO;QACL,KAAK,EAAE,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI;QAC9C,IAAI,EAAE,CAAC;QACP,MAAM;QACN,MAAM;QACN,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,OAAO,CAAC,CAAC,MAAM;QACxD,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,MAAM;QACpD,QAAQ,EAAE,MAAM,GAAG,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC;QAC1E,UAAU,EAAE;YACV,GAAG,EAAE,UAAU,CAAC,SAAS,EAAE,EAAE,CAAC;YAC9B,GAAG,EAAE,UAAU,CAAC,SAAS,EAAE,EAAE,CAAC;YAC9B,IAAI,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI;SAClG;QACD,MAAM,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE;QACnE,IAAI,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE;QACjE,KAAK,EAAE,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE;QAClD,aAAa,EAAE,IAAI,CAAC,MAAM,CACxB,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,YAAY,IAAI,CAAC,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC,MAAM,EAC1F,CAAC,CACF;QACD,MAAM,EAAE,aAAa,CAAC,IAAI,CAAC;KAC5B,CAAC;AACJ,CAAC;AAED,SAAS,aAAa,CAAC,IAAiB;IACtC,MAAM,GAAG,GAAkC,EAAE,CAAC;IAC9C,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;QACrB,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC;YACzB,IAAI,CAAC,CAAC,MAAM,KAAK,YAAY;gBAAE,SAAS;YACxC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;YACnF,IAAI,CAAC,CAAC,IAAI,KAAK,IAAI;gBAAE,CAAC,CAAC,MAAM,EAAE,CAAC;iBAC3B,IAAI,CAAC,CAAC,IAAI,KAAK,KAAK;gBAAE,CAAC,CAAC,MAAM,EAAE,CAAC;;gBACjC,CAAC,CAAC,aAAa,EAAE,CAAC;QACzB,CAAC;IACH,CAAC;IACD,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC;QAAE,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;IACxH,OAAO,GAAG,CAAC;AACb,CAAC;AAID,6EAA6E;AAC7E,MAAM,IAAI,GAAG,CAAC,CAAiB,EAAU,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAElF,MAAM,QAAQ,GAAG,CAAC,CAAS,EAAU,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC;AAEjG,SAAS,QAAQ,CAAC,IAAoB,EAAE,IAAoB;IAC1D,OAAO,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;AAC3C,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,QAAQ,GAAG,CAAC,IAAiB,EAAU,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC;AAWzG,SAAS,KAAK,CAAC,IAA+B,EAAE,IAA+B;IAC7E,MAAM,CAAC,GAAG,IAAI,IAAI,IAAI,CAAC;IACvB,MAAM,CAAC,GAAG,IAAI,IAAI,IAAI,CAAC;IACvB,MAAM,CAAC,GAAG,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACzD,OAAO,EAAE,IAAI,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;AAClG,CAAC;AAkED,kEAAkE;AAClE,SAAS,OAAO,CAAC,CAAO;IACtB,IAAI,CAAC,CAAC,KAAK;QAAE,OAAO,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;IAChE,MAAM,IAAI,GAAG,IAAI,GAAG,EAAkB,CAAC;IACvC,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI;QAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC;IACtD,OAAO,CAAC,GAAG,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;AACrD,CAAC;AAED,MAAM,UAAU,GAAG,CAAC,IAAiB,EAAiB,EAAE;IACtD,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC;IACnD,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AAC3F,CAAC,CAAC;AAEF,MAAM,MAAM,GAAG,CAAC,EAAwB,EAAiB,EAAE;IACzD,MAAM,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC;IAChE,OAAO,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AAChF,CAAC,CAAC;AAEF,+EAA+E;AAC/E,MAAM,QAAQ,GAAG,CAAC,CAA0B,EAAE,CAA0B,EAAW,EAAE,CAAC,SAAS,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,IAAI,IAAI,CAAC,CAAC;AAEtH;;;;;;;;;;GAUG;AACH,MAAM,UAAU,kBAAkB,CAAC,IAAU,EAAE,SAAe;IAC5D,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAChC,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC;IACrC,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,IAAI,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,kBAAkB,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACzG,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IAC7D,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC9F,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;IAC/D,MAAM,QAAQ,GAAG,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;IAEpE,MAAM,GAAG,GAAG,CAAC,CAAY,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,SAAS,CAAC,CAAC,MAAM,EAAE,CAAC;IAC7D,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IACpD,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IACpD,MAAM,IAAI,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAC7D,MAAM,IAAI,GAAG,CAAC,CAAY,EAAE,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;IAE1G,MAAM,IAAI,GAAoB,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QAC3C,MAAM,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QACnB,MAAM,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QACnB,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAqB,CAAC;QAC/D,OAAO;YACL,MAAM;YACN,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC;YACtB,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC;YACjF,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC/B,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACpC,aAAa,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC;YAC1D,UAAU,EAAE,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,UAAU,EAAE,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC;YAC/D,WAAW,EAAE,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,WAAW,EAAE,CAAC,EAAE,OAAO,CAAC,WAAW,CAAC;YAClE,IAAI,EAAE,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC,EAAE,OAAO,CAAC,IAAI,CAAC;YAC7C,KAAK,EAAE,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,EAAE,OAAO,CAAC,KAAK,CAAC;SACjD,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,MAAM,OAAO,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,QAAQ,EAAE,GAAG,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACrF,MAAM,KAAK,GAAqB,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;QACrD,MAAM,EAAE,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC;QACvD,MAAM,EAAE,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC;QACvD,MAAM,EAAE,GAAG,UAAU,CAAC,EAAE,CAAC,CAAC;QAC1B,MAAM,EAAE,GAAG,UAAU,CAAC,EAAE,CAAC,CAAC;QAC1B,MAAM,MAAM,GAAW,CAAC,EAAE,CAAC,MAAM;YAC/B,CAAC,CAAC,mBAAmB;YACrB,CAAC,CAAC,CAAC,EAAE,CAAC,MAAM;gBACV,CAAC,CAAC,cAAc;gBAChB,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QACnD,OAAO;YACL,MAAM;YACN,MAAM;YACN,YAAY,EAAE,EAAE;YAChB,iBAAiB,EAAE,EAAE;YACrB,UAAU,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,EAAE,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;YAC3G,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,EAAE,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC;YAC9G,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,EAAE,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;SAC1F,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,MAAM,EAAE,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAC;IAC/B,MAAM,EAAE,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAC;IAC/B,MAAM,MAAM,GAA2B;QACrC,QAAQ,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE,mBAAmB,EAAE,CAAC;KACnF,CAAC;IACF,KAAK,MAAM,CAAC,IAAI,IAAI;QAAE,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,CAAC,CAAO,EAAE,QAAgB,EAAE,EAAE,CAAC,CAAC;QAC5C,EAAE,EAAE,CAAC,CAAC,UAAU,EAAE,EAAE,IAAI,QAAQ;QAChC,GAAG,CAAC,CAAC,CAAC,UAAU,EAAE,eAAe,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,CAAC,CAAC,UAAU,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACzG,GAAG,CAAC,CAAC,CAAC,UAAU,EAAE,cAAc,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,CAAC,CAAC,UAAU,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACvG,CAAC,CAAC;IACH,OAAO;QACL,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,MAAM,CAAC;QACzB,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,WAAW,CAAC;QACxC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACvC,UAAU,EAAE,MAAM,CAAC,IAAI;QACvB,MAAM;QACN,KAAK;QACL,IAAI;QACJ,OAAO,EAAE;YACP,QAAQ,EAAE,KAAK,CAAC,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC;YACzC,WAAW,EAAE,KAAK,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC;YACxD,WAAW,EAAE,KAAK,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC;YACxD,YAAY,EAAE,KAAK,CAAC,EAAE,CAAC,MAAM,CAAC,UAAU,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC/D,UAAU,EAAE,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,UAAU,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC;YACzD,WAAW,EAAE,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,UAAU,EAAE,EAAE,CAAC,KAAK,CAAC,UAAU,CAAC;YAC5D,aAAa,EAAE,KAAK,CAAC,EAAE,CAAC,aAAa,EAAE,EAAE,CAAC,aAAa,CAAC;SACzD;KACF,CAAC;AACJ,CAAC"}

package/dist/types.d.ts

@@ -0,0 +1,265 @@
+/**
+ * The vocabulary.
+ *
+ *   Dataset     a versioned set of Cases for one workflow
+ *   Case        one input (a `workflow.start` payload) and what its Result should be
+ *   Version     a frozen cut of a dataset: each case pinned at one revision by sha256
+ *   Experiment  one dataset version × one workflow version × N repeats
+ *   Run         one case × one repeat inside an experiment — a RunRecord
+ *
+ * Everything here is plain JSON on the wire: ROC and the CLI read and write the
+ * same objects at the same keys (see layout.ts).
+ */
+export type Json = string | number | boolean | null | Json[] | {
+    [k: string]: Json;
+};
+/** Thresholds a run should stay under. Breaching one WARNS — it never fails a case. */
+export interface Limits {
+    maxDurationMs?: number;
+    maxTokens?: number;
+    maxSteps?: number;
+    maxCost?: number;
+}
+/** `workflows/<wf>/datasets/<dataset>/header.json` — mutable; each version embeds a snapshot. */
+export interface Header {
+    dataset: string;
+    workflowId: string;
+    description?: string;
+    limits?: Limits;
+    createdAt: string;
+    /** The experiment others are compared to. Mutable; set with `setBaseline`. */
+    baselineId?: string;
+}
+/** The value types a Result field may declare (the runner's `resultTypes`). */
+export type ResultFieldType = "string" | "number" | "boolean" | "object" | "array";
+/**
+ * One field of a workflow version's declared Result: a key of an `output`
+ * node's `params.result`, with its `resultTypes` entry when it has one. See
+ * `resultFieldsFromGraph`.
+ */
+export interface ResultField {
+    name: string;
+    /** Usually a {@link ResultFieldType}; any other word is treated as "untyped". */
+    type?: string;
+}
+/**
+ * How one expected field is matched.
+ *
+ * A plain value is deep equality, except that a plain OBJECT is a subset match
+ * (keys it does not mention are ignored) and a string such as `">= 0.8"` is a
+ * numeric band when the actual value is a number.
+ */
+export type Matcher = {
+    $eq: Json;
+} | {
+    $oneOf: Json[];
+} | {
+    $contains: Json;
+} | {
+    $regex: string;
+} | {
+    $between: [number, number];
+} | {
+    $gte: number;
+} | {
+    $lte: number;
+} | {
+    $exists: boolean;
+} | {
+    $approx: number;
+    $tol: number;
+};
+/** One input and the Result it should produce. Stored as write-once revisions. */
+export interface Case {
+    /** Human readable, `[A-Za-z0-9._-]{1,80}`. */
+    id: string;
+    /** The `workflow.start` payload. */
+    input: Record<string, Json>;
+    /** Keyed by Result field; dot paths reach into nested fields. */
+    expected?: Record<string, Json | Matcher>;
+    /** Per-case limits; each one overrides the header's. */
+    limits?: Limits;
+    tags?: string[];
+    notes?: string;
+    /** Where the case came from, when it was captured from a real run. */
+    source?: {
+        context: string;
+        workflowVersion?: string;
+        capturedAt: string;
+    };
+}
+/** One case as a version pins it. */
+export interface VersionEntry {
+    id: string;
+    rev: number;
+    sha256: string;
+}
+/** `workflows/<wf>/datasets/<dataset>/versions/v<n>.json` — written once. */
+export interface VersionManifest {
+    version: number;
+    header: Header;
+    cases: VersionEntry[];
+    createdAt: string;
+    note?: string;
+}
+/** What a run cost. Tokens and cost include agent-reported (A2A) figures. */
+export interface Metrics {
+    durationMs: number | null;
+    inputTokens: number;
+    outputTokens: number;
+    totalTokens: number;
+    cost: number;
+    steps: number;
+    modelCalls: number;
+    cpuUnits: number;
+    gpuUnits: number;
+}
+export interface Score {
+    /** The Result field (dot path) for a field score; `limit.<name>` for a limit score. */
+    name: string;
+    value: number | string | boolean | null;
+    /** null = not judged. */
+    pass: boolean | null;
+    /** `eval/field@1` | `eval/limit@1` | a future `judge/<model>@1`. */
+    scorer: string;
+    source: "eval" | "annotation";
+    detail?: string;
+}
+export type RunStatus = "done" | "failed" | "timeout" | "error";
+/**
+ * `workflows/<wf>/experiments/<expId>/runs/<caseId>.<repeat>.json` — written once.
+ * Records written before v2 carry `trial` instead of `repeat`; the parser maps it.
+ */
+export interface RunRecord {
+    caseId: string;
+    caseRev: number;
+    /** Which repeat of the case this run is; 1-based. */
+    repeat: number;
+    /** Which attempt of the experiment ran it (1 = the first; each resume adds one). Part of `context`. */
+    attempt?: number;
+    context: string;
+    runId: string;
+    /**
+     * `done` the workflow completed; `failed` it announced a failure; `timeout`
+     * it did not end in time; `error` the harness could not run it at all.
+     */
+    status: RunStatus;
+    output: Json | null;
+    /** The declared Result's outcome label, when the run carried one. */
+    outcome?: string;
+    error?: string;
+    /** The start event's server timestamp when the target read it, else when the harness published it. */
+    startedAt: string;
+    endedAt: string | null;
+    metrics: Metrics;
+    scores: Score[];
+    /** All field scores pass; null when the case has no expectations or the harness errored. */
+    pass: boolean | null;
+}
+export type ExperimentStatus = "running" | "done" | "cancelled" | "error";
+/** `workflows/<wf>/experiments/<expId>/meta.json` — rewritten as the experiment moves through its states. */
+export interface Experiment {
+    id: string;
+    dataset: string;
+    datasetVersion: number;
+    workflowId: string;
+    workflowVersion: string;
+    /** Whether the workflow's live deployment ran it (no version pin). */
+    live?: boolean;
+    /** How many times each case runs. */
+    repeats: number;
+    /**
+     * The Result the workflow version declares, when known. Expected fields it
+     * does not declare, or declares with an incompatible type, are scored n/a.
+     */
+    resultFields?: ResultField[];
+    /** 1 when created; each resume adds one, so its runs get contexts no earlier attempt used. Absent = 1. */
+    attempt?: number;
+    startedAt: string;
+    finishedAt?: string;
+    status: ExperimentStatus;
+    origin: "roc" | "cli";
+    createdBy?: string;
+    baselineId?: string;
+    note?: string;
+    /** Why the experiment ended in `error`. */
+    error?: string;
+    summary?: Summary;
+}
+/** How one expected field fared across an experiment's runs. */
+export interface FieldAccuracy {
+    passed: number;
+    failed: number;
+    /** Scores with `pass: null`: not in the version's Result, or its type changed. */
+    notApplicable: number;
+    /** passed / (passed + failed); null when nothing was judged. */
+    rate: number | null;
+}
+export interface Summary {
+    cases: number;
+    runs: number;
+    passed: number;
+    failed: number;
+    errored: number;
+    unscored: number;
+    passRate: number | null;
+    durationMs: {
+        p50: number | null;
+        p95: number | null;
+        mean: number | null;
+    };
+    tokens: {
+        total: number;
+        meanPerRun: number;
+    };
+    cost: {
+        total: number;
+        meanPerRun: number;
+    };
+    steps: {
+        meanPerRun: number;
+    };
+    limitBreaches: number;
+    /** Per expected field (dot path). Absent on summaries written before v2. */
+    fields: Record<string, FieldAccuracy>;
+}
+/** How a dataset's cases changed between two points (versions, or the experiments that ran them). */
+export interface DatasetDiff {
+    added: string[];
+    removed: string[];
+    /** Same id, another revision. */
+    edited: string[];
+}
+/** How a run ended, as the target saw it. */
+export interface RunEnd {
+    status: "done" | "failed" | "timeout";
+    output?: Json;
+    outcome?: string;
+    error?: string;
+    /** Server timestamp of the end event. */
+    endedAt?: string;
+    /**
+     * Server timestamp of the run's own `workflow.start` event, read back from
+     * the context. With `endedAt` it is the run's duration; neither is ever
+     * compared with the harness's local clock.
+     */
+    startedAt?: string;
+}
+/**
+ * Where experiments run. Injected: ROC implements it over the browser's vault
+ * and orchestrator clients, the CLI over Node's.
+ */
+export interface WorkflowTarget {
+    /** Publish `workflow.start` with `input` into `context`, targeted at the workflow agent. */
+    start(input: Record<string, Json>, context: string): Promise<void>;
+    /** Wait for `workflow.completed` / `workflow.failed` in `context`; report both events' server timestamps. */
+    waitForEnd(context: string, timeoutMs: number): Promise<RunEnd>;
+    /** What the runs in `context` cost. `durationMs` is the fallback when `waitForEnd` lacks a timestamp; server-measured or null. */
+    metrics(context: string): Promise<Metrics>;
+    /**
+     * Route runs whose context starts with `contextPrefix` to `version`, until
+     * the returned `release()` is called. Needed for a non-live version only.
+     */
+    pinVersion?(version: string, contextPrefix: string): Promise<() => Promise<void>>;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,MAAM,MAAM,IAAI,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,IAAI,GAAG,IAAI,EAAE,GAAG;IAAE,CAAC,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,CAAC;AAErF,uFAAuF;AACvF,MAAM,WAAW,MAAM;IACrB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,iGAAiG;AACjG,MAAM,WAAW,MAAM;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,8EAA8E;IAC9E,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,+EAA+E;AAC/E,MAAM,MAAM,eAAe,GAAG,QAAQ,GAAG,QAAQ,GAAG,SAAS,GAAG,QAAQ,GAAG,OAAO,CAAC;AAEnF;;;;GAIG;AACH,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,iFAAiF;IACjF,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED;;;;;;GAMG;AACH,MAAM,MAAM,OAAO,GACf;IAAE,GAAG,EAAE,IAAI,CAAA;CAAE,GACb;IAAE,MAAM,EAAE,IAAI,EAAE,CAAA;CAAE,GAClB;IAAE,SAAS,EAAE,IAAI,CAAA;CAAE,GACnB;IAAE,MAAM,EAAE,MAAM,CAAA;CAAE,GAClB;IAAE,QAAQ,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CAAE,GAC9B;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,GAChB;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,GAChB;IAAE,OAAO,EAAE,OAAO,CAAA;CAAE,GACpB;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC;AAEtC,kFAAkF;AAClF,MAAM,WAAW,IAAI;IACnB,8CAA8C;IAC9C,EAAE,EAAE,MAAM,CAAC;IACX,oCAAoC;IACpC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAC5B,iEAAiE;IACjE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,IAAI,GAAG,OAAO,CAAC,CAAC;IAC1C,wDAAwD;IACxD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,sEAAsE;IACtE,MAAM,CAAC,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,eAAe,CAAC,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAC;CAC5E;AAED,qCAAqC;AACrC,MAAM,WAAW,YAAY;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,6EAA6E;AAC7E,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,YAAY,EAAE,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,6EAA6E;AAC7E,MAAM,WAAW,OAAO;IACtB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,KAAK;IACpB,uFAAuF;IACvF,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,IAAI,CAAC;IACxC,yBAAyB;IACzB,IAAI,EAAE,OAAO,GAAG,IAAI,CAAC;IACrB,oEAAoE;IACpE,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,GAAG,YAAY,CAAC;IAC9B,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,MAAM,SAAS,GAAG,MAAM,GAAG,QAAQ,GAAG,SAAS,GAAG,OAAO,CAAC;AAEhE;;;GAGG;AACH,MAAM,WAAW,SAAS;IACxB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,qDAAqD;IACrD,MAAM,EAAE,MAAM,CAAC;IACf,uGAAuG;IACvG,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd;;;OAGG;IACH,MAAM,EAAE,SAAS,CAAC;IAClB,MAAM,EAAE,IAAI,GAAG,IAAI,CAAC;IACpB,qEAAqE;IACrE,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,sGAAsG;IACtG,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,OAAO,EAAE,OAAO,CAAC;IACjB,MAAM,EAAE,KAAK,EAAE,CAAC;IAChB,4FAA4F;IAC5F,IAAI,EAAE,OAAO,GAAG,IAAI,CAAC;CACtB;AAED,MAAM,MAAM,gBAAgB,GAAG,SAAS,GAAG,MAAM,GAAG,WAAW,GAAG,OAAO,CAAC;AAE1E,6GAA6G;AAC7G,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;IAChB,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,eAAe,EAAE,MAAM,CAAC;IACxB,sEAAsE;IACtE,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,qCAAqC;IACrC,OAAO,EAAE,MAAM,CAAC;IAChB;;;OAGG;IACH,YAAY,CAAC,EAAE,WAAW,EAAE,CAAC;IAC7B,0GAA0G;IAC1G,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,gBAAgB,CAAC;IACzB,MAAM,EAAE,KAAK,GAAG,KAAK,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,2CAA2C;IAC3C,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED,gEAAgE;AAChE,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,kFAAkF;IAClF,aAAa,EAAE,MAAM,CAAC;IACtB,gEAAgE;IAChE,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;CACrB;AAED,MAAM,WAAW,OAAO;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,UAAU,EAAE;QAAE,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE,CAAC;IAC5E,MAAM,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAC;IAC9C,IAAI,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAC;IAC5C,KAAK,EAAE;QAAE,UAAU,EAAE,MAAM,CAAA;KAAE,CAAC;IAC9B,aAAa,EAAE,MAAM,CAAC;IACtB,4EAA4E;IAC5E,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;CACvC;AAED,qGAAqG;AACrG,MAAM,WAAW,WAAW;IAC1B,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,iCAAiC;IACjC,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB;AAED,6CAA6C;AAC7C,MAAM,WAAW,MAAM;IACrB,MAAM,EAAE,MAAM,GAAG,QAAQ,GAAG,SAAS,CAAC;IACtC,MAAM,CAAC,EAAE,IAAI,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,yCAAyC;IACzC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,4FAA4F;IAC5F,KAAK,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACnE,6GAA6G;IAC7G,UAAU,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAChE,kIAAkI;IAClI,OAAO,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAC3C;;;OAGG;IACH,UAAU,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;CACnF"}

package/dist/types.js

@@ -0,0 +1,14 @@
+/**
+ * The vocabulary.
+ *
+ *   Dataset     a versioned set of Cases for one workflow
+ *   Case        one input (a `workflow.start` payload) and what its Result should be
+ *   Version     a frozen cut of a dataset: each case pinned at one revision by sha256
+ *   Experiment  one dataset version × one workflow version × N repeats
+ *   Run         one case × one repeat inside an experiment — a RunRecord
+ *
+ * Everything here is plain JSON on the wire: ROC and the CLI read and write the
+ * same objects at the same keys (see layout.ts).
+ */
+export {};
+//# sourceMappingURL=types.js.map

package/dist/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG"}