@swissspidy/belay-calibrate 0.1.0 → 0.1.1

package/dist/analyze.d.ts

@@ -0,0 +1,139 @@
+import type { CalibrationCurvePoint, CalibrationFile, CloudUsage, TargetMode } from '@swissspidy/belay-core';
+/** One labeled example after both runners have seen it. */
+export interface Sample {
+    id: string;
+    input: string;
+    /** Ground-truth label (for classifier tasks, the label string; "true"/"false" for binary). */
+    truth: string;
+    /** Local prediction, or `null` if the local runner failed / was unavailable (always escalated). */
+    local: {
+        label: string;
+        confidence: number;
+        correct: boolean; /** The parsed local answer. */
+        value?: unknown;
+    } | null;
+    /** Cloud prediction, or `null` if the cloud runner failed (counted as wrong). */
+    cloud: {
+        label: string;
+        correct: boolean;
+        /** The cloud model's own confidence, when its runner reports one (e.g. Jev). */
+        confidence?: number;
+        /** Measured cost of the call (from its token usage and the configured prices). */
+        cost?: number;
+        usage?: CloudUsage[];
+    } | null;
+}
+export interface AnalyzeOptions {
+    /**
+     * A cascade accuracy in (0, 1]; `'cloud'` for at least the cloud-only accuracy; `'max'` for the
+     * most accurate threshold. The smallest threshold that reaches it is recommended.
+     */
+    target: number | 'cloud' | 'max';
+    /** Labels in schema order (for per-label thresholds and the confusion matrix). */
+    labels?: string[];
+    /** Fit per-label thresholds (see ADR 0001, Decision 4). */
+    perLabel?: boolean;
+    /**
+     * Enables costs. Samples with a measured `cloud.cost` are priced at it; the rest (and failed
+     * cloud calls) at the mean measured cost, or at `cloudPerRun` when nothing was measured.
+     */
+    cost?: {
+        currency: string;
+        cloudPerRun?: number;
+    };
+    histogramBins?: number;
+}
+/** How the local and cloud answers compare on the runs the cascade keeps local. */
+export interface HeadToHead {
+    kept: number;
+    bothRight: number;
+    /** Local right, cloud wrong: each one is accuracy the cascade gains over cloud only. */
+    localOnlyRight: number;
+    /** Cloud right, local wrong: each one is accuracy the cascade loses. */
+    cloudOnlyRight: number;
+    bothWrong: number;
+}
+export interface CostSummary {
+    currency: string;
+    basis: 'measured' | 'estimate';
+    /** Mean cost of one cloud call. */
+    cloudPerRun: number;
+    /** Cloud calls with measured cost. */
+    measuredCalls: number;
+    tokensPerRun?: {
+        input: number;
+        output: number;
+        cacheRead: number;
+        cacheWrite: number;
+    };
+}
+export interface OptionStats {
+    label: string;
+    /** Examples with this true label. */
+    count: number;
+    localAccuracy: number | null;
+    cloudAccuracy: number | null;
+    /** At the recommended threshold(s). */
+    localShare: number;
+    cascadeAccuracy: number;
+    threshold: number;
+}
+export interface Analysis {
+    threshold: number;
+    thresholds: Record<string, number> | undefined;
+    /** The target as an accuracy, and how it was set. */
+    target: {
+        value: number;
+        mode: TargetMode;
+    };
+    targetMet: boolean;
+    headToHead: HeadToHead;
+    cost: CostSummary | undefined;
+    expected: CalibrationFile['expected'];
+    /** 95% Wilson score interval of the expected cascade accuracy. */
+    accuracyInterval: [number, number];
+    curve: CalibrationCurvePoint[];
+    confidenceHistogram: CalibrationFile['confidenceHistogram'];
+    confusion: NonNullable<CalibrationFile['confusion']> | undefined;
+    perOption: OptionStats[];
+    counts: {
+        total: number;
+        localFailed: number;
+        cloudFailed: number;
+    };
+    /** Local answers that were wrong but would be accepted at the recommended threshold, most confident first. */
+    confidentMistakes: Sample[];
+}
+/**
+ * The shortest decimal `t` with `lo < t <= hi`, so the threshold reads well (0.82 rather than
+ * 0.8123456) and sits strictly between two observed confidences, which keeps it from
+ * over-fitting to a single sample's exact score. `lo = -Infinity` yields 0.
+ */
+export declare function niceThreshold(lo: number, hi: number): number;
+/**
+ * Candidate thresholds: 0, then one per gap between consecutive distinct local confidences,
+ * then one above the maximum, so escalating everything is always a candidate. When a confidence
+ * is exactly 1 (common for deterministic checks) that one is above 1. Every threshold in the same
+ * gap routes the samples identically, so this set is exhaustive and deterministic.
+ */
+export declare function candidateThresholds(confidences: readonly number[]): number[];
+export declare function wilsonInterval(successes: number, n: number, z?: number): [number, number];
+export declare function analyze(samples: readonly Sample[], options: AnalyzeOptions): Analysis;
+export interface CrossValidation {
+    folds: number;
+    /** Cascade accuracy on held-out folds, each routed by thresholds fitted on the other folds. */
+    accuracy: number;
+    localShare: number;
+    /** Cloud-only accuracy on the same examples, for comparison. */
+    cloudAccuracy: number;
+    /** Cloud cost per 1,000 runs on the held-out folds (with costs). */
+    costPer1k?: number;
+}
+/**
+ * K-fold cross-validation of the threshold choice: fits the thresholds on K-1 folds and routes
+ * the held-out fold with them. The recommended threshold is fitted and scored on the same data,
+ * which flatters it; this estimate does not. Folds interleave by position, so it is deterministic.
+ * Returns undefined for datasets too small to split (fewer than 10 examples per fold).
+ */
+export declare function crossValidate(samples: readonly Sample[], options: AnalyzeOptions, folds?: number): CrossValidation | undefined;
+//# sourceMappingURL=analyze.d.ts.map

package/dist/analyze.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"analyze.d.ts","sourceRoot":"","sources":["../src/analyze.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,qBAAqB,EAAE,eAAe,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAE7G,2DAA2D;AAC3D,MAAM,WAAW,MAAM;IACrB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,8FAA8F;IAC9F,KAAK,EAAE,MAAM,CAAC;IACd,mGAAmG;IACnG,KAAK,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,OAAO,CAAC,CAAC,+BAA+B;QAAC,KAAK,CAAC,EAAE,OAAO,CAAA;KAAE,GAAG,IAAI,CAAC;IACvH,iFAAiF;IACjF,KAAK,EAAE;QACL,KAAK,EAAE,MAAM,CAAC;QACd,OAAO,EAAE,OAAO,CAAC;QACjB,gFAAgF;QAChF,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,kFAAkF;QAClF,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,KAAK,CAAC,EAAE,UAAU,EAAE,CAAC;KACtB,GAAG,IAAI,CAAC;CACV;AAED,MAAM,WAAW,cAAc;IAC7B;;;OAGG;IACH,MAAM,EAAE,MAAM,GAAG,OAAO,GAAG,KAAK,CAAC;IACjC,kFAAkF;IAClF,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAClB,2DAA2D;IAC3D,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB;;;OAGG;IACH,IAAI,CAAC,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAClD,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,mFAAmF;AACnF,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,wFAAwF;IACxF,cAAc,EAAE,MAAM,CAAC;IACvB,wEAAwE;IACxE,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,WAAW;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,UAAU,GAAG,UAAU,CAAC;IAC/B,mCAAmC;IACnC,WAAW,EAAE,MAAM,CAAC;IACpB,sCAAsC;IACtC,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAC;CACzF;AAED,MAAM,WAAW,WAAW;IAC1B,KAAK,EAAE,MAAM,CAAC;IACd,qCAAqC;IACrC,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,uCAAuC;IACvC,UAAU,EAAE,MAAM,CAAC;IACnB,eAAe,EAAE,MAAM,CAAC;IACxB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,QAAQ;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,SAAS,CAAC;IAC/C,qDAAqD;IACrD,MAAM,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,UAAU,CAAA;KAAE,CAAC;IAC5C,SAAS,EAAE,OAAO,CAAC;IACnB,UAAU,EAAE,UAAU,CAAC;IACvB,IAAI,EAAE,WAAW,GAAG,SAAS,CAAC;IAC9B,QAAQ,EAAE,eAAe,CAAC,UAAU,CAAC,CAAC;IACtC,kEAAkE;IAClE,gBAAgB,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACnC,KAAK,EAAE,qBAAqB,EAAE,CAAC;IAC/B,mBAAmB,EAAE,eAAe,CAAC,qBAAqB,CAAC,CAAC;IAC5D,SAAS,EAAE,WAAW,CAAC,eAAe,CAAC,WAAW,CAAC,CAAC,GAAG,SAAS,CAAC;IACjE,SAAS,EAAE,WAAW,EAAE,CAAC;IACzB,MAAM,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC;IACpE,8GAA8G;IAC9G,iBAAiB,EAAE,MAAM,EAAE,CAAC;CAC7B;AAID;;;;GAIG;AACH,wBAAgB,aAAa,CAAC,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,MAAM,CAS5D;AA6ED;;;;;GAKG;AACH,wBAAgB,mBAAmB,CAAC,WAAW,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,EAAE,CAU5E;AAED,wBAAgB,cAAc,CAAC,SAAS,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,SAAoB,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAOpG;AAED,wBAAgB,OAAO,CAAC,OAAO,EAAE,SAAS,MAAM,EAAE,EAAE,OAAO,EAAE,cAAc,GAAG,QAAQ,CAqHrF;AAED,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,+FAA+F;IAC/F,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,gEAAgE;IAChE,aAAa,EAAE,MAAM,CAAC;IACtB,oEAAoE;IACpE,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;;;;GAKG;AACH,wBAAgB,aAAa,CAAC,OAAO,EAAE,SAAS,MAAM,EAAE,EAAE,OAAO,EAAE,cAAc,EAAE,KAAK,SAAI,GAAG,eAAe,GAAG,SAAS,CA6BzH"}

package/dist/analyze.js

@@ -0,0 +1,265 @@
+const EPS = 1e-12;
+/**
+ * The shortest decimal `t` with `lo < t <= hi`, so the threshold reads well (0.82 rather than
+ * 0.8123456) and sits strictly between two observed confidences, which keeps it from
+ * over-fitting to a single sample's exact score. `lo = -Infinity` yields 0.
+ */
+export function niceThreshold(lo, hi) {
+    if (lo < 0)
+        return 0;
+    for (let d = 1; d <= 15; d++) {
+        const scale = 10 ** d;
+        let t = Math.ceil(lo * scale) / scale;
+        if (t <= lo)
+            t = (Math.round(lo * scale) + 1) / scale;
+        if (t > lo && t <= hi)
+            return t;
+    }
+    return hi;
+}
+const kept = (s, thresholdOf) => !!s.local && s.local.confidence >= thresholdOf(s);
+function evaluate(samples, thresholdOf, costOf = () => 0) {
+    let correct = 0;
+    let local = 0;
+    let localCorrect = 0;
+    let cloudCost = 0;
+    for (const s of samples) {
+        if (kept(s, thresholdOf)) {
+            local++;
+            if (s.local.correct) {
+                correct++;
+                localCorrect++;
+            }
+        }
+        else {
+            cloudCost += costOf(s);
+            if (s.cloud?.correct)
+                correct++;
+        }
+    }
+    const n = samples.length;
+    return {
+        accuracy: n ? correct / n : 0,
+        localShare: n ? local / n : 0,
+        localAccuracy: local ? localCorrect / local : null,
+        cloudCost,
+    };
+}
+function headToHead(samples, thresholdOf) {
+    const h = { kept: 0, bothRight: 0, localOnlyRight: 0, cloudOnlyRight: 0, bothWrong: 0 };
+    for (const s of samples) {
+        if (!kept(s, thresholdOf))
+            continue;
+        h.kept++;
+        const cloudRight = !!s.cloud?.correct;
+        if (s.local.correct)
+            cloudRight ? h.bothRight++ : h.localOnlyRight++;
+        else
+            cloudRight ? h.cloudOnlyRight++ : h.bothWrong++;
+    }
+    return h;
+}
+function summarizeCost(samples, cost) {
+    if (!cost)
+        return undefined;
+    const measured = samples.flatMap((s) => (s.cloud?.cost !== undefined ? [s.cloud] : []));
+    if (measured.length) {
+        const tokens = { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 };
+        for (const c of measured) {
+            for (const u of c.usage ?? []) {
+                tokens.input += u.inputTokens;
+                tokens.output += u.outputTokens;
+                tokens.cacheRead += u.cacheReadInputTokens ?? 0;
+                tokens.cacheWrite += u.cacheWriteInputTokens ?? 0;
+            }
+        }
+        const m = measured.length;
+        const round = (v) => Math.round((v / m) * 10) / 10;
+        return {
+            currency: cost.currency,
+            basis: 'measured',
+            cloudPerRun: measured.reduce((a, c) => a + c.cost, 0) / m,
+            measuredCalls: m,
+            tokensPerRun: { input: round(tokens.input), output: round(tokens.output), cacheRead: round(tokens.cacheRead), cacheWrite: round(tokens.cacheWrite) },
+        };
+    }
+    if (cost.cloudPerRun === undefined)
+        return undefined;
+    return { currency: cost.currency, basis: 'estimate', cloudPerRun: cost.cloudPerRun, measuredCalls: 0 };
+}
+/**
+ * Candidate thresholds: 0, then one per gap between consecutive distinct local confidences,
+ * then one above the maximum, so escalating everything is always a candidate. When a confidence
+ * is exactly 1 (common for deterministic checks) that one is above 1. Every threshold in the same
+ * gap routes the samples identically, so this set is exhaustive and deterministic.
+ */
+export function candidateThresholds(confidences) {
+    const distinct = [...new Set(confidences)].sort((a, b) => a - b);
+    const out = [0];
+    for (let i = 0; i < distinct.length; i++) {
+        const t = niceThreshold(i === 0 ? -Infinity : distinct[i - 1], distinct[i]);
+        if (t > out[out.length - 1])
+            out.push(t);
+    }
+    const max = distinct[distinct.length - 1];
+    if (max !== undefined)
+        out.push(max < 1 ? niceThreshold(max, 1) : niceThreshold(max, max + 1));
+    return out;
+}
+export function wilsonInterval(successes, n, z = 1.959963984540054) {
+    if (n === 0)
+        return [0, 1];
+    const p = successes / n;
+    const denom = 1 + (z * z) / n;
+    const center = (p + (z * z) / (2 * n)) / denom;
+    const half = (z * Math.sqrt((p * (1 - p)) / n + (z * z) / (4 * n * n))) / denom;
+    return [Math.max(0, center - half), Math.min(1, center + half)];
+}
+export function analyze(samples, options) {
+    const n = samples.length;
+    const confidences = samples.flatMap((s) => (s.local ? [s.local.confidence] : []));
+    const candidates = candidateThresholds(confidences);
+    const cost = summarizeCost(samples, options.cost);
+    const sampleCost = (s) => s.cloud?.cost ?? cost?.cloudPerRun ?? 0;
+    const per1k = (sum) => (cost && n ? Math.round((sum / n) * 1000 * 1e6) / 1e6 : undefined);
+    const curve = candidates.map((threshold) => {
+        const { cloudCost, ...e } = evaluate(samples, () => threshold, sampleCost);
+        const c = per1k(cloudCost);
+        return { threshold, ...e, ...(c !== undefined ? { costPer1k: c } : {}) };
+    });
+    const cloudAccuracy = n ? samples.filter((s) => s.cloud?.correct).length / n : null;
+    const mode = typeof options.target === 'number' ? 'fixed' : options.target;
+    const target = mode === 'fixed' ? options.target : mode === 'cloud' ? (cloudAccuracy ?? 0) : Math.max(...curve.map((p) => p.accuracy));
+    // Smallest threshold meeting the target maximizes local share (local share is non-increasing in t).
+    let best = curve.find((p) => p.accuracy >= target - EPS);
+    const targetMet = best !== undefined;
+    if (!best) {
+        best = curve.reduce((a, b) => (b.accuracy > a.accuracy + EPS ? b : a));
+    }
+    const threshold = best.threshold;
+    // Per-label thresholds: greedy in label order, each lowered as far as the target allows.
+    let thresholds;
+    if (options.perLabel && targetMet) {
+        const current = {};
+        const labels = options.labels ?? [...new Set(samples.flatMap((s) => (s.local ? [s.local.label] : [])))].sort();
+        const thresholdOf = (s) => (s.local && Object.hasOwn(current, s.local.label) ? current[s.local.label] : threshold);
+        for (const label of labels) {
+            const own = samples.flatMap((s) => (s.local?.label === label ? [s.local.confidence] : []));
+            for (const t of candidateThresholds(own)) {
+                if (t >= threshold)
+                    break;
+                current[label] = t;
+                if (evaluate(samples, thresholdOf).accuracy >= target - EPS)
+                    break;
+                delete current[label];
+            }
+        }
+        if (Object.keys(current).length)
+            thresholds = current;
+    }
+    const thresholdOf = (s) => s.local && thresholds && Object.hasOwn(thresholds, s.local.label) ? thresholds[s.local.label] : threshold;
+    const final = evaluate(samples, thresholdOf, sampleCost);
+    const cloudAnswered = samples.filter((s) => s.cloud);
+    const finalCost = per1k(final.cloudCost);
+    const cloudOnlyCost = per1k(samples.reduce((a, s) => a + sampleCost(s), 0));
+    const expected = {
+        accuracy: final.accuracy,
+        localShare: final.localShare,
+        localAccuracy: final.localAccuracy,
+        cloudAccuracy,
+        ...(finalCost !== undefined && cloudOnlyCost !== undefined
+            ? { costPer1k: finalCost, cloudOnlyCostPer1k: cloudOnlyCost, savingsPer1k: Math.round((cloudOnlyCost - finalCost) * 1e6) / 1e6 }
+            : {}),
+    };
+    // Histogram of local confidences.
+    const bins = options.histogramBins ?? 10;
+    const edges = Array.from({ length: bins + 1 }, (_, i) => Math.round((i / bins) * 1e6) / 1e6);
+    const counts = new Array(bins).fill(0);
+    for (const c of confidences)
+        counts[Math.min(bins - 1, Math.floor(c * bins))]++;
+    // Confusion matrix (local predictions only).
+    let confusion;
+    const labels = options.labels;
+    if (labels?.length) {
+        const index = new Map(labels.map((l, i) => [l, i]));
+        const matrix = labels.map(() => labels.map(() => 0));
+        for (const s of samples) {
+            const i = index.get(s.truth);
+            const j = s.local ? index.get(s.local.label) : undefined;
+            if (i !== undefined && j !== undefined)
+                matrix[i][j]++;
+        }
+        confusion = { labels, local: matrix };
+    }
+    const perOption = (labels ?? [...new Set(samples.map((s) => s.truth))].sort()).map((label) => {
+        const group = samples.filter((s) => s.truth === label);
+        const withLocal = group.filter((s) => s.local);
+        const withCloud = group.filter((s) => s.cloud);
+        const e = evaluate(group, thresholdOf);
+        return {
+            label,
+            count: group.length,
+            localAccuracy: withLocal.length ? withLocal.filter((s) => s.local.correct).length / withLocal.length : null,
+            cloudAccuracy: withCloud.length ? withCloud.filter((s) => s.cloud.correct).length / group.length : null,
+            localShare: e.localShare,
+            cascadeAccuracy: e.accuracy,
+            threshold: thresholds?.[label] ?? threshold,
+        };
+    });
+    const confidentMistakes = samples
+        .filter((s) => s.local && !s.local.correct && s.local.confidence >= thresholdOf(s))
+        .sort((a, b) => b.local.confidence - a.local.confidence || (a.id < b.id ? -1 : 1));
+    return {
+        threshold,
+        thresholds,
+        target: { value: target, mode },
+        targetMet,
+        headToHead: headToHead(samples, thresholdOf),
+        cost,
+        expected,
+        accuracyInterval: wilsonInterval(Math.round(final.accuracy * n), n),
+        curve,
+        confidenceHistogram: { edges, counts },
+        confusion,
+        perOption,
+        counts: { total: n, localFailed: n - confidences.length, cloudFailed: n - cloudAnswered.length },
+        confidentMistakes,
+    };
+}
+/**
+ * K-fold cross-validation of the threshold choice: fits the thresholds on K-1 folds and routes
+ * the held-out fold with them. The recommended threshold is fitted and scored on the same data,
+ * which flatters it; this estimate does not. Folds interleave by position, so it is deterministic.
+ * Returns undefined for datasets too small to split (fewer than 10 examples per fold).
+ */
+export function crossValidate(samples, options, folds = 5) {
+    const n = samples.length;
+    if (folds < 2 || n < folds * 10)
+        return undefined;
+    const cost = summarizeCost(samples, options.cost);
+    const sampleCost = (s) => s.cloud?.cost ?? cost?.cloudPerRun ?? 0;
+    let correct = 0;
+    let local = 0;
+    let cloudCorrect = 0;
+    let cloudCost = 0;
+    for (let k = 0; k < folds; k++) {
+        const train = samples.filter((_, i) => i % folds !== k);
+        const test = samples.filter((_, i) => i % folds === k);
+        const fit = analyze(train, options);
+        const thresholdOf = (s) => s.local && fit.thresholds && Object.hasOwn(fit.thresholds, s.local.label) ? fit.thresholds[s.local.label] : fit.threshold;
+        const e = evaluate(test, thresholdOf, sampleCost);
+        correct += e.accuracy * test.length;
+        local += e.localShare * test.length;
+        cloudCost += e.cloudCost;
+        cloudCorrect += test.filter((s) => s.cloud?.correct).length;
+    }
+    const round = (v) => Math.round(v * 1e9) / 1e9;
+    return {
+        folds,
+        accuracy: round(correct / n),
+        localShare: round(local / n),
+        cloudAccuracy: round(cloudCorrect / n),
+        ...(cost ? { costPer1k: Math.round((cloudCost / n) * 1000 * 1e6) / 1e6 } : {}),
+    };
+}
+//# sourceMappingURL=analyze.js.map

package/dist/analyze.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"analyze.js","sourceRoot":"","sources":["../src/analyze.ts"],"names":[],"mappings":"AA6FA,MAAM,GAAG,GAAG,KAAK,CAAC;AAElB;;;;GAIG;AACH,MAAM,UAAU,aAAa,CAAC,EAAU,EAAE,EAAU;IAClD,IAAI,EAAE,GAAG,CAAC;QAAE,OAAO,CAAC,CAAC;IACrB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;QAC7B,MAAM,KAAK,GAAG,EAAE,IAAI,CAAC,CAAC;QACtB,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,KAAK,CAAC,GAAG,KAAK,CAAC;QACtC,IAAI,CAAC,IAAI,EAAE;YAAE,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC;QACtD,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,EAAE;YAAE,OAAO,CAAC,CAAC;IAClC,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC;AAUD,MAAM,IAAI,GAAG,CAAC,CAAS,EAAE,WAAkC,EAAW,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,KAAK,CAAC,UAAU,IAAI,WAAW,CAAC,CAAC,CAAC,CAAC;AAE3H,SAAS,QAAQ,CAAC,OAA0B,EAAE,WAAkC,EAAE,MAAM,GAA0B,GAAG,EAAE,CAAC,CAAC;IACvH,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,IAAI,IAAI,CAAC,CAAC,EAAE,WAAW,CAAC,EAAE,CAAC;YACzB,KAAK,EAAE,CAAC;YACR,IAAI,CAAC,CAAC,KAAM,CAAC,OAAO,EAAE,CAAC;gBACrB,OAAO,EAAE,CAAC;gBACV,YAAY,EAAE,CAAC;YACjB,CAAC;QACH,CAAC;aAAM,CAAC;YACN,SAAS,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC;YACvB,IAAI,CAAC,CAAC,KAAK,EAAE,OAAO;gBAAE,OAAO,EAAE,CAAC;QAClC,CAAC;IACH,CAAC;IACD,MAAM,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;IACzB,OAAO;QACL,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAC7B,UAAU,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAC7B,aAAa,EAAE,KAAK,CAAC,CAAC,CAAC,YAAY,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI;QAClD,SAAS;KACV,CAAC;AACJ,CAAC;AAED,SAAS,UAAU,CAAC,OAA0B,EAAE,WAAkC;IAChF,MAAM,CAAC,GAAe,EAAE,IAAI,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC;IACpG,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,WAAW,CAAC;YAAE,SAAS;QACpC,CAAC,CAAC,IAAI,EAAE,CAAC;QACT,MAAM,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,OAAO,CAAC;QACtC,IAAI,CAAC,CAAC,KAAM,CAAC,OAAO;YAAE,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,cAAc,EAAE,CAAC;;YACjE,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC;IACvD,CAAC;IACD,OAAO,CAAC,CAAC;AACX,CAAC;AAED,SAAS,aAAa,CAAC,OAA0B,EAAE,IAA4B;IAC7E,IAAI,CAAC,IAAI;QAAE,OAAO,SAAS,CAAC;IAC5B,MAAM,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACxF,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;QACpB,MAAM,MAAM,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,CAAC;QACpE,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;YACzB,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,EAAE,CAAC;gBAC9B,MAAM,CAAC,KAAK,IAAI,CAAC,CAAC,WAAW,CAAC;gBAC9B,MAAM,CAAC,MAAM,IAAI,CAAC,CAAC,YAAY,CAAC;gBAChC,MAAM,CAAC,SAAS,IAAI,CAAC,CAAC,oBAAoB,IAAI,CAAC,CAAC;gBAChD,MAAM,CAAC,UAAU,IAAI,CAAC,CAAC,qBAAqB,IAAI,CAAC,CAAC;YACpD,CAAC;QACH,CAAC;QACD,MAAM,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC;QAC1B,MAAM,KAAK,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE,CAAC;QAC3D,OAAO;YACL,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,KAAK,EAAE,UAAU;YACjB,WAAW,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,IAAK,EAAE,CAAC,CAAC,GAAG,CAAC;YAC1D,aAAa,EAAE,CAAC;YAChB,YAAY,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,UAAU,EAAE,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE;SACrJ,CAAC;IACJ,CAAC;IACD,IAAI,IAAI,CAAC,WAAW,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IACrD,OAAO,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,KAAK,EAAE,UAAU,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,EAAE,aAAa,EAAE,CAAC,EAAE,CAAC;AACzG,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,mBAAmB,CAAC,WAA8B;IAChE,MAAM,QAAQ,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACjE,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;IAChB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACzC,MAAM,CAAC,GAAG,aAAa,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAE,EAAE,QAAQ,CAAC,CAAC,CAAE,CAAC,CAAC;QAC9E,IAAI,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAE;YAAE,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC5C,CAAC;IACD,MAAM,GAAG,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC1C,IAAI,GAAG,KAAK,SAAS;QAAE,GAAG,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,GAAG,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;IAC/F,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,SAAiB,EAAE,CAAS,EAAE,CAAC,GAAG,iBAAiB;IAChF,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC3B,MAAM,CAAC,GAAG,SAAS,GAAG,CAAC,CAAC;IACxB,MAAM,KAAK,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;IAC9B,MAAM,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC;IAC/C,MAAM,IAAI,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC;IAChF,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC;AAClE,CAAC;AAED,MAAM,UAAU,OAAO,CAAC,OAA0B,EAAE,OAAuB;IACzE,MAAM,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;IACzB,MAAM,WAAW,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAClF,MAAM,UAAU,GAAG,mBAAmB,CAAC,WAAW,CAAC,CAAC;IACpD,MAAM,IAAI,GAAG,aAAa,CAAC,OAAO,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IAClD,MAAM,UAAU,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,EAAE,IAAI,IAAI,IAAI,EAAE,WAAW,IAAI,CAAC,CAAC;IAC1E,MAAM,KAAK,GAAG,CAAC,GAAW,EAAE,EAAE,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,IAAI,GAAG,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IAElG,MAAM,KAAK,GAA4B,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE;QAClE,MAAM,EAAE,SAAS,EAAE,GAAG,CAAC,EAAE,GAAG,QAAQ,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;QAC3E,MAAM,CAAC,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC;QAC3B,OAAO,EAAE,SAAS,EAAE,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;IAC3E,CAAC,CAAC,CAAC;IAEH,MAAM,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IACpF,MAAM,IAAI,GAAe,OAAO,OAAO,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC;IACvF,MAAM,MAAM,GACV,IAAI,KAAK,OAAO,CAAC,CAAC,CAAE,OAAO,CAAC,MAAiB,CAAC,CAAC,CAAC,IAAI,KAAK,OAAO,CAAC,CAAC,CAAC,CAAC,aAAa,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;IAEtI,oGAAoG;IACpG,IAAI,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,IAAI,MAAM,GAAG,GAAG,CAAC,CAAC;IACzD,MAAM,SAAS,GAAG,IAAI,KAAK,SAAS,CAAC;IACrC,IAAI,CAAC,IAAI,EAAE,CAAC;QACV,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACzE,CAAC;IACD,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;IAEjC,yFAAyF;IACzF,IAAI,UAA8C,CAAC;IACnD,IAAI,OAAO,CAAC,QAAQ,IAAI,SAAS,EAAE,CAAC;QAClC,MAAM,OAAO,GAA2B,EAAE,CAAC;QAC3C,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,CAAC,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC/G,MAAM,WAAW,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QAC5H,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,GAAG,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,KAAK,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YAC3F,KAAK,MAAM,CAAC,IAAI,mBAAmB,CAAC,GAAG,CAAC,EAAE,CAAC;gBACzC,IAAI,CAAC,IAAI,SAAS;oBAAE,MAAM;gBAC1B,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBACnB,IAAI,QAAQ,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC,QAAQ,IAAI,MAAM,GAAG,GAAG;oBAAE,MAAM;gBACnE,OAAO,OAAO,CAAC,KAAK,CAAC,CAAC;YACxB,CAAC;QACH,CAAC;QACD,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM;YAAE,UAAU,GAAG,OAAO,CAAC;IACxD,CAAC;IAED,MAAM,WAAW,GAAG,CAAC,CAAS,EAAE,EAAE,CAChC,CAAC,CAAC,KAAK,IAAI,UAAU,IAAI,MAAM,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAE,CAAC,CAAC,CAAC,SAAS,CAAC;IAC7G,MAAM,KAAK,GAAG,QAAQ,CAAC,OAAO,EAAE,WAAW,EAAE,UAAU,CAAC,CAAC;IACzD,MAAM,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IACrD,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;IACzC,MAAM,aAAa,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAC5E,MAAM,QAAQ,GAAgC;QAC5C,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,UAAU,EAAE,KAAK,CAAC,UAAU;QAC5B,aAAa,EAAE,KAAK,CAAC,aAAa;QAClC,aAAa;QACb,GAAG,CAAC,SAAS,KAAK,SAAS,IAAI,aAAa,KAAK,SAAS;YACxD,CAAC,CAAC,EAAE,SAAS,EAAE,SAAS,EAAE,kBAAkB,EAAE,aAAa,EAAE,YAAY,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,aAAa,GAAG,SAAS,CAAC,GAAG,GAAG,CAAC,GAAG,GAAG,EAAE;YAChI,CAAC,CAAC,EAAE,CAAC;KACR,CAAC;IAEF,kCAAkC;IAClC,MAAM,IAAI,GAAG,OAAO,CAAC,aAAa,IAAI,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC;IAC7F,MAAM,MAAM,GAAG,IAAI,KAAK,CAAS,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC/C,KAAK,MAAM,CAAC,IAAI,WAAW;QAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAE,EAAE,CAAC;IAEjF,6CAA6C;IAC7C,IAAI,SAAgC,CAAC;IACrC,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAC9B,IAAI,MAAM,EAAE,MAAM,EAAE,CAAC;QACnB,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACpD,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACrD,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,MAAM,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;YAC7B,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;YACzD,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,SAAS;gBAAE,MAAM,CAAC,CAAC,CAAE,CAAC,CAAC,CAAE,EAAE,CAAC;QAC3D,CAAC;QACD,SAAS,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;IACxC,CAAC;IAED,MAAM,SAAS,GAAkB,CAAC,MAAM,IAAI,CAAC,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;QAC1G,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,KAAK,CAAC,CAAC;QACvD,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QAC/C,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QAC/C,MAAM,CAAC,GAAG,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;QACvC,OAAO;YACL,KAAK;YACL,KAAK,EAAE,KAAK,CAAC,MAAM;YACnB,aAAa,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAM,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI;YAC5G,aAAa,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAM,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI;YACxG,UAAU,EAAE,CAAC,CAAC,UAAU;YACxB,eAAe,EAAE,CAAC,CAAC,QAAQ;YAC3B,SAAS,EAAE,UAAU,EAAE,CAAC,KAAK,CAAC,IAAI,SAAS;SAC5C,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,MAAM,iBAAiB,GAAG,OAAO;SAC9B,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,IAAI,CAAC,CAAC,KAAK,CAAC,UAAU,IAAI,WAAW,CAAC,CAAC,CAAC,CAAC;SAClF,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAM,CAAC,UAAU,GAAG,CAAC,CAAC,KAAM,CAAC,UAAU,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAEvF,OAAO;QACL,SAAS;QACT,UAAU;QACV,MAAM,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE;QAC/B,SAAS;QACT,UAAU,EAAE,UAAU,CAAC,OAAO,EAAE,WAAW,CAAC;QAC5C,IAAI;QACJ,QAAQ;QACR,gBAAgB,EAAE,cAAc,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;QACnE,KAAK;QACL,mBAAmB,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE;QACtC,SAAS;QACT,SAAS;QACT,MAAM,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,GAAG,WAAW,CAAC,MAAM,EAAE,WAAW,EAAE,CAAC,GAAG,aAAa,CAAC,MAAM,EAAE;QAChG,iBAAiB;KAClB,CAAC;AACJ,CAAC;AAaD;;;;;GAKG;AACH,MAAM,UAAU,aAAa,CAAC,OAA0B,EAAE,OAAuB,EAAE,KAAK,GAAG,CAAC;IAC1F,MAAM,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;IACzB,IAAI,KAAK,GAAG,CAAC,IAAI,CAAC,GAAG,KAAK,GAAG,EAAE;QAAE,OAAO,SAAS,CAAC;IAClD,MAAM,IAAI,GAAG,aAAa,CAAC,OAAO,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IAClD,MAAM,UAAU,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,EAAE,IAAI,IAAI,IAAI,EAAE,WAAW,IAAI,CAAC,CAAC;IAC1E,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;QAC/B,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,KAAK,KAAK,CAAC,CAAC,CAAC;QACxD,MAAM,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,KAAK,KAAK,CAAC,CAAC,CAAC;QACvD,MAAM,GAAG,GAAG,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QACpC,MAAM,WAAW,GAAG,CAAC,CAAS,EAAE,EAAE,CAChC,CAAC,CAAC,KAAK,IAAI,GAAG,CAAC,UAAU,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAE,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC;QAC7H,MAAM,CAAC,GAAG,QAAQ,CAAC,IAAI,EAAE,WAAW,EAAE,UAAU,CAAC,CAAC;QAClD,OAAO,IAAI,CAAC,CAAC,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC;QACpC,KAAK,IAAI,CAAC,CAAC,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC;QACpC,SAAS,IAAI,CAAC,CAAC,SAAS,CAAC;QACzB,YAAY,IAAI,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC,MAAM,CAAC;IAC9D,CAAC;IACD,MAAM,KAAK,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,GAAG,CAAC,GAAG,GAAG,CAAC;IACvD,OAAO;QACL,KAAK;QACL,QAAQ,EAAE,KAAK,CAAC,OAAO,GAAG,CAAC,CAAC;QAC5B,UAAU,EAAE,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC;QAC5B,aAAa,EAAE,KAAK,CAAC,YAAY,GAAG,CAAC,CAAC;QACtC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,GAAG,IAAI,GAAG,GAAG,CAAC,GAAG,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC/E,CAAC;AACJ,CAAC"}

package/dist/bin.d.ts

@@ -0,0 +1,3 @@
+#!/usr/bin/env node
+export {};
+//# sourceMappingURL=bin.d.ts.map

package/dist/bin.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"bin.d.ts","sourceRoot":"","sources":["../src/bin.ts"],"names":[],"mappings":""}

package/dist/bin.js

@@ -0,0 +1,7 @@
+#!/usr/bin/env node
+import { main } from './cli.js';
+main(process.argv.slice(2)).then((code) => process.exit(code), (err) => {
+    process.stderr.write(`belay: ${err instanceof Error ? err.message : String(err)}\n`);
+    process.exit(1);
+});
+//# sourceMappingURL=bin.js.map

package/dist/bin.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"bin.js","sourceRoot":"","sources":["../src/bin.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,IAAI,EAAE,MAAM,UAAU,CAAC;AAEhC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAC9B,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAC5B,CAAC,GAAY,EAAE,EAAE;IACf,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACrF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CACF,CAAC"}

package/dist/browser.d.ts

@@ -0,0 +1,22 @@
+import type { TaskSchema } from '@swissspidy/belay-core';
+import type { BrowserConfig, BrowserJudgeSpec, BrowserRunnerSpec } from './config.js';
+import type { LocalBackend } from './evaluate.js';
+export interface BrowserBackendOptions {
+    task: string;
+    schema: TaskSchema;
+    context?: string;
+    local: BrowserRunnerSpec;
+    judge?: BrowserJudgeSpec;
+    browser?: BrowserConfig;
+    cacheDir: string;
+    log?: (message: string) => void;
+    /** Injected in tests: a script evaluated in the page before any other script (e.g. a stub `Classifier`). */
+    initScript?: string;
+}
+/**
+ * Runs a browser runner inside a real Chrome through a Playwright persistent context, optionally
+ * with an unpacked extension (the WebAI Studio extension polyfills the Classifier API). The
+ * profile persists in the cache directory, so a downloaded model is reused across runs.
+ */
+export declare function browserBackend(options: BrowserBackendOptions): Promise<LocalBackend>;
+//# sourceMappingURL=browser.d.ts.map

package/dist/browser.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"browser.d.ts","sourceRoot":"","sources":["../src/browser.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAA6B,UAAU,EAAE,MAAM,wBAAwB,CAAC;AACpF,OAAO,KAAK,EAAE,aAAa,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AACtF,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAElD,MAAM,WAAW,qBAAqB;IACpC,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,UAAU,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,iBAAiB,CAAC;IACzB,KAAK,CAAC,EAAE,gBAAgB,CAAC;IACzB,OAAO,CAAC,EAAE,aAAa,CAAC;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;IAChC,4GAA4G;IAC5G,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AA+HD;;;;GAIG;AACH,wBAAsB,cAAc,CAAC,OAAO,EAAE,qBAAqB,GAAG,OAAO,CAAC,YAAY,CAAC,CAqI1F"}