@cleocode/core 2026.9.22 → 2026.9.24

package/dist/decide/bench/dataset.js

@@ -0,0 +1,457 @@
+/**
+ * Build the System One benchmark dataset from CLEO's own history (T12495).
+ *
+ * The control group is CLEO dogfooded: every row is a question a decision
+ * site really asks, labelled from what the project's records already say.
+ *
+ * - `duplicateDetection` — task pairs. Positives: a `duplicates` relation, a
+ *   task cancelled with a reason that calls it a duplicate of a named task,
+ *   or a note that records a duplicate of a named task. Negatives: pairs of
+ *   tasks under the same parent with no relation between them.
+ * - `observationType` — observations whose stored type can only have come
+ *   from a caller: it is one of the offered options and differs from BOTH
+ *   the keyword default the writer applies today and the substring default
+ *   it applied before T12494. A type that agrees with a keyword default may
+ *   have been defaulted, so it is not used as a label.
+ * - `decisionContradiction` — decision pairs. Positives: a supersedes edge
+ *   (newer → older). Negatives: random pairs with no supersedes edge between
+ *   them. The newer decision's declared `supersedes` is never sent.
+ *
+ * Pair rows whose text gives the label away (one side names the other's id,
+ * or says "duplicate of T123" / "supersedes D12") are dropped
+ * ({@link leaksBenchLabel}).
+ *
+ * Every text field is redacted with the memory redaction patterns System One
+ * uses (`redactContent`) and clipped before it is stored, so the dataset file
+ * never holds a secret the provider would not have been sent anyway.
+ *
+ * Pure: the store is read through a {@link BenchSource}; randomness is seeded.
+ *
+ * @task T12495
+ * @epic T12486
+ */
+import { readFile, writeFile } from 'node:fs/promises';
+import { classifyObservationTypeByKeywords, OBSERVATION_TYPE_OPTIONS, } from '../../memory/observation-type-decision.js';
+import { redactContent } from '../../memory/redaction.js';
+import { redactThenClip } from '../site.js';
+import { createSeededRandom, DEFAULT_BENCH_SEED, shuffled } from './stats.js';
+import { BENCH_SITES, benchRowSchema, } from './types.js';
+/** Characters kept per text field in the dataset (the site builders clip further). */
+export const BENCH_TEXT_MAX_CHARS = 1_000;
+/** Default cap on rows per site. */
+export const DEFAULT_BENCH_MAX_ROWS_PER_SITE = 400;
+/** Default negatives sampled per positive. */
+export const DEFAULT_BENCH_NEGATIVES_PER_POSITIVE = 1;
+/** Negatives sampled even when a site has few or no positives. */
+export const MIN_BENCH_NEGATIVES = 10;
+/** Keyword default of the observation writer BEFORE T12494 (substring match). */
+const LEGACY_TYPE_KEYWORDS = [
+    { keywords: ['bug', 'fix', 'error', 'crash'], type: 'bugfix' },
+    { keywords: ['refactor', 'rename', 'extract', 'move'], type: 'refactor' },
+    { keywords: ['add', 'create', 'implement', 'new'], type: 'feature' },
+    { keywords: ['decide', 'chose', 'pick', 'instead'], type: 'decision' },
+    { keywords: ['update', 'change', 'modify', 'upgrade'], type: 'change' },
+];
+/**
+ * The type the pre-T12494 writer assigned when the caller gave none: the
+ * first keyword group with a SUBSTRING match, else `discovery`.
+ *
+ * @param text - Observation narrative.
+ * @returns The legacy default type.
+ */
+export function legacyKeywordObservationType(text) {
+    const lower = text.toLowerCase();
+    for (const { keywords, type } of LEGACY_TYPE_KEYWORDS) {
+        if (keywords.some((k) => lower.includes(k)))
+            return type;
+    }
+    return 'discovery';
+}
+/** Words that mark a cancellation reason or note as a duplicate verdict. */
+const DUPLICATE_WORDS = /\b(duplicat\w*|dup(?:e)?s?|dedup\w*|same as)\b/i;
+/** Task ids named in `text`. */
+function taskIdsIn(text) {
+    return [...new Set(text.match(/\bT\d{2,}\b/g) ?? [])];
+}
+/** A verdict word followed closely by a task or decision id: "duplicate of T123". */
+const GIVEAWAY = /\b(duplicat\w*|dup(?:e)?s?|dedup\w*|same as|supersed\w*|replac\w*|obsolete\w*)\b[^\n]{0,40}?\b[TD]\d{2,}\b/i;
+/**
+ * Whether a pair row's text gives its label away: either side names the
+ * other's id, or says "duplicate of T123" / "supersedes D12". Such rows are
+ * dropped, so a provider is never scored on reading the answer off the text.
+ *
+ * @param row - A dataset row.
+ * @returns True when the text leaks the label.
+ */
+export function leaksBenchLabel(row) {
+    if (row.site === 'observationType')
+        return false;
+    const [x, y] = row.site === 'duplicateDetection'
+        ? [row.input.a, row.input.b]
+        : [row.input.newer, row.input.older];
+    const textOf = (side) => 'title' in side ? `${side.title}\n${side.description}` : `${side.decision}\n${side.rationale}`;
+    const mentions = (text, id) => new RegExp(`\\b${id.replace(/[^\w]/g, '')}\\b`).test(text);
+    const xt = textOf(x);
+    const yt = textOf(y);
+    return mentions(xt, y.id) || mentions(yt, x.id) || GIVEAWAY.test(xt) || GIVEAWAY.test(yt);
+}
+/** Rows kept: those whose text does not leak the label. */
+function fair(row) {
+    return !leaksBenchLabel(row);
+}
+/** Unordered pair key. */
+function pairKey(a, b) {
+    return a < b ? `${a}+${b}` : `${b}+${a}`;
+}
+/** Redact then clip one text field. */
+function clean(text, redact) {
+    return redactThenClip(text ?? '', BENCH_TEXT_MAX_CHARS, redact);
+}
+/** Keep at most `max` rows, balancing positives and negatives. */
+function capBalanced(positives, negatives, max, random) {
+    if (positives.length + negatives.length <= max)
+        return [...positives, ...negatives];
+    const half = Math.floor(max / 2);
+    const keepPos = Math.min(positives.length, Math.max(half, max - negatives.length));
+    const keepNeg = Math.min(negatives.length, max - keepPos);
+    return [
+        ...shuffled(positives, random).slice(0, keepPos),
+        ...shuffled(negatives, random).slice(0, keepNeg),
+    ];
+}
+/** Number of negatives to sample for `positives` positives. */
+function negativeTarget(positives, perPositive) {
+    return Math.max(MIN_BENCH_NEGATIVES, Math.ceil(positives * perPositive));
+}
+/** Duplicate-detection rows. */
+function duplicateRows(tasks, opts, random) {
+    const byId = new Map(tasks.map((t) => [t.id, t]));
+    const linked = new Set();
+    for (const t of tasks)
+        for (const r of t.relates)
+            linked.add(pairKey(t.id, r.taskId));
+    const positives = new Map();
+    const addPositive = (aId, bId, rule, reason) => {
+        const a = byId.get(aId);
+        const b = byId.get(bId);
+        if (!a || !b || a.id === b.id || !a.title.trim() || !b.title.trim())
+            return;
+        const key = pairKey(a.id, b.id);
+        if (positives.has(key))
+            return;
+        const [first, second] = a.id < b.id ? [a, b] : [b, a];
+        positives.set(key, {
+            id: `duplicateDetection:${key}`,
+            site: 'duplicateDetection',
+            label: 'duplicate',
+            input: {
+                a: {
+                    id: first.id,
+                    title: clean(first.title, opts.redact),
+                    description: clean(first.description, opts.redact),
+                },
+                b: {
+                    id: second.id,
+                    title: clean(second.title, opts.redact),
+                    description: clean(second.description, opts.redact),
+                },
+            },
+            provenance: { sourceIds: [first.id, second.id], rule, reason },
+        });
+    };
+    for (const t of tasks) {
+        for (const r of t.relates) {
+            if (r.type !== 'duplicates')
+                continue;
+            addPositive(t.id, r.taskId, 'duplicates-relation', `${t.id} has a "duplicates" relation to ${r.taskId}${r.reason ? ` (${clean(r.reason, opts.redact)})` : ''}`);
+        }
+    }
+    for (const t of tasks) {
+        const reason = t.cancellationReason ?? '';
+        if (!reason || !DUPLICATE_WORDS.test(reason))
+            continue;
+        for (const other of taskIdsIn(reason)) {
+            addPositive(t.id, other, 'cancelled-duplicate-reason', `${t.id} was cancelled as a duplicate of ${other}: "${clean(reason, opts.redact)}"`);
+        }
+    }
+    for (const t of tasks) {
+        for (const note of t.notes) {
+            if (!DUPLICATE_WORDS.test(note))
+                continue;
+            for (const other of taskIdsIn(note)) {
+                addPositive(t.id, other, 'note-duplicate-reference', `a note on ${t.id} records a duplicate of ${other}: "${clean(note, opts.redact)}"`);
+            }
+        }
+    }
+    // Negatives: same-parent pairs with no relation of any kind.
+    const byParent = new Map();
+    for (const t of tasks) {
+        if (!t.parentId || !t.title.trim())
+            continue;
+        const siblings = byParent.get(t.parentId) ?? [];
+        siblings.push(t);
+        byParent.set(t.parentId, siblings);
+    }
+    const families = [...byParent.entries()]
+        .filter(([, kids]) => kids.length >= 2)
+        .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0));
+    const negatives = new Map();
+    const want = negativeTarget(positives.size, opts.negativesPerPositive);
+    for (let attempt = 0; negatives.size < want && attempt < want * 50; attempt++) {
+        const family = families[Math.floor(random() * families.length)];
+        if (!family)
+            break;
+        const [parentId, kids] = family;
+        const a = kids[Math.floor(random() * kids.length)];
+        const b = kids[Math.floor(random() * kids.length)];
+        if (!a || !b || a.id === b.id)
+            continue;
+        const key = pairKey(a.id, b.id);
+        if (positives.has(key) || negatives.has(key) || linked.has(key))
+            continue;
+        const [first, second] = a.id < b.id ? [a, b] : [b, a];
+        negatives.set(key, {
+            id: `duplicateDetection:${key}`,
+            site: 'duplicateDetection',
+            label: 'distinct',
+            input: {
+                a: {
+                    id: first.id,
+                    title: clean(first.title, opts.redact),
+                    description: clean(first.description, opts.redact),
+                },
+                b: {
+                    id: second.id,
+                    title: clean(second.title, opts.redact),
+                    description: clean(second.description, opts.redact),
+                },
+            },
+            provenance: {
+                sourceIds: [first.id, second.id],
+                rule: 'same-parent-unrelated',
+                reason: `${first.id} and ${second.id} share parent ${parentId} and have no relation between them`,
+            },
+        });
+    }
+    return capBalanced([...positives.values()].filter(fair), [...negatives.values()].filter(fair), opts.maxRowsPerSite, random);
+}
+/** Whether `value` is one of the offered observation types. */
+function isOfferedType(value) {
+    return OBSERVATION_TYPE_OPTIONS.some((o) => o === value);
+}
+/** Observation-type rows. */
+function observationRows(observations, opts, random) {
+    const rows = [];
+    for (const o of observations) {
+        const text = o.narrative.trim();
+        if (!text || !isOfferedType(o.type))
+            continue;
+        const current = classifyObservationTypeByKeywords(text);
+        const legacy = legacyKeywordObservationType(text);
+        if (o.type === current || o.type === legacy)
+            continue;
+        rows.push({
+            id: `observationType:${o.id}`,
+            site: 'observationType',
+            label: o.type,
+            input: { id: o.id, title: clean(o.title, opts.redact), text: clean(text, opts.redact) },
+            provenance: {
+                sourceIds: [o.id],
+                rule: 'explicit-type',
+                reason: `stored type "${o.type}" differs from both keyword defaults (current: ${current}, pre-T12494: ${legacy}), so a caller set it`,
+            },
+        });
+    }
+    if (rows.length <= opts.maxRowsPerSite)
+        return rows;
+    // Stratified down-sampling by type keeps rare types represented.
+    const byType = new Map();
+    for (const r of rows)
+        byType.set(r.label, [...(byType.get(r.label) ?? []), r]);
+    const out = [];
+    const types = [...byType.keys()].sort();
+    const perType = Math.max(1, Math.floor(opts.maxRowsPerSite / types.length));
+    for (const t of types)
+        out.push(...shuffled(byType.get(t) ?? [], random).slice(0, perType));
+    return out.slice(0, opts.maxRowsPerSite);
+}
+/** Decision-contradiction rows. */
+function contradictionRows(decisions, opts, random) {
+    const byId = new Map(decisions.map((d) => [d.id, d]));
+    const edges = new Map();
+    for (const d of decisions) {
+        if (d.supersedes && byId.has(d.supersedes) && d.supersedes !== d.id) {
+            edges.set(pairKey(d.id, d.supersedes), {
+                newer: d.id,
+                older: d.supersedes,
+                via: `${d.id}.supersedes = ${d.supersedes}`,
+            });
+        }
+    }
+    for (const d of decisions) {
+        if (d.supersededBy && byId.has(d.supersededBy) && d.supersededBy !== d.id) {
+            const key = pairKey(d.id, d.supersededBy);
+            if (!edges.has(key)) {
+                edges.set(key, {
+                    newer: d.supersededBy,
+                    older: d.id,
+                    via: `${d.id}.supersededBy = ${d.supersededBy}`,
+                });
+            }
+        }
+    }
+    const text = (d, withType) => ({
+        id: d.id,
+        ...(withType ? { type: d.type } : {}),
+        decision: clean(d.decision, opts.redact),
+        rationale: clean(d.rationale, opts.redact),
+    });
+    const positives = [];
+    for (const [key, e] of edges) {
+        const newer = byId.get(e.newer);
+        const older = byId.get(e.older);
+        if (!newer || !older)
+            continue;
+        positives.push({
+            id: `decisionContradiction:${e.newer}>${e.older}`,
+            site: 'decisionContradiction',
+            label: 'conflict',
+            input: { newer: text(newer, true), older: text(older, false) },
+            provenance: {
+                sourceIds: [e.newer, e.older],
+                rule: 'supersedes-edge',
+                reason: `supersedes edge ${e.via} (pair ${key}): the newer decision replaces the older one`,
+            },
+        });
+    }
+    const pool = [...decisions].sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
+    const negatives = new Map();
+    const want = negativeTarget(positives.length, opts.negativesPerPositive);
+    for (let attempt = 0; negatives.size < want && attempt < want * 50; attempt++) {
+        const a = pool[Math.floor(random() * pool.length)];
+        const b = pool[Math.floor(random() * pool.length)];
+        if (!a || !b || a.id === b.id)
+            continue;
+        const key = pairKey(a.id, b.id);
+        if (edges.has(key) || negatives.has(key))
+            continue;
+        negatives.set(key, {
+            id: `decisionContradiction:${a.id}>${b.id}`,
+            site: 'decisionContradiction',
+            label: 'compatible',
+            input: { newer: text(a, true), older: text(b, false) },
+            provenance: {
+                sourceIds: [a.id, b.id],
+                rule: 'random-unlinked-pair',
+                reason: `random pair with no supersedes edge between ${a.id} and ${b.id}`,
+            },
+        });
+    }
+    return capBalanced(positives.filter(fair), [...negatives.values()].filter(fair), opts.maxRowsPerSite, random);
+}
+/**
+ * Build the labelled dataset from a store.
+ *
+ * @param source - Store port (the accessors in production, records in tests).
+ * @param opts - Sites, seed, caps and redaction.
+ * @returns Rows, grouped by site in {@link BENCH_SITES} order.
+ */
+export async function buildBenchDataset(source, opts = {}) {
+    const sites = opts.sites ?? BENCH_SITES;
+    const resolved = {
+        maxRowsPerSite: opts.maxRowsPerSite ?? DEFAULT_BENCH_MAX_ROWS_PER_SITE,
+        negativesPerPositive: opts.negativesPerPositive ?? DEFAULT_BENCH_NEGATIVES_PER_POSITIVE,
+        redact: opts.redact ?? ((s) => redactContent(s).content),
+    };
+    const seed = opts.seed ?? DEFAULT_BENCH_SEED;
+    const rows = [];
+    // One generator per site, so adding a site never reshuffles another.
+    for (const [k, site] of BENCH_SITES.entries()) {
+        if (!sites.includes(site))
+            continue;
+        const random = createSeededRandom(seed + k);
+        if (site === 'duplicateDetection') {
+            rows.push(...duplicateRows(await source.tasks(), resolved, random));
+        }
+        else if (site === 'observationType') {
+            rows.push(...observationRows(await source.observations(), resolved, random));
+        }
+        else {
+            rows.push(...contradictionRows(await source.decisions(), resolved, random));
+        }
+    }
+    return rows;
+}
+/**
+ * Count rows per site and label.
+ *
+ * @param rows - Dataset rows.
+ * @returns The sizes.
+ */
+export function benchDatasetSizes(rows) {
+    const bySite = {};
+    for (const r of rows) {
+        const site = bySite[r.site] ?? {};
+        site[r.label] = (site[r.label] ?? 0) + 1;
+        bySite[r.site] = site;
+    }
+    return {
+        total: rows.length,
+        ownerVerified: rows.filter((r) => r.ownerVerified === true).length,
+        bySite,
+    };
+}
+/**
+ * Serialise rows as JSONL (one row per line, trailing newline).
+ *
+ * @param rows - Dataset rows.
+ * @returns The JSONL text.
+ */
+export function serializeBenchDataset(rows) {
+    return rows.map((r) => JSON.stringify(r)).join('\n') + (rows.length > 0 ? '\n' : '');
+}
+/**
+ * Parse and validate JSONL rows.
+ *
+ * @param text - JSONL text.
+ * @returns The rows.
+ * @throws Error naming the first invalid line.
+ */
+export function parseBenchDataset(text) {
+    const rows = [];
+    for (const [i, line] of text.split('\n').entries()) {
+        if (!line.trim())
+            continue;
+        let json;
+        try {
+            json = JSON.parse(line);
+        }
+        catch {
+            throw new Error(`dataset line ${i + 1} is not JSON`);
+        }
+        const parsed = benchRowSchema.safeParse(json);
+        if (!parsed.success)
+            throw new Error(`dataset line ${i + 1} is not a benchmark row`);
+        rows.push(parsed.data);
+    }
+    return rows;
+}
+/**
+ * Write rows to a JSONL file.
+ *
+ * @param path - Target file.
+ * @param rows - Dataset rows.
+ */
+export async function writeBenchDataset(path, rows) {
+    await writeFile(path, serializeBenchDataset(rows), { encoding: 'utf-8', mode: 0o600 });
+}
+/**
+ * Read rows from a JSONL file.
+ *
+ * @param path - Source file.
+ * @returns The rows.
+ */
+export async function readBenchDataset(path) {
+    return parseBenchDataset(await readFile(path, 'utf-8'));
+}
+//# sourceMappingURL=dataset.js.map

package/dist/decide/bench/dataset.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"dataset.js","sourceRoot":"","sources":["../../../src/decide/bench/dataset.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AAEH,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACvD,OAAO,EACL,iCAAiC,EACjC,wBAAwB,GACzB,MAAM,2CAA2C,CAAC;AACnD,OAAO,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAC1D,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAC5C,OAAO,EAAoB,kBAAkB,EAAE,kBAAkB,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAChG,OAAO,EACL,WAAW,EAOX,cAAc,GACf,MAAM,YAAY,CAAC;AAEpB,sFAAsF;AACtF,MAAM,CAAC,MAAM,oBAAoB,GAAG,KAAK,CAAC;AAE1C,oCAAoC;AACpC,MAAM,CAAC,MAAM,+BAA+B,GAAG,GAAG,CAAC;AAEnD,8CAA8C;AAC9C,MAAM,CAAC,MAAM,oCAAoC,GAAG,CAAC,CAAC;AAEtD,kEAAkE;AAClE,MAAM,CAAC,MAAM,mBAAmB,GAAG,EAAE,CAAC;AA0BtC,iFAAiF;AACjF,MAAM,oBAAoB,GAGpB;IACJ,EAAE,QAAQ,EAAE,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE;IAC9D,EAAE,QAAQ,EAAE,CAAC,UAAU,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE;IACzE,EAAE,QAAQ,EAAE,CAAC,KAAK,EAAE,QAAQ,EAAE,WAAW,EAAE,KAAK,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE;IACpE,EAAE,QAAQ,EAAE,CAAC,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE;IACtE,EAAE,QAAQ,EAAE,CAAC,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,SAAS,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE;CACxE,CAAC;AAEF;;;;;;GAMG;AACH,MAAM,UAAU,4BAA4B,CAAC,IAAY;IACvD,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;IACjC,KAAK,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,IAAI,oBAAoB,EAAE,CAAC;QACtD,IAAI,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;YAAE,OAAO,IAAI,CAAC;IAC3D,CAAC;IACD,OAAO,WAAW,CAAC;AACrB,CAAC;AAED,4EAA4E;AAC5E,MAAM,eAAe,GAAG,iDAAiD,CAAC;AAE1E,gCAAgC;AAChC,SAAS,SAAS,CAAC,IAAY;IAC7B,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;AACxD,CAAC;AAED,qFAAqF;AACrF,MAAM,QAAQ,GACZ,6GAA6G,CAAC;AAEhH;;;;;;;GAOG;AACH,MAAM,UAAU,eAAe,CAAC,GAAa;IAC3C,IAAI,GAAG,CAAC,IAAI,KAAK,iBAAiB;QAAE,OAAO,KAAK,CAAC;IACjD,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,GACV,GAAG,CAAC,IAAI,KAAK,oBAAoB;QAC/B,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;QAC5B,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACzC,MAAM,MAAM,GAAG,CAAC,IAAc,EAAU,EAAE,CACxC,OAAO,IAAI,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,QAAQ,KAAK,IAAI,CAAC,SAAS,EAAE,CAAC;IACjG,MAAM,QAAQ,GAAG,CAAC,IAAY,EAAE,EAAU,EAAW,EAAE,CACrD,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC7D,MAAM,EAAE,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;IACrB,MAAM,EAAE,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;IACrB,OAAO,QAAQ,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,IAAI,QAAQ,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,IAAI,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;AAC5F,CAAC;AAED,2DAA2D;AAC3D,SAAS,IAAI,CAAC,GAAa;IACzB,OAAO,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;AAC/B,CAAC;AAED,0BAA0B;AAC1B,SAAS,OAAO,CAAC,CAAS,EAAE,CAAS;IACnC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;AAC3C,CAAC;AAED,uCAAuC;AACvC,SAAS,KAAK,CAAC,IAA+B,EAAE,MAA6B;IAC3E,OAAO,cAAc,CAAC,IAAI,IAAI,EAAE,EAAE,oBAAoB,EAAE,MAAM,CAAC,CAAC;AAClE,CAAC;AAMD,kEAAkE;AAClE,SAAS,WAAW,CAClB,SAAuB,EACvB,SAAuB,EACvB,GAAW,EACX,MAAmB;IAEnB,IAAI,SAAS,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM,IAAI,GAAG;QAAE,OAAO,CAAC,GAAG,SAAS,EAAE,GAAG,SAAS,CAAC,CAAC;IACpF,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC;IACjC,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC;IACnF,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,MAAM,EAAE,GAAG,GAAG,OAAO,CAAC,CAAC;IAC1D,OAAO;QACL,GAAG,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC;QAChD,GAAG,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC;KACjD,CAAC;AACJ,CAAC;AAED,+DAA+D;AAC/D,SAAS,cAAc,CAAC,SAAiB,EAAE,WAAmB;IAC5D,OAAO,IAAI,CAAC,GAAG,CAAC,mBAAmB,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,GAAG,WAAW,CAAC,CAAC,CAAC;AAC3E,CAAC;AAED,gCAAgC;AAChC,SAAS,aAAa,CACpB,KAAiC,EACjC,IAAgE,EAChE,MAAmB;IAEnB,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAClD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAU,CAAC;IACjC,KAAK,MAAM,CAAC,IAAI,KAAK;QAAE,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,OAAO;YAAE,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;IAEtF,MAAM,SAAS,GAAG,IAAI,GAAG,EAAwB,CAAC;IAClD,MAAM,WAAW,GAAG,CAClB,GAAW,EACX,GAAW,EACX,IAAwC,EACxC,MAAc,EACR,EAAE;QACR,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACxB,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACxB,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE;YAAE,OAAO;QAC5E,MAAM,GAAG,GAAG,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;QAChC,IAAI,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,OAAO;QAC/B,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACtD,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE;YACjB,EAAE,EAAE,sBAAsB,GAAG,EAAE;YAC/B,IAAI,EAAE,oBAAoB;YAC1B,KAAK,EAAE,WAAW;YAClB,KAAK,EAAE;gBACL,CAAC,EAAE;oBACD,EAAE,EAAE,KAAK,CAAC,EAAE;oBACZ,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC;oBACtC,WAAW,EAAE,KAAK,CAAC,KAAK,CAAC,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC;iBACnD;gBACD,CAAC,EAAE;oBACD,EAAE,EAAE,MAAM,CAAC,EAAE;oBACb,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC;oBACvC,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC;iBACpD;aACF;YACD,UAAU,EAAE,EAAE,SAAS,EAAE,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE;SAC/D,CAAC,CAAC;IACL,CAAC,CAAC;IAEF,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC;YAC1B,IAAI,CAAC,CAAC,IAAI,KAAK,YAAY;gBAAE,SAAS;YACtC,WAAW,CACT,CAAC,CAAC,EAAE,EACJ,CAAC,CAAC,MAAM,EACR,qBAAqB,EACrB,GAAG,CAAC,CAAC,EAAE,mCAAmC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAC5G,CAAC;QACJ,CAAC;IACH,CAAC;IACD,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,MAAM,MAAM,GAAG,CAAC,CAAC,kBAAkB,IAAI,EAAE,CAAC;QAC1C,IAAI,CAAC,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,MAAM,CAAC;YAAE,SAAS;QACvD,KAAK,MAAM,KAAK,IAAI,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;YACtC,WAAW,CACT,CAAC,CAAC,EAAE,EACJ,KAAK,EACL,4BAA4B,EAC5B,GAAG,CAAC,CAAC,EAAE,oCAAoC,KAAK,MAAM,KAAK,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG,CACpF,CAAC;QACJ,CAAC;IACH,CAAC;IACD,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,KAAK,MAAM,IAAI,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC;YAC3B,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC;gBAAE,SAAS;YAC1C,KAAK,MAAM,KAAK,IAAI,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC;gBACpC,WAAW,CACT,CAAC,CAAC,EAAE,EACJ,KAAK,EACL,0BAA0B,EAC1B,aAAa,CAAC,CAAC,EAAE,2BAA2B,KAAK,MAAM,KAAK,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG,CACnF,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAED,6DAA6D;IAC7D,MAAM,QAAQ,GAAG,IAAI,GAAG,EAA6B,CAAC;IACtD,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,IAAI,CAAC,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE;YAAE,SAAS;QAC7C,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;QAChD,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACjB,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IACrC,CAAC;IACD,MAAM,QAAQ,GAAG,CAAC,GAAG,QAAQ,CAAC,OAAO,EAAE,CAAC;SACrC,MAAM,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC;SACtC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACpD,MAAM,SAAS,GAAG,IAAI,GAAG,EAAwB,CAAC;IAClD,MAAM,IAAI,GAAG,cAAc,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,CAAC,oBAAoB,CAAC,CAAC;IACvE,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,SAAS,CAAC,IAAI,GAAG,IAAI,IAAI,OAAO,GAAG,IAAI,GAAG,EAAE,EAAE,OAAO,EAAE,EAAE,CAAC;QAC9E,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;QAChE,IAAI,CAAC,MAAM;YAAE,MAAM;QACnB,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC,GAAG,MAAM,CAAC;QAChC,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;QACnD,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;QACnD,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,EAAE;YAAE,SAAS;QACxC,MAAM,GAAG,GAAG,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;QAChC,IAAI,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,SAAS;QAC1E,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACtD,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE;YACjB,EAAE,EAAE,sBAAsB,GAAG,EAAE;YAC/B,IAAI,EAAE,oBAAoB;YAC1B,KAAK,EAAE,UAAU;YACjB,KAAK,EAAE;gBACL,CAAC,EAAE;oBACD,EAAE,EAAE,KAAK,CAAC,EAAE;oBACZ,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC;oBACtC,WAAW,EAAE,KAAK,CAAC,KAAK,CAAC,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC;iBACnD;gBACD,CAAC,EAAE;oBACD,EAAE,EAAE,MAAM,CAAC,EAAE;oBACb,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC;oBACvC,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC;iBACpD;aACF;YACD,UAAU,EAAE;gBACV,SAAS,EAAE,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,EAAE,CAAC;gBAChC,IAAI,EAAE,uBAAuB;gBAC7B,MAAM,EAAE,GAAG,KAAK,CAAC,EAAE,QAAQ,MAAM,CAAC,EAAE,iBAAiB,QAAQ,oCAAoC;aAClG;SACF,CAAC,CAAC;IACL,CAAC;IACD,OAAO,WAAW,CAChB,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,EACpC,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,EACpC,IAAI,CAAC,cAAc,EACnB,MAAM,CACP,CAAC;AACJ,CAAC;AAED,+DAA+D;AAC/D,SAAS,aAAa,CAAC,KAAa;IAClC,OAAO,wBAAwB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC;AAC3D,CAAC;AAED,6BAA6B;AAC7B,SAAS,eAAe,CACtB,YAA+C,EAC/C,IAAgE,EAChE,MAAmB;IAEnB,MAAM,IAAI,GAAqB,EAAE,CAAC;IAClC,KAAK,MAAM,CAAC,IAAI,YAAY,EAAE,CAAC;QAC7B,MAAM,IAAI,GAAG,CAAC,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;QAChC,IAAI,CAAC,IAAI,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC;YAAE,SAAS;QAC9C,MAAM,OAAO,GAAG,iCAAiC,CAAC,IAAI,CAAC,CAAC;QACxD,MAAM,MAAM,GAAG,4BAA4B,CAAC,IAAI,CAAC,CAAC;QAClD,IAAI,CAAC,CAAC,IAAI,KAAK,OAAO,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM;YAAE,SAAS;QACtD,IAAI,CAAC,IAAI,CAAC;YACR,EAAE,EAAE,mBAAmB,CAAC,CAAC,EAAE,EAAE;YAC7B,IAAI,EAAE,iBAAiB;YACvB,KAAK,EAAE,CAAC,CAAC,IAAI;YACb,KAAK,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE;YACvF,UAAU,EAAE;gBACV,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACjB,IAAI,EAAE,eAAe;gBACrB,MAAM,EAAE,gBAAgB,CAAC,CAAC,IAAI,kDAAkD,OAAO,iBAAiB,MAAM,uBAAuB;aACtI;SACF,CAAC,CAAC;IACL,CAAC;IACD,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,cAAc;QAAE,OAAO,IAAI,CAAC;IACpD,iEAAiE;IACjE,MAAM,MAAM,GAAG,IAAI,GAAG,EAA4B,CAAC;IACnD,KAAK,MAAM,CAAC,IAAI,IAAI;QAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAC/E,MAAM,GAAG,GAAqB,EAAE,CAAC;IACjC,MAAM,KAAK,GAAG,CAAC,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IACxC,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,cAAc,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;IAC5E,KAAK,MAAM,CAAC,IAAI,KAAK;QAAE,GAAG,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;IAC5F,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,cAAc,CAAC,CAAC;AAC3C,CAAC;AAED,mCAAmC;AACnC,SAAS,iBAAiB,CACxB,SAAyC,EACzC,IAAgE,EAChE,MAAmB;IAEnB,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IACtD,MAAM,KAAK,GAAG,IAAI,GAAG,EAAyD,CAAC;IAC/E,KAAK,MAAM,CAAC,IAAI,SAAS,EAAE,CAAC;QAC1B,IAAI,CAAC,CAAC,UAAU,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,UAAU,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;YACpE,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,EAAE;gBACrC,KAAK,EAAE,CAAC,CAAC,EAAE;gBACX,KAAK,EAAE,CAAC,CAAC,UAAU;gBACnB,GAAG,EAAE,GAAG,CAAC,CAAC,EAAE,iBAAiB,CAAC,CAAC,UAAU,EAAE;aAC5C,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IACD,KAAK,MAAM,CAAC,IAAI,SAAS,EAAE,CAAC;QAC1B,IAAI,CAAC,CAAC,YAAY,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,YAAY,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;YAC1E,MAAM,GAAG,GAAG,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,YAAY,CAAC,CAAC;YAC1C,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;gBACpB,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE;oBACb,KAAK,EAAE,CAAC,CAAC,YAAY;oBACrB,KAAK,EAAE,CAAC,CAAC,EAAE;oBACX,GAAG,EAAE,GAAG,CAAC,CAAC,EAAE,mBAAmB,CAAC,CAAC,YAAY,EAAE;iBAChD,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IACD,MAAM,IAAI,GAAG,CAAC,CAAsB,EAAE,QAAiB,EAAE,EAAE,CAAC,CAAC;QAC3D,EAAE,EAAE,CAAC,CAAC,EAAE;QACR,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACrC,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC;QACxC,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC;KAC3C,CAAC,CAAC;IACH,MAAM,SAAS,GAAuB,EAAE,CAAC;IACzC,KAAK,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,IAAI,KAAK,EAAE,CAAC;QAC7B,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QAChC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QAChC,IAAI,CAAC,KAAK,IAAI,CAAC,KAAK;YAAE,SAAS;QAC/B,SAAS,CAAC,IAAI,CAAC;YACb,EAAE,EAAE,yBAAyB,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,KAAK,EAAE;YACjD,IAAI,EAAE,uBAAuB;YAC7B,KAAK,EAAE,UAAU;YACjB,KAAK,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,KAAK,CAAC,EAAE;YAC9D,UAAU,EAAE;gBACV,SAAS,EAAE,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC;gBAC7B,IAAI,EAAE,iBAAiB;gBACvB,MAAM,EAAE,mBAAmB,CAAC,CAAC,GAAG,UAAU,GAAG,8CAA8C;aAC5F;SACF,CAAC,CAAC;IACL,CAAC;IACD,MAAM,IAAI,GAAG,CAAC,GAAG,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACrF,MAAM,SAAS,GAAG,IAAI,GAAG,EAA4B,CAAC;IACtD,MAAM,IAAI,GAAG,cAAc,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,CAAC,oBAAoB,CAAC,CAAC;IACzE,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,SAAS,CAAC,IAAI,GAAG,IAAI,IAAI,OAAO,GAAG,IAAI,GAAG,EAAE,EAAE,OAAO,EAAE,EAAE,CAAC;QAC9E,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;QACnD,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;QACnD,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,EAAE;YAAE,SAAS;QACxC,MAAM,GAAG,GAAG,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;QAChC,IAAI,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,SAAS;QACnD,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE;YACjB,EAAE,EAAE,yBAAyB,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,EAAE,EAAE;YAC3C,IAAI,EAAE,uBAAuB;YAC7B,KAAK,EAAE,YAAY;YACnB,KAAK,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE;YACtD,UAAU,EAAE;gBACV,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC;gBACvB,IAAI,EAAE,sBAAsB;gBAC5B,MAAM,EAAE,+CAA+C,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC,EAAE,EAAE;aAC1E;SACF,CAAC,CAAC;IACL,CAAC;IACD,OAAO,WAAW,CAChB,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,EACtB,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,EACpC,IAAI,CAAC,cAAc,EACnB,MAAM,CACP,CAAC;AACJ,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,MAAmB,EACnB,OAAiC,EAAE;IAEnC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,WAAW,CAAC;IACxC,MAAM,QAAQ,GAAG;QACf,cAAc,EAAE,IAAI,CAAC,cAAc,IAAI,+BAA+B;QACtE,oBAAoB,EAAE,IAAI,CAAC,oBAAoB,IAAI,oCAAoC;QACvF,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;KACjE,CAAC;IACF,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,kBAAkB,CAAC;IAC7C,MAAM,IAAI,GAAe,EAAE,CAAC;IAC5B,qEAAqE;IACrE,KAAK,MAAM,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,WAAW,CAAC,OAAO,EAAE,EAAE,CAAC;QAC9C,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC;YAAE,SAAS;QACpC,MAAM,MAAM,GAAG,kBAAkB,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC;QAC5C,IAAI,IAAI,KAAK,oBAAoB,EAAE,CAAC;YAClC,IAAI,CAAC,IAAI,CAAC,GAAG,aAAa,CAAC,MAAM,MAAM,CAAC,KAAK,EAAE,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC;QACtE,CAAC;aAAM,IAAI,IAAI,KAAK,iBAAiB,EAAE,CAAC;YACtC,IAAI,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,MAAM,MAAM,CAAC,YAAY,EAAE,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC;QAC/E,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,IAAI,CAAC,GAAG,iBAAiB,CAAC,MAAM,MAAM,CAAC,SAAS,EAAE,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC;QAC9E,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,iBAAiB,CAAC,IAAyB;IACzD,MAAM,MAAM,GAA2C,EAAE,CAAC;IAC1D,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;QACrB,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;QAClC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;QACzC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;IACxB,CAAC;IACD,OAAO;QACL,KAAK,EAAE,IAAI,CAAC,MAAM;QAClB,aAAa,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,KAAK,IAAI,CAAC,CAAC,MAAM;QAClE,MAAM;KACP,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,qBAAqB,CAAC,IAAyB;IAC7D,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;AACvF,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,iBAAiB,CAAC,IAAY;IAC5C,MAAM,IAAI,GAAe,EAAE,CAAC;IAC5B,KAAK,MAAM,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC;QACnD,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;YAAE,SAAS;QAC3B,IAAI,IAAa,CAAC;QAClB,IAAI,CAAC;YACH,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,IAAI,KAAK,CAAC,gBAAgB,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;QACvD,CAAC;QACD,MAAM,MAAM,GAAG,cAAc,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QAC9C,IAAI,CAAC,MAAM,CAAC,OAAO;YAAE,MAAM,IAAI,KAAK,CAAC,gBAAgB,CAAC,GAAG,CAAC,yBAAyB,CAAC,CAAC;QACrF,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACzB,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,IAAY,EAAE,IAAyB;IAC7E,MAAM,SAAS,CAAC,IAAI,EAAE,qBAAqB,CAAC,IAAI,CAAC,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;AACzF,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,IAAY;IACjD,OAAO,iBAAiB,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC;AAC1D,CAAC"}

package/dist/decide/bench/index.d.ts

@@ -0,0 +1,23 @@
+/**
+ * System One accuracy benchmark (`cleo decide bench`): a labelled dataset
+ * from CLEO's own history, an owner spot-check, and a capped, repeated
+ * comparison of providers against each site's heuristic.
+ *
+ * Loaded on demand (`@cleocode/core/decide/bench/index.js`); the decide
+ * barrel does not re-export it, so the CLI's startup never pays for it.
+ *
+ * @task T12495
+ * @epic T12486
+ */
+export { BENCH_TEXT_MAX_CHARS, type BenchDatasetSizes, type BuildBenchDatasetOptions, benchDatasetSizes, buildBenchDataset, DEFAULT_BENCH_MAX_ROWS_PER_SITE, DEFAULT_BENCH_NEGATIVES_PER_POSITIVE, leaksBenchLabel, legacyKeywordObservationType, MIN_BENCH_NEGATIVES, parseBenchDataset, readBenchDataset, serializeBenchDataset, writeBenchDataset, } from './dataset.js';
+export { type BenchClassificationMetrics, type BenchLatency, type BenchPrediction, classificationMetrics, latencyOf, } from './metrics.js';
+export { BENCH_DATASET_FILE, BENCH_REPORT_FILE, BENCH_RESULTS_FILE, BENCH_SPOT_CHECK_FILE, type DecideBenchInput, DecideBenchInputError, type DecideBenchSummary, runDecideBenchOperation, } from './operation.js';
+export { BenchProfileError, BenchProfileInvalidError, type BenchProfileResolver, benchProfileEnvPrefix, createInterimProfileResolver, resolveBenchProfiles, } from './profiles.js';
+export { type BenchQuestion, benchQuestionFor } from './questions.js';
+export { renderBenchReport } from './report.js';
+export { type BenchAggregateRow, type BenchProviderRun, type BenchResults, type BenchRunRecord, type BenchSiteResult, type BenchSpendSummary, DEFAULT_BENCH_BATCH_SIZE, DEFAULT_BENCH_MAX_MICROS, HEURISTIC_PROVIDER, MAX_BENCH_BATCH_SIZE, MIN_BENCH_TIMEOUT_MS, type RunDecideBenchOptions, runDecideBench, } from './runner.js';
+export { createStoreBenchSource } from './source.js';
+export { applyBenchCorrections, type BenchCorrection, type BenchCorrectionsReceipt, type BenchSpotCheckFile, type BenchSpotCheckItem, DEFAULT_SPOT_CHECK_SIZE, parseBenchCorrections, readBenchCorrections, sampleBenchSpotCheck, } from './spot-check.js';
+export { type BenchRandom, type BenchSpread, createSeededRandom, DEFAULT_BENCH_SEED, percentile, shuffled, spreadOf, } from './stats.js';
+export { BENCH_LABEL_RULES, BENCH_SITES, type BenchConnection, type BenchDecisionRecord, type BenchLabelRule, type BenchObservationRecord, type BenchProvenance, type BenchRow, type BenchSite, type BenchSource, type BenchTaskRecord, benchRowSchema, CONTRADICTION_LABELS, DUPLICATE_LABELS, labelsForSite, positiveLabel, } from './types.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/decide/bench/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/decide/bench/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EACL,oBAAoB,EACpB,KAAK,iBAAiB,EACtB,KAAK,wBAAwB,EAC7B,iBAAiB,EACjB,iBAAiB,EACjB,+BAA+B,EAC/B,oCAAoC,EACpC,eAAe,EACf,4BAA4B,EAC5B,mBAAmB,EACnB,iBAAiB,EACjB,gBAAgB,EAChB,qBAAqB,EACrB,iBAAiB,GAClB,MAAM,cAAc,CAAC;AACtB,OAAO,EACL,KAAK,0BAA0B,EAC/B,KAAK,YAAY,EACjB,KAAK,eAAe,EACpB,qBAAqB,EACrB,SAAS,GACV,MAAM,cAAc,CAAC;AACtB,OAAO,EACL,kBAAkB,EAClB,iBAAiB,EACjB,kBAAkB,EAClB,qBAAqB,EACrB,KAAK,gBAAgB,EACrB,qBAAqB,EACrB,KAAK,kBAAkB,EACvB,uBAAuB,GACxB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EACL,iBAAiB,EACjB,wBAAwB,EACxB,KAAK,oBAAoB,EACzB,qBAAqB,EACrB,4BAA4B,EAC5B,oBAAoB,GACrB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,KAAK,aAAa,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AACtE,OAAO,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAChD,OAAO,EACL,KAAK,iBAAiB,EACtB,KAAK,gBAAgB,EACrB,KAAK,YAAY,EACjB,KAAK,cAAc,EACnB,KAAK,eAAe,EACpB,KAAK,iBAAiB,EACtB,wBAAwB,EACxB,wBAAwB,EACxB,kBAAkB,EAClB,oBAAoB,EACpB,oBAAoB,EACpB,KAAK,qBAAqB,EAC1B,cAAc,GACf,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,sBAAsB,EAAE,MAAM,aAAa,CAAC;AACrD,OAAO,EACL,qBAAqB,EACrB,KAAK,eAAe,EACpB,KAAK,uBAAuB,EAC5B,KAAK,kBAAkB,EACvB,KAAK,kBAAkB,EACvB,uBAAuB,EACvB,qBAAqB,EACrB,oBAAoB,EACpB,oBAAoB,GACrB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EACL,KAAK,WAAW,EAChB,KAAK,WAAW,EAChB,kBAAkB,EAClB,kBAAkB,EAClB,UAAU,EACV,QAAQ,EACR,QAAQ,GACT,MAAM,YAAY,CAAC;AACpB,OAAO,EACL,iBAAiB,EACjB,WAAW,EACX,KAAK,eAAe,EACpB,KAAK,mBAAmB,EACxB,KAAK,cAAc,EACnB,KAAK,sBAAsB,EAC3B,KAAK,eAAe,EACpB,KAAK,QAAQ,EACb,KAAK,SAAS,EACd,KAAK,WAAW,EAChB,KAAK,eAAe,EACpB,cAAc,EACd,oBAAoB,EACpB,gBAAgB,EAChB,aAAa,EACb,aAAa,GACd,MAAM,YAAY,CAAC"}

package/dist/decide/bench/index.js

@@ -0,0 +1,23 @@
+/**
+ * System One accuracy benchmark (`cleo decide bench`): a labelled dataset
+ * from CLEO's own history, an owner spot-check, and a capped, repeated
+ * comparison of providers against each site's heuristic.
+ *
+ * Loaded on demand (`@cleocode/core/decide/bench/index.js`); the decide
+ * barrel does not re-export it, so the CLI's startup never pays for it.
+ *
+ * @task T12495
+ * @epic T12486
+ */
+export { BENCH_TEXT_MAX_CHARS, benchDatasetSizes, buildBenchDataset, DEFAULT_BENCH_MAX_ROWS_PER_SITE, DEFAULT_BENCH_NEGATIVES_PER_POSITIVE, leaksBenchLabel, legacyKeywordObservationType, MIN_BENCH_NEGATIVES, parseBenchDataset, readBenchDataset, serializeBenchDataset, writeBenchDataset, } from './dataset.js';
+export { classificationMetrics, latencyOf, } from './metrics.js';
+export { BENCH_DATASET_FILE, BENCH_REPORT_FILE, BENCH_RESULTS_FILE, BENCH_SPOT_CHECK_FILE, DecideBenchInputError, runDecideBenchOperation, } from './operation.js';
+export { BenchProfileError, BenchProfileInvalidError, benchProfileEnvPrefix, createInterimProfileResolver, resolveBenchProfiles, } from './profiles.js';
+export { benchQuestionFor } from './questions.js';
+export { renderBenchReport } from './report.js';
+export { DEFAULT_BENCH_BATCH_SIZE, DEFAULT_BENCH_MAX_MICROS, HEURISTIC_PROVIDER, MAX_BENCH_BATCH_SIZE, MIN_BENCH_TIMEOUT_MS, runDecideBench, } from './runner.js';
+export { createStoreBenchSource } from './source.js';
+export { applyBenchCorrections, DEFAULT_SPOT_CHECK_SIZE, parseBenchCorrections, readBenchCorrections, sampleBenchSpotCheck, } from './spot-check.js';
+export { createSeededRandom, DEFAULT_BENCH_SEED, percentile, shuffled, spreadOf, } from './stats.js';
+export { BENCH_LABEL_RULES, BENCH_SITES, benchRowSchema, CONTRADICTION_LABELS, DUPLICATE_LABELS, labelsForSite, positiveLabel, } from './types.js';
+//# sourceMappingURL=index.js.map

package/dist/decide/bench/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/decide/bench/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EACL,oBAAoB,EAGpB,iBAAiB,EACjB,iBAAiB,EACjB,+BAA+B,EAC/B,oCAAoC,EACpC,eAAe,EACf,4BAA4B,EAC5B,mBAAmB,EACnB,iBAAiB,EACjB,gBAAgB,EAChB,qBAAqB,EACrB,iBAAiB,GAClB,MAAM,cAAc,CAAC;AACtB,OAAO,EAIL,qBAAqB,EACrB,SAAS,GACV,MAAM,cAAc,CAAC;AACtB,OAAO,EACL,kBAAkB,EAClB,iBAAiB,EACjB,kBAAkB,EAClB,qBAAqB,EAErB,qBAAqB,EAErB,uBAAuB,GACxB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EACL,iBAAiB,EACjB,wBAAwB,EAExB,qBAAqB,EACrB,4BAA4B,EAC5B,oBAAoB,GACrB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAsB,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AACtE,OAAO,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAChD,OAAO,EAOL,wBAAwB,EACxB,wBAAwB,EACxB,kBAAkB,EAClB,oBAAoB,EACpB,oBAAoB,EAEpB,cAAc,GACf,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,sBAAsB,EAAE,MAAM,aAAa,CAAC;AACrD,OAAO,EACL,qBAAqB,EAKrB,uBAAuB,EACvB,qBAAqB,EACrB,oBAAoB,EACpB,oBAAoB,GACrB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAGL,kBAAkB,EAClB,kBAAkB,EAClB,UAAU,EACV,QAAQ,EACR,QAAQ,GACT,MAAM,YAAY,CAAC;AACpB,OAAO,EACL,iBAAiB,EACjB,WAAW,EAUX,cAAc,EACd,oBAAoB,EACpB,gBAAgB,EAChB,aAAa,EACb,aAAa,GACd,MAAM,YAAY,CAAC"}

package/dist/decide/bench/metrics.d.ts

@@ -0,0 +1,55 @@
+/**
+ * Classification metrics for the System One benchmark (T12495).
+ *
+ * Two-label sites report precision, recall, F1 and false-positive rate for
+ * their positive class. The multi-class observation-type site reports macro
+ * averages over the labels present in the gold set (one-vs-rest per label).
+ *
+ * @task T12495
+ * @epic T12486
+ */
+/** One scored prediction. */
+export interface BenchPrediction {
+    /** Gold label. */
+    readonly gold: string;
+    /** Predicted label. */
+    readonly predicted: string;
+}
+/** Classification quality over a set of predictions. */
+export interface BenchClassificationMetrics {
+    /** Scored predictions. */
+    readonly n: number;
+    /** Share predicted correctly. */
+    readonly accuracy: number | null;
+    /** Precision (positive class, or macro average). */
+    readonly precision: number | null;
+    /** Recall (positive class, or macro average). */
+    readonly recall: number | null;
+    /** F1 (positive class, or macro average). */
+    readonly f1: number | null;
+    /** False-positive rate (positive class, or macro average one-vs-rest). */
+    readonly falsePositiveRate: number | null;
+}
+/**
+ * Score predictions.
+ *
+ * @param preds - Gold and predicted labels.
+ * @param positive - The positive class of a two-label site; `null` → macro averages.
+ * @returns The metrics (`null` where undefined, e.g. no positives predicted).
+ */
+export declare function classificationMetrics(preds: readonly BenchPrediction[], positive: string | null): BenchClassificationMetrics;
+/** p50 and p95 of latencies, in ms. */
+export interface BenchLatency {
+    /** Median. */
+    readonly p50Ms: number | null;
+    /** 95th percentile. */
+    readonly p95Ms: number | null;
+}
+/**
+ * Latency percentiles.
+ *
+ * @param samples - Latencies in ms.
+ * @returns p50 and p95 (nearest rank).
+ */
+export declare function latencyOf(samples: readonly number[]): BenchLatency;
+//# sourceMappingURL=metrics.d.ts.map

package/dist/decide/bench/metrics.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../../../src/decide/bench/metrics.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAIH,6BAA6B;AAC7B,MAAM,WAAW,eAAe;IAC9B,kBAAkB;IAClB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,uBAAuB;IACvB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B;AAED,wDAAwD;AACxD,MAAM,WAAW,0BAA0B;IACzC,0BAA0B;IAC1B,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC;IACnB,iCAAiC;IACjC,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,oDAAoD;IACpD,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,iDAAiD;IACjD,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,6CAA6C;IAC7C,QAAQ,CAAC,EAAE,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,0EAA0E;IAC1E,QAAQ,CAAC,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;CAC3C;AAoCD;;;;;;GAMG;AACH,wBAAgB,qBAAqB,CACnC,KAAK,EAAE,SAAS,eAAe,EAAE,EACjC,QAAQ,EAAE,MAAM,GAAG,IAAI,GACtB,0BAA0B,CAoC5B;AAED,uCAAuC;AACvC,MAAM,WAAW,YAAY;IAC3B,cAAc;IACd,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,uBAAuB;IACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;CAC/B;AAED;;;;;GAKG;AACH,wBAAgB,SAAS,CAAC,OAAO,EAAE,SAAS,MAAM,EAAE,GAAG,YAAY,CAElE"}