@namzu/sdk 62.12.0 → 62.14.0

package/dist/eval/compare.js

@@ -0,0 +1,324 @@
+import { bootstrapPairedDifference, exactSignTestP, pooledMeanAndRate, validateSeed, } from './bootstrap.js';
+import { UNKNOWN } from './disclosure.js';
+import { duplicateRun, groupRuns, passedEveryRun, tallyOf } from './report-stats.js';
+function isRecord(value) {
+    return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+/**
+ * Read the JSON `namzu eval --out` writes. Tolerant of a file from an older
+ * version: only `suites[].suite` and each case's name, mean and status are
+ * required; repeats, splits and disclosure are read when they are there.
+ * Throws an Error whose message names what is wrong, for a person to read.
+ */
+export function loadEvalReportFile(json, label) {
+    if (!isRecord(json) || !Array.isArray(json.suites)) {
+        throw new Error(`${label} is not an eval report: it has no "suites" list.`);
+    }
+    const suites = [];
+    for (const [index, entry] of json.suites.entries()) {
+        if (!isRecord(entry) || typeof entry.suite !== 'string' || !isRecord(entry.report)) {
+            throw new Error(`${label}: suites[${index}] needs a "suite" id and a "report".`);
+        }
+        const report = entry.report;
+        if (!Array.isArray(report.cases)) {
+            throw new Error(`${label}: suite "${entry.suite}" has no "cases" list.`);
+        }
+        const runs = [];
+        for (const [caseIndex, run] of report.cases.entries()) {
+            if (!isRecord(run) || typeof run.case !== 'string' || !Number.isFinite(run.mean)) {
+                throw new Error(`${label}: suite "${entry.suite}", case ${caseIndex} needs a name in "case" and a finite number in "mean".`);
+            }
+            runs.push(run);
+        }
+        const twice = duplicateRun(runs);
+        if (twice !== undefined) {
+            throw new Error(`${label}: suite "${entry.suite}" has the case "${twice}" more than once, so its runs cannot be told from one case's repeats. A case is named by its id; the report needs each case once per repeat.`);
+        }
+        suites.push({
+            id: entry.suite,
+            cases: groupRuns(runs),
+            repeats: typeof report.repeats === 'number' ? report.repeats : 1,
+            facts: isRecord(report.disclosure) ? report.disclosure : undefined,
+        });
+    }
+    return { label, suites };
+}
+const DIMENSION_ORDER = [
+    'harness',
+    'suites',
+    'model',
+    'route',
+    'effort',
+    'repeats',
+];
+function known(value) {
+    return typeof value === 'string' && value !== UNKNOWN && value.length > 0 ? value : null;
+}
+function canonical(value) {
+    if (value === undefined || value === null || value === UNKNOWN)
+        return null;
+    if (typeof value === 'string')
+        return value;
+    return JSON.stringify(value, (_key, v) => isRecord(v)
+        ? Object.fromEntries(Object.entries(v).sort(([x], [y]) => (x < y ? -1 : x > y ? 1 : 0)))
+        : v);
+}
+function rolesFact(value) {
+    if (!isRecord(value))
+        return null;
+    const parts = [];
+    for (const role of Object.keys(value).sort()) {
+        const who = value[role];
+        const provider = isRecord(who) ? known(who.provider) : null;
+        const model = isRecord(who) ? known(who.model) : null;
+        if (provider === null || model === null)
+            return null;
+        parts.push(`${role}=${provider}/${model}`);
+    }
+    return parts.length === 0 ? 'no model' : parts.join('; ');
+}
+function firstKnown(suites, field) {
+    for (const suite of suites) {
+        const value = known(suite.facts?.[field]);
+        if (value !== null)
+            return value;
+    }
+    return null;
+}
+function factsOf(file, suiteIds) {
+    const facts = [
+        {
+            dimension: 'harness',
+            key: 'harness version',
+            value: firstKnown(file.suites, 'harnessVersion'),
+        },
+        { dimension: 'harness', key: 'sdk version', value: firstKnown(file.suites, 'sdkVersion') },
+        { dimension: 'harness', key: 'commit', value: firstKnown(file.suites, 'commit') },
+    ];
+    for (const id of suiteIds) {
+        const suite = file.suites.find((s) => s.id === id);
+        if (!suite)
+            continue;
+        const f = suite.facts;
+        facts.push({ dimension: 'suites', key: `${id} version`, value: known(f?.suiteVersion) }, { dimension: 'model', key: `${id} mode`, value: known(f?.mode) }, { dimension: 'model', key: `${id} models`, value: rolesFact(f?.roles) }, { dimension: 'route', key: `${id} routing`, value: canonical(f?.routing) }, { dimension: 'effort', key: `${id} reasoning effort`, value: known(f?.reasoningEffort) }, 
+        // A file from before repeats existed ran every case once.
+        { dimension: 'repeats', key: `${id} repeats`, value: String(suite.repeats) });
+    }
+    return facts;
+}
+function compareDisclosure(a, b, sharedSuites) {
+    const left = new Map(factsOf(a, sharedSuites).map((f) => [f.key, f]));
+    const right = new Map(factsOf(b, sharedSuites).map((f) => [f.key, f]));
+    const differences = [];
+    const unknown = [];
+    for (const [key, fa] of left) {
+        const fb = right.get(key);
+        if (fa.value !== null && fb.value !== null) {
+            if (fa.value !== fb.value) {
+                differences.push({ dimension: fa.dimension, key, a: fa.value, b: fb.value });
+            }
+        }
+        else if (fa.value !== null || fb.value !== null) {
+            unknown.push({
+                dimension: fa.dimension,
+                key,
+                missingIn: fa.value === null ? 'a' : 'b',
+                value: (fa.value ?? fb.value),
+            });
+        }
+    }
+    const differingDimensions = DIMENSION_ORDER.filter((d) => differences.some((x) => x.dimension === d));
+    return {
+        differences,
+        unknown,
+        differingDimensions,
+        ...(differingDimensions.length > 1
+            ? {
+                warning: `More than one thing differs between these two runs (${differingDimensions.join(', ')}), so a difference in the scores cannot be attributed to any one of them.`,
+            }
+            : {}),
+    };
+}
+function sorted(refs) {
+    return refs.sort((x, y) => x.suite === y.suite ? (x.case < y.case ? -1 : 1) : x.suite < y.suite ? -1 : 1);
+}
+function signed(value, digits = 3) {
+    const text = value.toFixed(digits);
+    return value > 0 ? `+${text}` : text;
+}
+export function compareEvalReports(a, b, options = {}) {
+    const seed = validateSeed(options.seed);
+    const index = (file) => {
+        const map = new Map();
+        for (const suite of file.suites) {
+            for (const c of suite.cases) {
+                map.set(`${suite.id}\u0000${c.name}`, { ref: { suite: suite.id, case: c.name }, runs: c });
+            }
+        }
+        return map;
+    };
+    const left = index(a);
+    const right = index(b);
+    const onlyInA = [];
+    const onlyInB = [];
+    const notJudged = [];
+    const pairs = [];
+    let onlyBPasses = 0;
+    let onlyAPasses = 0;
+    let bothPass = 0;
+    let bothFail = 0;
+    for (const [key, l] of left) {
+        const r = right.get(key);
+        if (!r) {
+            onlyInA.push(l.ref);
+            continue;
+        }
+        const ta = tallyOf(l.runs.runs);
+        const tb = tallyOf(r.runs.runs);
+        if (!ta || !tb) {
+            notJudged.push(l.ref);
+            continue;
+        }
+        pairs.push({ a: ta, b: tb });
+        const passA = passedEveryRun(l.runs.runs);
+        const passB = passedEveryRun(r.runs.runs);
+        if (passA && passB)
+            bothPass++;
+        else if (!passA && !passB)
+            bothFail++;
+        else if (passB)
+            onlyBPasses++;
+        else
+            onlyAPasses++;
+    }
+    for (const [key, r] of right)
+        if (!left.has(key))
+            onlyInB.push(r.ref);
+    const aSide = pairs.map((p) => p.a);
+    const bSide = pairs.map((p) => p.b);
+    const numbers = (tallies) => {
+        if (tallies.length === 0)
+            return { mean: null, passRate: null };
+        const [mean, passRate] = pooledMeanAndRate(tallies);
+        return { mean, passRate };
+    };
+    const difference = bootstrapPairedDifference(pairs, seed);
+    const flips = {
+        onlyBPasses,
+        onlyAPasses,
+        bothPass,
+        bothFail,
+        pValue: exactSignTestP(onlyBPasses, onlyAPasses),
+        test: 'exact two-sided sign test on the cases that flipped',
+    };
+    const verdict = decide(difference.mean, pairs.length, flips);
+    const sharedSuites = a.suites.map((s) => s.id).filter((id) => b.suites.some((s) => s.id === id));
+    return {
+        seed,
+        a: { label: a.label, cases: left.size },
+        b: { label: b.label, cases: right.size },
+        onlyInA: sorted(onlyInA),
+        onlyInB: sorted(onlyInB),
+        notJudged: sorted(notJudged),
+        paired: pairs.length,
+        sideA: numbers(aSide),
+        sideB: numbers(bSide),
+        difference,
+        flips,
+        verdict,
+        disclosure: compareDisclosure(a, b, sharedSuites),
+    };
+}
+/** The sign test must be at least this convincing before the sentence names a winner. */
+const CLAIM_P = 0.05;
+/** Bounds are judged at the precision they are printed at. */
+function atPrintedPrecision(value) {
+    return Math.round(value * 1000) / 1000 || 0;
+}
+function decide(mean, paired, flips) {
+    const none = 'no difference can be claimed at this size';
+    if (!mean.computed) {
+        return { claim: 'none', sentence: none, detail: `no interval: ${mean.reason}.` };
+    }
+    const range = `[${mean.low.toFixed(3)}, ${mean.high.toFixed(3)}]`;
+    // A bound is zero when it prints as zero: sums that differ by rounding only
+    // (the same scores added in another order) must not read as a difference.
+    const low = atPrintedPrecision(mean.low);
+    const high = atPrintedPrecision(mean.high);
+    const excludesZero = (low > 0 || high < 0) && atPrintedPrecision(mean.estimate) !== 0;
+    if (!excludesZero) {
+        return {
+            claim: 'none',
+            sentence: none,
+            detail: `the 95% interval ${range} for the difference in mean score over ${paired} paired cases includes zero.`,
+        };
+    }
+    const flipped = flips.onlyAPasses + flips.onlyBPasses;
+    if (flipped > 0 && flips.pValue >= CLAIM_P) {
+        return {
+            claim: 'none',
+            sentence: none,
+            detail: `the 95% interval ${range} over ${paired} paired cases excludes zero, but the exact test on the ${flipped} case${flipped === 1 ? '' : 's'} that flipped gives p = ${flips.pValue.toFixed(4)}, not below ${CLAIM_P}; two checks that disagree are not a finding.`,
+        };
+    }
+    const better = low > 0;
+    const size = Math.abs(mean.estimate).toFixed(3);
+    return {
+        claim: better ? 'better' : 'worse',
+        sentence: `B is ${better ? 'better' : 'worse'} than A by ${size} in mean score, 95% interval ${range}`,
+        detail: `the interval over ${paired} paired cases excludes zero${flipped > 0 ? ` and the exact test on the flips agrees (p = ${flips.pValue.toFixed(4)})` : ''}.`,
+    };
+}
+function listCases(title, refs) {
+    if (refs.length === 0)
+        return [`${title}: none`];
+    const shown = refs.slice(0, 20).map((r) => `  ${r.suite} / ${r.case}`);
+    if (refs.length > shown.length)
+        shown.push(`  and ${refs.length - shown.length} more`);
+    return [`${title} (${refs.length}):`, ...shown];
+}
+function intervalText(interval, show) {
+    if (!interval.computed)
+        return `no interval: ${interval.reason}`;
+    return `95% paired interval [${show(interval.low)}, ${show(interval.high)}]`;
+}
+export function formatEvalComparison(c) {
+    const score = (n) => (n === null ? 'none' : n.toFixed(3));
+    const rate = (n) => (n === null ? 'none' : `${(n * 100).toFixed(1)}%`);
+    const points = (n) => `${signed(n * 100, 1)} points`;
+    const lines = [
+        `A: ${c.a.label} (${c.a.cases} cases)`,
+        `B: ${c.b.label} (${c.b.cases} cases)`,
+        `Cases are paired by suite and id; ${c.paired} paired${c.notJudged.length > 0 ? `, ${c.notJudged.length} more present on both but not judged on one side` : ''}. Seed ${c.seed}.`,
+        '',
+        ...listCases('Only in A', c.onlyInA),
+        ...listCases('Only in B', c.onlyInB),
+        '',
+        `Mean score: A ${score(c.sideA.mean)}, B ${score(c.sideB.mean)}${c.difference.mean.estimate !== null ? `, difference ${signed(c.difference.mean.estimate)}` : ''}; ${intervalText(c.difference.mean, (n) => signed(n))}`,
+        `Pass rate: A ${rate(c.sideA.passRate)}, B ${rate(c.sideB.passRate)}${c.difference.passRate.estimate !== null ? `, difference ${points(c.difference.passRate.estimate)}` : ''}; ${intervalText(c.difference.passRate, (n) => points(n))}`,
+        `Flips: ${c.flips.onlyBPasses} case${c.flips.onlyBPasses === 1 ? '' : 's'} passed on B and not on A, ${c.flips.onlyAPasses} passed on A and not on B (${c.flips.bothPass} passed on both, ${c.flips.bothFail} failed on both). A case passes when every one of its runs passed. ${c.flips.test}: p = ${c.flips.pValue.toFixed(4)}`,
+        '',
+        `${c.verdict.sentence}.`,
+        `  (${c.verdict.detail})`,
+        '',
+    ];
+    const d = c.disclosure;
+    if (d.differences.length === 0) {
+        lines.push('What the two reports say about how they were produced: no stated difference.');
+    }
+    else {
+        lines.push('What differs in how the two reports were produced:');
+        for (const x of d.differences)
+            lines.push(`  ${x.key}: A ${x.a}, B ${x.b}`);
+    }
+    if (d.unknown.length > 0) {
+        lines.push('Stated on one side only, so it cannot be compared:');
+        for (const x of d.unknown) {
+            lines.push(`  ${x.key}: ${x.missingIn === 'a' ? 'A does not say' : 'B does not say'}, the other says ${x.value}`);
+        }
+    }
+    if (d.warning)
+        lines.push('', `Warning: ${d.warning}`);
+    return lines.join('\n');
+}
+//# sourceMappingURL=compare.js.map

package/dist/eval/compare.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"compare.js","sourceRoot":"","sources":["../../src/eval/compare.ts"],"names":[],"mappings":"AAAA,OAAO,EAKN,yBAAyB,EACzB,cAAc,EACd,iBAAiB,EACjB,YAAY,GACZ,MAAM,gBAAgB,CAAA;AACvB,OAAO,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAA;AACzC,OAAO,EAAiB,YAAY,EAAE,SAAS,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAA;AA6BnG,SAAS,QAAQ,CAAC,KAAc;IAC/B,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;AAC5E,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,kBAAkB,CAAC,IAAa,EAAE,KAAa;IAC9D,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;QACpD,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,kDAAkD,CAAC,CAAA;IAC5E,CAAC;IACD,MAAM,MAAM,GAAkB,EAAE,CAAA;IAChC,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC;QACpD,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,OAAO,KAAK,CAAC,KAAK,KAAK,QAAQ,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;YACpF,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,YAAY,KAAK,sCAAsC,CAAC,CAAA;QACjF,CAAC;QACD,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAA;QAC3B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;YAClC,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,YAAY,KAAK,CAAC,KAAK,wBAAwB,CAAC,CAAA;QACzE,CAAC;QACD,MAAM,IAAI,GAAiB,EAAE,CAAA;QAC7B,KAAK,MAAM,CAAC,SAAS,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;YACvD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,OAAO,GAAG,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBAClF,MAAM,IAAI,KAAK,CACd,GAAG,KAAK,YAAY,KAAK,CAAC,KAAK,WAAW,SAAS,wDAAwD,CAC3G,CAAA;YACF,CAAC;YACD,IAAI,CAAC,IAAI,CAAC,GAA4B,CAAC,CAAA;QACxC,CAAC;QACD,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,CAAA;QAChC,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACzB,MAAM,IAAI,KAAK,CACd,GAAG,KAAK,YAAY,KAAK,CAAC,KAAK,mBAAmB,KAAK,8IAA8I,CACrM,CAAA;QACF,CAAC;QACD,MAAM,CAAC,IAAI,CAAC;YACX,EAAE,EAAE,KAAK,CAAC,KAAK;YACf,KAAK,EAAE,SAAS,CAAC,IAAI,CAAC;YACtB,OAAO,EAAE,OAAO,MAAM,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YAChE,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS;SAClE,CAAC,CAAA;IACH,CAAC;IACD,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,CAAA;AACzB,CAAC;AAMD,MAAM,eAAe,GAAmC;IACvD,SAAS;IACT,QAAQ;IACR,OAAO;IACP,OAAO;IACP,QAAQ;IACR,SAAS;CACT,CAAA;AASD,SAAS,KAAK,CAAC,KAAc;IAC5B,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,OAAO,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAA;AACzF,CAAC;AAED,SAAS,SAAS,CAAC,KAAc;IAChC,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,OAAO;QAAE,OAAO,IAAI,CAAA;IAC3E,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAA;IAC3C,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CACxC,QAAQ,CAAC,CAAC,CAAC;QACV,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACxF,CAAC,CAAC,CAAC,CACJ,CAAA;AACF,CAAC;AAED,SAAS,SAAS,CAAC,KAAc;IAChC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,IAAI,CAAA;IACjC,MAAM,KAAK,GAAa,EAAE,CAAA;IAC1B,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;QAC9C,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,CAAA;QACvB,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;QAC3D,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;QACrD,IAAI,QAAQ,KAAK,IAAI,IAAI,KAAK,KAAK,IAAI;YAAE,OAAO,IAAI,CAAA;QACpD,KAAK,CAAC,IAAI,CAAC,GAAG,IAAI,IAAI,QAAQ,IAAI,KAAK,EAAE,CAAC,CAAA;IAC3C,CAAC;IACD,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;AAC1D,CAAC;AAED,SAAS,UAAU,CAAC,MAA8B,EAAE,KAAa;IAChE,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC5B,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC,CAAA;QACzC,IAAI,KAAK,KAAK,IAAI;YAAE,OAAO,KAAK,CAAA;IACjC,CAAC;IACD,OAAO,IAAI,CAAA;AACZ,CAAC;AAED,SAAS,OAAO,CAAC,IAAsB,EAAE,QAA2B;IACnE,MAAM,KAAK,GAAW;QACrB;YACC,SAAS,EAAE,SAAS;YACpB,GAAG,EAAE,iBAAiB;YACtB,KAAK,EAAE,UAAU,CAAC,IAAI,CAAC,MAAM,EAAE,gBAAgB,CAAC;SAChD;QACD,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,EAAE,aAAa,EAAE,KAAK,EAAE,UAAU,CAAC,IAAI,CAAC,MAAM,EAAE,YAAY,CAAC,EAAE;QAC1F,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,EAAE,QAAQ,EAAE,KAAK,EAAE,UAAU,CAAC,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,EAAE;KACjF,CAAA;IACD,KAAK,MAAM,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC3B,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAA;QAClD,IAAI,CAAC,KAAK;YAAE,SAAQ;QACpB,MAAM,CAAC,GAAG,KAAK,CAAC,KAAK,CAAA;QACrB,KAAK,CAAC,IAAI,CACT,EAAE,SAAS,EAAE,QAAQ,EAAE,GAAG,EAAE,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,EAAE,YAAY,CAAC,EAAE,EAC5E,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,EAChE,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,EACvE,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,SAAS,CAAC,CAAC,EAAE,OAAO,CAAC,EAAE,EAC1E,EAAE,SAAS,EAAE,QAAQ,EAAE,GAAG,EAAE,GAAG,EAAE,mBAAmB,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,EAAE,eAAe,CAAC,EAAE;QACxF,0DAA0D;QAC1D,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,EAAE,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAC5E,CAAA;IACF,CAAC;IACD,OAAO,KAAK,CAAA;AACb,CAAC;AA2BD,SAAS,iBAAiB,CACzB,CAAmB,EACnB,CAAmB,EACnB,YAA+B;IAE/B,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;IACrE,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;IACtE,MAAM,WAAW,GAA2B,EAAE,CAAA;IAC9C,MAAM,OAAO,GAAwB,EAAE,CAAA;IACvC,KAAK,MAAM,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC;QAC9B,MAAM,EAAE,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAS,CAAA;QACjC,IAAI,EAAE,CAAC,KAAK,KAAK,IAAI,IAAI,EAAE,CAAC,KAAK,KAAK,IAAI,EAAE,CAAC;YAC5C,IAAI,EAAE,CAAC,KAAK,KAAK,EAAE,CAAC,KAAK,EAAE,CAAC;gBAC3B,WAAW,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,EAAE,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,KAAK,EAAE,CAAC,CAAA;YAC7E,CAAC;QACF,CAAC;aAAM,IAAI,EAAE,CAAC,KAAK,KAAK,IAAI,IAAI,EAAE,CAAC,KAAK,KAAK,IAAI,EAAE,CAAC;YACnD,OAAO,CAAC,IAAI,CAAC;gBACZ,SAAS,EAAE,EAAE,CAAC,SAAS;gBACvB,GAAG;gBACH,SAAS,EAAE,EAAE,CAAC,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG;gBACxC,KAAK,EAAE,CAAC,EAAE,CAAC,KAAK,IAAI,EAAE,CAAC,KAAK,CAAW;aACvC,CAAC,CAAA;QACH,CAAC;IACF,CAAC;IACD,MAAM,mBAAmB,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CACxD,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,CAAC,CAAC,CAC1C,CAAA;IACD,OAAO;QACN,WAAW;QACX,OAAO;QACP,mBAAmB;QACnB,GAAG,CAAC,mBAAmB,CAAC,MAAM,GAAG,CAAC;YACjC,CAAC,CAAC;gBACA,OAAO,EAAE,uDAAuD,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,2EAA2E;aACzK;YACF,CAAC,CAAC,EAAE,CAAC;KACN,CAAA;AACF,CAAC;AAmDD,SAAS,MAAM,CAAC,IAAe;IAC9B,OAAO,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACzB,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAC7E,CAAA;AACF,CAAC;AAED,SAAS,MAAM,CAAC,KAAa,EAAE,MAAM,GAAG,CAAC;IACxC,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;IAClC,OAAO,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAA;AACrC,CAAC;AAED,MAAM,UAAU,kBAAkB,CACjC,CAAmB,EACnB,CAAmB,EACnB,UAA6B,EAAE;IAE/B,MAAM,IAAI,GAAG,YAAY,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;IACvC,MAAM,KAAK,GAAG,CAAC,IAAsB,EAAE,EAAE;QACxC,MAAM,GAAG,GAAG,IAAI,GAAG,EAA4C,CAAA;QAC/D,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YACjC,KAAK,MAAM,CAAC,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;gBAC7B,GAAG,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,EAAE,SAAS,CAAC,CAAC,IAAI,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC,CAAA;YAC3F,CAAC;QACF,CAAC;QACD,OAAO,GAAG,CAAA;IACX,CAAC,CAAA;IACD,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;IACrB,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;IAEtB,MAAM,OAAO,GAAc,EAAE,CAAA;IAC7B,MAAM,OAAO,GAAc,EAAE,CAAA;IAC7B,MAAM,SAAS,GAAc,EAAE,CAAA;IAC/B,MAAM,KAAK,GAAkB,EAAE,CAAA;IAC/B,IAAI,WAAW,GAAG,CAAC,CAAA;IACnB,IAAI,WAAW,GAAG,CAAC,CAAA;IACnB,IAAI,QAAQ,GAAG,CAAC,CAAA;IAChB,IAAI,QAAQ,GAAG,CAAC,CAAA;IAEhB,KAAK,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;QAC7B,MAAM,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QACxB,IAAI,CAAC,CAAC,EAAE,CAAC;YACR,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAA;YACnB,SAAQ;QACT,CAAC;QACD,MAAM,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC/B,MAAM,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC/B,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC;YAChB,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAA;YACrB,SAAQ;QACT,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,CAAA;QAC5B,MAAM,KAAK,GAAG,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACzC,MAAM,KAAK,GAAG,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACzC,IAAI,KAAK,IAAI,KAAK;YAAE,QAAQ,EAAE,CAAA;aACzB,IAAI,CAAC,KAAK,IAAI,CAAC,KAAK;YAAE,QAAQ,EAAE,CAAA;aAChC,IAAI,KAAK;YAAE,WAAW,EAAE,CAAA;;YACxB,WAAW,EAAE,CAAA;IACnB,CAAC;IACD,KAAK,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,IAAI,KAAK;QAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAA;IAErE,MAAM,KAAK,GAAyB,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;IACzD,MAAM,KAAK,GAAyB,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;IACzD,MAAM,OAAO,GAAG,CAAC,OAA6B,EAAe,EAAE;QAC9D,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAA;QAC/D,MAAM,CAAC,IAAI,EAAE,QAAQ,CAAC,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAA;QACnD,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAA;IAC1B,CAAC,CAAA;IAED,MAAM,UAAU,GAAG,yBAAyB,CAAC,KAAK,EAAE,IAAI,CAAC,CAAA;IACzD,MAAM,KAAK,GAAU;QACpB,WAAW;QACX,WAAW;QACX,QAAQ;QACR,QAAQ;QACR,MAAM,EAAE,cAAc,CAAC,WAAW,EAAE,WAAW,CAAC;QAChD,IAAI,EAAE,qDAAqD;KAC3D,CAAA;IACD,MAAM,OAAO,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,CAAA;IAC5D,MAAM,YAAY,GAAG,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAA;IAEhG,OAAO;QACN,IAAI;QACJ,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,IAAI,CAAC,IAAI,EAAE;QACvC,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE;QACxC,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC;QACxB,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC;QACxB,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC;QAC5B,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC;QACrB,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC;QACrB,UAAU;QACV,KAAK;QACL,OAAO;QACP,UAAU,EAAE,iBAAiB,CAAC,CAAC,EAAE,CAAC,EAAE,YAAY,CAAC;KACjD,CAAA;AACF,CAAC;AAED,yFAAyF;AACzF,MAAM,OAAO,GAAG,IAAI,CAAA;AAEpB,8DAA8D;AAC9D,SAAS,kBAAkB,CAAC,KAAa;IACxC,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,CAAA;AAC5C,CAAC;AAED,SAAS,MAAM,CAAC,IAAc,EAAE,MAAc,EAAE,KAAY;IAC3D,MAAM,IAAI,GAAG,2CAA2C,CAAA;IACxD,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;QACpB,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,gBAAgB,IAAI,CAAC,MAAM,GAAG,EAAE,CAAA;IACjF,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAA;IACjE,4EAA4E;IAC5E,0EAA0E;IAC1E,MAAM,GAAG,GAAG,kBAAkB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;IACxC,MAAM,IAAI,GAAG,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAC1C,MAAM,YAAY,GAAG,CAAC,GAAG,GAAG,CAAC,IAAI,IAAI,GAAG,CAAC,CAAC,IAAI,kBAAkB,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAA;IACrF,IAAI,CAAC,YAAY,EAAE,CAAC;QACnB,OAAO;YACN,KAAK,EAAE,MAAM;YACb,QAAQ,EAAE,IAAI;YACd,MAAM,EAAE,oBAAoB,KAAK,0CAA0C,MAAM,8BAA8B;SAC/G,CAAA;IACF,CAAC;IACD,MAAM,OAAO,GAAG,KAAK,CAAC,WAAW,GAAG,KAAK,CAAC,WAAW,CAAA;IACrD,IAAI,OAAO,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,IAAI,OAAO,EAAE,CAAC;QAC5C,OAAO;YACN,KAAK,EAAE,MAAM;YACb,QAAQ,EAAE,IAAI;YACd,MAAM,EAAE,oBAAoB,KAAK,SAAS,MAAM,0DAA0D,OAAO,QAAQ,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,2BAA2B,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,OAAO,+CAA+C;SACxQ,CAAA;IACF,CAAC;IACD,MAAM,MAAM,GAAG,GAAG,GAAG,CAAC,CAAA;IACtB,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAA;IAC/C,OAAO;QACN,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO;QAClC,QAAQ,EAAE,QAAQ,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,cAAc,IAAI,gCAAgC,KAAK,EAAE;QACtG,MAAM,EAAE,qBAAqB,MAAM,8BAA8B,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,gDAAgD,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG;KACjK,CAAA;AACF,CAAC;AAED,SAAS,SAAS,CAAC,KAAa,EAAE,IAAwB;IACzD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,GAAG,KAAK,QAAQ,CAAC,CAAA;IAChD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,CAAA;IACtE,IAAI,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM;QAAE,KAAK,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,OAAO,CAAC,CAAA;IACtF,OAAO,CAAC,GAAG,KAAK,KAAK,IAAI,CAAC,MAAM,IAAI,EAAE,GAAG,KAAK,CAAC,CAAA;AAChD,CAAC;AAED,SAAS,YAAY,CAAC,QAAkB,EAAE,IAA2B;IACpE,IAAI,CAAC,QAAQ,CAAC,QAAQ;QAAE,OAAO,gBAAgB,QAAQ,CAAC,MAAM,EAAE,CAAA;IAChE,OAAO,wBAAwB,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAA;AAC7E,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,CAAiB;IACrD,MAAM,KAAK,GAAG,CAAC,CAAgB,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAA;IACxE,MAAM,IAAI,GAAG,CAAC,CAAgB,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAA;IACrF,MAAM,MAAM,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,GAAG,MAAM,CAAC,CAAC,GAAG,GAAG,EAAE,CAAC,CAAC,SAAS,CAAA;IAC5D,MAAM,KAAK,GAAG;QACb,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,SAAS;QACtC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,SAAS;QACtC,qCAAqC,CAAC,CAAC,MAAM,UAAU,CAAC,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,SAAS,CAAC,MAAM,kDAAkD,CAAC,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC,IAAI,GAAG;QACjL,EAAE;QACF,GAAG,SAAS,CAAC,WAAW,EAAE,CAAC,CAAC,OAAO,CAAC;QACpC,GAAG,SAAS,CAAC,WAAW,EAAE,CAAC,CAAC,OAAO,CAAC;QACpC,EAAE;QACF,iBAAiB,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,gBAAgB,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,YAAY,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE;QACxN,gBAAgB,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,gBAAgB,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,YAAY,CAAC,CAAC,CAAC,UAAU,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE;QACzO,UAAU,CAAC,CAAC,KAAK,CAAC,WAAW,QAAQ,CAAC,CAAC,KAAK,CAAC,WAAW,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,8BAA8B,CAAC,CAAC,KAAK,CAAC,WAAW,8BAA8B,CAAC,CAAC,KAAK,CAAC,QAAQ,oBAAoB,CAAC,CAAC,KAAK,CAAC,QAAQ,sEAAsE,CAAC,CAAC,KAAK,CAAC,IAAI,SAAS,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE;QAClU,EAAE;QACF,GAAG,CAAC,CAAC,OAAO,CAAC,QAAQ,GAAG;QACxB,MAAM,CAAC,CAAC,OAAO,CAAC,MAAM,GAAG;QACzB,EAAE;KACF,CAAA;IACD,MAAM,CAAC,GAAG,CAAC,CAAC,UAAU,CAAA;IACtB,IAAI,CAAC,CAAC,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAChC,KAAK,CAAC,IAAI,CAAC,8EAA8E,CAAC,CAAA;IAC3F,CAAC;SAAM,CAAC;QACP,KAAK,CAAC,IAAI,CAAC,oDAAoD,CAAC,CAAA;QAChE,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,WAAW;YAAE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;IAC5E,CAAC;IACD,IAAI,CAAC,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1B,KAAK,CAAC,IAAI,CAAC,oDAAoD,CAAC,CAAA;QAChE,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC;YAC3B,KAAK,CAAC,IAAI,CACT,KAAK,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,SAAS,KAAK,GAAG,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,gBAAgB,oBAAoB,CAAC,CAAC,KAAK,EAAE,CACrG,CAAA;QACF,CAAC;IACF,CAAC;IACD,IAAI,CAAC,CAAC,OAAO;QAAE,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,YAAY,CAAC,CAAC,OAAO,EAAE,CAAC,CAAA;IACtD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;AACxB,CAAC"}

package/dist/eval/disclosure.d.ts

@@ -0,0 +1,70 @@
+import type { EvalRunOptions } from './run-options.js';
+/**
+ * What a report says about how it was produced.
+ *
+ * A score without its provenance cannot be compared with another: two runs
+ * three points apart may differ by a model, a route, a reasoning effort or a
+ * harness version, and the number alone does not say which. Every fact here
+ * is either known or the literal string `unknown`. Nothing is guessed, and a
+ * report written before this existed has none of it and is read as unknown.
+ */
+export type RunMode = 
+/** The model is a script; nothing measures a vendor. */
+'scripted'
+/** A real provider answered. */
+ | 'real-model'
+/** No model was involved at all. */
+ | 'no-model' | 'unknown';
+export interface RoleDisclosure {
+    readonly provider: string;
+    readonly model: string;
+}
+/** The routing configuration, `off` when routing was not on. */
+export type RoutingDisclosure = 'off' | 'unknown' | Readonly<Record<string, unknown>>;
+/**
+ * What a suite declares about itself in its `runExperiment` config. Every
+ * field is optional; the ones left out are written `unknown`.
+ */
+export interface SuiteFacts {
+    readonly mode?: RunMode;
+    /** One entry per role that ran, such as `agent`, `judge`, `child`. */
+    readonly roles?: Readonly<Record<string, {
+        provider?: string;
+        model?: string;
+    }>>;
+    readonly routing?: RoutingDisclosure;
+    /** The reasoning effort the model ran at, or `not applicable`. */
+    readonly reasoningEffort?: string;
+    /** Version of the suite. The runner fills it from the suite's package when it can. */
+    readonly suiteVersion?: string;
+}
+export interface ReportDisclosure {
+    /** The program that ran the suites, for example the CLI's version. */
+    readonly harnessVersion: string;
+    readonly sdkVersion: string;
+    /** ISO time the experiment finished. */
+    readonly generatedAt: string;
+    /** Cases in the suite. */
+    readonly cases: number;
+    readonly repeats: number;
+    readonly seed: number;
+    readonly mode: RunMode;
+    /** `unknown` when the suite declared no roles; an empty object when no model ran. */
+    readonly roles: Readonly<Record<string, RoleDisclosure>> | 'unknown';
+    readonly routing: RoutingDisclosure;
+    readonly reasoningEffort: string;
+    readonly suiteVersion: string;
+    /** Source revision, when the runner could read one. */
+    readonly commit: string;
+}
+export declare const UNKNOWN = "unknown";
+/** The version of this SDK, read from its manifest; `unknown` when it cannot be read. */
+export declare function sdkVersion(): string;
+export declare function buildDisclosure(facts: SuiteFacts | undefined, counts: {
+    cases: number;
+    repeats: number;
+    seed: number;
+}, options: EvalRunOptions, generatedAt: string): ReportDisclosure;
+/** One plain line for a terminal. */
+export declare function describeDisclosure(d: ReportDisclosure): string;
+//# sourceMappingURL=disclosure.d.ts.map

package/dist/eval/disclosure.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"disclosure.d.ts","sourceRoot":"","sources":["../../src/eval/disclosure.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAA;AAEtD;;;;;;;;GAQG;AACH,MAAM,MAAM,OAAO;AAClB,wDAAwD;AACtD,UAAU;AACZ,gCAAgC;GAC9B,YAAY;AACd,oCAAoC;GAClC,UAAU,GACV,SAAS,CAAA;AAEZ,MAAM,WAAW,cAAc;IAC9B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;CACtB;AAED,gEAAgE;AAChE,MAAM,MAAM,iBAAiB,GAAG,KAAK,GAAG,SAAS,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAA;AAErF;;;GAGG;AACH,MAAM,WAAW,UAAU;IAC1B,QAAQ,CAAC,IAAI,CAAC,EAAE,OAAO,CAAA;IACvB,sEAAsE;IACtE,QAAQ,CAAC,KAAK,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE;QAAE,QAAQ,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC,CAAA;IAChF,QAAQ,CAAC,OAAO,CAAC,EAAE,iBAAiB,CAAA;IACpC,kEAAkE;IAClE,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,CAAA;IACjC,sFAAsF;IACtF,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAA;CAC9B;AAED,MAAM,WAAW,gBAAgB;IAChC,sEAAsE;IACtE,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAA;IAC/B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;IAC3B,wCAAwC;IACxC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAA;IAC5B,0BAA0B;IAC1B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAA;IACtB,qFAAqF;IACrF,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC,GAAG,SAAS,CAAA;IACpE,QAAQ,CAAC,OAAO,EAAE,iBAAiB,CAAA;IACnC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAA;IAChC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAA;IAC7B,uDAAuD;IACvD,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;CACvB;AAED,eAAO,MAAM,OAAO,YAAY,CAAA;AAIhC,yFAAyF;AACzF,wBAAgB,UAAU,IAAI,MAAM,CAcnC;AAMD,wBAAgB,eAAe,CAC9B,KAAK,EAAE,UAAU,GAAG,SAAS,EAC7B,MAAM,EAAE;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,EACxD,OAAO,EAAE,cAAc,EACvB,WAAW,EAAE,MAAM,GACjB,gBAAgB,CAuBlB;AAED,qCAAqC;AACrC,wBAAgB,kBAAkB,CAAC,CAAC,EAAE,gBAAgB,GAAG,MAAM,CAc9D"}

package/dist/eval/disclosure.js

@@ -0,0 +1,64 @@
+import { readFileSync } from 'node:fs';
+import { fileURLToPath } from 'node:url';
+export const UNKNOWN = 'unknown';
+let cachedSdkVersion;
+/** The version of this SDK, read from its manifest; `unknown` when it cannot be read. */
+export function sdkVersion() {
+    if (cachedSdkVersion !== undefined)
+        return cachedSdkVersion;
+    try {
+        // `src/eval/` and `dist/eval/` are both two levels below the package.
+        const path = fileURLToPath(new URL('../../package.json', import.meta.url));
+        const manifest = JSON.parse(readFileSync(path, 'utf8'));
+        cachedSdkVersion =
+            typeof manifest.version === 'string' && manifest.version.length > 0
+                ? manifest.version
+                : UNKNOWN;
+    }
+    catch {
+        cachedSdkVersion = UNKNOWN;
+    }
+    return cachedSdkVersion;
+}
+function text(value) {
+    return value !== undefined && value.length > 0 ? value : UNKNOWN;
+}
+export function buildDisclosure(facts, counts, options, generatedAt) {
+    let roles = UNKNOWN;
+    if (facts?.roles !== undefined) {
+        const declared = {};
+        for (const [role, who] of Object.entries(facts.roles)) {
+            declared[role] = { provider: text(who.provider), model: text(who.model) };
+        }
+        roles = declared;
+    }
+    return {
+        harnessVersion: text(options.harnessVersion),
+        sdkVersion: sdkVersion(),
+        generatedAt,
+        cases: counts.cases,
+        repeats: counts.repeats,
+        seed: counts.seed,
+        mode: facts?.mode ?? UNKNOWN,
+        roles,
+        routing: facts?.routing ?? UNKNOWN,
+        reasoningEffort: text(facts?.reasoningEffort),
+        suiteVersion: text(facts?.suiteVersion ?? options.suiteVersion),
+        commit: text(options.commit),
+    };
+}
+/** One plain line for a terminal. */
+export function describeDisclosure(d) {
+    const roles = d.roles === UNKNOWN
+        ? 'models unknown'
+        : Object.keys(d.roles).length === 0
+            ? 'no model'
+            : Object.entries(d.roles)
+                .map(([role, who]) => `${role} ${who.provider}/${who.model}`)
+                .join(', ');
+    const routing = d.routing === 'off' || d.routing === UNKNOWN
+        ? `routing ${d.routing}`
+        : `routing ${JSON.stringify(d.routing)}`;
+    return `ran as ${d.mode}: ${roles}; ${routing}; effort ${d.reasoningEffort}; suite ${d.suiteVersion}`;
+}
+//# sourceMappingURL=disclosure.js.map

package/dist/eval/disclosure.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"disclosure.js","sourceRoot":"","sources":["../../src/eval/disclosure.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAA;AACtC,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAA;AAgExC,MAAM,CAAC,MAAM,OAAO,GAAG,SAAS,CAAA;AAEhC,IAAI,gBAAoC,CAAA;AAExC,yFAAyF;AACzF,MAAM,UAAU,UAAU;IACzB,IAAI,gBAAgB,KAAK,SAAS;QAAE,OAAO,gBAAgB,CAAA;IAC3D,IAAI,CAAC;QACJ,sEAAsE;QACtE,MAAM,IAAI,GAAG,aAAa,CAAC,IAAI,GAAG,CAAC,oBAAoB,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAA;QAC1E,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAA0B,CAAA;QAChF,gBAAgB;YACf,OAAO,QAAQ,CAAC,OAAO,KAAK,QAAQ,IAAI,QAAQ,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC;gBAClE,CAAC,CAAC,QAAQ,CAAC,OAAO;gBAClB,CAAC,CAAC,OAAO,CAAA;IACZ,CAAC;IAAC,MAAM,CAAC;QACR,gBAAgB,GAAG,OAAO,CAAA;IAC3B,CAAC;IACD,OAAO,gBAAgB,CAAA;AACxB,CAAC;AAED,SAAS,IAAI,CAAC,KAAyB;IACtC,OAAO,KAAK,KAAK,SAAS,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAA;AACjE,CAAC;AAED,MAAM,UAAU,eAAe,CAC9B,KAA6B,EAC7B,MAAwD,EACxD,OAAuB,EACvB,WAAmB;IAEnB,IAAI,KAAK,GAA8B,OAAO,CAAA;IAC9C,IAAI,KAAK,EAAE,KAAK,KAAK,SAAS,EAAE,CAAC;QAChC,MAAM,QAAQ,GAAmC,EAAE,CAAA;QACnD,KAAK,MAAM,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;YACvD,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,EAAE,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAA;QAC1E,CAAC;QACD,KAAK,GAAG,QAAQ,CAAA;IACjB,CAAC;IACD,OAAO;QACN,cAAc,EAAE,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC;QAC5C,UAAU,EAAE,UAAU,EAAE;QACxB,WAAW;QACX,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,OAAO,EAAE,MAAM,CAAC,OAAO;QACvB,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,IAAI,EAAE,KAAK,EAAE,IAAI,IAAI,OAAO;QAC5B,KAAK;QACL,OAAO,EAAE,KAAK,EAAE,OAAO,IAAI,OAAO;QAClC,eAAe,EAAE,IAAI,CAAC,KAAK,EAAE,eAAe,CAAC;QAC7C,YAAY,EAAE,IAAI,CAAC,KAAK,EAAE,YAAY,IAAI,OAAO,CAAC,YAAY,CAAC;QAC/D,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;KAC5B,CAAA;AACF,CAAC;AAED,qCAAqC;AACrC,MAAM,UAAU,kBAAkB,CAAC,CAAmB;IACrD,MAAM,KAAK,GACV,CAAC,CAAC,KAAK,KAAK,OAAO;QAClB,CAAC,CAAC,gBAAgB;QAClB,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,MAAM,KAAK,CAAC;YAClC,CAAC,CAAC,UAAU;YACZ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC;iBACtB,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,CAAC,GAAG,IAAI,IAAI,GAAG,CAAC,QAAQ,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC;iBAC5D,IAAI,CAAC,IAAI,CAAC,CAAA;IAChB,MAAM,OAAO,GACZ,CAAC,CAAC,OAAO,KAAK,KAAK,IAAI,CAAC,CAAC,OAAO,KAAK,OAAO;QAC3C,CAAC,CAAC,WAAW,CAAC,CAAC,OAAO,EAAE;QACxB,CAAC,CAAC,WAAW,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAA;IAC1C,OAAO,UAAU,CAAC,CAAC,IAAI,KAAK,KAAK,KAAK,OAAO,YAAY,CAAC,CAAC,eAAe,WAAW,CAAC,CAAC,YAAY,EAAE,CAAA;AACtG,CAAC"}

package/dist/eval/experiment.d.ts

@@ -1,3 +1,4 @@
+import { type SuiteFacts } from './disclosure.js';
 import type { CaseResult, EvalCase, EvalTurn, ExperimentReport, Scorer } from './types.js';
 export interface ExperimentConfig<TInput = unknown> {
     name: string;
@@ -29,7 +30,29 @@ export interface ExperimentConfig<TInput = unknown> {
     passThreshold?: number;
     /** Cases to run at once. Default 1 — deterministic ordering by default. */
     concurrency?: number;
+    /** Called once per run of a case; with repeats, once per repeat. */
     onCaseFinish?: (result: CaseResult) => void;
+    /**
+     * Times every case runs, from 1 to 100. Default 1.
+     *
+     * One run of a case says little about a model that does not answer the
+     * same way twice. The report keeps every run and says per case how many
+     * passed, and per suite how many cases passed every time. An
+     * `EvalRunOptions` scope opened by `namzu eval --repeat` wins over this.
+     */
+    repeats?: number;
+    /**
+     * Seed for the bootstrap intervals, an integer from 0 to 4294967295.
+     * Default 1, so the same cases give the same interval on every machine.
+     * Written into the report.
+     */
+    seed?: number;
+    /**
+     * What the suite knows about how it runs: scripted or a real model, which
+     * provider and model fill each role, routing, reasoning effort. Whatever is
+     * left out is written `unknown` in the report, never guessed.
+     */
+    disclosure?: SuiteFacts;
 }
 /**
  * Run a dataset and score it.

package/dist/eval/experiment.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"experiment.d.ts","sourceRoot":"","sources":["../../src/eval/experiment.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACX,UAAU,EAEV,QAAQ,EACR,QAAQ,EACR,gBAAgB,EAEhB,MAAM,EACN,MAAM,YAAY,CAAA;AAMnB,MAAM,WAAW,gBAAgB,CAAC,MAAM,GAAG,OAAO;IACjD,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,SAAS,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAA;IAClC,4DAA4D;IAC5D,OAAO,EAAE,SAAS,MAAM,EAAE,CAAA;IAC1B;;;;OAIG;IACH,GAAG,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,WAAW,KAAK,OAAO,CAAC,QAAQ,CAAC,CAAA;IAC1F;;;;;;;;;;;;;OAaG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,kEAAkE;IAClE,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,2EAA2E;IAC3E,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,YAAY,CAAC,EAAE,CAAC,MAAM,EAAE,UAAU,KAAK,IAAI,CAAA;CAC3C;AAED;;;;;;;;;;;GAWG;AACH,wBAAsB,aAAa,CAAC,MAAM,EACzC,MAAM,EAAE,gBAAgB,CAAC,MAAM,CAAC,GAC9B,OAAO,CAAC,gBAAgB,CAAC,CA2C3B;AAwPD,wBAAgB,YAAY,CAAC,MAAM,EAAE,gBAAgB,GAAG,MAAM,CA6D7D"}
+{"version":3,"file":"experiment.d.ts","sourceRoot":"","sources":["../../src/eval/experiment.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,UAAU,EAAuC,MAAM,iBAAiB,CAAA;AAUtF,OAAO,KAAK,EACX,UAAU,EAEV,QAAQ,EACR,QAAQ,EACR,gBAAgB,EAEhB,MAAM,EACN,MAAM,YAAY,CAAA;AAQnB,MAAM,WAAW,gBAAgB,CAAC,MAAM,GAAG,OAAO;IACjD,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,SAAS,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAA;IAClC,4DAA4D;IAC5D,OAAO,EAAE,SAAS,MAAM,EAAE,CAAA;IAC1B;;;;OAIG;IACH,GAAG,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,WAAW,KAAK,OAAO,CAAC,QAAQ,CAAC,CAAA;IAC1F;;;;;;;;;;;;;OAaG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,kEAAkE;IAClE,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,2EAA2E;IAC3E,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,oEAAoE;IACpE,YAAY,CAAC,EAAE,CAAC,MAAM,EAAE,UAAU,KAAK,IAAI,CAAA;IAC3C;;;;;;;OAOG;IACH,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB;;;;OAIG;IACH,IAAI,CAAC,EAAE,MAAM,CAAA;IACb;;;;OAIG;IACH,UAAU,CAAC,EAAE,UAAU,CAAA;CACvB;AAED;;;;;;;;;;;GAWG;AACH,wBAAsB,aAAa,CAAC,MAAM,EACzC,MAAM,EAAE,gBAAgB,CAAC,MAAM,CAAC,GAC9B,OAAO,CAAC,gBAAgB,CAAC,CAsE3B;AA2QD,wBAAgB,YAAY,CAAC,MAAM,EAAE,gBAAgB,GAAG,MAAM,CA4F7D"}