@plumloom/cli 0.1.3 → 0.1.5

package/dist/github/reliability-report.js

@@ -0,0 +1,341 @@
+import { createHash } from 'node:crypto';
+import { redactText } from '../auth/redact.js';
+import { extractJudgmentEvidence, extractScenarioRepeatabilityEvidence, } from '../reliability/evidence.js';
+const GITHUB_COMMENT_LIMIT = 65_536;
+const REPORT_LIMIT = 65_000;
+const MAX_CELL_CHARS = 280;
+function sanitizedText(input, maxChars) {
+    const withoutControls = Array.from(redactText(input), (character) => {
+        const codePoint = character.codePointAt(0) ?? 0;
+        return codePoint <= 31 || (codePoint >= 127 && codePoint <= 159) ? ' ' : character;
+    }).join('');
+    const clipped = withoutControls.length > maxChars
+        ? `${withoutControls.slice(0, maxChars - 1)}…`
+        : withoutControls;
+    return clipped.replaceAll('&', '&amp;').replaceAll('<', '&lt;').replaceAll('>', '&gt;');
+}
+function clean(input, maxChars = 1_000) {
+    return sanitizedText(input, maxChars).replace(/([\\`*_[\]{}()#+.!|])/gu, '\\$1');
+}
+function cell(input) {
+    return clean(input.replace(/[\r\n]+/gu, ' '), MAX_CELL_CHARS);
+}
+function number(input) {
+    return Number.isInteger(input) ? String(input) : input.toFixed(3).replace(/0+$/u, '');
+}
+function safeRunUrl(input) {
+    if (!input)
+        return undefined;
+    try {
+        const url = new URL(input);
+        return url.protocol === 'https:' ? url.toString() : undefined;
+    }
+    catch {
+        return undefined;
+    }
+}
+function contextLabel(context) {
+    switch (context) {
+        case 'scenario':
+            return 'Live scenario';
+        case 'conversation':
+            return 'Frozen conversation';
+        case 'agent_trace':
+            return 'Frozen agent trace';
+        default:
+            return 'Unknown';
+    }
+}
+function standardSummary(standard) {
+    if (!standard?.rubricApplied)
+        return 'Not available';
+    const parts = ['Rubric'];
+    if (standard.judgeModel) {
+        parts.push(`judge ${standard.judgeModel}${standard.judgeModelPinned ? ' (pinned)' : ''}`);
+    }
+    if (standard.qualityStandardStatus === 'not_configured') {
+        parts.push('QS: Not configured');
+    }
+    else if (standard.qualityStandardStatus === 'not_available') {
+        parts.push('QS: Not available');
+    }
+    else {
+        const name = standard.qualityStandardName ?? 'applied';
+        const version = standard.qualityStandardVersion
+            ? `; version ${standard.qualityStandardVersion}`
+            : '';
+        const anchors = standard.anchorCount === undefined ? '' : `; ${standard.anchorCount} anchors`;
+        parts.push(`QS: ${name}${version}${anchors}`);
+    }
+    return parts.join('; ');
+}
+function judgmentLine(evidence) {
+    const count = evidence.assessmentCount;
+    if (count === 1)
+        return `K=1; Agreement not measured`;
+    if (count === undefined)
+        return 'K: Not available';
+    if (evidence.passVotes !== undefined) {
+        return `K=${count}; ${evidence.passVotes}/${count} pass votes`;
+    }
+    if (evidence.agreedAssessments !== undefined) {
+        return `K=${count}; ${evidence.agreedAssessments}/${count} assessments agreed`;
+    }
+    return `K=${count}; agreement evidence Not available`;
+}
+function judgmentSummary(entries) {
+    if (entries.length === 0)
+        return 'Not available';
+    if (entries.length === 1 && entries[0])
+        return judgmentLine(entries[0]);
+    return entries.map((entry) => `${entry.scope}: ${judgmentLine(entry)}`).join('; ');
+}
+function repeatabilitySummary(evaluation) {
+    if (evaluation.context === 'conversation' || evaluation.context === 'agent_trace') {
+        return 'N/A: frozen input; behavioral repeatability not measured';
+    }
+    if (evaluation.context === 'unknown')
+        return 'Not available';
+    if (evaluation.decision === 'ERROR' && evaluation.repeatability?.statistics.length === 0) {
+        return 'Error';
+    }
+    const evidence = evaluation.repeatability;
+    if (!evidence)
+        return 'Not available';
+    const trials = evidence.completedTrials;
+    const hasStatistics = evidence.statistics.length > 0;
+    const insufficient = trials === 1 || evidence.statistics.some((entry) => entry.insufficientEvidence === true);
+    if (insufficient) {
+        return `${trials === undefined ? 'Trial count Not available' : `${trials} trial`}; Insufficient evidence`;
+    }
+    if (trials !== undefined) {
+        return `${trials} trials; statistics ${hasStatistics ? 'available' : 'Not available'}`;
+    }
+    return hasStatistics ? 'Trial count Not available; statistics available' : 'Not available';
+}
+function reportEvaluations(input) {
+    const summaryByIndex = new Map(input.summary.evals.map((entry) => [entry.index, entry]));
+    return [...input.gate.perEvalDecisions]
+        .sort((left, right) => left.index - right.index)
+        .map((decision) => {
+        const summary = summaryByIndex.get(decision.index);
+        const executionContext = summary?.execution?.contextType;
+        const decisionContext = decision.contextType;
+        const context = executionContext && decisionContext && executionContext !== decisionContext
+            ? 'unknown'
+            : (executionContext ?? decisionContext ?? 'unknown');
+        const results = summary?.results;
+        const statusRaw = summary?.execution?.statusRaw;
+        return {
+            index: decision.index,
+            name: decision.evaluationName ?? decision.inputFile,
+            context,
+            decision: decision.decision,
+            checks: decision.checks,
+            reason: decision.reason,
+            ...(summary?.execution?.standard === undefined
+                ? {}
+                : { standard: summary.execution.standard }),
+            judgment: extractJudgmentEvidence(context === 'unknown' ? undefined : context, results, statusRaw),
+            ...(context === 'scenario'
+                ? { repeatability: extractScenarioRepeatabilityEvidence(results, statusRaw) }
+                : {}),
+        };
+    });
+}
+function contextCounts(evaluations) {
+    const counts = {
+        scenario: 0,
+        conversation: 0,
+        agent_trace: 0,
+        unknown: 0,
+    };
+    for (const evaluation of evaluations)
+        counts[evaluation.context] += 1;
+    return [
+        `${counts.scenario} live scenario`,
+        `${counts.conversation} frozen conversation`,
+        `${counts.agent_trace} frozen agent trace`,
+        ...(counts.unknown > 0 ? [`${counts.unknown} unknown`] : []),
+    ].join(' · ');
+}
+function header(input, evaluations) {
+    const runUrl = safeRunUrl(input.actionsRunUrl);
+    return [
+        input.marker,
+        `## Autoeval Reliability Report — ${input.gate.suiteDecision}`,
+        '',
+        `${evaluations.length} evaluations · ${contextCounts(evaluations)}${runUrl ? ` · [GitHub Actions run](${runUrl})` : ''}`,
+        '',
+    ];
+}
+function legend() {
+    return [
+        '**Standard:** instructions, Quality Standard, and calibration anchors used.',
+        '**Judgment:** evidence from multiple judge assessments.',
+        '**Repeatability:** performance variation across independent executions of the system being evaluated.',
+        '',
+    ];
+}
+function table(evaluations) {
+    const lines = [
+        '| Evaluation | Context | Gate | Standard | Judgment | Repeatability |',
+        '| --- | --- | --- | --- | --- | --- |',
+    ];
+    for (const evaluation of evaluations) {
+        lines.push(`| ${cell(evaluation.name)} | ${cell(contextLabel(evaluation.context))} | **${evaluation.decision}** | ${cell(standardSummary(evaluation.standard))} | ${cell(judgmentSummary(evaluation.judgment))} | ${cell(repeatabilitySummary(evaluation))} |`);
+    }
+    lines.push('');
+    return lines;
+}
+function statisticLine(statistic) {
+    if (statistic.hasData === false)
+        return `${statistic.scope}: Not measured`;
+    if (statistic.insufficientEvidence === true) {
+        const sample = statistic.sampleSize === undefined ? '' : ` (n=${statistic.sampleSize})`;
+        return `${statistic.scope}: Insufficient evidence${sample}`;
+    }
+    const fields = [];
+    if (statistic.mean !== undefined)
+        fields.push(`mean ${number(statistic.mean)}`);
+    if (statistic.standardDeviation !== undefined) {
+        fields.push(`SD ${number(statistic.standardDeviation)}`);
+    }
+    if (statistic.ci95Lower !== undefined && statistic.ci95Upper !== undefined) {
+        fields.push(`95% CI ${number(statistic.ci95Lower)}–${number(statistic.ci95Upper)}`);
+    }
+    else if (statistic.ci95HalfWidth !== undefined) {
+        fields.push(`reported 95% CI half-width ±${number(statistic.ci95HalfWidth)}`);
+    }
+    if (statistic.sampleSize !== undefined)
+        fields.push(`n=${statistic.sampleSize}`);
+    return `${statistic.scope}: ${fields.length === 0 ? 'Not available' : fields.join('; ')}`;
+}
+function missingEvidence(evaluation) {
+    const missing = [];
+    if (!evaluation.standard?.rubricApplied)
+        missing.push('Standard metadata: Not available.');
+    else if (evaluation.standard.qualityStandardStatus === 'not_available') {
+        missing.push('Quality Standard metadata: Not available for this run.');
+    }
+    if (evaluation.judgment.length === 0)
+        missing.push('Judgment evidence: Not available.');
+    else if (evaluation.judgment.some((entry) => (entry.assessmentCount ?? 0) > 1 &&
+        entry.passVotes === undefined &&
+        entry.agreedAssessments === undefined)) {
+        missing.push('Judge agreement evidence: Not available at the reported granularity.');
+    }
+    if (evaluation.context === 'scenario' && evaluation.repeatability?.statistics.length === 0) {
+        missing.push('Repeatability statistics: Not available.');
+    }
+    if (evaluation.context === 'unknown') {
+        missing.push('Authoritative context: Not available; repeatability was not inferred.');
+    }
+    return missing;
+}
+function details(evaluation, actionsRunUrl) {
+    const lines = [
+        '<details>',
+        `<summary>${evaluation.decision} — ${sanitizedText(evaluation.name, 200)} (${contextLabel(evaluation.context)})</summary>`,
+        '',
+        `- **Standard:** ${clean(standardSummary(evaluation.standard), 500)}`,
+    ];
+    if (evaluation.judgment.length === 0) {
+        lines.push('- **Judgment:** Not available');
+    }
+    else {
+        lines.push('- **Judgment:**');
+        for (const entry of evaluation.judgment) {
+            lines.push(`  - ${clean(entry.scope)}: ${clean(judgmentLine(entry))}`);
+        }
+    }
+    if (evaluation.context === 'scenario') {
+        lines.push(`- **Repeatability:** ${clean(repeatabilitySummary(evaluation))}`);
+        for (const statistic of evaluation.repeatability?.statistics ?? []) {
+            lines.push(`  - ${clean(statisticLine(statistic), 700)}`);
+        }
+    }
+    else {
+        lines.push(`- **Repeatability:** ${clean(repeatabilitySummary(evaluation))}`);
+    }
+    if (evaluation.checks.length === 0) {
+        lines.push(`- **Gate checks:** ${clean(evaluation.reason, 700)}`);
+    }
+    else {
+        lines.push('- **Gate checks:**');
+        for (const check of evaluation.checks) {
+            lines.push(`  - ${clean(check.label, 200)}: ${clean(check.actual, 200)}; required ${clean(check.threshold, 200)} → **${check.decision}**${check.decision === 'PASS' ? '' : ` — ${clean(check.reason, 500)}`}`);
+        }
+    }
+    const missing = missingEvidence(evaluation);
+    if (missing.length > 0) {
+        lines.push('- **Missing evidence:**');
+        for (const item of missing)
+            lines.push(`  - ${item}`);
+    }
+    if (actionsRunUrl)
+        lines.push(`- [Full diagnostics in GitHub Actions](${actionsRunUrl})`);
+    lines.push('', '</details>', '');
+    return lines;
+}
+function renderFull(input, evaluations) {
+    const runUrl = safeRunUrl(input.actionsRunUrl);
+    const priority = { ERROR: 3, FAIL: 2, INCONCLUSIVE: 1, PASS: 0 };
+    const detailOrder = [...evaluations].sort((left, right) => priority[right.decision] - priority[left.decision] || left.index - right.index);
+    return [
+        ...header(input, evaluations),
+        ...legend(),
+        ...table(evaluations),
+        ...detailOrder.flatMap((evaluation) => details(evaluation, runUrl)),
+    ].join('\n');
+}
+function renderSummary(input, evaluations) {
+    return [...header(input, evaluations), ...legend(), ...table(evaluations)].join('\n');
+}
+function renderCritical(input, evaluations) {
+    const critical = evaluations.filter((evaluation) => evaluation.decision !== 'PASS');
+    return [
+        ...header(input, evaluations),
+        'Detailed evidence was omitted to stay within the GitHub comment size limit.',
+        '',
+        ...(critical.length === 0
+            ? ['All evaluations passed. See the Actions run for full diagnostics.', '']
+            : table(critical)),
+    ].join('\n');
+}
+function renderMinimal(input, evaluations) {
+    const runUrl = safeRunUrl(input.actionsRunUrl);
+    return [
+        input.marker,
+        `## Autoeval Reliability Report — ${input.gate.suiteDecision}`,
+        '',
+        `${evaluations.length} evaluations · ${contextCounts(evaluations)}`,
+        ...(runUrl ? ['', `[Full diagnostics in GitHub Actions](${runUrl})`] : []),
+        '',
+        'Evaluation details were omitted to stay within the GitHub comment size limit.',
+    ].join('\n');
+}
+export function createReliabilityReportMarker(identity) {
+    const digest = createHash('sha256').update(identity).digest('hex').slice(0, 24);
+    return `<!-- autoeval-reliability-report:${digest} -->`;
+}
+/** Deterministic, context-aware Markdown over already-sanitized suite evidence. */
+export function renderReliabilityReport(input) {
+    const evaluations = reportEvaluations(input);
+    const candidates = [
+        ['full', renderFull(input, evaluations)],
+        ['summary', renderSummary(input, evaluations)],
+        ['critical', renderCritical(input, evaluations)],
+        ['minimal', renderMinimal(input, evaluations)],
+    ];
+    for (const [detailLevel, markdown] of candidates) {
+        if (markdown.length <= REPORT_LIMIT)
+            return { markdown, detailLevel };
+    }
+    const fallback = renderMinimal(input, []);
+    if (fallback.length > GITHUB_COMMENT_LIMIT) {
+        throw new Error('Autoeval reliability report could not fit within the GitHub comment limit.');
+    }
+    return { markdown: fallback, detailLevel: 'minimal' };
+}
+//# sourceMappingURL=reliability-report.js.map

package/dist/github/reliability-report.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"reliability-report.js","sourceRoot":"","sources":["../../src/github/reliability-report.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAIzC,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAE/C,OAAO,EACL,uBAAuB,EACvB,oCAAoC,GAIrC,MAAM,4BAA4B,CAAC;AAEpC,MAAM,oBAAoB,GAAG,MAAM,CAAC;AACpC,MAAM,YAAY,GAAG,MAAM,CAAC;AAC5B,MAAM,cAAc,GAAG,GAAG,CAAC;AA4B3B,SAAS,aAAa,CAAC,KAAa,EAAE,QAAgB;IACpD,MAAM,eAAe,GAAG,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC,SAAS,EAAE,EAAE;QAClE,MAAM,SAAS,GAAG,SAAS,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAChD,OAAO,SAAS,IAAI,EAAE,IAAI,CAAC,SAAS,IAAI,GAAG,IAAI,SAAS,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC;IACrF,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACZ,MAAM,OAAO,GACX,eAAe,CAAC,MAAM,GAAG,QAAQ;QAC/B,CAAC,CAAC,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,GAAG,CAAC,CAAC,GAAG;QAC9C,CAAC,CAAC,eAAe,CAAC;IACtB,OAAO,OAAO,CAAC,UAAU,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,UAAU,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,UAAU,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;AAC1F,CAAC;AAED,SAAS,KAAK,CAAC,KAAa,EAAE,QAAQ,GAAG,KAAK;IAC5C,OAAO,aAAa,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,OAAO,CAAC,yBAAyB,EAAE,MAAM,CAAC,CAAC;AACnF,CAAC;AAED,SAAS,IAAI,CAAC,KAAa;IACzB,OAAO,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,WAAW,EAAE,GAAG,CAAC,EAAE,cAAc,CAAC,CAAC;AAChE,CAAC;AAED,SAAS,MAAM,CAAC,KAAa;IAC3B,OAAO,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;AACxF,CAAC;AAED,SAAS,UAAU,CAAC,KAAyB;IAC3C,IAAI,CAAC,KAAK;QAAE,OAAO,SAAS,CAAC;IAC7B,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;QAC3B,OAAO,GAAG,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;IAChE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED,SAAS,YAAY,CAAC,OAAsB;IAC1C,QAAQ,OAAO,EAAE,CAAC;QAChB,KAAK,UAAU;YACb,OAAO,eAAe,CAAC;QACzB,KAAK,cAAc;YACjB,OAAO,qBAAqB,CAAC;QAC/B,KAAK,aAAa;YAChB,OAAO,oBAAoB,CAAC;QAC9B;YACE,OAAO,SAAS,CAAC;IACrB,CAAC;AACH,CAAC;AAED,SAAS,eAAe,CAAC,QAAyC;IAChE,IAAI,CAAC,QAAQ,EAAE,aAAa;QAAE,OAAO,eAAe,CAAC;IACrD,MAAM,KAAK,GAAG,CAAC,QAAQ,CAAC,CAAC;IACzB,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC;QACxB,KAAK,CAAC,IAAI,CAAC,SAAS,QAAQ,CAAC,UAAU,GAAG,QAAQ,CAAC,gBAAgB,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC5F,CAAC;IACD,IAAI,QAAQ,CAAC,qBAAqB,KAAK,gBAAgB,EAAE,CAAC;QACxD,KAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;IACnC,CAAC;SAAM,IAAI,QAAQ,CAAC,qBAAqB,KAAK,eAAe,EAAE,CAAC;QAC9D,KAAK,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;IAClC,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,GAAG,QAAQ,CAAC,mBAAmB,IAAI,SAAS,CAAC;QACvD,MAAM,OAAO,GAAG,QAAQ,CAAC,sBAAsB;YAC7C,CAAC,CAAC,aAAa,QAAQ,CAAC,sBAAsB,EAAE;YAChD,CAAC,CAAC,EAAE,CAAC;QACP,MAAM,OAAO,GAAG,QAAQ,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,QAAQ,CAAC,WAAW,UAAU,CAAC;QAC9F,KAAK,CAAC,IAAI,CAAC,OAAO,IAAI,GAAG,OAAO,GAAG,OAAO,EAAE,CAAC,CAAC;IAChD,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,SAAS,YAAY,CAAC,QAA0B;IAC9C,MAAM,KAAK,GAAG,QAAQ,CAAC,eAAe,CAAC;IACvC,IAAI,KAAK,KAAK,CAAC;QAAE,OAAO,6BAA6B,CAAC;IACtD,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,kBAAkB,CAAC;IACnD,IAAI,QAAQ,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;QACrC,OAAO,KAAK,KAAK,KAAK,QAAQ,CAAC,SAAS,IAAI,KAAK,aAAa,CAAC;IACjE,CAAC;IACD,IAAI,QAAQ,CAAC,iBAAiB,KAAK,SAAS,EAAE,CAAC;QAC7C,OAAO,KAAK,KAAK,KAAK,QAAQ,CAAC,iBAAiB,IAAI,KAAK,qBAAqB,CAAC;IACjF,CAAC;IACD,OAAO,KAAK,KAAK,oCAAoC,CAAC;AACxD,CAAC;AAED,SAAS,eAAe,CAAC,OAAoC;IAC3D,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,eAAe,CAAC;IACjD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC;QAAE,OAAO,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;IACxE,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,GAAG,KAAK,CAAC,KAAK,KAAK,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACrF,CAAC;AAED,SAAS,oBAAoB,CAAC,UAA4B;IACxD,IAAI,UAAU,CAAC,OAAO,KAAK,cAAc,IAAI,UAAU,CAAC,OAAO,KAAK,aAAa,EAAE,CAAC;QAClF,OAAO,0DAA0D,CAAC;IACpE,CAAC;IACD,IAAI,UAAU,CAAC,OAAO,KAAK,SAAS;QAAE,OAAO,eAAe,CAAC;IAC7D,IAAI,UAAU,CAAC,QAAQ,KAAK,OAAO,IAAI,UAAU,CAAC,aAAa,EAAE,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzF,OAAO,OAAO,CAAC;IACjB,CAAC;IACD,MAAM,QAAQ,GAAG,UAAU,CAAC,aAAa,CAAC;IAC1C,IAAI,CAAC,QAAQ;QAAE,OAAO,eAAe,CAAC;IACtC,MAAM,MAAM,GAAG,QAAQ,CAAC,eAAe,CAAC;IACxC,MAAM,aAAa,GAAG,QAAQ,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC;IACrD,MAAM,YAAY,GAChB,MAAM,KAAK,CAAC,IAAI,QAAQ,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,oBAAoB,KAAK,IAAI,CAAC,CAAC;IAC3F,IAAI,YAAY,EAAE,CAAC;QACjB,OAAO,GAAG,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,2BAA2B,CAAC,CAAC,CAAC,GAAG,MAAM,QAAQ,yBAAyB,CAAC;IAC5G,CAAC;IACD,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;QACzB,OAAO,GAAG,MAAM,uBAAuB,aAAa,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC;IACzF,CAAC;IACD,OAAO,aAAa,CAAC,CAAC,CAAC,iDAAiD,CAAC,CAAC,CAAC,eAAe,CAAC;AAC7F,CAAC;AAED,SAAS,iBAAiB,CAAC,KAA6B;IACtD,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;IACzF,OAAO,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC;SACpC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC;SAC/C,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE;QAChB,MAAM,OAAO,GAAG,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QACnD,MAAM,gBAAgB,GAAG,OAAO,EAAE,SAAS,EAAE,WAAW,CAAC;QACzD,MAAM,eAAe,GAAG,QAAQ,CAAC,WAAW,CAAC;QAC7C,MAAM,OAAO,GACX,gBAAgB,IAAI,eAAe,IAAI,gBAAgB,KAAK,eAAe;YACzE,CAAC,CAAC,SAAS;YACX,CAAC,CAAC,CAAC,gBAAgB,IAAI,eAAe,IAAI,SAAS,CAAC,CAAC;QACzD,MAAM,OAAO,GAAG,OAAO,EAAE,OAAO,CAAC;QACjC,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,EAAE,SAAS,CAAC;QAChD,OAAO;YACL,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,IAAI,EAAE,QAAQ,CAAC,cAAc,IAAI,QAAQ,CAAC,SAAS;YACnD,OAAO;YACP,QAAQ,EAAE,QAAQ,CAAC,QAAQ;YAC3B,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,GAAG,CAAC,OAAO,EAAE,SAAS,EAAE,QAAQ,KAAK,SAAS;gBAC5C,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,EAAE,QAAQ,EAAE,OAAO,CAAC,SAAS,CAAC,QAAQ,EAAE,CAAC;YAC7C,QAAQ,EAAE,uBAAuB,CAC/B,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,EAC3C,OAAO,EACP,SAAS,CACV;YACD,GAAG,CAAC,OAAO,KAAK,UAAU;gBACxB,CAAC,CAAC,EAAE,aAAa,EAAE,oCAAoC,CAAC,OAAO,EAAE,SAAS,CAAC,EAAE;gBAC7E,CAAC,CAAC,EAAE,CAAC;SACR,CAAC;IACJ,CAAC,CAAC,CAAC;AACP,CAAC;AAED,SAAS,aAAa,CAAC,WAAwC;IAC7D,MAAM,MAAM,GAAkC;QAC5C,QAAQ,EAAE,CAAC;QACX,YAAY,EAAE,CAAC;QACf,WAAW,EAAE,CAAC;QACd,OAAO,EAAE,CAAC;KACX,CAAC;IACF,KAAK,MAAM,UAAU,IAAI,WAAW;QAAE,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IACtE,OAAO;QACL,GAAG,MAAM,CAAC,QAAQ,gBAAgB;QAClC,GAAG,MAAM,CAAC,YAAY,sBAAsB;QAC5C,GAAG,MAAM,CAAC,WAAW,qBAAqB;QAC1C,GAAG,CAAC,MAAM,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,OAAO,UAAU,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;KAC7D,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AAChB,CAAC;AAED,SAAS,MAAM,CAAC,KAA6B,EAAE,WAAwC;IACrF,MAAM,MAAM,GAAG,UAAU,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;IAC/C,OAAO;QACL,KAAK,CAAC,MAAM;QACZ,oCAAoC,KAAK,CAAC,IAAI,CAAC,aAAa,EAAE;QAC9D,EAAE;QACF,GAAG,WAAW,CAAC,MAAM,kBAAkB,aAAa,CAAC,WAAW,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,2BAA2B,MAAM,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACxH,EAAE;KACH,CAAC;AACJ,CAAC;AAED,SAAS,MAAM;IACb,OAAO;QACL,6EAA6E;QAC7E,yDAAyD;QACzD,uGAAuG;QACvG,EAAE;KACH,CAAC;AACJ,CAAC;AAED,SAAS,KAAK,CAAC,WAAwC;IACrD,MAAM,KAAK,GAAG;QACZ,uEAAuE;QACvE,uCAAuC;KACxC,CAAC;IACF,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;QACrC,KAAK,CAAC,IAAI,CACR,KAAK,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,QAAQ,UAAU,CAAC,QAAQ,QAAQ,IAAI,CAAC,eAAe,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,IAAI,CAAC,eAAe,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,IAAI,CAAC,oBAAoB,CAAC,UAAU,CAAC,CAAC,IAAI,CACpP,CAAC;IACJ,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,aAAa,CAAC,SAA8D;IACnF,IAAI,SAAS,CAAC,OAAO,KAAK,KAAK;QAAE,OAAO,GAAG,SAAS,CAAC,KAAK,gBAAgB,CAAC;IAC3E,IAAI,SAAS,CAAC,oBAAoB,KAAK,IAAI,EAAE,CAAC;QAC5C,MAAM,MAAM,GAAG,SAAS,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,SAAS,CAAC,UAAU,GAAG,CAAC;QACxF,OAAO,GAAG,SAAS,CAAC,KAAK,0BAA0B,MAAM,EAAE,CAAC;IAC9D,CAAC;IACD,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,SAAS,CAAC,IAAI,KAAK,SAAS;QAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAChF,IAAI,SAAS,CAAC,iBAAiB,KAAK,SAAS,EAAE,CAAC;QAC9C,MAAM,CAAC,IAAI,CAAC,MAAM,MAAM,CAAC,SAAS,CAAC,iBAAiB,CAAC,EAAE,CAAC,CAAC;IAC3D,CAAC;IACD,IAAI,SAAS,CAAC,SAAS,KAAK,SAAS,IAAI,SAAS,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;QAC3E,MAAM,CAAC,IAAI,CAAC,UAAU,MAAM,CAAC,SAAS,CAAC,SAAS,CAAC,IAAI,MAAM,CAAC,SAAS,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;IACtF,CAAC;SAAM,IAAI,SAAS,CAAC,aAAa,KAAK,SAAS,EAAE,CAAC;QACjD,MAAM,CAAC,IAAI,CAAC,+BAA+B,MAAM,CAAC,SAAS,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;IAChF,CAAC;IACD,IAAI,SAAS,CAAC,UAAU,KAAK,SAAS;QAAE,MAAM,CAAC,IAAI,CAAC,KAAK,SAAS,CAAC,UAAU,EAAE,CAAC,CAAC;IACjF,OAAO,GAAG,SAAS,CAAC,KAAK,KAAK,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;AAC5F,CAAC;AAED,SAAS,eAAe,CAAC,UAA4B;IACnD,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,aAAa;QAAE,OAAO,CAAC,IAAI,CAAC,mCAAmC,CAAC,CAAC;SACtF,IAAI,UAAU,CAAC,QAAQ,CAAC,qBAAqB,KAAK,eAAe,EAAE,CAAC;QACvE,OAAO,CAAC,IAAI,CAAC,wDAAwD,CAAC,CAAC;IACzE,CAAC;IACD,IAAI,UAAU,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,IAAI,CAAC,mCAAmC,CAAC,CAAC;SACnF,IACH,UAAU,CAAC,QAAQ,CAAC,IAAI,CACtB,CAAC,KAAK,EAAE,EAAE,CACR,CAAC,KAAK,CAAC,eAAe,IAAI,CAAC,CAAC,GAAG,CAAC;QAChC,KAAK,CAAC,SAAS,KAAK,SAAS;QAC7B,KAAK,CAAC,iBAAiB,KAAK,SAAS,CACxC,EACD,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,sEAAsE,CAAC,CAAC;IACvF,CAAC;IACD,IAAI,UAAU,CAAC,OAAO,KAAK,UAAU,IAAI,UAAU,CAAC,aAAa,EAAE,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3F,OAAO,CAAC,IAAI,CAAC,0CAA0C,CAAC,CAAC;IAC3D,CAAC;IACD,IAAI,UAAU,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;QACrC,OAAO,CAAC,IAAI,CAAC,uEAAuE,CAAC,CAAC;IACxF,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,OAAO,CAAC,UAA4B,EAAE,aAAiC;IAC9E,MAAM,KAAK,GAAG;QACZ,WAAW;QACX,YAAY,UAAU,CAAC,QAAQ,MAAM,aAAa,CAAC,UAAU,CAAC,IAAI,EAAE,GAAG,CAAC,KAAK,YAAY,CAAC,UAAU,CAAC,OAAO,CAAC,aAAa;QAC1H,EAAE;QACF,mBAAmB,KAAK,CAAC,eAAe,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,GAAG,CAAC,EAAE;KACtE,CAAC;IACF,IAAI,UAAU,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACrC,KAAK,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;IAC9C,CAAC;SAAM,CAAC;QACN,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;QAC9B,KAAK,MAAM,KAAK,IAAI,UAAU,CAAC,QAAQ,EAAE,CAAC;YACxC,KAAK,CAAC,IAAI,CAAC,OAAO,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;QACzE,CAAC;IACH,CAAC;IAED,IAAI,UAAU,CAAC,OAAO,KAAK,UAAU,EAAE,CAAC;QACtC,KAAK,CAAC,IAAI,CAAC,wBAAwB,KAAK,CAAC,oBAAoB,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC,CAAC;QAC9E,KAAK,MAAM,SAAS,IAAI,UAAU,CAAC,aAAa,EAAE,UAAU,IAAI,EAAE,EAAE,CAAC;YACnE,KAAK,CAAC,IAAI,CAAC,OAAO,KAAK,CAAC,aAAa,CAAC,SAAS,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QAC5D,CAAC;IACH,CAAC;SAAM,CAAC;QACN,KAAK,CAAC,IAAI,CAAC,wBAAwB,KAAK,CAAC,oBAAoB,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC,CAAC;IAChF,CAAC;IAED,IAAI,UAAU,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACnC,KAAK,CAAC,IAAI,CAAC,sBAAsB,KAAK,CAAC,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;IACpE,CAAC;SAAM,CAAC;QACN,KAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;QACjC,KAAK,MAAM,KAAK,IAAI,UAAU,CAAC,MAAM,EAAE,CAAC;YACtC,KAAK,CAAC,IAAI,CACR,OAAO,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,KAAK,KAAK,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,CAAC,cAAc,KAAK,CAAC,KAAK,CAAC,SAAS,EAAE,GAAG,CAAC,QAAQ,KAAK,CAAC,QAAQ,KAAK,KAAK,CAAC,QAAQ,KAAK,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,EAAE,CACnM,CAAC;QACJ,CAAC;IACH,CAAC;IAED,MAAM,OAAO,GAAG,eAAe,CAAC,UAAU,CAAC,CAAC;IAC5C,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,KAAK,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAC;QACtC,KAAK,MAAM,IAAI,IAAI,OAAO;YAAE,KAAK,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC;IACxD,CAAC;IACD,IAAI,aAAa;QAAE,KAAK,CAAC,IAAI,CAAC,0CAA0C,aAAa,GAAG,CAAC,CAAC;IAC1F,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,YAAY,EAAE,EAAE,CAAC,CAAC;IACjC,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,UAAU,CACjB,KAA6B,EAC7B,WAAwC;IAExC,MAAM,MAAM,GAAG,UAAU,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;IAC/C,MAAM,QAAQ,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAW,CAAC;IAC1E,MAAM,WAAW,GAAG,CAAC,GAAG,WAAW,CAAC,CAAC,IAAI,CACvC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC,KAAK,CAChG,CAAC;IACF,OAAO;QACL,GAAG,MAAM,CAAC,KAAK,EAAE,WAAW,CAAC;QAC7B,GAAG,MAAM,EAAE;QACX,GAAG,KAAK,CAAC,WAAW,CAAC;QACrB,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,OAAO,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;KACpE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACf,CAAC;AAED,SAAS,aAAa,CACpB,KAA6B,EAC7B,WAAwC;IAExC,OAAO,CAAC,GAAG,MAAM,CAAC,KAAK,EAAE,WAAW,CAAC,EAAE,GAAG,MAAM,EAAE,EAAE,GAAG,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACxF,CAAC;AAED,SAAS,cAAc,CACrB,KAA6B,EAC7B,WAAwC;IAExC,MAAM,QAAQ,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,UAAU,CAAC,QAAQ,KAAK,MAAM,CAAC,CAAC;IACpF,OAAO;QACL,GAAG,MAAM,CAAC,KAAK,EAAE,WAAW,CAAC;QAC7B,6EAA6E;QAC7E,EAAE;QACF,GAAG,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;YACvB,CAAC,CAAC,CAAC,mEAAmE,EAAE,EAAE,CAAC;YAC3E,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;KACrB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACf,CAAC;AAED,SAAS,aAAa,CACpB,KAA6B,EAC7B,WAAwC;IAExC,MAAM,MAAM,GAAG,UAAU,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;IAC/C,OAAO;QACL,KAAK,CAAC,MAAM;QACZ,oCAAoC,KAAK,CAAC,IAAI,CAAC,aAAa,EAAE;QAC9D,EAAE;QACF,GAAG,WAAW,CAAC,MAAM,kBAAkB,aAAa,CAAC,WAAW,CAAC,EAAE;QACnE,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,wCAAwC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1E,EAAE;QACF,+EAA+E;KAChF,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACf,CAAC;AAED,MAAM,UAAU,6BAA6B,CAAC,QAAgB;IAC5D,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAChF,OAAO,oCAAoC,MAAM,MAAM,CAAC;AAC1D,CAAC;AAED,mFAAmF;AACnF,MAAM,UAAU,uBAAuB,CAAC,KAA6B;IACnE,MAAM,WAAW,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAC;IAC7C,MAAM,UAAU,GAAyD;QACvE,CAAC,MAAM,EAAE,UAAU,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;QACxC,CAAC,SAAS,EAAE,aAAa,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;QAC9C,CAAC,UAAU,EAAE,cAAc,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;QAChD,CAAC,SAAS,EAAE,aAAa,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;KAC/C,CAAC;IACF,KAAK,MAAM,CAAC,WAAW,EAAE,QAAQ,CAAC,IAAI,UAAU,EAAE,CAAC;QACjD,IAAI,QAAQ,CAAC,MAAM,IAAI,YAAY;YAAE,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,CAAC;IACxE,CAAC;IACD,MAAM,QAAQ,GAAG,aAAa,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IAC1C,IAAI,QAAQ,CAAC,MAAM,GAAG,oBAAoB,EAAE,CAAC;QAC3C,MAAM,IAAI,KAAK,CAAC,4EAA4E,CAAC,CAAC;IAChG,CAAC;IACD,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,WAAW,EAAE,SAAS,EAAE,CAAC;AACxD,CAAC"}

package/dist/reliability/evidence.d.ts

@@ -0,0 +1,44 @@
+import type { ParsedConfiguredRunInput } from '../configured-run/validation.js';
+import type { EvaluationContextType, EvaluationResults } from '../domain/types.js';
+import type { JsonObject } from '../domain/common.js';
+export type QualityStandardStatus = 'applied' | 'not_configured' | 'not_available';
+export type RunStandardEvidence = {
+    rubricApplied: boolean;
+    judgeModel?: string;
+    judgeModelPinned: boolean;
+    qualityStandardStatus: QualityStandardStatus;
+    qualityStandardName?: string;
+    qualityStandardVersion?: string;
+    anchorCount?: number;
+};
+export type JudgmentEvidence = {
+    scope: string;
+    assessmentCount?: number;
+    passVotes?: number;
+    agreedAssessments?: number;
+};
+export type RepeatabilityStatistic = {
+    scope: string;
+    mean?: number;
+    standardDeviation?: number;
+    ci95Lower?: number;
+    ci95Upper?: number;
+    ci95HalfWidth?: number;
+    sampleSize?: number;
+    hasData?: boolean;
+    insufficientEvidence?: boolean;
+};
+export type ScenarioRepeatabilityEvidence = {
+    completedTrials?: number;
+    plannedTrials?: number;
+    statistics: RepeatabilityStatistic[];
+};
+/**
+ * Extract only public-safe standard metadata from the exact configuration used
+ * by the run. Rubric prose and anchor contents are deliberately never retained.
+ */
+export declare function extractRunStandardEvidence(configuredRun: ParsedConfiguredRunInput, exactConfiguration?: JsonObject): RunStandardEvidence;
+export declare function extractJudgmentEvidence(contextType: EvaluationContextType | undefined, results: EvaluationResults | undefined, statusRaw: JsonObject | undefined): JudgmentEvidence[];
+/** Report backend statistics at their native model/metric and scenario/model grain. */
+export declare function extractScenarioRepeatabilityEvidence(results: EvaluationResults | undefined, statusRaw: JsonObject | undefined): ScenarioRepeatabilityEvidence;
+//# sourceMappingURL=evidence.d.ts.map

package/dist/reliability/evidence.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"evidence.d.ts","sourceRoot":"","sources":["../../src/reliability/evidence.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,wBAAwB,EAAE,MAAM,iCAAiC,CAAC;AAChF,OAAO,KAAK,EAAE,qBAAqB,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AACnF,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEtD,MAAM,MAAM,qBAAqB,GAAG,SAAS,GAAG,gBAAgB,GAAG,eAAe,CAAC;AAEnF,MAAM,MAAM,mBAAmB,GAAG;IAChC,aAAa,EAAE,OAAO,CAAC;IACvB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,gBAAgB,EAAE,OAAO,CAAC;IAC1B,qBAAqB,EAAE,qBAAqB,CAAC;IAC7C,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,CAAC;AAEF,MAAM,MAAM,gBAAgB,GAAG;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B,CAAC;AAEF,MAAM,MAAM,sBAAsB,GAAG;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,oBAAoB,CAAC,EAAE,OAAO,CAAC;CAChC,CAAC;AAEF,MAAM,MAAM,6BAA6B,GAAG;IAC1C,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,sBAAsB,EAAE,CAAC;CACtC,CAAC;AA+FF;;;GAGG;AACH,wBAAgB,0BAA0B,CACxC,aAAa,EAAE,wBAAwB,EACvC,kBAAkB,CAAC,EAAE,UAAU,GAC9B,mBAAmB,CA2BrB;AAkCD,wBAAgB,uBAAuB,CACrC,WAAW,EAAE,qBAAqB,GAAG,SAAS,EAC9C,OAAO,EAAE,iBAAiB,GAAG,SAAS,EACtC,SAAS,EAAE,UAAU,GAAG,SAAS,GAChC,gBAAgB,EAAE,CAgBpB;AAsED,uFAAuF;AACvF,wBAAgB,oCAAoC,CAClD,OAAO,EAAE,iBAAiB,GAAG,SAAS,EACtC,SAAS,EAAE,UAAU,GAAG,SAAS,GAChC,6BAA6B,CAuC/B"}