@tenphi/akno-core 0.1.0

package/dist/bench/ranking-review.d.ts

@@ -0,0 +1,93 @@
+import { type RankingCorpus } from './ranking-corpus.ts';
+export declare const RANKING_REVIEW_PACKET_SCHEMA_VERSION = "ranking-corpus-review-packet-v1";
+export declare const RANKING_REVIEW_EVIDENCE_SCHEMA_VERSION = "ranking-corpus-review-evidence-v1";
+export type RankingReviewMark = 'pending' | 'pass' | 'issue';
+export type RankingReviewerKind = 'human' | 'model';
+export interface RankingReviewSource {
+    id: string;
+    text: string;
+    sourceKind: 'page' | 'document';
+    instructionBearing: boolean;
+    review: RankingReviewMark;
+}
+export interface RankingReviewCase {
+    id: string;
+    split: 'development' | 'test';
+    category: string;
+    query: string;
+    intent: string;
+    candidates: Array<{
+        id: string;
+        text: string;
+        sourceKind: 'page' | 'document';
+        instructionBearing: boolean;
+        grade: 0 | 1 | 2 | 3;
+    }>;
+    review: RankingReviewMark;
+}
+export interface RankingReviewIssue {
+    scope: 'corpus' | 'source' | 'case';
+    id: string | null;
+    description: string;
+}
+export interface RankingReviewDecision {
+    verdict: 'pending' | 'approved' | 'changes_requested';
+    reviewerKind: RankingReviewerKind | null;
+    reviewedAt: string | null;
+    independence: {
+        didNotAuthorCorpus: boolean;
+        didNotTuneRuntimeContract: boolean;
+        reviewedWithoutBenchmarkOutputs: boolean;
+    };
+    checks: {
+        inventedContent: boolean;
+        sourceClarity: boolean;
+        queryIntentAlignment: boolean;
+        poolAndJudgmentAccuracy: boolean;
+        splitIsolation: boolean;
+    };
+    issues: RankingReviewIssue[];
+}
+export interface RankingReviewPacket {
+    kind: 'ranking_corpus_review_packet';
+    schemaVersion: string;
+    createdAt: string;
+    corpus: {
+        version: string;
+        fingerprint: string;
+        sources: number;
+        queries: number;
+        judgments: number;
+    };
+    packetFingerprint: string;
+    instructions: string[];
+    sources: RankingReviewSource[];
+    cases: RankingReviewCase[];
+    review: RankingReviewDecision;
+}
+/** Content-free proof copied into a matrix after the complete packet validates. */
+export interface RankingReviewEvidence {
+    kind: 'ranking_corpus_review_evidence';
+    schemaVersion: string;
+    corpusVersion: string;
+    corpusFingerprint: string;
+    packetFingerprint: string;
+    receiptFingerprint: string;
+    reviewerKind: RankingReviewerKind;
+    reviewedAt: string;
+    sourceReviews: number;
+    caseReviews: number;
+    independenceConfirmed: true;
+    checksConfirmed: true;
+}
+export declare function rankingCorpusFingerprint(corpus?: RankingCorpus): string;
+export declare function createRankingReviewPacket(now?: Date): RankingReviewPacket;
+export declare function rebaseRankingReviewPacket(previousValue: unknown, now?: Date): RankingReviewPacket;
+export declare function reviewPacketFingerprint(packet: RankingReviewPacket): string;
+export declare function completeRankingReview(packetValue: unknown): RankingReviewEvidence;
+export declare function rankingReviewEvidenceMatches(evidence: RankingReviewEvidence | null, corpus: {
+    version: string;
+    fingerprint?: string | null;
+    sources: number;
+}): boolean;
+//# sourceMappingURL=ranking-review.d.ts.map

package/dist/bench/ranking-review.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ranking-review.d.ts","sourceRoot":"","sources":["../../src/bench/ranking-review.ts"],"names":[],"mappings":"AACA,OAAO,EAAkB,KAAK,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAEzE,eAAO,MAAM,oCAAoC,oCAAoC,CAAC;AACtF,eAAO,MAAM,sCAAsC,sCAAsC,CAAC;AAE1F,MAAM,MAAM,iBAAiB,GAAG,SAAS,GAAG,MAAM,GAAG,OAAO,CAAC;AAC7D,MAAM,MAAM,mBAAmB,GAAG,OAAO,GAAG,OAAO,CAAC;AAEpD,MAAM,WAAW,mBAAmB;IAClC,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,GAAG,UAAU,CAAC;IAChC,kBAAkB,EAAE,OAAO,CAAC;IAC5B,MAAM,EAAE,iBAAiB,CAAC;CAC3B;AAED,MAAM,WAAW,iBAAiB;IAChC,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,aAAa,GAAG,MAAM,CAAC;IAC9B,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,KAAK,CAAC;QAChB,EAAE,EAAE,MAAM,CAAC;QACX,IAAI,EAAE,MAAM,CAAC;QACb,UAAU,EAAE,MAAM,GAAG,UAAU,CAAC;QAChC,kBAAkB,EAAE,OAAO,CAAC;QAC5B,KAAK,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;KACtB,CAAC,CAAC;IACH,MAAM,EAAE,iBAAiB,CAAC;CAC3B;AAED,MAAM,WAAW,kBAAkB;IACjC,KAAK,EAAE,QAAQ,GAAG,QAAQ,GAAG,MAAM,CAAC;IACpC,EAAE,EAAE,MAAM,GAAG,IAAI,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,qBAAqB;IACpC,OAAO,EAAE,SAAS,GAAG,UAAU,GAAG,mBAAmB,CAAC;IACtD,YAAY,EAAE,mBAAmB,GAAG,IAAI,CAAC;IACzC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,YAAY,EAAE;QACZ,kBAAkB,EAAE,OAAO,CAAC;QAC5B,yBAAyB,EAAE,OAAO,CAAC;QACnC,+BAA+B,EAAE,OAAO,CAAC;KAC1C,CAAC;IACF,MAAM,EAAE;QACN,eAAe,EAAE,OAAO,CAAC;QACzB,aAAa,EAAE,OAAO,CAAC;QACvB,oBAAoB,EAAE,OAAO,CAAC;QAC9B,uBAAuB,EAAE,OAAO,CAAC;QACjC,cAAc,EAAE,OAAO,CAAC;KACzB,CAAC;IACF,MAAM,EAAE,kBAAkB,EAAE,CAAC;CAC9B;AAED,MAAM,WAAW,mBAAmB;IAClC,IAAI,EAAE,8BAA8B,CAAC;IACrC,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE;QACN,OAAO,EAAE,MAAM,CAAC;QAChB,WAAW,EAAE,MAAM,CAAC;QACpB,OAAO,EAAE,MAAM,CAAC;QAChB,OAAO,EAAE,MAAM,CAAC;QAChB,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,iBAAiB,EAAE,MAAM,CAAC;IAC1B,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,OAAO,EAAE,mBAAmB,EAAE,CAAC;IAC/B,KAAK,EAAE,iBAAiB,EAAE,CAAC;IAC3B,MAAM,EAAE,qBAAqB,CAAC;CAC/B;AAED,mFAAmF;AACnF,MAAM,WAAW,qBAAqB;IACpC,IAAI,EAAE,gCAAgC,CAAC;IACvC,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,YAAY,EAAE,mBAAmB,CAAC;IAClC,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,qBAAqB,EAAE,IAAI,CAAC;IAC5B,eAAe,EAAE,IAAI,CAAC;CACvB;AAED,wBAAgB,wBAAwB,CAAC,MAAM,GAAE,aAA8B,GAAG,MAAM,CAyBvF;AAED,wBAAgB,yBAAyB,CAAC,GAAG,OAAa,GAAG,mBAAmB,CAuE/E;AAED,wBAAgB,yBAAyB,CAAC,aAAa,EAAE,OAAO,EAAE,GAAG,OAAa,GAAG,mBAAmB,CAmDvG;AAED,wBAAgB,uBAAuB,CAAC,MAAM,EAAE,mBAAmB,GAAG,MAAM,CAU3E;AAED,wBAAgB,qBAAqB,CAAC,WAAW,EAAE,OAAO,GAAG,qBAAqB,CAyDjF;AAED,wBAAgB,4BAA4B,CAC1C,QAAQ,EAAE,qBAAqB,GAAG,IAAI,EACtC,MAAM,EAAE;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,GACxE,OAAO,CAgBT"}

package/dist/bench/ranking-review.js

@@ -0,0 +1,249 @@
+import { createHash } from 'node:crypto';
+import { RANKING_CORPUS } from "./ranking-corpus.js";
+export const RANKING_REVIEW_PACKET_SCHEMA_VERSION = 'ranking-corpus-review-packet-v1';
+export const RANKING_REVIEW_EVIDENCE_SCHEMA_VERSION = 'ranking-corpus-review-evidence-v1';
+export function rankingCorpusFingerprint(corpus = RANKING_CORPUS) {
+    return sha256(JSON.stringify({
+        version: corpus.version,
+        candidates: Object.values(corpus.candidates)
+            .sort((left, right) => left.id.localeCompare(right.id))
+            .map((candidate) => ({
+            id: candidate.id,
+            text: candidate.text,
+            sourceKind: candidate.sourceKind,
+            instructionBearing: candidate.instructionBearing ?? false,
+        })),
+        cases: [...corpus.cases]
+            .sort((left, right) => left.id.localeCompare(right.id))
+            .map((benchCase) => ({
+            id: benchCase.id,
+            split: benchCase.split,
+            category: benchCase.category,
+            query: benchCase.query,
+            intent: benchCase.intent,
+            pool: benchCase.pool,
+            judgments: benchCase.pool.map((id) => [id, benchCase.judgments[id]]),
+        })),
+    }));
+}
+export function createRankingReviewPacket(now = new Date()) {
+    const packet = {
+        kind: 'ranking_corpus_review_packet',
+        schemaVersion: RANKING_REVIEW_PACKET_SCHEMA_VERSION,
+        createdAt: now.toISOString(),
+        corpus: {
+            version: RANKING_CORPUS.version,
+            fingerprint: rankingCorpusFingerprint(),
+            sources: Object.keys(RANKING_CORPUS.candidates).length,
+            queries: RANKING_CORPUS.cases.length,
+            judgments: RANKING_CORPUS.cases.reduce((sum, benchCase) => sum + benchCase.pool.length, 0),
+        },
+        packetFingerprint: '',
+        instructions: [
+            'Review this packet in a separate session from corpus authorship and runtime prompt tuning.',
+            'Do not consult benchmark outputs while deciding whether a source, query, pool, or grade is correct.',
+            'Each case repeats its frozen candidate texts in rank order; verify every grade, then mark the case pass or issue.',
+            'Mark every source and case pass or issue; describe every issue and request changes when any remain.',
+            'Approve only after every global check and independence attestation is true.',
+            'Return the completed packet; Akno copies only its content-free receipt into the ranking matrix.',
+        ],
+        sources: Object.values(RANKING_CORPUS.candidates)
+            .sort((left, right) => left.id.localeCompare(right.id))
+            .map((candidate) => ({
+            id: candidate.id,
+            text: candidate.text,
+            sourceKind: candidate.sourceKind,
+            instructionBearing: candidate.instructionBearing ?? false,
+            review: 'pending',
+        })),
+        cases: [...RANKING_CORPUS.cases]
+            .sort((left, right) => left.id.localeCompare(right.id))
+            .map((benchCase) => ({
+            id: benchCase.id,
+            split: benchCase.split,
+            category: benchCase.category,
+            query: benchCase.query,
+            intent: benchCase.intent,
+            candidates: benchCase.pool.map((id) => {
+                const candidate = RANKING_CORPUS.candidates[id];
+                return {
+                    id,
+                    text: candidate.text,
+                    sourceKind: candidate.sourceKind,
+                    instructionBearing: candidate.instructionBearing ?? false,
+                    grade: benchCase.judgments[id],
+                };
+            }),
+            review: 'pending',
+        })),
+        review: {
+            verdict: 'pending',
+            reviewerKind: null,
+            reviewedAt: null,
+            independence: {
+                didNotAuthorCorpus: false,
+                didNotTuneRuntimeContract: false,
+                reviewedWithoutBenchmarkOutputs: false,
+            },
+            checks: {
+                inventedContent: false,
+                sourceClarity: false,
+                queryIntentAlignment: false,
+                poolAndJudgmentAccuracy: false,
+                splitIsolation: false,
+            },
+            issues: [],
+        },
+    };
+    packet.packetFingerprint = reviewPacketFingerprint(packet);
+    return packet;
+}
+export function rebaseRankingReviewPacket(previousValue, now = new Date()) {
+    if (!isReviewPacket(previousValue))
+        throw new Error('invalid previous ranking review packet shape');
+    const previous = previousValue;
+    if (previous.schemaVersion !== RANKING_REVIEW_PACKET_SCHEMA_VERSION) {
+        throw new Error('unsupported previous ranking review packet schema');
+    }
+    if (reviewPacketFingerprint(previous) !== previous.packetFingerprint) {
+        throw new Error('previous ranking review packet content does not match its fingerprint');
+    }
+    const current = createRankingReviewPacket(now);
+    const previousSources = new Map(previous.sources.map((source) => [source.id, source]));
+    const previousCases = new Map(previous.cases.map((benchCase) => [benchCase.id, benchCase]));
+    current.sources = current.sources.map((source) => {
+        const before = previousSources.get(source.id);
+        return {
+            ...source,
+            review: before?.review === 'pass' && reviewEntryContent(before) === reviewEntryContent(source)
+                ? 'pass'
+                : 'pending',
+        };
+    });
+    current.cases = current.cases.map((benchCase) => {
+        const before = previousCases.get(benchCase.id);
+        return {
+            ...benchCase,
+            review: before?.review === 'pass' && reviewEntryContent(before) === reviewEntryContent(benchCase)
+                ? 'pass'
+                : 'pending',
+        };
+    });
+    const pendingSources = new Set(current.sources.filter((source) => source.review !== 'pass').map((source) => source.id));
+    const pendingCases = new Set(current.cases.filter((benchCase) => benchCase.review !== 'pass').map((benchCase) => benchCase.id));
+    current.instructions = [
+        `Exact prior decisions carried forward: ${current.sources.length - pendingSources.size} sources and ${current.cases.length - pendingCases.size} cases.`,
+        'Review every pending entry against the corrected content and remove an issue only after verifying its resolution.',
+        ...current.instructions,
+    ];
+    current.review.issues = previous.review.issues.filter((issue) => issue.scope === 'corpus' ||
+        (issue.scope === 'source' && issue.id !== null && pendingSources.has(issue.id)) ||
+        (issue.scope === 'case' && issue.id !== null && pendingCases.has(issue.id)));
+    return current;
+}
+export function reviewPacketFingerprint(packet) {
+    return sha256(JSON.stringify({
+        kind: packet.kind,
+        schemaVersion: packet.schemaVersion,
+        corpus: packet.corpus,
+        sources: packet.sources.map(({ review: _review, ...source }) => source),
+        cases: packet.cases.map(({ review: _review, ...benchCase }) => benchCase),
+    }));
+}
+export function completeRankingReview(packetValue) {
+    if (!isReviewPacket(packetValue))
+        throw new Error('invalid ranking review packet shape');
+    const packet = packetValue;
+    if (packet.schemaVersion !== RANKING_REVIEW_PACKET_SCHEMA_VERSION) {
+        throw new Error('unsupported ranking review packet schema');
+    }
+    if (reviewPacketFingerprint(packet) !== packet.packetFingerprint) {
+        throw new Error('ranking review packet content does not match its fingerprint');
+    }
+    const expected = createRankingReviewPacket(new Date(0));
+    if (packet.packetFingerprint !== expected.packetFingerprint ||
+        packet.corpus.version !== expected.corpus.version ||
+        packet.corpus.fingerprint !== expected.corpus.fingerprint) {
+        throw new Error('ranking review packet does not match the current corpus');
+    }
+    if (packet.review.verdict !== 'approved')
+        throw new Error('ranking review verdict is not approved');
+    if (packet.review.reviewerKind !== 'human' && packet.review.reviewerKind !== 'model') {
+        throw new Error('ranking review must identify the reviewer kind');
+    }
+    if (!packet.review.reviewedAt || Number.isNaN(Date.parse(packet.review.reviewedAt))) {
+        throw new Error('ranking review needs a valid reviewedAt timestamp');
+    }
+    if (Object.values(packet.review.independence).some((value) => value !== true)) {
+        throw new Error('ranking review independence attestations are incomplete');
+    }
+    if (Object.values(packet.review.checks).some((value) => value !== true)) {
+        throw new Error('ranking review corpus checks are incomplete');
+    }
+    const pendingSource = packet.sources.find((source) => source.review !== 'pass');
+    if (pendingSource)
+        throw new Error(`ranking source ${pendingSource.id} is not approved`);
+    const pendingCase = packet.cases.find((benchCase) => benchCase.review !== 'pass');
+    if (pendingCase)
+        throw new Error(`ranking case ${pendingCase.id} is not approved`);
+    if (packet.review.issues.length > 0)
+        throw new Error('ranking review still contains unresolved issues');
+    return {
+        kind: 'ranking_corpus_review_evidence',
+        schemaVersion: RANKING_REVIEW_EVIDENCE_SCHEMA_VERSION,
+        corpusVersion: packet.corpus.version,
+        corpusFingerprint: packet.corpus.fingerprint,
+        packetFingerprint: packet.packetFingerprint,
+        receiptFingerprint: sha256(JSON.stringify({
+            packet: packet.packetFingerprint,
+            sourceReviews: packet.sources.map((source) => [source.id, source.review]),
+            caseReviews: packet.cases.map((benchCase) => [benchCase.id, benchCase.review]),
+            review: packet.review,
+        })),
+        reviewerKind: packet.review.reviewerKind,
+        reviewedAt: new Date(packet.review.reviewedAt).toISOString(),
+        sourceReviews: packet.sources.length,
+        caseReviews: packet.cases.length,
+        independenceConfirmed: true,
+        checksConfirmed: true,
+    };
+}
+export function rankingReviewEvidenceMatches(evidence, corpus) {
+    return Boolean(evidence &&
+        evidence.kind === 'ranking_corpus_review_evidence' &&
+        evidence.schemaVersion === RANKING_REVIEW_EVIDENCE_SCHEMA_VERSION &&
+        evidence.corpusVersion === corpus.version &&
+        evidence.corpusFingerprint === corpus.fingerprint &&
+        evidence.sourceReviews === corpus.sources &&
+        evidence.caseReviews === RANKING_CORPUS.cases.length &&
+        evidence.independenceConfirmed === true &&
+        evidence.checksConfirmed === true &&
+        (evidence.reviewerKind === 'human' || evidence.reviewerKind === 'model') &&
+        !Number.isNaN(Date.parse(evidence.reviewedAt)) &&
+        /^[a-f0-9]{64}$/.test(evidence.packetFingerprint) &&
+        /^[a-f0-9]{64}$/.test(evidence.receiptFingerprint));
+}
+function isReviewPacket(value) {
+    if (!value || typeof value !== 'object')
+        return false;
+    const packet = value;
+    return Boolean(packet.kind === 'ranking_corpus_review_packet' &&
+        packet.corpus &&
+        typeof packet.corpus === 'object' &&
+        typeof packet.packetFingerprint === 'string' &&
+        Array.isArray(packet.sources) &&
+        Array.isArray(packet.cases) &&
+        packet.review &&
+        typeof packet.review === 'object' &&
+        packet.review.independence &&
+        packet.review.checks &&
+        Array.isArray(packet.review.issues));
+}
+function sha256(value) {
+    return createHash('sha256').update(value).digest('hex');
+}
+function reviewEntryContent(value) {
+    const { review: _review, ...content } = value;
+    return JSON.stringify(content);
+}
+//# sourceMappingURL=ranking-review.js.map

package/dist/bench/ranking-review.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ranking-review.js","sourceRoot":"","sources":["../../src/bench/ranking-review.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,cAAc,EAAsB,MAAM,qBAAqB,CAAC;AAEzE,MAAM,CAAC,MAAM,oCAAoC,GAAG,iCAAiC,CAAC;AACtF,MAAM,CAAC,MAAM,sCAAsC,GAAG,mCAAmC,CAAC;AAwF1F,MAAM,UAAU,wBAAwB,CAAC,SAAwB,cAAc;IAC7E,OAAO,MAAM,CACX,IAAI,CAAC,SAAS,CAAC;QACb,OAAO,EAAE,MAAM,CAAC,OAAO;QACvB,UAAU,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC;aACzC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,aAAa,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;aACtD,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;YACnB,EAAE,EAAE,SAAS,CAAC,EAAE;YAChB,IAAI,EAAE,SAAS,CAAC,IAAI;YACpB,UAAU,EAAE,SAAS,CAAC,UAAU;YAChC,kBAAkB,EAAE,SAAS,CAAC,kBAAkB,IAAI,KAAK;SAC1D,CAAC,CAAC;QACL,KAAK,EAAE,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC;aACrB,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,aAAa,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;aACtD,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;YACnB,EAAE,EAAE,SAAS,CAAC,EAAE;YAChB,KAAK,EAAE,SAAS,CAAC,KAAK;YACtB,QAAQ,EAAE,SAAS,CAAC,QAAQ;YAC5B,KAAK,EAAE,SAAS,CAAC,KAAK;YACtB,MAAM,EAAE,SAAS,CAAC,MAAM;YACxB,IAAI,EAAE,SAAS,CAAC,IAAI;YACpB,SAAS,EAAE,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,SAAS,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC;SACrE,CAAC,CAAC;KACN,CAAC,CACH,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,yBAAyB,CAAC,GAAG,GAAG,IAAI,IAAI,EAAE;IACxD,MAAM,MAAM,GAAwB;QAClC,IAAI,EAAE,8BAA8B;QACpC,aAAa,EAAE,oCAAoC;QACnD,SAAS,EAAE,GAAG,CAAC,WAAW,EAAE;QAC5B,MAAM,EAAE;YACN,OAAO,EAAE,cAAc,CAAC,OAAO;YAC/B,WAAW,EAAE,wBAAwB,EAAE;YACvC,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,CAAC,MAAM;YACtD,OAAO,EAAE,cAAc,CAAC,KAAK,CAAC,MAAM;YACpC,SAAS,EAAE,cAAc,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,SAAS,EAAE,EAAE,CAAC,GAAG,GAAG,SAAS,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;SAC3F;QACD,iBAAiB,EAAE,EAAE;QACrB,YAAY,EAAE;YACZ,4FAA4F;YAC5F,qGAAqG;YACrG,mHAAmH;YACnH,qGAAqG;YACrG,6EAA6E;YAC7E,iGAAiG;SAClG;QACD,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,UAAU,CAAC;aAC9C,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,aAAa,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;aACtD,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;YACnB,EAAE,EAAE,SAAS,CAAC,EAAE;YAChB,IAAI,EAAE,SAAS,CAAC,IAAI;YACpB,UAAU,EAAE,SAAS,CAAC,UAAU;YAChC,kBAAkB,EAAE,SAAS,CAAC,kBAAkB,IAAI,KAAK;YACzD,MAAM,EAAE,SAAS;SAClB,CAAC,CAAC;QACL,KAAK,EAAE,CAAC,GAAG,cAAc,CAAC,KAAK,CAAC;aAC7B,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,aAAa,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;aACtD,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;YACnB,EAAE,EAAE,SAAS,CAAC,EAAE;YAChB,KAAK,EAAE,SAAS,CAAC,KAAK;YACtB,QAAQ,EAAE,SAAS,CAAC,QAAQ;YAC5B,KAAK,EAAE,SAAS,CAAC,KAAK;YACtB,MAAM,EAAE,SAAS,CAAC,MAAM;YACxB,UAAU,EAAE,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE;gBACpC,MAAM,SAAS,GAAG,cAAc,CAAC,UAAU,CAAC,EAAE,CAAE,CAAC;gBACjD,OAAO;oBACL,EAAE;oBACF,IAAI,EAAE,SAAS,CAAC,IAAI;oBACpB,UAAU,EAAE,SAAS,CAAC,UAAU;oBAChC,kBAAkB,EAAE,SAAS,CAAC,kBAAkB,IAAI,KAAK;oBACzD,KAAK,EAAE,SAAS,CAAC,SAAS,CAAC,EAAE,CAAE;iBAChC,CAAC;YACJ,CAAC,CAAC;YACF,MAAM,EAAE,SAAS;SAClB,CAAC,CAAC;QACL,MAAM,EAAE;YACN,OAAO,EAAE,SAAS;YAClB,YAAY,EAAE,IAAI;YAClB,UAAU,EAAE,IAAI;YAChB,YAAY,EAAE;gBACZ,kBAAkB,EAAE,KAAK;gBACzB,yBAAyB,EAAE,KAAK;gBAChC,+BAA+B,EAAE,KAAK;aACvC;YACD,MAAM,EAAE;gBACN,eAAe,EAAE,KAAK;gBACtB,aAAa,EAAE,KAAK;gBACpB,oBAAoB,EAAE,KAAK;gBAC3B,uBAAuB,EAAE,KAAK;gBAC9B,cAAc,EAAE,KAAK;aACtB;YACD,MAAM,EAAE,EAAE;SACX;KACF,CAAC;IACF,MAAM,CAAC,iBAAiB,GAAG,uBAAuB,CAAC,MAAM,CAAC,CAAC;IAC3D,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,yBAAyB,CAAC,aAAsB,EAAE,GAAG,GAAG,IAAI,IAAI,EAAE;IAChF,IAAI,CAAC,cAAc,CAAC,aAAa,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;IACpG,MAAM,QAAQ,GAAG,aAAa,CAAC;IAC/B,IAAI,QAAQ,CAAC,aAAa,KAAK,oCAAoC,EAAE,CAAC;QACpE,MAAM,IAAI,KAAK,CAAC,mDAAmD,CAAC,CAAC;IACvE,CAAC;IACD,IAAI,uBAAuB,CAAC,QAAQ,CAAC,KAAK,QAAQ,CAAC,iBAAiB,EAAE,CAAC;QACrE,MAAM,IAAI,KAAK,CAAC,uEAAuE,CAAC,CAAC;IAC3F,CAAC;IAED,MAAM,OAAO,GAAG,yBAAyB,CAAC,GAAG,CAAC,CAAC;IAC/C,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC;IACvF,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,EAAE,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC;IAC5F,OAAO,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;QAC/C,MAAM,MAAM,GAAG,eAAe,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QAC9C,OAAO;YACL,GAAG,MAAM;YACT,MAAM,EACJ,MAAM,EAAE,MAAM,KAAK,MAAM,IAAI,kBAAkB,CAAC,MAAM,CAAC,KAAK,kBAAkB,CAAC,MAAM,CAAC;gBACpF,CAAC,CAAC,MAAM;gBACR,CAAC,CAAC,SAAS;SAChB,CAAC;IACJ,CAAC,CAAC,CAAC;IACH,OAAO,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE;QAC9C,MAAM,MAAM,GAAG,aAAa,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAC/C,OAAO;YACL,GAAG,SAAS;YACZ,MAAM,EACJ,MAAM,EAAE,MAAM,KAAK,MAAM,IAAI,kBAAkB,CAAC,MAAM,CAAC,KAAK,kBAAkB,CAAC,SAAS,CAAC;gBACvF,CAAC,CAAC,MAAM;gBACR,CAAC,CAAC,SAAS;SAChB,CAAC;IACJ,CAAC,CAAC,CAAC;IACH,MAAM,cAAc,GAAG,IAAI,GAAG,CAC5B,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,CACxF,CAAC;IACF,MAAM,YAAY,GAAG,IAAI,GAAG,CAC1B,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,EAAE,CAAC,CAClG,CAAC;IACF,OAAO,CAAC,YAAY,GAAG;QACrB,0CAA0C,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,cAAc,CAAC,IAAI,gBAAgB,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,YAAY,CAAC,IAAI,SAAS;QACvJ,mHAAmH;QACnH,GAAG,OAAO,CAAC,YAAY;KACxB,CAAC;IACF,OAAO,CAAC,MAAM,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CACnD,CAAC,KAAK,EAAE,EAAE,CACR,KAAK,CAAC,KAAK,KAAK,QAAQ;QACxB,CAAC,KAAK,CAAC,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,EAAE,KAAK,IAAI,IAAI,cAAc,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAC/E,CAAC,KAAK,CAAC,KAAK,KAAK,MAAM,IAAI,KAAK,CAAC,EAAE,KAAK,IAAI,IAAI,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAC9E,CAAC;IACF,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,MAA2B;IACjE,OAAO,MAAM,CACX,IAAI,CAAC,SAAS,CAAC;QACb,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,aAAa,EAAE,MAAM,CAAC,aAAa;QACnC,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,EAAE,EAAE,EAAE,CAAC,MAAM,CAAC;QACvE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,SAAS,EAAE,EAAE,EAAE,CAAC,SAAS,CAAC;KAC1E,CAAC,CACH,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,WAAoB;IACxD,IAAI,CAAC,cAAc,CAAC,WAAW,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;IACzF,MAAM,MAAM,GAAG,WAAW,CAAC;IAC3B,IAAI,MAAM,CAAC,aAAa,KAAK,oCAAoC,EAAE,CAAC;QAClE,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;IAC9D,CAAC;IACD,IAAI,uBAAuB,CAAC,MAAM,CAAC,KAAK,MAAM,CAAC,iBAAiB,EAAE,CAAC;QACjE,MAAM,IAAI,KAAK,CAAC,8DAA8D,CAAC,CAAC;IAClF,CAAC;IACD,MAAM,QAAQ,GAAG,yBAAyB,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IACxD,IACE,MAAM,CAAC,iBAAiB,KAAK,QAAQ,CAAC,iBAAiB;QACvD,MAAM,CAAC,MAAM,CAAC,OAAO,KAAK,QAAQ,CAAC,MAAM,CAAC,OAAO;QACjD,MAAM,CAAC,MAAM,CAAC,WAAW,KAAK,QAAQ,CAAC,MAAM,CAAC,WAAW,EACzD,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;IAC7E,CAAC;IACD,IAAI,MAAM,CAAC,MAAM,CAAC,OAAO,KAAK,UAAU;QAAE,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;IACpG,IAAI,MAAM,CAAC,MAAM,CAAC,YAAY,KAAK,OAAO,IAAI,MAAM,CAAC,MAAM,CAAC,YAAY,KAAK,OAAO,EAAE,CAAC;QACrF,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;IACpE,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC;QACpF,MAAM,IAAI,KAAK,CAAC,mDAAmD,CAAC,CAAC;IACvE,CAAC;IACD,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,IAAI,CAAC,EAAE,CAAC;QAC9E,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;IAC7E,CAAC;IACD,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,IAAI,CAAC,EAAE,CAAC;QACxE,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;IACjE,CAAC;IACD,MAAM,aAAa,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC;IAChF,IAAI,aAAa;QAAE,MAAM,IAAI,KAAK,CAAC,kBAAkB,aAAa,CAAC,EAAE,kBAAkB,CAAC,CAAC;IACzF,MAAM,WAAW,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC;IAClF,IAAI,WAAW;QAAE,MAAM,IAAI,KAAK,CAAC,gBAAgB,WAAW,CAAC,EAAE,kBAAkB,CAAC,CAAC;IACnF,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;IAExG,OAAO;QACL,IAAI,EAAE,gCAAgC;QACtC,aAAa,EAAE,sCAAsC;QACrD,aAAa,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO;QACpC,iBAAiB,EAAE,MAAM,CAAC,MAAM,CAAC,WAAW;QAC5C,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;QAC3C,kBAAkB,EAAE,MAAM,CACxB,IAAI,CAAC,SAAS,CAAC;YACb,MAAM,EAAE,MAAM,CAAC,iBAAiB;YAChC,aAAa,EAAE,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;YACzE,WAAW,EAAE,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,EAAE,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;YAC9E,MAAM,EAAE,MAAM,CAAC,MAAM;SACtB,CAAC,CACH;QACD,YAAY,EAAE,MAAM,CAAC,MAAM,CAAC,YAAY;QACxC,UAAU,EAAE,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,WAAW,EAAE;QAC5D,aAAa,EAAE,MAAM,CAAC,OAAO,CAAC,MAAM;QACpC,WAAW,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM;QAChC,qBAAqB,EAAE,IAAI;QAC3B,eAAe,EAAE,IAAI;KACtB,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,4BAA4B,CAC1C,QAAsC,EACtC,MAAyE;IAEzE,OAAO,OAAO,CACZ,QAAQ;QACR,QAAQ,CAAC,IAAI,KAAK,gCAAgC;QAClD,QAAQ,CAAC,aAAa,KAAK,sCAAsC;QACjE,QAAQ,CAAC,aAAa,KAAK,MAAM,CAAC,OAAO;QACzC,QAAQ,CAAC,iBAAiB,KAAK,MAAM,CAAC,WAAW;QACjD,QAAQ,CAAC,aAAa,KAAK,MAAM,CAAC,OAAO;QACzC,QAAQ,CAAC,WAAW,KAAK,cAAc,CAAC,KAAK,CAAC,MAAM;QACpD,QAAQ,CAAC,qBAAqB,KAAK,IAAI;QACvC,QAAQ,CAAC,eAAe,KAAK,IAAI;QACjC,CAAC,QAAQ,CAAC,YAAY,KAAK,OAAO,IAAI,QAAQ,CAAC,YAAY,KAAK,OAAO,CAAC;QACxE,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QAC9C,gBAAgB,CAAC,IAAI,CAAC,QAAQ,CAAC,iBAAiB,CAAC;QACjD,gBAAgB,CAAC,IAAI,CAAC,QAAQ,CAAC,kBAAkB,CAAC,CACnD,CAAC;AACJ,CAAC;AAED,SAAS,cAAc,CAAC,KAAc;IACpC,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IACtD,MAAM,MAAM,GAAG,KAAqC,CAAC;IACrD,OAAO,OAAO,CACZ,MAAM,CAAC,IAAI,KAAK,8BAA8B;QAC9C,MAAM,CAAC,MAAM;QACb,OAAO,MAAM,CAAC,MAAM,KAAK,QAAQ;QACjC,OAAO,MAAM,CAAC,iBAAiB,KAAK,QAAQ;QAC5C,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC;QAC7B,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC;QAC3B,MAAM,CAAC,MAAM;QACb,OAAO,MAAM,CAAC,MAAM,KAAK,QAAQ;QACjC,MAAM,CAAC,MAAM,CAAC,YAAY;QAC1B,MAAM,CAAC,MAAM,CAAC,MAAM;QACpB,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CACpC,CAAC;AACJ,CAAC;AAED,SAAS,MAAM,CAAC,KAAa;IAC3B,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC1D,CAAC;AAED,SAAS,kBAAkB,CAAC,KAA8C;IACxE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,OAAO,EAAE,GAAG,KAAK,CAAC;IAC9C,OAAO,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;AACjC,CAAC"}

package/dist/bench/ranking.d.ts

@@ -0,0 +1,122 @@
+import type { AknoConfig, ReasoningEffort } from '../config/schema.ts';
+import { type ModelUsage } from '../models/client.ts';
+import { type RankingBenchSplit, type RankingCategory, type RelevanceGrade } from './ranking-corpus.ts';
+export type { RankingBenchSplit, RankingCategory } from './ranking-corpus.ts';
+export type RankingBenchSystem = 'fusion' | 'native' | 'llm';
+export type RankingCandidateCount = 10 | 20 | 40;
+export type RankingExcerptChars = 400 | 800 | 1600;
+export interface RankingBenchOptions {
+    system: RankingBenchSystem;
+    split?: RankingBenchSplit;
+    candidateCount?: RankingCandidateCount;
+    excerptChars?: RankingExcerptChars;
+    concurrency?: number;
+    provider?: string;
+    model?: string;
+    reasoningEffort?: ReasoningEffort;
+}
+export interface RankingQualityMetrics {
+    ndcgAt10: number;
+    mrrAt10: number;
+    successAt1: number;
+    successAt3: number;
+    precisionAt5: number;
+    gradeZeroAboveGradeThree: number;
+}
+export interface RankingQualificationMetrics {
+    answerRetention: number;
+    supportRetention: number;
+    marginalRetention: number;
+    irrelevantRejection: number;
+    retainedPrecision: number;
+    instructionNegativeRejection: number;
+}
+export interface RankingCategoryReport {
+    category: RankingCategory;
+    queries: number;
+    quality: RankingQualityMetrics;
+    fusionBaseline: RankingQualityMetrics;
+    ndcgDeltaFromFusion: number;
+}
+export interface RankingTokenUsage {
+    reportedQueries: number;
+    inputTokens: number | null;
+    outputTokens: number | null;
+    totalTokens: number | null;
+    cachedInputTokens: number | null;
+    reasoningOutputTokens: number | null;
+}
+export interface RankingQueryReport {
+    queryId: string;
+    category: RankingCategory;
+    order: string[];
+    rejected: string[] | null;
+    latencyMs: number;
+    endpointRequests: number;
+    usage: ModelUsage | null;
+    fallback: string | null;
+}
+export interface RankingBenchReport {
+    passed: boolean;
+    development: true;
+    releaseEligible: false;
+    system: RankingBenchSystem;
+    split: RankingBenchSplit;
+    provider: string | null;
+    model: string | null;
+    reasoningEffort: ReasoningEffort | null;
+    promptVersion: string | null;
+    schemaVersion: string | null;
+    corpus: {
+        queries: number;
+        sources: number;
+        judgments: number;
+        categories: number;
+        version: string;
+        fingerprint: string;
+    };
+    quality: RankingQualityMetrics;
+    fusionBaseline: RankingQualityMetrics;
+    ndcgDeltaFromFusion: number;
+    byCategory: RankingCategoryReport[];
+    queries: RankingQueryReport[];
+    qualification: RankingQualificationMetrics | null;
+    validResponseRate: number;
+    fallbackQueries: string[];
+    p50LatencyMs: number;
+    p95LatencyMs: number;
+    maxLatencyMs: number;
+    execution: {
+        requests: number;
+        concurrency: number;
+        candidateCount: RankingCandidateCount;
+        excerptChars: RankingExcerptChars;
+        maxExcerptChars: number;
+        endpointRequests: number;
+        extraEndpointRequests: number;
+        tokenUsage: RankingTokenUsage | null;
+    };
+    calibration: {
+        basis: 'auto' | 'none';
+        threshold: number | null;
+        lowestAnswerScore: number | null;
+        lowestSupportScore: number | null;
+        highestIrrelevantScore: number | null;
+        error: string | null;
+    };
+    failures: {
+        queryId: string;
+        error: string;
+    }[];
+}
+export declare function runRankingBench(config: AknoConfig, options: RankingBenchOptions): Promise<RankingBenchReport>;
+export declare function validateRankingCorpus(): void;
+export declare function qualityFor(benchCase: {
+    grades: RelevanceGrade[];
+}, order: number[]): RankingQualityMetrics;
+export declare function qualificationFor(judgments: {
+    grade: RelevanceGrade;
+    rejected: boolean;
+    instructionBearing?: boolean;
+}[]): RankingQualificationMetrics;
+//# sourceMappingURL=ranking.d.ts.map

package/dist/bench/ranking.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ranking.d.ts","sourceRoot":"","sources":["../../src/bench/ranking.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,eAAe,EAAqB,MAAM,qBAAqB,CAAC;AAC1F,OAAO,EAAe,KAAK,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAUnE,OAAO,EAIL,KAAK,iBAAiB,EAEtB,KAAK,eAAe,EACpB,KAAK,cAAc,EACpB,MAAM,qBAAqB,CAAC;AAG7B,YAAY,EAAE,iBAAiB,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAC9E,MAAM,MAAM,kBAAkB,GAAG,QAAQ,GAAG,QAAQ,GAAG,KAAK,CAAC;AAC7D,MAAM,MAAM,qBAAqB,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC;AACjD,MAAM,MAAM,mBAAmB,GAAG,GAAG,GAAG,GAAG,GAAG,IAAI,CAAC;AAEnD,MAAM,WAAW,mBAAmB;IAClC,MAAM,EAAE,kBAAkB,CAAC;IAC3B,KAAK,CAAC,EAAE,iBAAiB,CAAC;IAC1B,cAAc,CAAC,EAAE,qBAAqB,CAAC;IACvC,YAAY,CAAC,EAAE,mBAAmB,CAAC;IACnC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,eAAe,CAAC,EAAE,eAAe,CAAC;CACnC;AAED,MAAM,WAAW,qBAAqB;IACpC,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,wBAAwB,EAAE,MAAM,CAAC;CAClC;AAED,MAAM,WAAW,2BAA2B;IAC1C,eAAe,EAAE,MAAM,CAAC;IACxB,gBAAgB,EAAE,MAAM,CAAC;IACzB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,4BAA4B,EAAE,MAAM,CAAC;CACtC;AAED,MAAM,WAAW,qBAAqB;IACpC,QAAQ,EAAE,eAAe,CAAC;IAC1B,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,qBAAqB,CAAC;IAC/B,cAAc,EAAE,qBAAqB,CAAC;IACtC,mBAAmB,EAAE,MAAM,CAAC;CAC7B;AAED,MAAM,WAAW,iBAAiB;IAChC,eAAe,EAAE,MAAM,CAAC;IACxB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,qBAAqB,EAAE,MAAM,GAAG,IAAI,CAAC;CACtC;AAED,MAAM,WAAW,kBAAkB;IACjC,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,eAAe,CAAC;IAC1B,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,QAAQ,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;IAC1B,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC;IACzB,KAAK,EAAE,UAAU,GAAG,IAAI,CAAC;IACzB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;CACzB;AAED,MAAM,WAAW,kBAAkB;IACjC,MAAM,EAAE,OAAO,CAAC;IAChB,WAAW,EAAE,IAAI,CAAC;IAClB,eAAe,EAAE,KAAK,CAAC;IACvB,MAAM,EAAE,kBAAkB,CAAC;IAC3B,KAAK,EAAE,iBAAiB,CAAC;IACzB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,eAAe,EAAE,eAAe,GAAG,IAAI,CAAC;IACxC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,MAAM,EAAE;QACN,OAAO,EAAE,MAAM,CAAC;QAChB,OAAO,EAAE,MAAM,CAAC;QAChB,SAAS,EAAE,MAAM,CAAC;QAClB,UAAU,EAAE,MAAM,CAAC;QACnB,OAAO,EAAE,MAAM,CAAC;QAChB,WAAW,EAAE,MAAM,CAAC;KACrB,CAAC;IACF,OAAO,EAAE,qBAAqB,CAAC;IAC/B,cAAc,EAAE,qBAAqB,CAAC;IACtC,mBAAmB,EAAE,MAAM,CAAC;IAC5B,UAAU,EAAE,qBAAqB,EAAE,CAAC;IACpC,OAAO,EAAE,kBAAkB,EAAE,CAAC;IAC9B,aAAa,EAAE,2BAA2B,GAAG,IAAI,CAAC;IAClD,iBAAiB,EAAE,MAAM,CAAC;IAC1B,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE;QACT,QAAQ,EAAE,MAAM,CAAC;QACjB,WAAW,EAAE,MAAM,CAAC;QACpB,cAAc,EAAE,qBAAqB,CAAC;QACtC,YAAY,EAAE,mBAAmB,CAAC;QAClC,eAAe,EAAE,MAAM,CAAC;QACxB,gBAAgB,EAAE,MAAM,CAAC;QACzB,qBAAqB,EAAE,MAAM,CAAC;QAC9B,UAAU,EAAE,iBAAiB,GAAG,IAAI,CAAC;KACtC,CAAC;IACF,WAAW,EAAE;QACX,KAAK,EAAE,MAAM,GAAG,MAAM,CAAC;QACvB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;QACzB,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;QACjC,kBAAkB,EAAE,MAAM,GAAG,IAAI,CAAC;QAClC,sBAAsB,EAAE,MAAM,GAAG,IAAI,CAAC;QACtC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;KACtB,CAAC;IACF,QAAQ,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CAChD;AAuBD,wBAAsB,eAAe,CACnC,MAAM,EAAE,UAAU,EAClB,OAAO,EAAE,mBAAmB,GAC3B,OAAO,CAAC,kBAAkB,CAAC,CA6I7B;AAED,wBAAgB,qBAAqB,IAAI,IAAI,CAoE5C;AAkSD,wBAAgB,UAAU,CAAC,SAAS,EAAE;IAAE,MAAM,EAAE,cAAc,EAAE,CAAA;CAAE,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,qBAAqB,CAiB1G;AAkBD,wBAAgB,gBAAgB,CAC9B,SAAS,EAAE;IACT,KAAK,EAAE,cAAc,CAAC;IACtB,QAAQ,EAAE,OAAO,CAAC;IAClB,kBAAkB,CAAC,EAAE,OAAO,CAAC;CAC9B,EAAE,GACF,2BAA2B,CA6C7B"}