@tenphi/akno-core 0.1.0

package/dist/bench/ranking-latency.js

@@ -0,0 +1,130 @@
+import { runRankingBench, } from "./ranking.js";
+export const RANKING_LATENCY_SCHEMA_VERSION = 'ranking-latency-v1';
+export const RANKING_INTERACTIVE_P95_TARGET_MS = 4_000;
+export async function runRankingLatencyBench(config, options) {
+    const split = options.split ?? 'development';
+    const loadConcurrency = normalizeLoadConcurrency(options.loadConcurrency ?? 4);
+    const common = {
+        system: 'llm',
+        split,
+        candidateCount: options.candidateCount,
+        excerptChars: options.excerptChars,
+        provider: options.provider,
+        model: options.model,
+        reasoningEffort: options.reasoningEffort,
+    };
+    options.onProgress?.({ profile: 'interactive', concurrency: 1 });
+    const interactiveRun = await runRankingBench(config, { ...common, concurrency: 1 });
+    options.onProgress?.({ profile: 'loaded', concurrency: loadConcurrency });
+    const loadedRun = await runRankingBench(config, { ...common, concurrency: loadConcurrency });
+    const interactive = profile(interactiveRun);
+    const loaded = profile(loadedRun);
+    return refreshRankingLatencyReport({
+        kind: 'ranking_latency',
+        schemaVersion: RANKING_LATENCY_SCHEMA_VERSION,
+        createdAt: new Date().toISOString(),
+        development: true,
+        releaseEligible: false,
+        split,
+        corpus: interactiveRun.corpus,
+        provider: interactiveRun.provider,
+        model: interactiveRun.model,
+        reasoningEffort: interactiveRun.reasoningEffort,
+        promptVersion: interactiveRun.promptVersion,
+        schemaVersionContract: interactiveRun.schemaVersion,
+        candidateCount: options.candidateCount,
+        excerptChars: options.excerptChars,
+        thresholds: {
+            interactiveP95LatencyMs: RANKING_INTERACTIVE_P95_TARGET_MS,
+        },
+        interactive,
+        loaded,
+        checks: [],
+        blockers: [],
+        passed: false,
+    });
+}
+/** Re-derive thresholds and verdicts without rerunning provider calls. */
+export function refreshRankingLatencyReport(report) {
+    if (report.schemaVersion !== RANKING_LATENCY_SCHEMA_VERSION) {
+        throw new Error('unsupported ranking latency artifact schema');
+    }
+    const checks = evaluateRankingLatency(report.interactive, report.loaded, report.thresholds.interactiveP95LatencyMs);
+    const blockers = checks.filter((entry) => !entry.passed).map((entry) => entry.id);
+    return {
+        ...report,
+        checks,
+        blockers,
+        passed: blockers.length === 0,
+    };
+}
+export function evaluateRankingLatency(interactive, loaded, interactiveP95TargetMs = RANKING_INTERACTIVE_P95_TARGET_MS) {
+    return [
+        check('interactive_cold', interactive.cold.validResponseRate === 1 && interactive.cold.endpointRequests >= 1, interactive.cold.validResponseRate === 1 && interactive.cold.endpointRequests >= 1, 'successful fresh-client call with a physical endpoint receipt'),
+        check('interactive_integrity', interactive.warm.validResponseRate === 1 &&
+            interactive.warm.endpointRequests === interactive.warm.samples, interactive.warm.validResponseRate === 1 &&
+            interactive.warm.endpointRequests === interactive.warm.samples, 'all warm responses valid with one endpoint request each'),
+        check('interactive_latency', interactive.warm.p95LatencyMs <= interactiveP95TargetMs, interactive.warm.p95LatencyMs, `at most ${interactiveP95TargetMs}ms warm single-flight p95`),
+        check('loaded_cold', loaded.cold.validResponseRate === 1 && loaded.cold.endpointRequests >= 1, loaded.cold.validResponseRate === 1 && loaded.cold.endpointRequests >= 1, 'successful fresh-client call with a physical endpoint receipt'),
+        check('loaded_integrity', loaded.warm.validResponseRate === 1 && loaded.warm.endpointRequests === loaded.warm.samples, loaded.warm.validResponseRate === 1 && loaded.warm.endpointRequests === loaded.warm.samples, 'all warm responses valid with one endpoint request each'),
+    ];
+}
+function profile(report) {
+    // `runRankingBench` deliberately serializes the first LLM case before starting its workers. That
+    // call is therefore the fresh-process compatibility receipt; every later query uses the learned
+    // dialect and measures the profile named by `execution.concurrency`.
+    return {
+        concurrency: report.execution.concurrency,
+        cold: metrics(report.queries.slice(0, 1)),
+        warm: metrics(report.queries.slice(1)),
+    };
+}
+function metrics(queries) {
+    const samples = queries.length;
+    const fallbackCount = queries.filter((query) => query.fallback !== null).length;
+    const latencies = queries.map((query) => query.latencyMs).filter((latency) => latency > 0);
+    const endpointRequests = queries.reduce((sum, query) => sum + query.endpointRequests, 0);
+    return {
+        samples,
+        validResponseRate: samples === 0 ? 0 : (samples - fallbackCount) / samples,
+        fallbackCount,
+        p50LatencyMs: percentile(latencies, 0.5),
+        p95LatencyMs: percentile(latencies, 0.95),
+        maxLatencyMs: latencies.length === 0 ? 0 : Math.max(...latencies),
+        endpointRequests,
+        extraEndpointRequests: Math.max(0, endpointRequests - samples),
+        tokenUsage: aggregateTokenUsage(queries),
+    };
+}
+function aggregateTokenUsage(queries) {
+    const usages = queries.flatMap((query) => (query.usage ? [query.usage] : []));
+    if (usages.length === 0)
+        return null;
+    return {
+        reportedQueries: usages.length,
+        inputTokens: sumReported(usages.map((usage) => usage.inputTokens)),
+        outputTokens: sumReported(usages.map((usage) => usage.outputTokens)),
+        totalTokens: sumReported(usages.map((usage) => usage.totalTokens)),
+        cachedInputTokens: sumReported(usages.map((usage) => usage.cachedInputTokens)),
+        reasoningOutputTokens: sumReported(usages.map((usage) => usage.reasoningOutputTokens)),
+    };
+}
+function sumReported(values) {
+    return values.every((value) => value === null || value === undefined)
+        ? null
+        : values.reduce((sum, value) => sum + (value ?? 0), 0);
+}
+function check(id, passed, actual, target) {
+    return { id, passed, actual, target };
+}
+function normalizeLoadConcurrency(value) {
+    return Number.isFinite(value) ? Math.max(2, Math.min(16, Math.floor(value))) : 4;
+}
+function percentile(values, quantile) {
+    if (values.length === 0)
+        return 0;
+    const sorted = [...values].sort((left, right) => left - right);
+    const index = Math.max(0, Math.ceil(quantile * sorted.length) - 1);
+    return sorted[index];
+}
+//# sourceMappingURL=ranking-latency.js.map

package/dist/bench/ranking-latency.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ranking-latency.js","sourceRoot":"","sources":["../../src/bench/ranking-latency.ts"],"names":[],"mappings":"AACA,OAAO,EACL,eAAe,GAMhB,MAAM,cAAc,CAAC;AAEtB,MAAM,CAAC,MAAM,8BAA8B,GAAG,oBAAoB,CAAC;AACnE,MAAM,CAAC,MAAM,iCAAiC,GAAG,KAAK,CAAC;AAuEvD,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAC1C,MAAkB,EAClB,OAA8B;IAE9B,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,aAAa,CAAC;IAC7C,MAAM,eAAe,GAAG,wBAAwB,CAAC,OAAO,CAAC,eAAe,IAAI,CAAC,CAAC,CAAC;IAC/E,MAAM,MAAM,GAAG;QACb,MAAM,EAAE,KAAc;QACtB,KAAK;QACL,cAAc,EAAE,OAAO,CAAC,cAAc;QACtC,YAAY,EAAE,OAAO,CAAC,YAAY;QAClC,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,eAAe,EAAE,OAAO,CAAC,eAAe;KACzC,CAAC;IAEF,OAAO,CAAC,UAAU,EAAE,CAAC,EAAE,OAAO,EAAE,aAAa,EAAE,WAAW,EAAE,CAAC,EAAE,CAAC,CAAC;IACjE,MAAM,cAAc,GAAG,MAAM,eAAe,CAAC,MAAM,EAAE,EAAE,GAAG,MAAM,EAAE,WAAW,EAAE,CAAC,EAAE,CAAC,CAAC;IACpF,OAAO,CAAC,UAAU,EAAE,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,eAAe,EAAE,CAAC,CAAC;IAC1E,MAAM,SAAS,GAAG,MAAM,eAAe,CAAC,MAAM,EAAE,EAAE,GAAG,MAAM,EAAE,WAAW,EAAE,eAAe,EAAE,CAAC,CAAC;IAC7F,MAAM,WAAW,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC;IAC5C,MAAM,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC;IAClC,OAAO,2BAA2B,CAAC;QACjC,IAAI,EAAE,iBAAiB;QACvB,aAAa,EAAE,8BAA8B;QAC7C,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,WAAW,EAAE,IAAI;QACjB,eAAe,EAAE,KAAK;QACtB,KAAK;QACL,MAAM,EAAE,cAAc,CAAC,MAAM;QAC7B,QAAQ,EAAE,cAAc,CAAC,QAAQ;QACjC,KAAK,EAAE,cAAc,CAAC,KAAK;QAC3B,eAAe,EAAE,cAAc,CAAC,eAAe;QAC/C,aAAa,EAAE,cAAc,CAAC,aAAa;QAC3C,qBAAqB,EAAE,cAAc,CAAC,aAAa;QACnD,cAAc,EAAE,OAAO,CAAC,cAAc;QACtC,YAAY,EAAE,OAAO,CAAC,YAAY;QAClC,UAAU,EAAE;YACV,uBAAuB,EAAE,iCAAiC;SAC3D;QACD,WAAW;QACX,MAAM;QACN,MAAM,EAAE,EAAE;QACV,QAAQ,EAAE,EAAE;QACZ,MAAM,EAAE,KAAK;KACd,CAAC,CAAC;AACL,CAAC;AAED,0EAA0E;AAC1E,MAAM,UAAU,2BAA2B,CAAC,MAA4B;IACtE,IAAI,MAAM,CAAC,aAAa,KAAK,8BAA8B,EAAE,CAAC;QAC5D,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;IACjE,CAAC;IACD,MAAM,MAAM,GAAG,sBAAsB,CACnC,MAAM,CAAC,WAAW,EAClB,MAAM,CAAC,MAAM,EACb,MAAM,CAAC,UAAU,CAAC,uBAAuB,CAC1C,CAAC;IACF,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAClF,OAAO;QACL,GAAG,MAAM;QACT,MAAM;QACN,QAAQ;QACR,MAAM,EAAE,QAAQ,CAAC,MAAM,KAAK,CAAC;KAC9B,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,sBAAsB,CACpC,WAAkC,EAClC,MAA6B,EAC7B,sBAAsB,GAAG,iCAAiC;IAE1D,OAAO;QACL,KAAK,CACH,kBAAkB,EAClB,WAAW,CAAC,IAAI,CAAC,iBAAiB,KAAK,CAAC,IAAI,WAAW,CAAC,IAAI,CAAC,gBAAgB,IAAI,CAAC,EAClF,WAAW,CAAC,IAAI,CAAC,iBAAiB,KAAK,CAAC,IAAI,WAAW,CAAC,IAAI,CAAC,gBAAgB,IAAI,CAAC,EAClF,+DAA+D,CAChE;QACD,KAAK,CACH,uBAAuB,EACvB,WAAW,CAAC,IAAI,CAAC,iBAAiB,KAAK,CAAC;YACtC,WAAW,CAAC,IAAI,CAAC,gBAAgB,KAAK,WAAW,CAAC,IAAI,CAAC,OAAO,EAChE,WAAW,CAAC,IAAI,CAAC,iBAAiB,KAAK,CAAC;YACtC,WAAW,CAAC,IAAI,CAAC,gBAAgB,KAAK,WAAW,CAAC,IAAI,CAAC,OAAO,EAChE,yDAAyD,CAC1D;QACD,KAAK,CACH,qBAAqB,EACrB,WAAW,CAAC,IAAI,CAAC,YAAY,IAAI,sBAAsB,EACvD,WAAW,CAAC,IAAI,CAAC,YAAY,EAC7B,WAAW,sBAAsB,2BAA2B,CAC7D;QACD,KAAK,CACH,aAAa,EACb,MAAM,CAAC,IAAI,CAAC,iBAAiB,KAAK,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,gBAAgB,IAAI,CAAC,EACxE,MAAM,CAAC,IAAI,CAAC,iBAAiB,KAAK,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,gBAAgB,IAAI,CAAC,EACxE,+DAA+D,CAChE;QACD,KAAK,CACH,kBAAkB,EAClB,MAAM,CAAC,IAAI,CAAC,iBAAiB,KAAK,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,gBAAgB,KAAK,MAAM,CAAC,IAAI,CAAC,OAAO,EAC3F,MAAM,CAAC,IAAI,CAAC,iBAAiB,KAAK,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,gBAAgB,KAAK,MAAM,CAAC,IAAI,CAAC,OAAO,EAC3F,yDAAyD,CAC1D;KACF,CAAC;AACJ,CAAC;AAED,SAAS,OAAO,CAAC,MAA0B;IACzC,iGAAiG;IACjG,gGAAgG;IAChG,qEAAqE;IACrE,OAAO;QACL,WAAW,EAAE,MAAM,CAAC,SAAS,CAAC,WAAW;QACzC,IAAI,EAAE,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACzC,IAAI,EAAE,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;KACvC,CAAC;AACJ,CAAC;AAED,SAAS,OAAO,CAAC,OAAsC;IACrD,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC;IAC/B,MAAM,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC,MAAM,CAAC;IAChF,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC;IAC3F,MAAM,gBAAgB,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC;IACzF,OAAO;QACL,OAAO;QACP,iBAAiB,EAAE,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,GAAG,aAAa,CAAC,GAAG,OAAO;QAC1E,aAAa;QACb,YAAY,EAAE,UAAU,CAAC,SAAS,EAAE,GAAG,CAAC;QACxC,YAAY,EAAE,UAAU,CAAC,SAAS,EAAE,IAAI,CAAC;QACzC,YAAY,EAAE,SAAS,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC;QACjE,gBAAgB;QAChB,qBAAqB,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,gBAAgB,GAAG,OAAO,CAAC;QAC9D,UAAU,EAAE,mBAAmB,CAAC,OAAO,CAAC;KACzC,CAAC;AACJ,CAAC;AAED,SAAS,mBAAmB,CAAC,OAAsC;IACjE,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC9E,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACrC,OAAO;QACL,eAAe,EAAE,MAAM,CAAC,MAAM;QAC9B,WAAW,EAAE,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;QAClE,YAAY,EAAE,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;QACpE,WAAW,EAAE,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;QAClE,iBAAiB,EAAE,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;QAC9E,qBAAqB,EAAE,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC;KACvF,CAAC;AACJ,CAAC;AAED,SAAS,WAAW,CAAC,MAAwC;IAC3D,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS,CAAC;QACnE,CAAC,CAAC,IAAI;QACN,CAAC,CAAC,MAAM,CAAC,MAAM,CAAS,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,KAAK,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AACnE,CAAC;AAED,SAAS,KAAK,CACZ,EAA6B,EAC7B,MAAe,EACf,MAAqC,EACrC,MAAc;IAEd,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;AACxC,CAAC;AAED,SAAS,wBAAwB,CAAC,KAAa;IAC7C,OAAO,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACnF,CAAC;AAED,SAAS,UAAU,CAAC,MAAgB,EAAE,QAAgB;IACpD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAClC,MAAM,MAAM,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,GAAG,KAAK,CAAC,CAAC;IAC/D,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;IACnE,OAAO,MAAM,CAAC,KAAK,CAAE,CAAC;AACxB,CAAC"}

package/dist/bench/ranking-matrix.d.ts

@@ -0,0 +1,141 @@
+import type { AknoConfig, ReasoningEffort } from '../config/schema.ts';
+import type { RankingEndToEndReport } from './ranking-end-to-end.ts';
+import { type RankingLatencyReport } from './ranking-latency.ts';
+import { type RankingBenchReport, type RankingBenchSplit, type RankingCandidateCount, type RankingCategoryReport, type RankingExcerptChars, type RankingQualityMetrics, type RankingQualificationMetrics, type RankingTokenUsage } from './ranking.ts';
+import { type RankingReviewEvidence } from './ranking-review.ts';
+export declare const RANKING_MATRIX_SCHEMA_VERSION = "ranking-matrix-v9";
+export declare const RANKING_MATRIX_VARIANT_IDS: readonly ["llm-none-c10", "llm-none-c20", "llm-none-c40", "llm-low-c20"];
+export type RankingMatrixVariantId = (typeof RANKING_MATRIX_VARIANT_IDS)[number];
+export interface RankingMatrixOptions {
+    split?: RankingBenchSplit;
+    provider?: string;
+    model?: string;
+    runs?: number;
+    concurrency?: number;
+    excerptChars?: RankingExcerptChars;
+    includeNative?: boolean;
+    /** Run only named repeated LLM variants; useful for development diagnostics, never release selection. */
+    variants?: RankingMatrixVariantId[];
+    onProgress?: (progress: RankingMatrixProgress) => void;
+}
+export interface RankingMatrixProgress {
+    variant: string;
+    run: number;
+    runs: number;
+}
+export interface RankingMatrixRun {
+    quality: RankingQualityMetrics;
+    validResponseRate: number;
+    p50LatencyMs: number;
+    p95LatencyMs: number;
+    maxLatencyMs: number;
+    topThree: Record<string, string[]>;
+    fallbackQueries: string[];
+    execution: RankingBenchReport['execution'];
+}
+interface RankingMatrixExecution {
+    requests: number;
+    endpointRequests: number;
+    extraEndpointRequests: number;
+    tokenUsage: RankingTokenUsage | null;
+}
+export interface RankingMatrixVariant {
+    id: string;
+    system: 'fusion' | 'native' | 'llm';
+    provider: string | null;
+    model: string | null;
+    reasoningEffort: ReasoningEffort | null;
+    promptVersion: string | null;
+    schemaVersion: string | null;
+    candidateCount: RankingCandidateCount;
+    excerptChars: RankingExcerptChars;
+    runCount: number;
+    /** True when at least one valid response produced finite measurements; release safety is separate. */
+    comparisonEligible: boolean;
+    quality: RankingQualityMetrics;
+    fusionBaseline: RankingQualityMetrics;
+    ndcgDeltaFromFusion: number;
+    byCategory: RankingCategoryReport[];
+    qualification: RankingQualificationMetrics | null;
+    validResponseRate: number;
+    fallbackPreserved: boolean;
+    p50LatencyMs: number;
+    p95LatencyMs: number;
+    maxLatencyMs: number;
+    medianTop3Overlap: number | null;
+    execution: RankingMatrixExecution;
+    runs: RankingMatrixRun[];
+}
+export interface RankingMatrixSelection {
+    variantId: string;
+    candidateCount: RankingCandidateCount;
+    reasoningEffort: ReasoningEffort;
+    rationale: string;
+}
+export interface RankingEndToEndEvidence {
+    split: RankingBenchSplit;
+    corpusVersion: string;
+    corpusFingerprint: string;
+    candidateCount: RankingCandidateCount;
+    retrievalPoolCount: number | null;
+    candidateSelectionVersion: string | null;
+    directAnswerFusionPoolRecall: number | null;
+    directAnswerCandidateRecall: number;
+    directAnswerRankedRecall: number;
+    candidateDegradedQueries: number;
+    rankedDegradedQueries: number;
+    rerankFallbackRate: number;
+    embeddingProvider: string | null;
+    embeddingModel: string | null;
+    embeddingDimensions: number | null;
+    embeddingAvailable: boolean;
+    totalChunks: number;
+    embeddedChunks: number;
+    rerankerProvider: string | null;
+    rerankerModel: string | null;
+    rerankerAvailable: boolean;
+    reasoningEffort: ReasoningEffort;
+    promptVersion: string;
+    schemaVersion: string;
+}
+export interface RankingReleaseCheck {
+    id: 'held_out_split' | 'independent_review' | 'persisted_artifact' | 'runtime_contract' | 'end_to_end_configuration' | 'end_to_end_candidate_recall' | 'end_to_end_ranked_recall' | 'end_to_end_integrity' | 'latency_configuration' | 'latency_integrity' | 'five_runs' | 'ndcg_gain' | 'category_regression' | 'exact_entity_mrr' | 'valid_responses' | 'fallback_preserved' | 'instruction_safety' | 'top3_stability' | 'latency' | 'cheapest_equivalent_effort';
+    passed: boolean;
+    actual: string | number | boolean | null;
+    target: string;
+}
+export interface RankingReleaseGate {
+    passed: boolean;
+    checks: RankingReleaseCheck[];
+    blockers: string[];
+}
+export interface RankingMatrixReport {
+    kind: 'ranking_matrix';
+    schemaVersion: string;
+    createdAt: string;
+    split: RankingBenchSplit;
+    corpus: RankingBenchReport['corpus'];
+    requestedRuns: number;
+    concurrency: number;
+    /** Non-null when this is deliberately diagnostic rather than a preset-selection matrix. */
+    targetedVariants: RankingMatrixVariantId[] | null;
+    variants: RankingMatrixVariant[];
+    selection: RankingMatrixSelection | null;
+    reviewEvidence: RankingReviewEvidence | null;
+    endToEndEvidence: RankingEndToEndEvidence | null;
+    latencyEvidence: RankingLatencyReport | null;
+    artifactPersisted: boolean;
+    releaseEligible: boolean;
+    releaseGate: RankingReleaseGate;
+}
+export declare function runRankingMatrix(config: AknoConfig, options?: RankingMatrixOptions): Promise<RankingMatrixReport>;
+export declare function attachRankingReviewEvidence(matrix: RankingMatrixReport, evidence: RankingReviewEvidence): RankingMatrixReport;
+export declare function attachRankingLatencyEvidence(matrix: RankingMatrixReport, report: RankingLatencyReport): RankingMatrixReport;
+export declare function markRankingMatrixPersisted(report: RankingMatrixReport): RankingMatrixReport;
+export declare function attachRankingEndToEndEvidence(matrix: RankingMatrixReport, report: RankingEndToEndReport): RankingMatrixReport;
+/** Re-derive every decision field from stored measurements after a gate implementation changes. */
+export declare function refreshRankingMatrixReport(report: RankingMatrixReport): RankingMatrixReport;
+export declare function evaluateRankingRelease(report: RankingMatrixReport): RankingReleaseGate;
+export declare function medianTop3Overlap(rankings: string[][]): number | null;
+export {};
+//# sourceMappingURL=ranking-matrix.d.ts.map

package/dist/bench/ranking-matrix.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ranking-matrix.d.ts","sourceRoot":"","sources":["../../src/bench/ranking-matrix.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAIvE,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,yBAAyB,CAAC;AACrE,OAAO,EAIL,KAAK,oBAAoB,EAC1B,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAEL,KAAK,kBAAkB,EACvB,KAAK,iBAAiB,EACtB,KAAK,qBAAqB,EAC1B,KAAK,qBAAqB,EAC1B,KAAK,mBAAmB,EACxB,KAAK,qBAAqB,EAC1B,KAAK,2BAA2B,EAChC,KAAK,iBAAiB,EACvB,MAAM,cAAc,CAAC;AACtB,OAAO,EAAgC,KAAK,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AAE/F,eAAO,MAAM,6BAA6B,sBAAsB,CAAC;AACjE,eAAO,MAAM,0BAA0B,0EAK7B,CAAC;AACX,MAAM,MAAM,sBAAsB,GAAG,CAAC,OAAO,0BAA0B,CAAC,CAAC,MAAM,CAAC,CAAC;AAEjF,MAAM,WAAW,oBAAoB;IACnC,KAAK,CAAC,EAAE,iBAAiB,CAAC;IAC1B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,mBAAmB,CAAC;IACnC,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,yGAAyG;IACzG,QAAQ,CAAC,EAAE,sBAAsB,EAAE,CAAC;IACpC,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,qBAAqB,KAAK,IAAI,CAAC;CACxD;AAED,MAAM,WAAW,qBAAqB;IACpC,OAAO,EAAE,MAAM,CAAC;IAChB,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,qBAAqB,CAAC;IAC/B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;IACnC,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,SAAS,EAAE,kBAAkB,CAAC,WAAW,CAAC,CAAC;CAC5C;AAED,UAAU,sBAAsB;IAC9B,QAAQ,EAAE,MAAM,CAAC;IACjB,gBAAgB,EAAE,MAAM,CAAC;IACzB,qBAAqB,EAAE,MAAM,CAAC;IAC9B,UAAU,EAAE,iBAAiB,GAAG,IAAI,CAAC;CACtC;AAED,MAAM,WAAW,oBAAoB;IACnC,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,QAAQ,GAAG,QAAQ,GAAG,KAAK,CAAC;IACpC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,eAAe,EAAE,eAAe,GAAG,IAAI,CAAC;IACxC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,cAAc,EAAE,qBAAqB,CAAC;IACtC,YAAY,EAAE,mBAAmB,CAAC;IAClC,QAAQ,EAAE,MAAM,CAAC;IACjB,sGAAsG;IACtG,kBAAkB,EAAE,OAAO,CAAC;IAC5B,OAAO,EAAE,qBAAqB,CAAC;IAC/B,cAAc,EAAE,qBAAqB,CAAC;IACtC,mBAAmB,EAAE,MAAM,CAAC;IAC5B,UAAU,EAAE,qBAAqB,EAAE,CAAC;IACpC,aAAa,EAAE,2BAA2B,GAAG,IAAI,CAAC;IAClD,iBAAiB,EAAE,MAAM,CAAC;IAC1B,iBAAiB,EAAE,OAAO,CAAC;IAC3B,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,SAAS,EAAE,sBAAsB,CAAC;IAClC,IAAI,EAAE,gBAAgB,EAAE,CAAC;CAC1B;AAED,MAAM,WAAW,sBAAsB;IACrC,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,EAAE,qBAAqB,CAAC;IACtC,eAAe,EAAE,eAAe,CAAC;IACjC,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,uBAAuB;IACtC,KAAK,EAAE,iBAAiB,CAAC;IACzB,aAAa,EAAE,MAAM,CAAC;IACtB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,cAAc,EAAE,qBAAqB,CAAC;IACtC,kBAAkB,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,yBAAyB,EAAE,MAAM,GAAG,IAAI,CAAC;IACzC,4BAA4B,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5C,2BAA2B,EAAE,MAAM,CAAC;IACpC,wBAAwB,EAAE,MAAM,CAAC;IACjC,wBAAwB,EAAE,MAAM,CAAC;IACjC,qBAAqB,EAAE,MAAM,CAAC;IAC9B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,kBAAkB,EAAE,OAAO,CAAC;IAC5B,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;IACvB,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,iBAAiB,EAAE,OAAO,CAAC;IAC3B,eAAe,EAAE,eAAe,CAAC;IACjC,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,mBAAmB;IAClC,EAAE,EACE,gBAAgB,GAChB,oBAAoB,GACpB,oBAAoB,GACpB,kBAAkB,GAClB,0BAA0B,GAC1B,6BAA6B,GAC7B,0BAA0B,GAC1B,sBAAsB,GACtB,uBAAuB,GACvB,mBAAmB,GACnB,WAAW,GACX,WAAW,GACX,qBAAqB,GACrB,kBAAkB,GAClB,iBAAiB,GACjB,oBAAoB,GACpB,oBAAoB,GACpB,gBAAgB,GAChB,SAAS,GACT,4BAA4B,CAAC;IACjC,MAAM,EAAE,OAAO,CAAC;IAChB,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,IAAI,CAAC;IACzC,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,kBAAkB;IACjC,MAAM,EAAE,OAAO,CAAC;IAChB,MAAM,EAAE,mBAAmB,EAAE,CAAC;IAC9B,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB;AAED,MAAM,WAAW,mBAAmB;IAClC,IAAI,EAAE,gBAAgB,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,iBAAiB,CAAC;IACzB,MAAM,EAAE,kBAAkB,CAAC,QAAQ,CAAC,CAAC;IACrC,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,2FAA2F;IAC3F,gBAAgB,EAAE,sBAAsB,EAAE,GAAG,IAAI,CAAC;IAClD,QAAQ,EAAE,oBAAoB,EAAE,CAAC;IACjC,SAAS,EAAE,sBAAsB,GAAG,IAAI,CAAC;IACzC,cAAc,EAAE,qBAAqB,GAAG,IAAI,CAAC;IAC7C,gBAAgB,EAAE,uBAAuB,GAAG,IAAI,CAAC;IACjD,eAAe,EAAE,oBAAoB,GAAG,IAAI,CAAC;IAC7C,iBAAiB,EAAE,OAAO,CAAC;IAC3B,eAAe,EAAE,OAAO,CAAC;IACzB,WAAW,EAAE,kBAAkB,CAAC;CACjC;AAUD,wBAAsB,gBAAgB,CACpC,MAAM,EAAE,UAAU,EAClB,OAAO,GAAE,oBAAyB,GACjC,OAAO,CAAC,mBAAmB,CAAC,CA8E9B;AAED,wBAAgB,2BAA2B,CACzC,MAAM,EAAE,mBAAmB,EAC3B,QAAQ,EAAE,qBAAqB,GAC9B,mBAAmB,CAKrB;AAED,wBAAgB,4BAA4B,CAC1C,MAAM,EAAE,mBAAmB,EAC3B,MAAM,EAAE,oBAAoB,GAC3B,mBAAmB,CA2BrB;AAED,wBAAgB,0BAA0B,CAAC,MAAM,EAAE,mBAAmB,GAAG,mBAAmB,CAO3F;AAED,wBAAgB,6BAA6B,CAC3C,MAAM,EAAE,mBAAmB,EAC3B,MAAM,EAAE,qBAAqB,GAC5B,mBAAmB,CA8DrB;AAED,mGAAmG;AACnG,wBAAgB,0BAA0B,CAAC,MAAM,EAAE,mBAAmB,GAAG,mBAAmB,CAqB3F;AAED,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,mBAAmB,GAAG,kBAAkB,CA6HtF;AAED,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE,GAAG,MAAM,GAAG,IAAI,CAWrE"}