@tenphi/akno-core 0.1.0

package/dist/bench/answer.d.ts

@@ -0,0 +1,156 @@
+import type { AnswerOutput, DegradedReason } from '@tenphi/akno-protocol';
+import type { AknoConfig, ReasoningEffort } from '../config/schema.ts';
+import { type AnswerBenchCategory, type AnswerBenchSplit } from './answer-corpus.ts';
+export declare const ANSWER_BENCH_SCHEMA_VERSION = "answer-benchmark-v3";
+export interface AnswerBenchOptions {
+    split?: AnswerBenchSplit;
+    runs?: number;
+    concurrency?: number;
+    embeddingProvider?: string;
+    embeddingModel?: string;
+    embeddingDimensions?: number;
+    provider?: string;
+    model?: string;
+    reasoningEffort?: ReasoningEffort;
+    onProgress?: (progress: {
+        run: number;
+        runs: number;
+        done: number;
+        total: number;
+    }) => void;
+}
+export interface AnswerBenchCaseReport {
+    id: string;
+    category: AnswerBenchCategory;
+    executed: boolean;
+    status: AnswerOutput['status'] | null;
+    outcome: AnswerOutput['outcome'] | null;
+    degraded: DegradedReason[];
+    outcomeCorrect: boolean;
+    answerPresenceCorrect: boolean;
+    requiredFacts: number;
+    supportedFacts: number;
+    forbiddenTextDetected: boolean;
+    citedSources: string[];
+    requiredCitations: number;
+    supportedCitations: number;
+    citationPrecisionCorrect: boolean;
+    citationRecallCorrect: boolean;
+    relatedSources: string[];
+    retrievalRecallCorrect: boolean;
+    latencyMs: number;
+    evidenceTokens: number;
+    answerTokens: number;
+    modelCalls: number;
+    usageReportedCalls: number;
+    providerInputTokens: number;
+    providerOutputTokens: number;
+    providerTotalTokens: number;
+    passed: boolean;
+    error: string | null;
+}
+export interface AnswerBenchReport {
+    kind: 'invented_answer_benchmark';
+    schemaVersion: string;
+    createdAt: string;
+    development: boolean;
+    artifactPersisted: boolean;
+    releaseEligible: boolean;
+    passed: boolean;
+    split: AnswerBenchSplit;
+    corpus: {
+        version: string;
+        fingerprint: string;
+        cases: number;
+        sources: number;
+        categories: number;
+        frozen: boolean;
+        independentlyReviewed: boolean;
+    };
+    thresholds: {
+        executionRate: number;
+        outcomeAccuracy: number;
+        expectedFactAccuracy: number;
+        citationPrecision: number;
+        citationRecall: number;
+        retrievalRecall: number;
+        abstentionAccuracy: number;
+        privacyLeakRate: number;
+        degradedRate: number;
+        verificationFailureRate: number;
+        stableCaseRate: number;
+        minimumRunPassRate: number;
+        p95LatencyMs: number;
+    };
+    embedding: {
+        provider: string | null;
+        model: string | null;
+        available: boolean;
+        totalChunks: number;
+        embeddedChunks: number;
+    };
+    answerModel: {
+        provider: string | null;
+        model: string | null;
+        reasoningEffort: ReasoningEffort | null;
+        available: boolean;
+        generationPromptVersion: string;
+        verifierPromptVersion: string;
+    };
+    execution: {
+        concurrency: number;
+        operations: number;
+        p50LatencyMs: number;
+        p95LatencyMs: number;
+        maxLatencyMs: number;
+        evidenceTokens: number;
+        answerTokens: number;
+        modelCalls: number;
+        usageReportedCalls: number;
+        providerInputTokens: number;
+        providerOutputTokens: number;
+        providerTotalTokens: number;
+    };
+    metrics: {
+        executionRate: number;
+        outcomeAccuracy: number;
+        expectedFactAccuracy: number;
+        citationPrecision: number;
+        citationRecall: number;
+        retrievalRecall: number;
+        abstentionAccuracy: number;
+        privacyLeakRate: number;
+        degradedRate: number;
+        verificationFailureRate: number;
+        mixedRetrievalPassed: boolean;
+    };
+    stability: {
+        requestedRuns: number;
+        completedRuns: number;
+        stableCaseRate: number | null;
+        minimumRunPassRate: number;
+        flakyCaseIds: string[];
+    };
+    runs: AnswerBenchRunSummary[];
+    cases: AnswerBenchCaseReport[];
+    blockers: string[];
+    releaseBlockers: string[];
+}
+export interface AnswerBenchRunSummary {
+    run: number;
+    passed: boolean;
+    casesPassed: number;
+    casesTotal: number;
+    p95LatencyMs: number;
+    modelCalls: number;
+    providerTotalTokens: number;
+}
+/**
+ * Runs the production answer path against only the tracked invented corpus. The temporary
+ * knowledge base and derived index are removed before returning, and the report contains no
+ * questions, evidence, generated prose, paths, slugs, or provider errors.
+ */
+export declare function runAnswerBench(config: AknoConfig, options?: AnswerBenchOptions): Promise<AnswerBenchReport>;
+/** Marks the exact report that is about to be stored; an output flag cannot be inferred after the fact. */
+export declare function markAnswerBenchPersisted(report: AnswerBenchReport): AnswerBenchReport;
+//# sourceMappingURL=answer.d.ts.map

package/dist/bench/answer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"answer.d.ts","sourceRoot":"","sources":["../../src/bench/answer.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAC1E,OAAO,KAAK,EAAa,UAAU,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAKlF,OAAO,EAOL,KAAK,mBAAmB,EAExB,KAAK,gBAAgB,EACtB,MAAM,oBAAoB,CAAC;AAE5B,eAAO,MAAM,2BAA2B,wBAAwB,CAAC;AAIjE,MAAM,WAAW,kBAAkB;IACjC,KAAK,CAAC,EAAE,gBAAgB,CAAC;IACzB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,KAAK,IAAI,CAAC;CAC7F;AAED,MAAM,WAAW,qBAAqB;IACpC,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,mBAAmB,CAAC;IAC9B,QAAQ,EAAE,OAAO,CAAC;IAClB,MAAM,EAAE,YAAY,CAAC,QAAQ,CAAC,GAAG,IAAI,CAAC;IACtC,OAAO,EAAE,YAAY,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC;IACxC,QAAQ,EAAE,cAAc,EAAE,CAAC;IAC3B,cAAc,EAAE,OAAO,CAAC;IACxB,qBAAqB,EAAE,OAAO,CAAC;IAC/B,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,qBAAqB,EAAE,OAAO,CAAC;IAC/B,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,wBAAwB,EAAE,OAAO,CAAC;IAClC,qBAAqB,EAAE,OAAO,CAAC;IAC/B,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,sBAAsB,EAAE,OAAO,CAAC;IAChC,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,MAAM,EAAE,OAAO,CAAC;IAChB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;CACtB;AAED,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,2BAA2B,CAAC;IAClC,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,OAAO,CAAC;IACrB,iBAAiB,EAAE,OAAO,CAAC;IAC3B,eAAe,EAAE,OAAO,CAAC;IACzB,MAAM,EAAE,OAAO,CAAC;IAChB,KAAK,EAAE,gBAAgB,CAAC;IACxB,MAAM,EAAE;QACN,OAAO,EAAE,MAAM,CAAC;QAChB,WAAW,EAAE,MAAM,CAAC;QACpB,KAAK,EAAE,MAAM,CAAC;QACd,OAAO,EAAE,MAAM,CAAC;QAChB,UAAU,EAAE,MAAM,CAAC;QACnB,MAAM,EAAE,OAAO,CAAC;QAChB,qBAAqB,EAAE,OAAO,CAAC;KAChC,CAAC;IACF,UAAU,EAAE;QACV,aAAa,EAAE,MAAM,CAAC;QACtB,eAAe,EAAE,MAAM,CAAC;QACxB,oBAAoB,EAAE,MAAM,CAAC;QAC7B,iBAAiB,EAAE,MAAM,CAAC;QAC1B,cAAc,EAAE,MAAM,CAAC;QACvB,eAAe,EAAE,MAAM,CAAC;QACxB,kBAAkB,EAAE,MAAM,CAAC;QAC3B,eAAe,EAAE,MAAM,CAAC;QACxB,YAAY,EAAE,MAAM,CAAC;QACrB,uBAAuB,EAAE,MAAM,CAAC;QAChC,cAAc,EAAE,MAAM,CAAC;QACvB,kBAAkB,EAAE,MAAM,CAAC;QAC3B,YAAY,EAAE,MAAM,CAAC;KACtB,CAAC;IACF,SAAS,EAAE;QACT,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;QACxB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;QACrB,SAAS,EAAE,OAAO,CAAC;QACnB,WAAW,EAAE,MAAM,CAAC;QACpB,cAAc,EAAE,MAAM,CAAC;KACxB,CAAC;IACF,WAAW,EAAE;QACX,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;QACxB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;QACrB,eAAe,EAAE,eAAe,GAAG,IAAI,CAAC;QACxC,SAAS,EAAE,OAAO,CAAC;QACnB,uBAAuB,EAAE,MAAM,CAAC;QAChC,qBAAqB,EAAE,MAAM,CAAC;KAC/B,CAAC;IACF,SAAS,EAAE;QACT,WAAW,EAAE,MAAM,CAAC;QACpB,UAAU,EAAE,MAAM,CAAC;QACnB,YAAY,EAAE,MAAM,CAAC;QACrB,YAAY,EAAE,MAAM,CAAC;QACrB,YAAY,EAAE,MAAM,CAAC;QACrB,cAAc,EAAE,MAAM,CAAC;QACvB,YAAY,EAAE,MAAM,CAAC;QACrB,UAAU,EAAE,MAAM,CAAC;QACnB,kBAAkB,EAAE,MAAM,CAAC;QAC3B,mBAAmB,EAAE,MAAM,CAAC;QAC5B,oBAAoB,EAAE,MAAM,CAAC;QAC7B,mBAAmB,EAAE,MAAM,CAAC;KAC7B,CAAC;IACF,OAAO,EAAE;QACP,aAAa,EAAE,MAAM,CAAC;QACtB,eAAe,EAAE,MAAM,CAAC;QACxB,oBAAoB,EAAE,MAAM,CAAC;QAC7B,iBAAiB,EAAE,MAAM,CAAC;QAC1B,cAAc,EAAE,MAAM,CAAC;QACvB,eAAe,EAAE,MAAM,CAAC;QACxB,kBAAkB,EAAE,MAAM,CAAC;QAC3B,eAAe,EAAE,MAAM,CAAC;QACxB,YAAY,EAAE,MAAM,CAAC;QACrB,uBAAuB,EAAE,MAAM,CAAC;QAChC,oBAAoB,EAAE,OAAO,CAAC;KAC/B,CAAC;IACF,SAAS,EAAE;QACT,aAAa,EAAE,MAAM,CAAC;QACtB,aAAa,EAAE,MAAM,CAAC;QACtB,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;QAC9B,kBAAkB,EAAE,MAAM,CAAC;QAC3B,YAAY,EAAE,MAAM,EAAE,CAAC;KACxB,CAAC;IACF,IAAI,EAAE,qBAAqB,EAAE,CAAC;IAC9B,KAAK,EAAE,qBAAqB,EAAE,CAAC;IAC/B,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,eAAe,EAAE,MAAM,EAAE,CAAC;CAC3B;AAED,MAAM,WAAW,qBAAqB;IACpC,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,OAAO,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,mBAAmB,EAAE,MAAM,CAAC;CAC7B;AAMD;;;;GAIG;AACH,wBAAsB,cAAc,CAClC,MAAM,EAAE,UAAU,EAClB,OAAO,GAAE,kBAAuB,GAC/B,OAAO,CAAC,iBAAiB,CAAC,CAwG5B;AAuID,2GAA2G;AAC3G,wBAAgB,wBAAwB,CAAC,MAAM,EAAE,iBAAiB,GAAG,iBAAiB,CAErF"}