@tenphi/akno-core 0.12.0 → 0.13.0

package/dist/bench/language.d.ts

@@ -0,0 +1,371 @@
+import type { AknoConfig } from '../config/schema.ts';
+import { type LanguageCase } from './language-corpus.ts';
+export interface LanguageBenchOptions {
+    split: LanguageCase['split'];
+    corpus?: 'v1' | 'v2' | 'v3' | 'v4' | 'v5' | 'v6' | 'v7' | 'v8' | 'v9' | 'v10' | 'v11' | 'v12' | 'v13' | 'v14' | 'v15' | 'v16' | 'v17' | 'v18' | 'v19' | 'v20' | 'v21' | 'v22';
+    runs?: number;
+    caseIds?: string[];
+    onProgress?: (id: string, done: number, total: number) => void;
+}
+/** Runs only invented inputs. Provider permissions/settings come from the caller's configured roles. */
+export declare function runLanguageBench(config: AknoConfig, options: LanguageBenchOptions): Promise<{
+    schemaVersion: string;
+    createdAt: string;
+    corpusVersion: string;
+    corpusFingerprint: string;
+    selectedCaseIds: string[];
+    runs: number;
+    setup: string;
+    queryMatrix: string[];
+    split: "development" | "held-out";
+    knowledgeLanguage: string;
+    proseProjectionVersion: string;
+    memoryViewVersion: string;
+    answerPromptVersion: string;
+    answerVerifierVersion: string;
+    retentionPromptVersion: string;
+    retentionVerifierVersion: string;
+    models: {
+        retention: string | null;
+        answer: string | null;
+        embedding: string | null;
+        expansion: string | null;
+    };
+    modelOutputTokenLimits: {
+        answer: number | null;
+        retention: number | null;
+    };
+    indexModelDerivation: boolean;
+    independentlyReviewed: boolean;
+    releaseEligible: boolean;
+    adjudication: string;
+    thresholds: {
+        usefulQualifiedAnswerCoverage: number;
+        unsupportedRetainedOutputs?: number | undefined;
+        unsupportedNonnullAnswers?: number | undefined;
+        unsafeFactualPromotion: 0;
+        acceptedLanguageViolations: 0;
+        translationQualificationErrors: 0;
+        sourceByteChanges: 0;
+        usefulRetentionCoverage: 0.8;
+        qualifiedRetrievalCoverage: 0.8;
+        availabilityFailureRate: 0.05;
+    };
+    metrics: {
+        availabilityFailures: {
+            numerator: number;
+            denominator: number;
+            rate: number | null;
+        };
+        answerOperationFailures: {
+            numerator: number;
+            denominator: number;
+            rate: number | null;
+        };
+        producedAnswers: {
+            numerator: number;
+            denominator: number;
+            rate: number | null;
+        };
+        producedAnswersOverRetained: {
+            numerator: number;
+            denominator: number;
+            rate: number | null;
+        };
+        answerReasons: Record<string, number>;
+        routingReasons: Record<string, number>;
+        holdStages: Record<string, number>;
+        independentlyJudgedQualifiedAnswers: {
+            numerator: null;
+            denominator: number;
+            rate: null;
+        };
+        independentlyJustifiedAbstentions: {
+            numerator: null;
+            denominator: number;
+            rate: null;
+        };
+        languagePolicyRejections: {
+            numerator: number;
+            denominator: number;
+            rate: number | null;
+        };
+        acceptedLanguageViolations: {
+            numerator: null;
+            denominator: number;
+            rate: null;
+        };
+        translationQualificationErrors: {
+            numerator: null;
+            denominator: number;
+            rate: null;
+        };
+        unsafeFactualPromotion: {
+            numerator: null;
+            denominator: number;
+            rate: null;
+        };
+        noncanonicalEligibilityFlags: {
+            numerator: number;
+            denominator: number;
+            rate: number | null;
+        };
+        usefulRetentionCoverage: {
+            numerator: number;
+            denominator: number;
+            rate: number | null;
+        };
+        expectedRetentionMisses: {
+            numerator: number;
+            denominator: number;
+            rate: number | null;
+        };
+        falseHolds: {
+            numerator: number;
+            denominator: number;
+            rate: number | null;
+        } | {
+            numerator: null;
+            denominator: number;
+            rate: null;
+        };
+        expectedSafeHolds: {
+            numerator: number;
+            denominator: number;
+            rate: number | null;
+        };
+        typedSemanticsMatch: {
+            numerator: number;
+            denominator: number;
+            rate: number | null;
+        };
+        qualifiedRetrievalCoverage: {
+            numerator: number;
+            denominator: number;
+            rate: number | null;
+        };
+        ordinaryProseQualification: {
+            numerator: number;
+            denominator: number;
+            rate: number | null;
+        };
+        sourceByteChanges: {
+            numerator: number;
+            denominator: number;
+            rate: number | null;
+        };
+    };
+    byLanguage: {
+        [k: string]: number;
+    };
+    byScenario: Record<string, number>;
+    cases: ({
+        retentionAvailabilityFailure: boolean;
+        availabilityFailure: boolean;
+        languageRejected: boolean;
+        retention: {
+            status: "unavailable" | "ok" | "empty" | "degraded";
+            outcome: "ok" | "held" | "noop" | "replayed" | "revision_conflict" | undefined;
+            reason: "language_mismatch" | "conflict" | "routing_uncertain" | "no_writable_destination" | "source_unavailable" | "discourse_uncertain" | "context_too_large" | "time_unresolved" | "noncanonical_without_context" | "support_limit" | "placement_degraded" | "apply_failed" | "validation_failed" | "language_policy_required" | null;
+            candidates: {
+                outcome: "not_found" | "held" | "retracted" | "duplicate" | "written" | "support_added";
+                reason: "language_mismatch" | "conflict" | "routing_uncertain" | "no_writable_destination" | "source_unavailable" | "discourse_uncertain" | "context_too_large" | "time_unresolved" | "noncanonical_without_context" | "support_limit" | "placement_degraded" | "apply_failed" | "validation_failed" | "language_policy_required" | null;
+                holdStage: "verification" | "apply" | "validation" | "placement" | null;
+                routingReason: "existing_selected" | "new_selected" | "read_only_match" | "no_admitted_destination" | "ownership_uncertain" | "model_unavailable" | "model_failed" | "invalid_model_response" | null;
+            }[] | undefined;
+            usage: {
+                extraction: {
+                    model: string;
+                    latency_ms: number;
+                    input_tokens: number | null;
+                    output_tokens: number | null;
+                    total_tokens: number | null;
+                } | null;
+                verification: {
+                    model: string;
+                    latency_ms: number;
+                    input_tokens: number | null;
+                    output_tokens: number | null;
+                    total_tokens: number | null;
+                } | null;
+                placement: {
+                    model: string;
+                    latency_ms: number;
+                    input_tokens: number | null;
+                    output_tokens: number | null;
+                    total_tokens: number | null;
+                }[];
+                repair?: {
+                    model: string;
+                    latency_ms: number;
+                    input_tokens: number | null;
+                    output_tokens: number | null;
+                    total_tokens: number | null;
+                } | undefined;
+            } | null;
+            degraded: ("language_mismatch" | "language_check_failed" | "prose_discourse_unresolved" | "no_embedding_model" | "no_reranker" | "no_derive_model" | "no_expansion_model" | "embedding_failed" | "rerank_failed" | "derive_failed" | "retain_verification_failed" | "retain_apply_failed" | "partial_temporal_index" | "partial_memory_index" | "timeline_range_limited" | "expansion_failed" | "no_vector_index" | "partial_index" | "no_document_text" | "document_source_missing" | "source_conflict" | "no_graph_index" | "no_entity_index" | "entity_resolution_failed" | "partial_graph_index" | "graph_traversal_limited" | "no_answer_model" | "answer_failed" | "answer_verification_failed" | "answer_truncated")[];
+        };
+        retainedItems: number;
+        noncanonicalEligibilityFlag: boolean;
+        semanticsMatch: boolean;
+        ordinaryCorrect: boolean;
+        ordinaryInspection: {
+            status: "unavailable" | "ok" | "empty" | "degraded";
+            results: number;
+        };
+        bytesStable: boolean;
+        replayOutcome: "ok" | "held" | "noop" | "replayed" | "revision_conflict" | undefined;
+        queries: {
+            queryLanguage: "en" | "ru";
+            explicitView: boolean;
+            requestedAnswerLanguage: "en" | "ru";
+            inferredView: "all" | "factual" | "history" | "planning" | "reports" | "questions" | "discussion";
+            recallStatus: "unavailable" | "ok" | "empty" | "degraded";
+            recallDegraded: ("language_mismatch" | "language_check_failed" | "prose_discourse_unresolved" | "no_embedding_model" | "no_reranker" | "no_derive_model" | "no_expansion_model" | "embedding_failed" | "rerank_failed" | "derive_failed" | "retain_verification_failed" | "retain_apply_failed" | "partial_temporal_index" | "partial_memory_index" | "timeline_range_limited" | "expansion_failed" | "no_vector_index" | "partial_index" | "no_document_text" | "document_source_missing" | "source_conflict" | "no_graph_index" | "no_entity_index" | "entity_resolution_failed" | "partial_graph_index" | "graph_traversal_limited" | "no_answer_model" | "answer_failed" | "answer_verification_failed" | "answer_truncated")[];
+            retainedEvidence: number;
+            reviewRetrieval: {
+                text: string;
+                qualification: {
+                    status: "qualified";
+                    id: string;
+                    level: 1;
+                    kind: "claim" | "question" | "plan" | "decision" | "preference" | "event";
+                    subject: string;
+                    source_role: "unknown" | "external" | "user" | "assistant";
+                    commitment: "none" | "tentative" | "asserted" | "hypothetical" | "counterfactual";
+                    disposition: "completed" | "rejected" | "active" | "proposed" | "accepted" | "resolved" | "cancelled" | "superseded";
+                    polarity: "affirmed" | "negated";
+                    basis: "self_attested" | "source_report" | "cited_evidence" | "system_record";
+                    answer_eligible: boolean;
+                    current_eligible: boolean;
+                    source_speaker?: string | undefined;
+                    temporal?: {
+                        time: {
+                            precision: "unknown" | "instant" | "day" | "month" | "year";
+                            relation: "occurred" | "valid" | "scheduled" | "due";
+                            status: "tentative" | "scheduled" | "actual" | "planned";
+                            start?: string | undefined;
+                            until?: string | undefined;
+                            timezone?: string | undefined;
+                            mentioned_at?: string | undefined;
+                            recurrence?: {
+                                frequency: "daily" | "weekly" | "monthly" | "yearly";
+                                interval?: number | undefined;
+                                weekdays?: ("mo" | "tu" | "we" | "th" | "fr" | "sa" | "su")[] | undefined;
+                                until?: string | undefined;
+                            } | undefined;
+                        };
+                        clock_relation: "past" | "today" | "current_period" | "ongoing" | "future" | "overdue" | "undated";
+                        actionable: boolean;
+                    } | undefined;
+                };
+            }[];
+            contextStatus: "unavailable" | "ok" | "empty" | "degraded";
+            contextDegraded: ("language_mismatch" | "language_check_failed" | "prose_discourse_unresolved" | "no_embedding_model" | "no_reranker" | "no_derive_model" | "no_expansion_model" | "embedding_failed" | "rerank_failed" | "derive_failed" | "retain_verification_failed" | "retain_apply_failed" | "partial_temporal_index" | "partial_memory_index" | "timeline_range_limited" | "expansion_failed" | "no_vector_index" | "partial_index" | "no_document_text" | "document_source_missing" | "source_conflict" | "no_graph_index" | "no_entity_index" | "entity_resolution_failed" | "partial_graph_index" | "graph_traversal_limited" | "no_answer_model" | "answer_failed" | "answer_verification_failed" | "answer_truncated")[];
+            contextActivated: boolean;
+            contextActivation: {
+                activated: boolean;
+                basis: "none" | "exact" | "semantic" | "qualified";
+                candidates: number;
+                selected: number;
+                qualification_run: boolean;
+                reference_resolution: "unresolved" | "not_needed" | "resolved" | "ambiguous";
+            } | null;
+            answerOutcome: "not_found" | "partial" | "complete" | "not_answered";
+            answerReason: "no_results" | "evidence_unavailable" | "retrieval_incomplete" | "no_eligible_evidence" | "generation_unavailable" | "generation_failed" | "invalid_draft" | "empty_draft" | "draft_rejected" | "verification_unavailable" | "verification_rejected" | "answered" | null;
+            answerValidation: {
+                generated_blocks: number;
+                passed_guards: number;
+                verified_blocks: number | null;
+                rejection_counts: Partial<Record<"attribution" | "discourse" | "language" | "citation" | "protected_value" | "semantic_support", number>>;
+            } | null;
+            answerDegraded: ("language_mismatch" | "language_check_failed" | "prose_discourse_unresolved" | "no_embedding_model" | "no_reranker" | "no_derive_model" | "no_expansion_model" | "embedding_failed" | "rerank_failed" | "derive_failed" | "retain_verification_failed" | "retain_apply_failed" | "partial_temporal_index" | "partial_memory_index" | "timeline_range_limited" | "expansion_failed" | "no_vector_index" | "partial_index" | "no_document_text" | "document_source_missing" | "source_conflict" | "no_graph_index" | "no_entity_index" | "entity_resolution_failed" | "partial_graph_index" | "graph_traversal_limited" | "no_answer_model" | "answer_failed" | "answer_verification_failed" | "answer_truncated")[];
+            answerModelUsage: {
+                generation: {
+                    model: string;
+                    latency_ms: number;
+                    input_tokens: number | null;
+                    output_tokens: number | null;
+                    total_tokens: number | null;
+                } | null;
+                verification: {
+                    model: string;
+                    latency_ms: number;
+                    input_tokens: number | null;
+                    output_tokens: number | null;
+                    total_tokens: number | null;
+                } | null;
+            };
+            reviewAnswer: string | null;
+        }[];
+        reviewKnowledge: {
+            text: string;
+            qualification: {
+                status: "qualified";
+                id: string;
+                level: 1;
+                kind: "claim" | "question" | "plan" | "decision" | "preference" | "event";
+                subject: string;
+                source_role: "unknown" | "external" | "user" | "assistant";
+                commitment: "none" | "tentative" | "asserted" | "hypothetical" | "counterfactual";
+                disposition: "completed" | "rejected" | "active" | "proposed" | "accepted" | "resolved" | "cancelled" | "superseded";
+                polarity: "affirmed" | "negated";
+                basis: "self_attested" | "source_report" | "cited_evidence" | "system_record";
+                answer_eligible: boolean;
+                current_eligible: boolean;
+                source_speaker?: string | undefined;
+                temporal?: {
+                    time: {
+                        precision: "unknown" | "instant" | "day" | "month" | "year";
+                        relation: "occurred" | "valid" | "scheduled" | "due";
+                        status: "tentative" | "scheduled" | "actual" | "planned";
+                        start?: string | undefined;
+                        until?: string | undefined;
+                        timezone?: string | undefined;
+                        mentioned_at?: string | undefined;
+                        recurrence?: {
+                            frequency: "daily" | "weekly" | "monthly" | "yearly";
+                            interval?: number | undefined;
+                            weekdays?: ("mo" | "tu" | "we" | "th" | "fr" | "sa" | "su")[] | undefined;
+                            until?: string | undefined;
+                        } | undefined;
+                    };
+                    clock_relation: "past" | "today" | "current_period" | "ongoing" | "future" | "overdue" | "undated";
+                    actionable: boolean;
+                } | undefined;
+            };
+        }[];
+        error: null;
+        id: string;
+        run: number;
+        admission: string;
+        reviewExpectation: string | null;
+        language: "en" | "ru" | "mixed";
+        scenario: string;
+        expectedHold: boolean;
+        nonfactualExpected: boolean;
+    } | {
+        retentionAvailabilityFailure: boolean;
+        availabilityFailure: boolean;
+        languageRejected: boolean;
+        retention: null;
+        retainedItems: number;
+        noncanonicalEligibilityFlag: boolean;
+        semanticsMatch: boolean;
+        ordinaryCorrect: boolean;
+        bytesStable: null;
+        replayOutcome: null;
+        queries: never[];
+        reviewKnowledge: never[];
+        error: string;
+        id: string;
+        run: number;
+        admission: string;
+        reviewExpectation: string | null;
+        language: "en" | "ru" | "mixed";
+        scenario: string;
+        expectedHold: boolean;
+        nonfactualExpected: boolean;
+    })[];
+}>;
+//# sourceMappingURL=language.d.ts.map

package/dist/bench/language.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"language.d.ts","sourceRoot":"","sources":["../../src/bench/language.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,UAAU,EAAa,MAAM,qBAAqB,CAAC;AA6BjE,OAAO,EAA4C,KAAK,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAEnG,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,YAAY,CAAC,OAAO,CAAC,CAAC;IAC7B,MAAM,CAAC,EACH,IAAI,GACJ,IAAI,GACJ,IAAI,GACJ,IAAI,GACJ,IAAI,GACJ,IAAI,GACJ,IAAI,GACJ,IAAI,GACJ,IAAI,GACJ,KAAK,GACL,KAAK,GACL,KAAK,GACL,KAAK,GACL,KAAK,GACL,KAAK,GACL,KAAK,GACL,KAAK,GACL,KAAK,GACL,KAAK,GACL,KAAK,GACL,KAAK,GACL,KAAK,CAAC;IACV,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,UAAU,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;CAChE;AAED,wGAAwG;AACxG,wBAAsB,gBAAgB,CAAC,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,oBAAoB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6LvF"}