@sanity/ailf 2.0.0 → 2.0.1

package/dist/scripts/calculate-scores.d.ts

@@ -0,0 +1,166 @@
+/**
+ * Calculate-scores.ts
+ *
+ * Reads Promptfoo evaluation output and computes the AI Literacy Score
+ * for each feature area. Each dimension is scored on a uniform 0–100 scale:
+ *
+ *   Task Completion    (0–100)  — Can the LLM implement the feature?
+ *   Code Correctness   (0–100)  — Is the code idiomatic and correct?
+ *   Doc Coverage       (0–100)  — Did docs provide the needed info?
+ *
+ * Dimensions are combined into a weighted composite (0–100) using weights
+ * from config/rubrics.yaml (default: Task×0.50 + Code×0.25 + Docs×0.25).
+ *
+ * Additionally compares with-docs vs without-docs scores to calculate
+ * the "Doc Lift" — how much documentation helps vs parametric knowledge.
+ *
+ * When tests are run with the InstrumentedProvider (agent-observer),
+ * this script also aggregates and reports agent behavior data: which
+ * documentation pages were visited, what searches were performed, and
+ * overall network activity patterns.
+ */
+import "dotenv/config";
+import type { FeatureScore, GraderJudgment, PerModelEntry } from "../pipeline/types.js";
+export interface ComponentResult {
+    assertion?: {
+        type: string;
+        value?: string;
+        /** Structured metadata propagated from rubric templates (Approach 5). */
+        metadata?: Record<string, unknown>;
+    };
+    pass: boolean;
+    reason?: string;
+    score?: number;
+}
+export interface PromptfooResultsWrapper {
+    results: RawTestResult[];
+    stats: {
+        successes: number;
+        failures: number;
+        tokenUsage?: {
+            assertions?: {
+                completion: number;
+                prompt: number;
+                total: number;
+            };
+            completion: number;
+            prompt: number;
+            total: number;
+        };
+    };
+}
+export interface RawPromptfooFile {
+    [key: string]: unknown;
+    config?: {
+        defaultTest?: {
+            options?: {
+                provider?: string;
+                rubricProvider?: string;
+            };
+        };
+    };
+    results: PromptfooResultsWrapper;
+}
+export interface RawTestResult {
+    cost?: number;
+    error?: string;
+    gradingResult: null | {
+        componentResults: ComponentResult[];
+        pass: boolean;
+    };
+    metadata?: Record<string, unknown>;
+    provider?: {
+        id?: string;
+        label?: string;
+    };
+    response: {
+        output: string;
+    };
+    testCase?: {
+        description?: string;
+        vars?: Record<string, string>;
+    };
+    vars: Record<string, string>;
+}
+export interface TestResult {
+    cost: number;
+    description: string;
+    gradingResult: {
+        componentResults: ComponentResult[];
+        pass: boolean;
+    };
+    metadata?: Record<string, unknown>;
+    /** Provider identifier (e.g., "openai:gpt-4o") */
+    providerId?: string;
+    /** Provider label (e.g., "GPT-4o") */
+    providerLabel?: string;
+    response: {
+        output: string;
+    };
+    vars: Record<string, string>;
+}
+export interface UrlMetadata {
+    otherUrls: string[];
+    sanityUrlCount: number;
+    sanityUrls: string[];
+    totalUrlCount: number;
+}
+/**
+ * Calculate scores grouped by model. Each model gets its own FeatureScore[]
+ * and model-level aggregates.
+ *
+ * Uses the provider.id from Promptfoo results to identify models.
+ * Falls back to provider.label, then "unknown" if neither is available.
+ *
+ * @returns Record keyed by model ID, or null if only one model was used
+ *          (per-model breakdown is redundant when there's only one model).
+ */
+export declare function calculateScoresPerModel(resultsPath: string, weights: Record<string, number>): null | PerModelEntry[];
+export declare function classifyRubric(component: ComponentResult): "codeCorrectness" | "docCoverage" | "taskCompletion" | null;
+export declare function detectFeatureArea(description: string): string;
+/**
+ * Extract grader judgments (reason text + scores) from evaluation results.
+ *
+ * This preserves the grader's natural language reasoning for downstream
+ * analysis (failure mode classification, gap analysis). Each llm-rubric
+ * assertion produces one GraderJudgment entry.
+ *
+ * Phase 3a prerequisite: structured judgment data for failure mode extraction.
+ */
+export declare function extractGraderJudgments(resultsPath: string): GraderJudgment[];
+/**
+ * Finds the URL-extraction assertion result in a test's componentResults
+ * and parses the structured JSON from its `reason` field.
+ */
+export declare function extractUrlMetadata(test: TestResult): null | UrlMetadata;
+export declare function parseRubricScore(component: ComponentResult): number;
+/**
+ * Score agentic evaluation results. In agentic mode, all test entries are
+ * gold-only (no baseline entries — the .expanded.agentic.yaml fix ensures this).
+ * The model retrieves docs via tools, so all results map to "actual" scores.
+ *
+ * Returns a record keyed by feature area with the composite actual score.
+ */
+interface ActualScoreEntry {
+    actualScore: number;
+    codeCorrectness: number;
+    docCoverage: number;
+    taskCompletion: number;
+    testCount: number;
+    totalCost: number;
+}
+export declare function scoreAgenticResults(resultsPath: string, weights: Record<string, number>): Record<string, ActualScoreEntry>;
+/**
+ * Merge baseline FeatureScore[] with agentic actual scores to produce
+ * the full three-layer decomposition.
+ *
+ * The merge is per feature area. For each area:
+ * - If baseline data exists: floor, ceiling, docLift, docQualityGap are populated
+ * - If agentic data exists: actualScore is populated
+ * - If both exist: retrievalGap and infrastructureEfficiency are computed
+ *
+ * @param baselineScores  Floor/ceiling scores from baseline evaluation (may be empty)
+ * @param agenticScores   Actual scores from agentic evaluation (may be empty)
+ */
+export declare function mergeScores(baselineScores: FeatureScore[], agenticScores: Record<string, ActualScoreEntry>): FeatureScore[];
+export {};