@sanity/ailf 2.0.1 → 2.1.0

package/dist/scripts/calculate-scores.d.ts

@@ -1,166 +0,0 @@
-/**
- * Calculate-scores.ts
- *
- * Reads Promptfoo evaluation output and computes the AI Literacy Score
- * for each feature area. Each dimension is scored on a uniform 0–100 scale:
- *
- *   Task Completion    (0–100)  — Can the LLM implement the feature?
- *   Code Correctness   (0–100)  — Is the code idiomatic and correct?
- *   Doc Coverage       (0–100)  — Did docs provide the needed info?
- *
- * Dimensions are combined into a weighted composite (0–100) using weights
- * from config/rubrics.yaml (default: Task×0.50 + Code×0.25 + Docs×0.25).
- *
- * Additionally compares with-docs vs without-docs scores to calculate
- * the "Doc Lift" — how much documentation helps vs parametric knowledge.
- *
- * When tests are run with the InstrumentedProvider (agent-observer),
- * this script also aggregates and reports agent behavior data: which
- * documentation pages were visited, what searches were performed, and
- * overall network activity patterns.
- */
-import "dotenv/config";
-import type { FeatureScore, GraderJudgment, PerModelEntry } from "../pipeline/types.js";
-export interface ComponentResult {
-    assertion?: {
-        type: string;
-        value?: string;
-        /** Structured metadata propagated from rubric templates (Approach 5). */
-        metadata?: Record<string, unknown>;
-    };
-    pass: boolean;
-    reason?: string;
-    score?: number;
-}
-export interface PromptfooResultsWrapper {
-    results: RawTestResult[];
-    stats: {
-        successes: number;
-        failures: number;
-        tokenUsage?: {
-            assertions?: {
-                completion: number;
-                prompt: number;
-                total: number;
-            };
-            completion: number;
-            prompt: number;
-            total: number;
-        };
-    };
-}
-export interface RawPromptfooFile {
-    [key: string]: unknown;
-    config?: {
-        defaultTest?: {
-            options?: {
-                provider?: string;
-                rubricProvider?: string;
-            };
-        };
-    };
-    results: PromptfooResultsWrapper;
-}
-export interface RawTestResult {
-    cost?: number;
-    error?: string;
-    gradingResult: null | {
-        componentResults: ComponentResult[];
-        pass: boolean;
-    };
-    metadata?: Record<string, unknown>;
-    provider?: {
-        id?: string;
-        label?: string;
-    };
-    response: {
-        output: string;
-    };
-    testCase?: {
-        description?: string;
-        vars?: Record<string, string>;
-    };
-    vars: Record<string, string>;
-}
-export interface TestResult {
-    cost: number;
-    description: string;
-    gradingResult: {
-        componentResults: ComponentResult[];
-        pass: boolean;
-    };
-    metadata?: Record<string, unknown>;
-    /** Provider identifier (e.g., "openai:gpt-4o") */
-    providerId?: string;
-    /** Provider label (e.g., "GPT-4o") */
-    providerLabel?: string;
-    response: {
-        output: string;
-    };
-    vars: Record<string, string>;
-}
-export interface UrlMetadata {
-    otherUrls: string[];
-    sanityUrlCount: number;
-    sanityUrls: string[];
-    totalUrlCount: number;
-}
-/**
- * Calculate scores grouped by model. Each model gets its own FeatureScore[]
- * and model-level aggregates.
- *
- * Uses the provider.id from Promptfoo results to identify models.
- * Falls back to provider.label, then "unknown" if neither is available.
- *
- * @returns Record keyed by model ID, or null if only one model was used
- *          (per-model breakdown is redundant when there's only one model).
- */
-export declare function calculateScoresPerModel(resultsPath: string, weights: Record<string, number>): null | PerModelEntry[];
-export declare function classifyRubric(component: ComponentResult): "codeCorrectness" | "docCoverage" | "taskCompletion" | null;
-export declare function detectFeatureArea(description: string): string;
-/**
- * Extract grader judgments (reason text + scores) from evaluation results.
- *
- * This preserves the grader's natural language reasoning for downstream
- * analysis (failure mode classification, gap analysis). Each llm-rubric
- * assertion produces one GraderJudgment entry.
- *
- * Phase 3a prerequisite: structured judgment data for failure mode extraction.
- */
-export declare function extractGraderJudgments(resultsPath: string): GraderJudgment[];
-/**
- * Finds the URL-extraction assertion result in a test's componentResults
- * and parses the structured JSON from its `reason` field.
- */
-export declare function extractUrlMetadata(test: TestResult): null | UrlMetadata;
-export declare function parseRubricScore(component: ComponentResult): number;
-/**
- * Score agentic evaluation results. In agentic mode, all test entries are
- * gold-only (no baseline entries — the .expanded.agentic.yaml fix ensures this).
- * The model retrieves docs via tools, so all results map to "actual" scores.
- *
- * Returns a record keyed by feature area with the composite actual score.
- */
-interface ActualScoreEntry {
-    actualScore: number;
-    codeCorrectness: number;
-    docCoverage: number;
-    taskCompletion: number;
-    testCount: number;
-    totalCost: number;
-}
-export declare function scoreAgenticResults(resultsPath: string, weights: Record<string, number>): Record<string, ActualScoreEntry>;
-/**
- * Merge baseline FeatureScore[] with agentic actual scores to produce
- * the full three-layer decomposition.
- *
- * The merge is per feature area. For each area:
- * - If baseline data exists: floor, ceiling, docLift, docQualityGap are populated
- * - If agentic data exists: actualScore is populated
- * - If both exist: retrievalGap and infrastructureEfficiency are computed
- *
- * @param baselineScores  Floor/ceiling scores from baseline evaluation (may be empty)
- * @param agenticScores   Actual scores from agentic evaluation (may be empty)
- */
-export declare function mergeScores(baselineScores: FeatureScore[], agenticScores: Record<string, ActualScoreEntry>): FeatureScore[];
-export {};