@sanity/ailf 0.1.0

package/dist/_vendor/ailf-core/services/scoring.js

@@ -0,0 +1,222 @@
+/**
+ * core/services/scoring.ts — Pure scoring functions.
+ *
+ * These functions operate on typed inputs and produce typed outputs
+ * with zero I/O (no filesystem, network, or console access).
+ *
+ * Extracted from packages/eval/src/lib/calculate-scores.ts during
+ * the Ports & Adapters migration (Phase 4e).
+ */
+// ---------------------------------------------------------------------------
+// Rubric classification
+// ---------------------------------------------------------------------------
+/**
+ * Classify a grading component into a scoring dimension.
+ *
+ * Prefers structured metadata (Approach 5) over heuristic string matching.
+ * Returns null if the component doesn't map to a known dimension.
+ */
+export function classifyRubric(component) {
+    // Prefer structured metadata (Approach 5) over heuristic matching
+    const metadata = component.assertion?.metadata;
+    if (metadata?.dimension) {
+        switch (metadata.dimension) {
+            case "code-correctness":
+                return "codeCorrectness";
+            case "doc-coverage":
+                return "docCoverage";
+            case "task-completion":
+                return "taskCompletion";
+            default:
+                return null;
+        }
+    }
+    // Fallback: heuristic name matching (for backward compatibility)
+    const value = (component.assertion?.value ?? "").toLowerCase();
+    if (value.includes("task completion")) {
+        return "taskCompletion";
+    }
+    if (value.includes("code correctness")) {
+        return "codeCorrectness";
+    }
+    if (value.includes("documentation coverage") ||
+        value.includes("hallucinate")) {
+        return "docCoverage";
+    }
+    return null;
+}
+// ---------------------------------------------------------------------------
+// Feature area detection
+// ---------------------------------------------------------------------------
+/**
+ * Detect the feature area from a test description string.
+ *
+ * Uses keyword matching on the lowercased description.
+ */
+export function detectFeatureArea(description) {
+    const desc = description.toLowerCase();
+    if (desc.includes("studio")) {
+        return "studio-setup";
+    }
+    if (desc.includes("visual") ||
+        desc.includes("presentation") ||
+        desc.includes("live preview")) {
+        return "visual-editing";
+    }
+    if (desc.includes("function") || desc.includes("webhook")) {
+        return "functions";
+    }
+    if (desc.startsWith("groq")) {
+        return "groq";
+    }
+    if (desc.includes("next") || desc.includes("app router")) {
+        return "nextjs-live";
+    }
+    if (desc.includes("remix") ||
+        desc.includes("nuxt") ||
+        desc.includes("svelte")) {
+        return "frameworks";
+    }
+    return "other";
+}
+// ---------------------------------------------------------------------------
+// Score parsing
+// ---------------------------------------------------------------------------
+/**
+ * Extract a numeric score (0–100) from a grading component result.
+ *
+ * Tries: direct score field → JSON-parsed reason → bare number in reason.
+ */
+export function parseRubricScore(component) {
+    // Direct score field
+    if (typeof component.score === "number") {
+        return component.score;
+    }
+    // Try to extract from reason (LLM rubric returns JSON)
+    if (component.reason) {
+        try {
+            const parsed = JSON.parse(component.reason);
+            const obj = parsed;
+            if (typeof obj.score === "number") {
+                return obj.score;
+            }
+        }
+        catch {
+            // Try to find a bare number
+            const match = component.reason.match(/(\d+)/);
+            if (match) {
+                return parseInt(match[1], 10);
+            }
+        }
+    }
+    return 0;
+}
+// ---------------------------------------------------------------------------
+// URL metadata extraction
+// ---------------------------------------------------------------------------
+/**
+ * Extract URL metadata from a test result's JavaScript assertion output.
+ *
+ * Returns null if the test doesn't contain URL extraction data.
+ */
+export function extractUrlMetadata(test) {
+    for (const comp of test.gradingResult.componentResults) {
+        if (comp.assertion?.type !== "javascript") {
+            continue;
+        }
+        if (!comp.reason) {
+            continue;
+        }
+        try {
+            const parsed = JSON.parse(comp.reason);
+            const obj = parsed;
+            if (Array.isArray(obj.sanityUrls) &&
+                typeof obj.totalUrlCount === "number") {
+                return parsed;
+            }
+        }
+        catch {
+            // Not the URL-extraction assertion - skip
+        }
+    }
+    return null;
+}
+// ---------------------------------------------------------------------------
+// Score merging (three-layer decomposition)
+// ---------------------------------------------------------------------------
+/**
+ * Merge baseline FeatureScore[] with agentic actual scores to produce
+ * the full three-layer decomposition.
+ *
+ * The merge is per feature area. For each area:
+ * - If baseline data exists: floor, ceiling, docLift, docQualityGap are populated
+ * - If agentic data exists: actualScore is populated
+ * - If both exist: retrievalGap and infrastructureEfficiency are computed
+ *
+ * @param baselineScores  Floor/ceiling scores from baseline evaluation (may be empty)
+ * @param agenticScores   Actual scores from agentic evaluation (may be empty)
+ */
+export function mergeScores(baselineScores, agenticScores) {
+    // Build a map of baseline scores by feature
+    const baselineMap = new Map();
+    for (const score of baselineScores) {
+        baselineMap.set(score.feature, score);
+    }
+    // Collect all feature areas from both sources
+    const allFeatures = new Set([
+        ...baselineScores.map((s) => s.feature),
+        ...Object.keys(agenticScores),
+    ]);
+    const merged = [];
+    for (const feature of allFeatures) {
+        const baseline = baselineMap.get(feature);
+        const agentic = agenticScores[feature];
+        if (baseline && agentic) {
+            // Both data sources — full decomposition
+            const retrievalGap = baseline.ceilingScore - agentic.actualScore;
+            const negativeDocLift = baseline.docLift < 0;
+            // Infrastructure efficiency: actual / ceiling
+            // Null when ceiling ≤ 0 or negative Doc Lift (the metric is meaningless
+            // when docs hurt performance — see evaluation-ceiling.md)
+            let infrastructureEfficiency = null;
+            if (!negativeDocLift && baseline.ceilingScore > 0) {
+                infrastructureEfficiency = agentic.actualScore / baseline.ceilingScore;
+            }
+            // Inverted retrieval gap: agents outperform injected docs.
+            // This happens when Doc Lift is negative AND actual > ceiling.
+            // It means retrieval failure is masking a doc quality problem.
+            const invertedRetrievalGap = negativeDocLift && retrievalGap < 0;
+            merged.push({
+                ...baseline,
+                actualScore: agentic.actualScore,
+                infrastructureEfficiency,
+                invertedRetrievalGap: invertedRetrievalGap || undefined,
+                retrievalGap,
+                totalCost: baseline.totalCost + agentic.totalCost,
+            });
+        }
+        else if (baseline) {
+            // Baseline only — no agentic data (partial summary)
+            merged.push({ ...baseline });
+        }
+        else if (agentic) {
+            // Agentic only — no baseline data (partial summary)
+            merged.push({
+                actualScore: agentic.actualScore,
+                ceilingScore: 0,
+                codeCorrectness: agentic.codeCorrectness,
+                docCoverage: agentic.docCoverage,
+                docLift: 0,
+                docQualityGap: 100,
+                feature,
+                floorScore: 0,
+                negativeDocLift: false,
+                taskCompletion: agentic.taskCompletion,
+                testCount: agentic.testCount,
+                totalCost: agentic.totalCost,
+                totalScore: agentic.actualScore,
+            });
+        }
+    }
+    return merged.sort((a, b) => a.feature.localeCompare(b.feature));
+}