@sanity/ailf 0.3.1 → 0.4.1

package/dist/_vendor/ailf-core/services/scoring.d.ts

@@ -25,7 +25,19 @@ export declare function detectFeatureArea(description: string): string;
 /**
  * Extract a numeric score (0–100) from a grading component result.
  *
- * Tries: direct score field → JSON-parsed reason → bare number in reason.
+ * Tries (in order):
+ *   1. JSON-parsed reason (grader's native 0–100 scale — most reliable)
+ *   2. Direct score field (may be Promptfoo-normalized to 0–1)
+ *   3. Bare number in reason text
+ *
+ * Promptfoo's `llm-rubric` assertion normalizes `component.score` to
+ * the 0–1 range for some providers (notably GPT models, ~50% of the
+ * time) while leaving others in the grader's native 0–100 range. The
+ * `reason` field always contains the raw grader JSON, so we prefer it.
+ *
+ * When falling back to `component.score`, values in (0, 1] are rescaled
+ * to 0–100 since the rubric explicitly requests a 0–100 score and a
+ * true score of 0 or 1 out of 100 is vanishingly unlikely.
  */
 export declare function parseRubricScore(component: ComponentResult): number;
 /**

package/dist/_vendor/ailf-core/services/scoring.js

@@ -85,14 +85,22 @@ export function detectFeatureArea(description) {
 /**
  * Extract a numeric score (0–100) from a grading component result.
  *
- * Tries: direct score field → JSON-parsed reason → bare number in reason.
+ * Tries (in order):
+ *   1. JSON-parsed reason (grader's native 0–100 scale — most reliable)
+ *   2. Direct score field (may be Promptfoo-normalized to 0–1)
+ *   3. Bare number in reason text
+ *
+ * Promptfoo's `llm-rubric` assertion normalizes `component.score` to
+ * the 0–1 range for some providers (notably GPT models, ~50% of the
+ * time) while leaving others in the grader's native 0–100 range. The
+ * `reason` field always contains the raw grader JSON, so we prefer it.
+ *
+ * When falling back to `component.score`, values in (0, 1] are rescaled
+ * to 0–100 since the rubric explicitly requests a 0–100 score and a
+ * true score of 0 or 1 out of 100 is vanishingly unlikely.
  */
 export function parseRubricScore(component) {
-    // Direct score field
-    if (typeof component.score === "number") {
-        return component.score;
-    }
-    // Try to extract from reason (LLM rubric returns JSON)
+    // 1. Prefer reason-extracted score — always in the grader's native 0–100 scale
     if (component.reason) {
         try {
             const parsed = JSON.parse(component.reason);
@@ -102,15 +110,38 @@ export function parseRubricScore(component) {
             }
         }
         catch {
-            // Try to find a bare number
-            const match = component.reason.match(/(\d+)/);
-            if (match) {
-                return parseInt(match[1], 10);
-            }
+            // Not valid JSON — fall through to direct score or bare number extraction
+        }
+    }
+    // 2. Direct score field — may be Promptfoo-normalized to 0–1
+    if (typeof component.score === "number") {
+        return normalizeScore(component.score);
+    }
+    // 3. Last resort: bare number in reason text
+    if (component.reason) {
+        const match = component.reason.match(/(\d+)/);
+        if (match) {
+            return parseInt(match[1], 10);
         }
     }
     return 0;
 }
+/**
+ * Normalize a score that may be in either the 0–1 or 0–100 range.
+ *
+ * Promptfoo's `llm-rubric` assertion inconsistently normalizes
+ * `component.score` to 0–1 for some providers. Since the rubric
+ * explicitly requests scores on a 0–100 scale:
+ *   - Scores in (0, 1] are rescaled to 0–100 (e.g., 0.95 → 95)
+ *   - Score of exactly 0 stays 0 (genuine zero)
+ *   - Scores > 1 are already on the 0–100 scale
+ */
+function normalizeScore(score) {
+    if (score > 0 && score <= 1) {
+        return Math.round(score * 100);
+    }
+    return score;
+}
 // ---------------------------------------------------------------------------
 // URL metadata extraction
 // ---------------------------------------------------------------------------

package/dist/pipeline/calculate-scores.d.ts

@@ -83,6 +83,14 @@ export declare function extractGraderJudgments(resultsPath: string): GraderJudgm
  * Returns a record keyed by feature area with the composite actual score.
  */
 export declare function scoreAgenticResults(resultsPath: string, weights: Record<string, number>): Record<string, ActualScoreEntry>;
+/**
+ * Score agentic results broken down by model.
+ *
+ * Same logic as `scoreAgenticResults` but groups by provider first,
+ * producing a map of model → feature → ActualScoreEntry.
+ * Used to enrich the per-model breakdown with actual scores in full mode.
+ */
+export declare function scoreAgenticResultsPerModel(resultsPath: string, weights: Record<string, number>): Record<string, Record<string, ActualScoreEntry>>;
 /** Options for the calculate-scores main() function. */
 export interface CalculateScoresOptions {
     /** Allowed origins for source isolation reporting */

package/dist/pipeline/calculate-scores.js

@@ -648,6 +648,69 @@ export function scoreAgenticResults(resultsPath, weights) {
     }
     return entries;
 }
+/**
+ * Score agentic results broken down by model.
+ *
+ * Same logic as `scoreAgenticResults` but groups by provider first,
+ * producing a map of model → feature → ActualScoreEntry.
+ * Used to enrich the per-model breakdown with actual scores in full mode.
+ */
+export function scoreAgenticResultsPerModel(resultsPath, weights) {
+    const results = readAndNormalizeResults(resultsPath);
+    const wTask = weights["task-completion"] ?? 0.5;
+    const wCode = weights["code-correctness"] ?? 0.25;
+    const wDoc = weights["doc-coverage"] ?? 0.25;
+    // Group by model, then feature
+    const byModel = {};
+    for (const result of results) {
+        const modelId = result.providerId ?? result.providerLabel ?? "unknown";
+        const feature = result.vars.__featureArea || detectFeatureArea(result.description);
+        if (!byModel[modelId])
+            byModel[modelId] = {};
+        if (!byModel[modelId][feature])
+            byModel[modelId][feature] = [];
+        byModel[modelId][feature].push(result);
+    }
+    const perModel = {};
+    for (const [modelId, features] of Object.entries(byModel)) {
+        perModel[modelId] = {};
+        for (const [feature, featureResults] of Object.entries(features)) {
+            let totalTask = 0;
+            let totalCode = 0;
+            let totalDoc = 0;
+            let featureCost = 0;
+            const count = featureResults.length || 1;
+            for (const test of featureResults) {
+                featureCost += test.cost;
+                for (const comp of test.gradingResult.componentResults) {
+                    if (comp.assertion?.type !== "llm-rubric")
+                        continue;
+                    const score = parseRubricScore(comp);
+                    const kind = classifyRubric(comp);
+                    if (kind === "taskCompletion")
+                        totalTask += score;
+                    else if (kind === "codeCorrectness")
+                        totalCode += score;
+                    else if (kind === "docCoverage")
+                        totalDoc += score;
+                }
+            }
+            const avgTask = totalTask / count;
+            const avgCode = totalCode / count;
+            const avgDoc = totalDoc / count;
+            const actualScore = Math.round(avgTask * wTask + avgCode * wCode + avgDoc * wDoc);
+            perModel[modelId][feature] = {
+                actualScore,
+                codeCorrectness: Math.round(avgCode),
+                docCoverage: Math.round(avgDoc),
+                taskCompletion: Math.round(avgTask),
+                testCount: featureResults.length,
+                totalCost: featureCost,
+            };
+        }
+    }
+    return perModel;
+}
 // ---------------------------------------------------------------------------
 // Score merging — combine baseline floor/ceiling with agentic actual
 // ---------------------------------------------------------------------------
@@ -736,6 +799,16 @@ export function calculateAndWriteScores(options) {
         });
         scores = mergeScores(baselineScores, agenticScores);
         evaluationMode = "full";
+        // Merge agentic actual scores into the per-model breakdown
+        if (perModel) {
+            const agenticPerModel = scoreAgenticResultsPerModel(agenticResultsPath, rubricConfig.weights);
+            for (const entry of perModel) {
+                const modelAgentic = agenticPerModel[entry.modelId];
+                if (modelAgentic) {
+                    entry.scores = mergeScores(entry.scores, modelAgentic);
+                }
+            }
+        }
         // Aggregate agent behavior and source isolation from agentic results
         agentBehavior = aggregateAgentBehavior(agenticResultsPath);
         sourceIsolation = aggregateSourceIsolation(agenticResultsPath, options?.allowedOrigins);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sanity/ailf",
-  "version": "0.3.1",
+  "version": "0.4.1",
   "private": false,
   "publishConfig": {
     "access": "restricted"
@@ -40,8 +40,8 @@
     "@types/node": "^22.13.1",
     "tsx": "^4.19.2",
     "typescript": "^5.7.3",
-    "@sanity/ailf-core": "0.1.0",
     "@sanity/ailf-tasks": "0.1.4",
+    "@sanity/ailf-core": "0.1.0",
     "@sanity/ailf-shared": "0.1.0"
   },
   "scripts": {