@sanity/ailf 0.4.0 → 0.4.1

package/dist/_vendor/ailf-core/services/scoring.d.ts

@@ -25,7 +25,19 @@ export declare function detectFeatureArea(description: string): string;
 /**
  * Extract a numeric score (0–100) from a grading component result.
  *
- * Tries: direct score field → JSON-parsed reason → bare number in reason.
+ * Tries (in order):
+ *   1. JSON-parsed reason (grader's native 0–100 scale — most reliable)
+ *   2. Direct score field (may be Promptfoo-normalized to 0–1)
+ *   3. Bare number in reason text
+ *
+ * Promptfoo's `llm-rubric` assertion normalizes `component.score` to
+ * the 0–1 range for some providers (notably GPT models, ~50% of the
+ * time) while leaving others in the grader's native 0–100 range. The
+ * `reason` field always contains the raw grader JSON, so we prefer it.
+ *
+ * When falling back to `component.score`, values in (0, 1] are rescaled
+ * to 0–100 since the rubric explicitly requests a 0–100 score and a
+ * true score of 0 or 1 out of 100 is vanishingly unlikely.
  */
 export declare function parseRubricScore(component: ComponentResult): number;
 /**

package/dist/_vendor/ailf-core/services/scoring.js

@@ -85,14 +85,22 @@ export function detectFeatureArea(description) {
 /**
  * Extract a numeric score (0–100) from a grading component result.
  *
- * Tries: direct score field → JSON-parsed reason → bare number in reason.
+ * Tries (in order):
+ *   1. JSON-parsed reason (grader's native 0–100 scale — most reliable)
+ *   2. Direct score field (may be Promptfoo-normalized to 0–1)
+ *   3. Bare number in reason text
+ *
+ * Promptfoo's `llm-rubric` assertion normalizes `component.score` to
+ * the 0–1 range for some providers (notably GPT models, ~50% of the
+ * time) while leaving others in the grader's native 0–100 range. The
+ * `reason` field always contains the raw grader JSON, so we prefer it.
+ *
+ * When falling back to `component.score`, values in (0, 1] are rescaled
+ * to 0–100 since the rubric explicitly requests a 0–100 score and a
+ * true score of 0 or 1 out of 100 is vanishingly unlikely.
  */
 export function parseRubricScore(component) {
-    // Direct score field
-    if (typeof component.score === "number") {
-        return component.score;
-    }
-    // Try to extract from reason (LLM rubric returns JSON)
+    // 1. Prefer reason-extracted score — always in the grader's native 0–100 scale
     if (component.reason) {
         try {
             const parsed = JSON.parse(component.reason);
@@ -102,15 +110,38 @@ export function parseRubricScore(component) {
             }
         }
         catch {
-            // Try to find a bare number
-            const match = component.reason.match(/(\d+)/);
-            if (match) {
-                return parseInt(match[1], 10);
-            }
+            // Not valid JSON — fall through to direct score or bare number extraction
+        }
+    }
+    // 2. Direct score field — may be Promptfoo-normalized to 0–1
+    if (typeof component.score === "number") {
+        return normalizeScore(component.score);
+    }
+    // 3. Last resort: bare number in reason text
+    if (component.reason) {
+        const match = component.reason.match(/(\d+)/);
+        if (match) {
+            return parseInt(match[1], 10);
         }
     }
     return 0;
 }
+/**
+ * Normalize a score that may be in either the 0–1 or 0–100 range.
+ *
+ * Promptfoo's `llm-rubric` assertion inconsistently normalizes
+ * `component.score` to 0–1 for some providers. Since the rubric
+ * explicitly requests scores on a 0–100 scale:
+ *   - Scores in (0, 1] are rescaled to 0–100 (e.g., 0.95 → 95)
+ *   - Score of exactly 0 stays 0 (genuine zero)
+ *   - Scores > 1 are already on the 0–100 scale
+ */
+function normalizeScore(score) {
+    if (score > 0 && score <= 1) {
+        return Math.round(score * 100);
+    }
+    return score;
+}
 // ---------------------------------------------------------------------------
 // URL metadata extraction
 // ---------------------------------------------------------------------------

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sanity/ailf",
-  "version": "0.4.0",
+  "version": "0.4.1",
   "private": false,
   "publishConfig": {
     "access": "restricted"
@@ -40,9 +40,9 @@
     "@types/node": "^22.13.1",
     "tsx": "^4.19.2",
     "typescript": "^5.7.3",
+    "@sanity/ailf-tasks": "0.1.4",
     "@sanity/ailf-core": "0.1.0",
-    "@sanity/ailf-shared": "0.1.0",
-    "@sanity/ailf-tasks": "0.1.4"
+    "@sanity/ailf-shared": "0.1.0"
   },
   "scripts": {
     "build": "tsc && tsx scripts/bundle-workspace-deps.ts",