@sanity/ailf 0.3.0 → 0.4.0

package/config/models.yaml

@@ -46,10 +46,11 @@ models:
       max_tokens: 4096
     modes: [baseline, observed, agentic-naive, agentic-optimized]
   - id: openai:chat:gpt-5.4
-    label: GPT 5.4 (high)
+    label: GPT 5.4
     config:
-      reasoning_effort: "high"
+      reasoning_effort: "medium"
       max_output_tokens: 4096
+      maxRetries: 1
     modes: [baseline, observed, agentic-naive, agentic-optimized]
 
   # ── Anthropic ───────────────────────────────────────────────

package/dist/pipeline/calculate-scores.d.ts

@@ -83,6 +83,14 @@ export declare function extractGraderJudgments(resultsPath: string): GraderJudgm
  * Returns a record keyed by feature area with the composite actual score.
  */
 export declare function scoreAgenticResults(resultsPath: string, weights: Record<string, number>): Record<string, ActualScoreEntry>;
+/**
+ * Score agentic results broken down by model.
+ *
+ * Same logic as `scoreAgenticResults` but groups by provider first,
+ * producing a map of model → feature → ActualScoreEntry.
+ * Used to enrich the per-model breakdown with actual scores in full mode.
+ */
+export declare function scoreAgenticResultsPerModel(resultsPath: string, weights: Record<string, number>): Record<string, Record<string, ActualScoreEntry>>;
 /** Options for the calculate-scores main() function. */
 export interface CalculateScoresOptions {
     /** Allowed origins for source isolation reporting */

package/dist/pipeline/calculate-scores.js

@@ -648,6 +648,69 @@ export function scoreAgenticResults(resultsPath, weights) {
     }
     return entries;
 }
+/**
+ * Score agentic results broken down by model.
+ *
+ * Same logic as `scoreAgenticResults` but groups by provider first,
+ * producing a map of model → feature → ActualScoreEntry.
+ * Used to enrich the per-model breakdown with actual scores in full mode.
+ */
+export function scoreAgenticResultsPerModel(resultsPath, weights) {
+    const results = readAndNormalizeResults(resultsPath);
+    const wTask = weights["task-completion"] ?? 0.5;
+    const wCode = weights["code-correctness"] ?? 0.25;
+    const wDoc = weights["doc-coverage"] ?? 0.25;
+    // Group by model, then feature
+    const byModel = {};
+    for (const result of results) {
+        const modelId = result.providerId ?? result.providerLabel ?? "unknown";
+        const feature = result.vars.__featureArea || detectFeatureArea(result.description);
+        if (!byModel[modelId])
+            byModel[modelId] = {};
+        if (!byModel[modelId][feature])
+            byModel[modelId][feature] = [];
+        byModel[modelId][feature].push(result);
+    }
+    const perModel = {};
+    for (const [modelId, features] of Object.entries(byModel)) {
+        perModel[modelId] = {};
+        for (const [feature, featureResults] of Object.entries(features)) {
+            let totalTask = 0;
+            let totalCode = 0;
+            let totalDoc = 0;
+            let featureCost = 0;
+            const count = featureResults.length || 1;
+            for (const test of featureResults) {
+                featureCost += test.cost;
+                for (const comp of test.gradingResult.componentResults) {
+                    if (comp.assertion?.type !== "llm-rubric")
+                        continue;
+                    const score = parseRubricScore(comp);
+                    const kind = classifyRubric(comp);
+                    if (kind === "taskCompletion")
+                        totalTask += score;
+                    else if (kind === "codeCorrectness")
+                        totalCode += score;
+                    else if (kind === "docCoverage")
+                        totalDoc += score;
+                }
+            }
+            const avgTask = totalTask / count;
+            const avgCode = totalCode / count;
+            const avgDoc = totalDoc / count;
+            const actualScore = Math.round(avgTask * wTask + avgCode * wCode + avgDoc * wDoc);
+            perModel[modelId][feature] = {
+                actualScore,
+                codeCorrectness: Math.round(avgCode),
+                docCoverage: Math.round(avgDoc),
+                taskCompletion: Math.round(avgTask),
+                testCount: featureResults.length,
+                totalCost: featureCost,
+            };
+        }
+    }
+    return perModel;
+}
 // ---------------------------------------------------------------------------
 // Score merging — combine baseline floor/ceiling with agentic actual
 // ---------------------------------------------------------------------------
@@ -736,6 +799,16 @@ export function calculateAndWriteScores(options) {
         });
         scores = mergeScores(baselineScores, agenticScores);
         evaluationMode = "full";
+        // Merge agentic actual scores into the per-model breakdown
+        if (perModel) {
+            const agenticPerModel = scoreAgenticResultsPerModel(agenticResultsPath, rubricConfig.weights);
+            for (const entry of perModel) {
+                const modelAgentic = agenticPerModel[entry.modelId];
+                if (modelAgentic) {
+                    entry.scores = mergeScores(entry.scores, modelAgentic);
+                }
+            }
+        }
         // Aggregate agent behavior and source isolation from agentic results
         agentBehavior = aggregateAgentBehavior(agenticResultsPath);
         sourceIsolation = aggregateSourceIsolation(agenticResultsPath, options?.allowedOrigins);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sanity/ailf",
-  "version": "0.3.0",
+  "version": "0.4.0",
   "private": false,
   "publishConfig": {
     "access": "restricted"
@@ -40,9 +40,9 @@
     "@types/node": "^22.13.1",
     "tsx": "^4.19.2",
     "typescript": "^5.7.3",
+    "@sanity/ailf-core": "0.1.0",
     "@sanity/ailf-shared": "0.1.0",
-    "@sanity/ailf-tasks": "0.1.4",
-    "@sanity/ailf-core": "0.1.0"
+    "@sanity/ailf-tasks": "0.1.4"
   },
   "scripts": {
     "build": "tsc && tsx scripts/bundle-workspace-deps.ts",