@sanity/ailf 0.2.0 → 0.3.1

package/dist/pipeline/calculate-scores.js

@@ -27,6 +27,7 @@
 import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
 import { join } from "path";
 import { calculateCost } from "../agent-observer/pricing.js";
+import { ConsoleLogger } from "../adapters/loggers/index.js";
 import { checkResultsExist } from "./checks.js";
 import { loadRubricTemplates } from "./expand-tasks.js";
 import { loadSource } from "../sources.js";
@@ -375,63 +376,63 @@ function extractGraderCost(resultsPath) {
 /**
  * Prints a formatted report of agent behavior observations.
  */
-function printAgentBehaviorReport(agentBehavior) {
-    console.log("-".repeat(80));
-    console.log("AGENT BEHAVIOR OBSERVATION");
-    console.log("-".repeat(80));
-    console.log();
+function printAgentBehaviorReport(agentBehavior, log) {
+    log.info("-".repeat(80));
+    log.info("AGENT BEHAVIOR OBSERVATION");
+    log.info("-".repeat(80));
+    log.info("");
     // Summary table
     const h = "| Feature Area        | Tests | Doc Pages | Searches | Net (ms) |";
     const sep = "|---------------------|-------|-----------|----------|----------|";
-    console.log(h);
-    console.log(sep);
+    log.info(h);
+    log.info(sep);
     for (const ab of agentBehavior) {
-        console.log(`| ${ab.feature.padEnd(19)} | ` +
+        log.info(`| ${ab.feature.padEnd(19)} | ` +
             `${ab.tasksWithBehaviorData.toString().padStart(5)} | ` +
             `${ab.avgDocPagesVisited.toFixed(1).padStart(9)} | ` +
             `${ab.avgSearchesPerformed.toFixed(1).padStart(8)} | ` +
             `${Math.round(ab.avgNetworkTimeMs).toString().padStart(8)} |`);
     }
-    console.log();
+    log.info("");
     // Doc pages visited
-    console.log("  Doc pages visited:");
+    log.info("  Doc pages visited:");
     for (const ab of agentBehavior) {
         if (ab.docSlugsVisited.length === 0) {
-            console.log(`    ${ab.feature}: (none)`);
+            log.info(`    ${ab.feature}: (none)`);
         }
         else {
-            console.log(`    ${ab.feature}:`);
+            log.info(`    ${ab.feature}:`);
             for (const slug of ab.docSlugsVisited) {
-                console.log(`      - /docs/${slug}`);
+                log.info(`      - /docs/${slug}`);
             }
         }
     }
-    console.log();
+    log.info("");
     // Search queries
     const hasSearches = agentBehavior.some((ab) => ab.searchQueries.length > 0);
     if (hasSearches) {
-        console.log("  Search queries:");
+        log.info("  Search queries:");
         for (const ab of agentBehavior) {
             if (ab.searchQueries.length === 0) {
                 continue;
             }
-            console.log(`    ${ab.feature}:`);
+            log.info(`    ${ab.feature}:`);
             for (const q of ab.searchQueries) {
-                console.log(`      - "${q}"`);
+                log.info(`      - "${q}"`);
             }
         }
-        console.log();
+        log.info("");
     }
     // External domains
     const allExternalDomains = [
         ...new Set(agentBehavior.flatMap((ab) => ab.externalDomains)),
     ];
     if (allExternalDomains.length > 0) {
-        console.log("  External domains contacted:");
+        log.info("  External domains contacted:");
         for (const d of allExternalDomains) {
-            console.log(`    - ${d}`);
+            log.info(`    - ${d}`);
         }
-        console.log();
+        log.info("");
     }
 }
 // ---------------------------------------------------------------------------
@@ -444,9 +445,15 @@ function printAgentBehaviorReport(agentBehavior) {
  * Reads the raw Promptfoo output file and normalizes each result so that
  * `description` is always a top-level field (pulled from `testCase` if needed).
  */
-function readAndNormalizeResults(resultsPath) {
+function readAndNormalizeResults(resultsPath, log) {
+    const _log = log ?? new ConsoleLogger();
     const file = JSON.parse(readFileSync(resultsPath, "utf-8"));
     const wrapper = file.results ?? file;
+    _log.debug("Reading results file", {
+        path: resultsPath,
+        resultCount: wrapper.results.length,
+        stats: wrapper.stats,
+    });
     const all = wrapper.results.map((r) => ({
         cost: r.cost ?? 0,
         description: r.testCase?.description ?? "unknown",
@@ -463,15 +470,20 @@ function readAndNormalizeResults(resultsPath) {
     // Promptfoo sets gradingResult to null when a test errors before grading.
     const valid = all.filter((r) => r.gradingResult !== null);
     const skipped = all.length - valid.length;
+    _log.debug("Filtered null gradingResults", {
+        totalResults: all.length,
+        validResults: valid.length,
+        skippedCount: skipped,
+    });
     if (skipped > 0) {
-        console.warn(`  ⚠ Skipping ${skipped} of ${all.length} result(s) with null gradingResult (errored tests):`);
+        _log.warn(`⚠ Skipping ${skipped} of ${all.length} result(s) with null gradingResult (errored tests):`);
         for (const r of all) {
             if (r.gradingResult === null) {
                 const providerLabel = r.provider ? `[${r.provider}] ` : "";
                 const errorMsg = r.error
                     ? r.error.slice(0, 150)
                     : "unknown error (no error field in result)";
-                console.warn(`    ✗ ${providerLabel}"${r.description}" — ${errorMsg}`);
+                _log.warn(`✗ ${providerLabel}"${r.description}" — ${errorMsg}`);
             }
         }
     }
@@ -555,7 +567,7 @@ function scoreResults(results, weights, modelId) {
         const ceilingScore = Math.round(withDocsTotal);
         const floorScore = Math.round(withoutDocsScore);
         const docLift = ceilingScore - floorScore;
-        scores.push({
+        const featureScore = {
             ceilingScore,
             codeCorrectness: Math.round(avgCode),
             docCoverage: Math.round(avgDoc),
@@ -569,7 +581,8 @@ function scoreResults(results, weights, modelId) {
             testCount: data.withDocs.length,
             totalCost: featureCost,
             totalScore: ceilingScore,
-        });
+        };
+        scores.push(featureScore);
     }
     return scores.sort((a, b) => a.feature.localeCompare(b.feature));
 }
@@ -654,6 +667,7 @@ export function scoreAgenticResults(resultsPath, weights) {
 const CRITICAL_THRESHOLD = 40;
 export function calculateAndWriteScores(options) {
     const ROOT = options.rootDir;
+    const log = options.logger ?? new ConsoleLogger();
     const sourceName = options.source;
     // Pre-resolved source wins over name-based lookup
     let source = options.resolvedSource;
@@ -662,7 +676,7 @@ export function calculateAndWriteScores(options) {
             source = loadSource(sourceName);
         }
         catch {
-            console.warn(`  [warn] Could not load source "${sourceName}", proceeding without source metadata`);
+            log.warn(`[warn] Could not load source "${sourceName}", proceeding without source metadata`);
         }
     }
     // Determine mode — controls which result files are read
@@ -679,13 +693,23 @@ export function calculateAndWriteScores(options) {
             .join("; ");
         throw new Error(`Results validation failed: ${details}. Run 'pnpm eval' first to generate results.`);
     }
-    console.log(`Reading results from: ${baselineResultsPath}`);
+    log.info(`Reading results from: ${baselineResultsPath}`);
     if (source) {
-        console.log(`Source: ${sourceName} (${source.baseUrl})`);
+        log.info(`Source: ${sourceName} (${source.baseUrl})`);
     }
     // Load dimension weights from rubrics.yaml
     const rubricConfig = loadRubricTemplates(ROOT);
+    log.debug("Loaded rubric weights", { weights: rubricConfig.weights });
     const baselineScores = calculateScores(baselineResultsPath, rubricConfig.weights);
+    log.debug("Baseline scores calculated", {
+        featureCount: baselineScores.length,
+        features: baselineScores.map((s) => ({
+            feature: s.feature,
+            ceilingScore: s.ceilingScore,
+            floorScore: s.floorScore,
+            docLift: s.docLift,
+        })),
+    });
     const perModel = calculateScoresPerModel(baselineResultsPath, rubricConfig.weights);
     const urlRefs = aggregateUrlReferences(baselineResultsPath);
     const sourceVerification = buildSourceVerification(ROOT, source, {
@@ -700,8 +724,16 @@ export function calculateAndWriteScores(options) {
     let sourceIsolation = null;
     let evaluationMode;
     if (mode === "full" && existsSync(agenticResultsPath)) {
-        console.log(`\nReading agentic results from: ${agenticResultsPath}`);
+        log.info(`\nReading agentic results from: ${agenticResultsPath}`);
         const agenticScores = scoreAgenticResults(agenticResultsPath, rubricConfig.weights);
+        log.debug("Agentic scores calculated", {
+            featureCount: Object.keys(agenticScores).length,
+            features: Object.entries(agenticScores).map(([f, s]) => ({
+                feature: f,
+                actualScore: s.actualScore,
+                testCount: s.testCount,
+            })),
+        });
         scores = mergeScores(baselineScores, agenticScores);
         evaluationMode = "full";
         // Aggregate agent behavior and source isolation from agentic results
@@ -728,12 +760,12 @@ export function calculateAndWriteScores(options) {
         sourceIsolation = aggregateSourceIsolation(baselineResultsPath, options?.allowedOrigins);
         evaluationMode = mode === "observed" ? "observed" : "baseline";
     }
-    const summary = printReport(scores, urlRefs, source, agentBehavior, graderCost, perModel, sourceIsolation, sourceVerification, evaluationMode);
+    const summary = printReport(scores, urlRefs, source, agentBehavior, graderCost, perModel, sourceIsolation, sourceVerification, evaluationMode, log);
     // Persist
     const outDir = join(ROOT, "results", "latest");
     mkdirSync(outDir, { recursive: true });
     writeFileSync(join(outDir, "score-summary.json"), JSON.stringify(summary, null, 2));
-    console.log("Score summary written to results/latest/score-summary.json");
+    log.info("Score summary written to results/latest/score-summary.json");
     // Extract and persist grader judgments (Phase 3a: failure mode extraction)
     const judgments = extractGraderJudgments(baselineResultsPath);
     // In full mode, also extract judgments from agentic results
@@ -743,20 +775,60 @@ export function calculateAndWriteScores(options) {
     }
     if (judgments.length > 0) {
         writeFileSync(join(outDir, "grader-judgments.json"), JSON.stringify(judgments, null, 2));
-        console.log(`Grader judgments written to results/latest/grader-judgments.json (${judgments.length} judgments)`);
+        log.info(`Grader judgments written to results/latest/grader-judgments.json (${judgments.length} judgments)`);
     }
-    return { belowCritical: summary.belowCritical };
+    // Compute test summary from the raw results file
+    const testSummary = computeTestSummary(baselineResultsPath);
+    return { belowCritical: summary.belowCritical, testSummary };
+}
+/**
+ * Compute a TestSummary from a raw Promptfoo results file.
+ * Counts total, passed, failed, and errored tests with error details.
+ */
+function computeTestSummary(resultsPath) {
+    const file = JSON.parse(readFileSync(resultsPath, "utf-8"));
+    const wrapper = file.results ?? file;
+    const rawResults = wrapper.results;
+    let passed = 0;
+    let failed = 0;
+    let errored = 0;
+    const errors = [];
+    for (const r of rawResults) {
+        if (r.gradingResult === null || r.gradingResult === undefined) {
+            errored++;
+            errors.push({
+                model: r.provider?.label ?? r.provider?.id ?? "unknown",
+                task: r.testCase?.description ?? "unknown",
+                error: r.error
+                    ? r.error.slice(0, 200)
+                    : "unknown error (null gradingResult)",
+            });
+        }
+        else if (r.gradingResult.pass) {
+            passed++;
+        }
+        else {
+            failed++;
+        }
+    }
+    return {
+        total: rawResults.length,
+        passed,
+        failed,
+        errored,
+        ...(errors.length > 0 ? { errors } : {}),
+    };
 }
-function printPerModelReport(perModel) {
-    console.log("-".repeat(80));
-    console.log("PER-MODEL BREAKDOWN");
-    console.log("-".repeat(80));
-    console.log();
+function printPerModelReport(perModel, log) {
+    log.info("-".repeat(80));
+    log.info("PER-MODEL BREAKDOWN");
+    log.info("-".repeat(80));
+    log.info("");
     // Model summary table
     const h = "| Model                          | Avg Score | Avg Lift | Tests |     Cost |";
     const sep = "|--------------------------------|-----------|----------|-------|----------|";
-    console.log(h);
-    console.log(sep);
+    log.info(h);
+    log.info(sep);
     const sorted = [...perModel].sort((a, b) => b.overall.avgScore - a.overall.avgScore);
     for (const entry of sorted) {
         const displayName = entry.label || entry.modelId;
@@ -766,63 +838,64 @@ function printPerModelReport(perModel) {
         const liftStr = entry.overall.avgDocLift >= 0
             ? `+${entry.overall.avgDocLift.toFixed(1)}`
             : entry.overall.avgDocLift.toFixed(1);
-        console.log(`| ${displayName.padEnd(30)} | ` +
+        log.info(`| ${displayName.padEnd(30)} | ` +
             `${entry.overall.avgScore.toFixed(1).padStart(9)} | ` +
             `${liftStr.padStart(8)} | ` +
             `${entry.overall.testCount.toString().padStart(5)} | ` +
             `${costStr.padStart(8)} |`);
     }
-    console.log();
+    log.info("");
     // Per-model × per-area breakdown
     for (const entry of sorted) {
         const displayName = entry.label || entry.modelId;
-        console.log(`  ${displayName} (${entry.modelId}):`);
+        log.info(`  ${displayName} (${entry.modelId}):`);
         const areaH = "    | Feature Area        | Task | Code | Docs | Total | Lift |";
         const areaSep = "    |---------------------|------|------|------|-------|------|";
-        console.log(areaH);
-        console.log(areaSep);
+        log.info(areaH);
+        log.info(areaSep);
         for (const s of entry.scores) {
             const lift = s.docLift >= 0 ? `+${s.docLift}` : `${s.docLift}`;
-            console.log(`    | ${s.feature.padEnd(19)} | ` +
+            log.info(`    | ${s.feature.padEnd(19)} | ` +
                 `${s.taskCompletion.toString().padStart(4)} | ` +
                 `${s.codeCorrectness.toString().padStart(4)} | ` +
                 `${s.docCoverage.toString().padStart(4)} | ` +
                 `${s.totalScore.toString().padStart(5)} | ` +
                 `${lift.padStart(4)} |`);
         }
-        console.log();
+        log.info("");
     }
     // Cost-per-quality-point
     const modelsWithCost = sorted.filter((e) => e.overall.cost && e.overall.cost > 0);
     if (modelsWithCost.length > 0) {
-        console.log("  Cost per quality point:");
+        log.info("  Cost per quality point:");
         for (const entry of modelsWithCost) {
             const displayName = entry.label;
             const costPerPoint = entry.overall.avgScore > 0
                 ? (entry.overall.cost ?? 0) / entry.overall.avgScore
                 : 0;
-            console.log(`    ${displayName}: $${costPerPoint.toFixed(6)}/point (score: ${entry.overall.avgScore.toFixed(1)}, cost: $${(entry.overall.cost ?? 0).toFixed(4)})`);
+            log.info(`    ${displayName}: $${costPerPoint.toFixed(6)}/point (score: ${entry.overall.avgScore.toFixed(1)}, cost: $${(entry.overall.cost ?? 0).toFixed(4)})`);
         }
-        console.log();
+        log.info("");
     }
 }
 // ---------------------------------------------------------------------------
 // Main
 // ---------------------------------------------------------------------------
-function printReport(scores, urlRefs, source, agentBehavior, graderCost, perModel, sourceIsolation, sourceVerification, evaluationMode) {
-    console.log("\n" + "=".repeat(80));
-    console.log("                    SANITY AI LITERACY SCORE REPORT");
-    console.log("=".repeat(80));
-    console.log();
+function printReport(scores, urlRefs, source, agentBehavior, graderCost, perModel, sourceIsolation, sourceVerification, evaluationMode, log) {
+    const _log = log ?? new ConsoleLogger();
+    _log.info("\n" + "=".repeat(80));
+    _log.info("                    SANITY AI LITERACY SCORE REPORT");
+    _log.info("=".repeat(80));
+    _log.info("");
     // Table header
     const h = "| Feature Area        | Task | Code | Docs | Total | w/o Docs | Doc Lift |";
     const sep = "|---------------------|------|------|------|-------|----------|----------|";
-    console.log(h);
-    console.log(sep);
+    _log.info(h);
+    _log.info(sep);
     for (const s of scores) {
         const status = s.totalScore < CRITICAL_THRESHOLD ? "!!" : "ok";
         const lift = s.docLift > 0 ? `+${s.docLift}` : `${s.docLift}`;
-        console.log(`| ${status} ${s.feature.padEnd(17)} | ` +
+        _log.info(`| ${status} ${s.feature.padEnd(17)} | ` +
             `${s.taskCompletion.toString().padStart(4)} | ` +
             `${s.codeCorrectness.toString().padStart(4)} | ` +
             `${s.docCoverage.toString().padStart(4)} | ` +
@@ -830,7 +903,7 @@ function printReport(scores, urlRefs, source, agentBehavior, graderCost, perMode
             `${s.floorScore.toString().padStart(8)} | ` +
             `${lift.padStart(8)} |`);
     }
-    console.log();
+    _log.info("");
     // OKR status
     const belowCritical = scores.filter((s) => s.totalScore < CRITICAL_THRESHOLD);
     const lowestScore = scores.reduce((min, s) => s.totalScore < min.totalScore ? s : min);
@@ -844,69 +917,69 @@ function printReport(scores, urlRefs, source, agentBehavior, graderCost, perMode
         area: s.feature,
         docLift: s.docLift,
     }));
-    console.log("-".repeat(80));
-    console.log("OKR STATUS");
-    console.log("-".repeat(80));
-    console.log();
+    _log.info("-".repeat(80));
+    _log.info("OKR STATUS");
+    _log.info("-".repeat(80));
+    _log.info("");
     if (belowCritical.length === 0) {
-        console.log("  KR1: PASS -- All areas above critical threshold (>=40)");
+        _log.info("  KR1: PASS -- All areas above critical threshold (>=40)");
     }
     else {
-        console.log("  KR1: FAIL -- Areas below critical threshold:");
-        belowCritical.forEach((s) => console.log(`        - ${s.feature}: ${s.totalScore}`));
+        _log.info("  KR1: FAIL -- Areas below critical threshold:");
+        belowCritical.forEach((s) => _log.info(`        - ${s.feature}: ${s.totalScore}`));
     }
-    console.log();
-    console.log(`  Lowest area: ${lowestScore.feature} (${lowestScore.totalScore})`);
-    console.log(`  Target: +15 points improvement`);
-    console.log();
-    console.log(`  Avg score:     ${avgScore.toFixed(1)}`);
-    console.log(`  Avg doc lift:  +${avgLift.toFixed(1)} points`);
-    console.log(`  (Doc lift = how much docs help vs parametric knowledge alone)`);
-    console.log();
+    _log.info("");
+    _log.info(`  Lowest area: ${lowestScore.feature} (${lowestScore.totalScore})`);
+    _log.info(`  Target: +15 points improvement`);
+    _log.info("");
+    _log.info(`  Avg score:     ${avgScore.toFixed(1)}`);
+    _log.info(`  Avg doc lift:  +${avgLift.toFixed(1)} points`);
+    _log.info(`  (Doc lift = how much docs help vs parametric knowledge alone)`);
+    _log.info("");
     // Ceiling decomposition
-    console.log("-".repeat(80));
-    console.log("CEILING DECOMPOSITION");
-    console.log("-".repeat(80));
-    console.log();
+    _log.info("-".repeat(80));
+    _log.info("CEILING DECOMPOSITION");
+    _log.info("-".repeat(80));
+    _log.info("");
     const ceilH = "| Feature Area        | Floor | Ceiling | Doc Lift | Quality Gap |";
     const ceilSep = "|---------------------|-------|---------|----------|-------------|";
-    console.log(ceilH);
-    console.log(ceilSep);
+    _log.info(ceilH);
+    _log.info(ceilSep);
     for (const s of scores) {
         const liftStr = s.docLift >= 0 ? `+${s.docLift}` : `${s.docLift}`;
         const liftFlag = s.negativeDocLift ? " 🚨" : "";
-        console.log(`| ${s.feature.padEnd(19)} | ` +
+        _log.info(`| ${s.feature.padEnd(19)} | ` +
             `${s.floorScore.toString().padStart(5)} | ` +
             `${s.ceilingScore.toString().padStart(7)} | ` +
             `${liftStr.padStart(8)}${liftFlag} | ` +
             `${s.docQualityGap.toString().padStart(11)} |`);
     }
-    console.log();
+    _log.info("");
     if (negativeDocLiftAreas.length > 0) {
-        console.log("  🚨 NEGATIVE DOC LIFT DETECTED:");
+        _log.info("  🚨 NEGATIVE DOC LIFT DETECTED:");
         for (const { area, docLift } of negativeDocLiftAreas) {
             const s = scores.find((sc) => sc.feature === area);
-            console.log(`     ${area}: Doc Lift = ${docLift} (floor: ${s.floorScore}, ceiling: ${s.ceilingScore})`);
+            _log.info(`     ${area}: Doc Lift = ${docLift} (floor: ${s.floorScore}, ceiling: ${s.ceilingScore})`);
         }
-        console.log("     Documentation is HURTING model performance for these areas.");
-        console.log("     See docs/design-docs/scenario-matrix/evaluation-ceiling.md");
-        console.log();
+        _log.info("     Documentation is HURTING model performance for these areas.");
+        _log.info("     See docs/design-docs/scenario-matrix/evaluation-ceiling.md");
+        _log.info("");
     }
     else {
-        console.log("  ✅ No areas with negative Doc Lift detected.");
-        console.log();
+        _log.info("  ✅ No areas with negative Doc Lift detected.");
+        _log.info("");
     }
     // Three-layer decomposition (only when actual scores are present)
     const hasActualScores = scores.some((s) => s.actualScore !== undefined);
     if (hasActualScores) {
-        console.log("-".repeat(80));
-        console.log("THREE-LAYER DECOMPOSITION (floor → ceiling → actual)");
-        console.log("-".repeat(80));
-        console.log();
+        _log.info("-".repeat(80));
+        _log.info("THREE-LAYER DECOMPOSITION (floor → ceiling → actual)");
+        _log.info("-".repeat(80));
+        _log.info("");
         const decompH = "| Feature Area        | Floor | Ceiling | Actual | Doc Lift | Ret. Gap | Infra % |";
         const decompSep = "|---------------------|-------|---------|--------|----------|----------|---------|";
-        console.log(decompH);
-        console.log(decompSep);
+        _log.info(decompH);
+        _log.info(decompSep);
         for (const s of scores) {
             const liftStr = s.docLift >= 0 ? `+${s.docLift}` : `${s.docLift}`;
             const actualStr = s.actualScore !== undefined ? s.actualScore.toString() : "—";
@@ -919,7 +992,7 @@ function printReport(scores, urlRefs, source, agentBehavior, graderCost, perMode
                 ? `${Math.round(s.infrastructureEfficiency * 100)}%`
                 : "—";
             const flag = s.invertedRetrievalGap ? " 🔄" : "";
-            console.log(`| ${s.feature.padEnd(19)} | ` +
+            _log.info(`| ${s.feature.padEnd(19)} | ` +
                 `${s.floorScore.toString().padStart(5)} | ` +
                 `${s.ceilingScore.toString().padStart(7)} | ` +
                 `${actualStr.padStart(6)} | ` +
@@ -927,10 +1000,10 @@ function printReport(scores, urlRefs, source, agentBehavior, graderCost, perMode
                 `${(gapStr + flag).padStart(8)} | ` +
                 `${infraStr.padStart(7)} |`);
         }
-        console.log();
-        console.log("  Doc Lift = ceiling − floor  |  Ret. Gap = ceiling − actual  |  Infra = actual / ceiling");
-        console.log("  🔄 = inverted retrieval gap (agents avoid bad docs → higher actual than ceiling)");
-        console.log();
+        _log.info("");
+        _log.info("  Doc Lift = ceiling − floor  |  Ret. Gap = ceiling − actual  |  Infra = actual / ceiling");
+        _log.info("  🔄 = inverted retrieval gap (agents avoid bad docs → higher actual than ceiling)");
+        _log.info("");
     }
     // Cost summary
     const totalCost = scores.reduce((sum, s) => sum + s.totalCost, 0);
@@ -938,66 +1011,66 @@ function printReport(scores, urlRefs, source, agentBehavior, graderCost, perMode
     const graderCostTotal = graderCost?.cost ?? 0;
     const combinedCost = totalCost + graderCostTotal;
     if (totalCost > 0 || graderCostTotal > 0) {
-        console.log("-".repeat(80));
-        console.log("COST SUMMARY");
-        console.log("-".repeat(80));
-        console.log();
-        console.log(`  Provider cost:        $${totalCost.toFixed(4)}`);
+        _log.info("-".repeat(80));
+        _log.info("COST SUMMARY");
+        _log.info("-".repeat(80));
+        _log.info("");
+        _log.info(`  Provider cost:        $${totalCost.toFixed(4)}`);
         if (graderCostTotal > 0) {
             const graderLabel = graderCost?.model ?? "unknown";
-            console.log(`  Grader cost:          $${graderCostTotal.toFixed(4)}  (${graderLabel}, ${(graderCost?.totalTokens ?? 0).toLocaleString()} tokens)`);
+            _log.info(`  Grader cost:          $${graderCostTotal.toFixed(4)}  (${graderLabel}, ${(graderCost?.totalTokens ?? 0).toLocaleString()} tokens)`);
         }
-        console.log(`  Total cost:           $${combinedCost.toFixed(4)}`);
-        console.log(`  Avg cost per test:    $${(combinedCost / (totalTests || 1)).toFixed(4)}`);
-        console.log();
+        _log.info(`  Total cost:           $${combinedCost.toFixed(4)}`);
+        _log.info(`  Avg cost per test:    $${(combinedCost / (totalTests || 1)).toFixed(4)}`);
+        _log.info("");
         const costHeader = "| Feature Area        | Tests |     Cost | Avg/Test |";
         const costSep = "|---------------------|-------|----------|----------|";
-        console.log(costHeader);
-        console.log(costSep);
+        _log.info(costHeader);
+        _log.info(costSep);
         for (const s of scores) {
             const avgCost = s.testCount > 0 ? s.totalCost / s.testCount : 0;
-            console.log(`| ${s.feature.padEnd(19)} | ` +
+            _log.info(`| ${s.feature.padEnd(19)} | ` +
                 `${s.testCount.toString().padStart(5)} | ` +
                 `$${s.totalCost.toFixed(4).padStart(7)} | ` +
                 `$${avgCost.toFixed(4).padStart(7)} |`);
         }
-        console.log();
+        _log.info("");
     }
     // Per-model breakdown
     if (perModel) {
-        printPerModelReport(perModel);
+        printPerModelReport(perModel, _log);
     }
     // URL References
-    printUrlReport(urlRefs);
+    printUrlReport(urlRefs, _log);
     // Agent Behavior (only present when run with instrumented provider)
     if (agentBehavior && agentBehavior.length > 0) {
-        printAgentBehaviorReport(agentBehavior);
+        printAgentBehaviorReport(agentBehavior, _log);
     }
     // Source verification (unified report for all modes)
     if (sourceVerification || sourceIsolation) {
-        console.log("-".repeat(80));
-        console.log("📋 SOURCE VERIFICATION");
-        console.log("-".repeat(80));
+        _log.info("-".repeat(80));
+        _log.info("📋 SOURCE VERIFICATION");
+        _log.info("-".repeat(80));
         if (sourceVerification) {
-            console.log(`  Source:  ${sourceVerification.source}`);
-            console.log(`  Mode:    ${sourceVerification.mode}`);
+            _log.info(`  Source:  ${sourceVerification.source}`);
+            _log.info(`  Mode:    ${sourceVerification.mode}`);
             if (sourceVerification.allowedOrigins) {
-                console.log(`  Sandbox: ${sourceVerification.allowedOrigins.join(", ")}`);
+                _log.info(`  Sandbox: ${sourceVerification.allowedOrigins.join(", ")}`);
             }
             if (sourceVerification.searchMode) {
-                console.log(`  Search:  ${sourceVerification.searchMode}`);
+                _log.info(`  Search:  ${sourceVerification.searchMode}`);
             }
             // URL fetch results (baseline mode with direct URLs)
             if (sourceVerification.urlFetch) {
                 const uf = sourceVerification.urlFetch;
-                console.log();
-                console.log(`  URL fetch: ${uf.totalFetched} fetched, ${uf.totalFailed} failed`);
+                _log.info("");
+                _log.info(`  URL fetch: ${uf.totalFetched} fetched, ${uf.totalFailed} failed`);
                 for (const f of uf.fetchedUrls) {
-                    console.log(`    ✅ ${f.url} (via ${f.method})`);
+                    _log.info(`    ✅ ${f.url} (via ${f.method})`);
                 }
                 for (const f of uf.failures) {
                     // oxlint-disable-next-line @typescript-eslint/prefer-nullish-coalescing -- empty string means no error info
-                    console.log(`    ⚠️  ${f.url}: ${f.error || "unknown error"}`);
+                    _log.info(`    ⚠️  ${f.url}: ${f.error || "unknown error"}`);
                 }
             }
         }
@@ -1005,22 +1078,22 @@ function printReport(scores, urlRefs, source, agentBehavior, graderCost, perMode
         if (sourceIsolation) {
             const pct = Math.round(sourceIsolation.isolationScore * 100);
             const icon = sourceIsolation.offOrigin === 0 ? "✅" : "⚠️";
-            console.log();
-            console.log(`  Agent isolation: ${icon} ${pct}% (${sourceIsolation.onOrigin}/${sourceIsolation.total} on-origin)`);
+            _log.info("");
+            _log.info(`  Agent isolation: ${icon} ${pct}% (${sourceIsolation.onOrigin}/${sourceIsolation.total} on-origin)`);
             if (sourceIsolation.offOrigin > 0) {
-                console.log(`  Off-origin fetches: ${sourceIsolation.offOrigin}`);
+                _log.info(`  Off-origin fetches: ${sourceIsolation.offOrigin}`);
                 for (const url of sourceIsolation.offOriginUrls.slice(0, 10)) {
-                    console.log(`    • ${url}`);
+                    _log.info(`    • ${url}`);
                 }
             }
             if (Object.keys(sourceIsolation.originBreakdown).length > 0) {
-                console.log("  Origin breakdown:");
+                _log.info("  Origin breakdown:");
                 for (const [origin, count] of Object.entries(sourceIsolation.originBreakdown).sort((a, b) => b[1] - a[1])) {
-                    console.log(`    ${origin}: ${count}`);
+                    _log.info(`    ${origin}: ${count}`);
                 }
             }
         }
-        console.log();
+        _log.info("");
     }
     // Build overall agent behavior stats for summary
     const overallAgentBehavior = agentBehavior && agentBehavior.length > 0
@@ -1102,31 +1175,31 @@ function printReport(scores, urlRefs, source, agentBehavior, graderCost, perMode
         urlReferences: urlRefs,
     };
 }
-function printUrlReport(urlRefs) {
-    console.log("-".repeat(80));
-    console.log("URL REFERENCES");
-    console.log("-".repeat(80));
-    console.log();
+function printUrlReport(urlRefs, log) {
+    log.info("-".repeat(80));
+    log.info("URL REFERENCES");
+    log.info("-".repeat(80));
+    log.info("");
     for (const ref of urlRefs) {
         const goldUrls = Object.entries(ref.gold.urls).sort((a, b) => b[1] - a[1]);
         const baselineUrls = Object.entries(ref.baseline.urls).sort((a, b) => b[1] - a[1]);
         if (goldUrls.length > 0) {
-            console.log(`  ${ref.feature} (gold):`);
+            log.info(`  ${ref.feature} (gold):`);
             for (const [url, count] of goldUrls) {
                 const suffix = count > 1 ? ` (${count} tests)` : "";
-                console.log(`    ${url}${suffix}`);
+                log.info(`    ${url}${suffix}`);
             }
         }
         if (baselineUrls.length > 0) {
-            console.log(`  ${ref.feature} (baseline):`);
+            log.info(`  ${ref.feature} (baseline):`);
             for (const [url, count] of baselineUrls) {
                 const suffix = count > 1 ? ` (${count} tests)` : "";
-                console.log(`    ${url}${suffix} [parametric]`);
+                log.info(`    ${url}${suffix} [parametric]`);
             }
         }
         if (goldUrls.length === 0 && baselineUrls.length === 0) {
-            console.log(`  ${ref.feature}: no URLs referenced`);
+            log.info(`  ${ref.feature}: no URLs referenced`);
         }
-        console.log();
+        log.info("");
     }
 }