@sanity/ailf 0.1.34 → 0.3.0

package/dist/pipeline/coverage-audit.d.ts

@@ -9,6 +9,7 @@
  *
  * @see docs/exec-plans/scenario-matrix-implementation/phase-3-gap-analysis.md
  */
+import type { Logger } from "../_vendor/ailf-core/index.d.ts";
 import type { CoverageAuditReport, ProductFeature } from "./types.js";
 /**
  * Count unique document slugs referenced across all tasks.
@@ -32,7 +33,7 @@ export declare function formatCoverageMarkdown(report: CoverageAuditReport): str
 /**
  * Load and validate the feature registry from config/features.yaml.
  */
-export declare function loadFeatureRegistry(rootDir: string): null | ProductFeature[];
+export declare function loadFeatureRegistry(rootDir: string, logger?: Logger): null | ProductFeature[];
 /**
  * Run the coverage audit and produce a structured report.
  */

package/dist/pipeline/coverage-audit.js

@@ -12,6 +12,7 @@
 import { existsSync, readFileSync } from "fs";
 import { join } from "path";
 import { load } from "js-yaml";
+import { ConsoleLogger } from "../adapters/loggers/index.js";
 import { FeatureRegistrySchema } from "./schemas.js";
 import { resolveMappings } from "./resolve-mappings.js";
 // ---------------------------------------------------------------------------
@@ -113,7 +114,8 @@ export function formatCoverageMarkdown(report) {
 /**
  * Load and validate the feature registry from config/features.yaml.
  */
-export function loadFeatureRegistry(rootDir) {
+export function loadFeatureRegistry(rootDir, logger) {
+    const log = logger ?? new ConsoleLogger();
     const filePath = join(rootDir, "config", "features.yaml");
     if (!existsSync(filePath)) {
         return null;
@@ -122,9 +124,9 @@ export function loadFeatureRegistry(rootDir) {
     const parsed = load(raw);
     const result = FeatureRegistrySchema.safeParse(parsed);
     if (!result.success) {
-        console.error("❌ config/features.yaml validation failed:");
+        log.error("❌ config/features.yaml validation failed:");
         for (const issue of result.error.issues) {
-            console.error(`  ${issue.path.join(".")}: ${issue.message}`);
+            log.error(`  ${issue.path.join(".")}: ${issue.message}`);
         }
         return null;
     }

package/dist/pipeline/expand-tasks.d.ts

@@ -41,6 +41,7 @@
  *   2. Baseline entry — sets docs: "", adds transform, uses abbreviated rubric
  */
 import type { TaskDefinition } from "../_vendor/ailf-core/index.d.ts";
+import type { Logger } from "../_vendor/ailf-core/index.d.ts";
 import { type RubricConfig } from "./schemas.js";
 import type { FilterOptions } from "./types.js";
 /** Any assertion entry (templated or value-based). */
@@ -198,7 +199,7 @@ export declare function isTemplatedAssert(entry: AssertEntry): entry is Template
  *
  * Returns the expanded entries grouped by source file.
  */
-export declare function loadAndExpandTasks(rootDir: string, filter?: FilterOptions, mode?: "agentic" | "baseline"): {
+export declare function loadAndExpandTasks(rootDir: string, filter?: FilterOptions, mode?: "agentic" | "baseline", logger?: Logger): {
     /** All expanded test entries, in order. */
     entries: ExpandedTestEntry[];
     /** Statistics about what was processed. */

package/dist/pipeline/expand-tasks.js

@@ -43,6 +43,7 @@
 import { existsSync, readFileSync, readdirSync } from "fs";
 import { resolve } from "path";
 import { load } from "js-yaml";
+import { ConsoleLogger } from "../adapters/loggers/index.js";
 import { RubricConfigSchema } from "./schemas.js";
 // ---------------------------------------------------------------------------
 // Rubric template loading and assembly
@@ -313,9 +314,13 @@ export function isTemplatedAssert(entry) {
  *
  * Returns the expanded entries grouped by source file.
  */
-export function loadAndExpandTasks(rootDir, filter, mode = "baseline") {
+export function loadAndExpandTasks(rootDir, filter, mode = "baseline", logger) {
+    const log = logger ?? new ConsoleLogger();
     const tasksDir = resolve(rootDir, "tasks");
     if (!existsSync(tasksDir)) {
+        log.debug("Tasks directory not found, returning empty (tasks may come from Content Lake)", {
+            tasksDir,
+        });
         // tasks/ may not exist when task definitions come from Content Lake
         return {
             entries: [],
@@ -332,13 +337,25 @@ export function loadAndExpandTasks(rootDir, filter, mode = "baseline") {
     let yamlFiles = readdirSync(tasksDir)
         .filter((f) => (f.endsWith(".yaml") || f.endsWith(".yml")) && !f.startsWith("."))
         .sort();
+    log.debug("Discovered task YAML files", {
+        directory: tasksDir,
+        fileCount: yamlFiles.length,
+        files: yamlFiles,
+    });
     // Apply area filter — area name = filename stem (e.g., "groq" matches "groq.yaml")
     if (filter?.areas && filter.areas.length > 0) {
         const allowedAreas = new Set(filter.areas.map((a) => a.toLowerCase()));
+        const beforeCount = yamlFiles.length;
         yamlFiles = yamlFiles.filter((f) => {
             const stem = f.replace(/\.ya?ml$/, "").toLowerCase();
             return allowedAreas.has(stem);
         });
+        log.debug("Applied area filter", {
+            allowedAreas: [...allowedAreas],
+            beforeCount,
+            afterCount: yamlFiles.length,
+            matchedFiles: yamlFiles,
+        });
     }
     const entries = [];
     let singleDefinitions = 0;
@@ -353,10 +370,17 @@ export function loadAndExpandTasks(rootDir, filter, mode = "baseline") {
         // Apply task ID filter
         if (filter?.taskIds && filter.taskIds.length > 0) {
             const allowedIds = new Set(filter.taskIds);
+            const beforeCount = parsed.length;
             parsed = parsed.filter((entry) => typeof entry === "object" &&
                 entry !== null &&
                 "id" in entry &&
                 allowedIds.has(entry.id));
+            log.debug("Applied task ID filter", {
+                file,
+                allowedIds: [...allowedIds],
+                beforeCount,
+                afterCount: parsed.length,
+            });
         }
         for (const entry of parsed) {
             if (isSingleTaskDefinition(entry)) {
@@ -370,8 +394,15 @@ export function loadAndExpandTasks(rootDir, filter, mode = "baseline") {
         }
     }
     if (filter?.areas || filter?.taskIds) {
-        console.log(`  Filter: ${filter.areas ? `areas=[${filter.areas.join(", ")}]` : ""}${filter.areas && filter.taskIds ? ", " : ""}${filter.taskIds ? `tasks=[${filter.taskIds.join(", ")}]` : ""}`);
+        log.info(`Filter: ${filter.areas ? `areas=[${filter.areas.join(", ")}]` : ""}${filter.areas && filter.taskIds ? ", " : ""}${filter.taskIds ? `tasks=[${filter.taskIds.join(", ")}]` : ""}`);
     }
+    log.debug("Task expansion complete", {
+        totalFiles: yamlFiles.length,
+        singleDefinitions,
+        legacyEntries,
+        expandedTotal: entries.length,
+        mode,
+    });
     return {
         entries,
         stats: {

package/dist/pipeline/generate-configs.d.ts

@@ -18,7 +18,7 @@
  * @see config/models.yaml — the central model registry
  * @see docs/exec-plans/eliminate-lib-layer.md
  */
-import { type TaskDefinition } from "../_vendor/ailf-core/index.d.ts";
+import { type Logger, type TaskDefinition } from "../_vendor/ailf-core/index.d.ts";
 import type { FilterOptions } from "./types.js";
 import { type ResolvedSourceConfig } from "../sources.js";
 export { extractModelName, extractProvider, mergeConfig, modelMatchesMode, } from "../_vendor/ailf-core/index.d.ts";
@@ -49,6 +49,8 @@ export interface GenerateConfigsOptions {
     allowedOrigins?: string[];
     /** Filter to specific feature areas or task IDs */
     filter?: FilterOptions;
+    /** Logger instance (defaults to ConsoleLogger) */
+    logger?: Logger;
     /** Pre-resolved source config (skips loadSource() call) */
     resolvedSource?: ResolvedSourceConfig;
     /** Root directory of the eval package (required) */

package/dist/pipeline/generate-configs.js

@@ -22,6 +22,7 @@ import { extractModelName, extractProvider, mergeConfig, modelMatchesMode, } fro
 import { existsSync, readFileSync, readdirSync, writeFileSync } from "fs";
 import { resolve } from "path";
 import { dump, load } from "js-yaml";
+import { ConsoleLogger } from "../adapters/loggers/index.js";
 import { expandTaskDefinitions, loadAndExpandTasks } from "./expand-tasks.js";
 import { validateModelsYaml } from "./validate.js";
 import { loadSource } from "../sources.js";
@@ -260,31 +261,34 @@ function generateObservedConfig(models, tests, prompts) {
  */
 export function generateConfigs(options) {
     const { rootDir } = options;
+    const log = options.logger ?? new ConsoleLogger();
     // Validate config/models.yaml before generating configs
     const modelIssues = validateModelsYaml(rootDir);
     const modelErrors = modelIssues.filter((i) => i.severity === "error");
     if (modelErrors.length > 0) {
-        console.error("❌ config/models.yaml validation failed:");
-        for (const e of modelErrors) {
-            console.error(`  ERROR: ${e.message}`);
-            if (e.path) {
-                console.error(`         at ${e.path}`);
-            }
-        }
-        console.error("\nFix config/models.yaml before generating configs. Run 'pnpm validate' for details.");
-        process.exit(1);
+        const details = modelErrors
+            .map((e) => (e.path ? `${e.message} (at ${e.path})` : e.message))
+            .join("; ");
+        throw new Error(`config/models.yaml validation failed: ${details}. Run 'pnpm validate' for details.`);
     }
-    console.log("Loading config/models.yaml...");
+    log.info("Loading config/models.yaml...");
     const models = loadModels(rootDir);
     const activeModels = models.models.filter((m) => m.id && m.label);
-    console.log(`  Found ${activeModels.length} active model(s):`);
+    log.debug("Models loaded from config/models.yaml", {
+        totalModels: models.models.length,
+        activeModels: activeModels.length,
+        modelIds: activeModels.map((m) => m.id),
+        graderId: models.grader.id,
+        maxConcurrency: models.maxConcurrency,
+    });
+    log.info(`  Found ${activeModels.length} active model(s):`);
     for (const m of activeModels) {
         // oxlint-disable-next-line @typescript-eslint/prefer-nullish-coalescing -- empty array join → "all"
         const modes = m.modes?.join(", ") || "all";
-        console.log(`    - ${m.label} (${m.id}) → [${modes}]`);
+        log.info(`    - ${m.label} (${m.id}) → [${modes}]`);
     }
     // oxlint-disable-next-line @typescript-eslint/prefer-nullish-coalescing -- empty label falls through to id
-    console.log(`  Grader: ${models.grader.label || models.grader.id}`);
+    log.info(`  Grader: ${models.grader.label || models.grader.id}`);
     // Build filter from options
     const filter = options.filter?.areas || options.filter?.taskIds
         ? options.filter
@@ -295,20 +299,24 @@ export function generateConfigs(options) {
     let agenticEntries;
     if (options.tasks) {
         // TaskSource path — tasks already loaded and filtered by the adapter
+        log.debug("Expanding tasks from TaskSource adapter", {
+            taskCount: options.tasks.length,
+            taskIds: options.tasks.map((t) => t.id),
+        });
         const baselineResult = expandTaskDefinitions(options.tasks, rootDir, "baseline");
         entries = baselineResult.entries;
-        console.log(`  Expanded ${baselineResult.stats.totalTasks} task(s) → ${baselineResult.stats.expandedTotal} test entries (from TaskSource)`);
+        log.info(`  Expanded ${baselineResult.stats.totalTasks} task(s) → ${baselineResult.stats.expandedTotal} test entries (from TaskSource)`);
         const agenticResult = expandTaskDefinitions(options.tasks, rootDir, "agentic");
         agenticEntries = agenticResult.entries;
-        console.log(`  Agentic: ${agenticResult.stats.expandedTotal} entries (gold only, no baseline)`);
+        log.info(`  Agentic: ${agenticResult.stats.expandedTotal} entries (gold only, no baseline)`);
     }
     else {
         // Legacy path — read from tasks/*.yaml files
-        const { entries: baselineEntries, stats } = loadAndExpandTasks(rootDir, filter, "baseline");
+        const { entries: baselineEntries, stats } = loadAndExpandTasks(rootDir, filter, "baseline", log);
         entries = baselineEntries;
-        console.log(`  Expanded ${stats.singleDefinitions} task(s) → ${stats.expandedTotal} test entries`);
+        log.info(`  Expanded ${stats.singleDefinitions} task(s) → ${stats.expandedTotal} test entries`);
         if (stats.legacyEntries > 0) {
-            console.log(`  ⚠ ${stats.legacyEntries} legacy (paired) entries passed through unchanged`);
+            log.info(`  ⚠ ${stats.legacyEntries} legacy (paired) entries passed through unchanged`);
         }
         if (filter) {
             const parts = [];
@@ -318,11 +326,11 @@ export function generateConfigs(options) {
             if (filter.taskIds) {
                 parts.push(`tasks: ${filter.taskIds.join(", ")}`);
             }
-            console.log(`  Scoped to: ${parts.join("; ")}`);
+            log.info(`  Scoped to: ${parts.join("; ")}`);
         }
-        const { entries: agenticFromYaml, stats: agenticStats } = loadAndExpandTasks(rootDir, filter, "agentic");
+        const { entries: agenticFromYaml, stats: agenticStats } = loadAndExpandTasks(rootDir, filter, "agentic", log);
         agenticEntries = agenticFromYaml;
-        console.log(`  Agentic: ${agenticStats.expandedTotal} entries (gold only, no baseline)`);
+        log.info(`  Agentic: ${agenticStats.expandedTotal} entries (gold only, no baseline)`);
     }
     // Write expanded tasks to generated files for Promptfoo to consume
     const expandedPath = resolve(rootDir, "tasks", ".expanded.yaml");
@@ -333,7 +341,7 @@ export function generateConfigs(options) {
         quotingType: "'",
     });
     writeFileSync(expandedPath, `# .expanded.yaml\n#\n# AUTO-GENERATED — do not edit directly.\n# Source: tasks/*.yaml (single-definition format)\n# Run: pnpm generate-configs\n\n${expandedYaml}`, "utf-8");
-    console.log(`  ✓ tasks/.expanded.yaml (${entries.length} entries)`);
+    log.info(`  ✓ tasks/.expanded.yaml (${entries.length} entries)`);
     const agenticExpandedPath = resolve(rootDir, "tasks", ".expanded.agentic.yaml");
     const agenticExpandedYaml = dump(agenticEntries, {
         forceQuotes: false,
@@ -342,46 +350,52 @@ export function generateConfigs(options) {
         quotingType: "'",
     });
     writeFileSync(agenticExpandedPath, `# .expanded.agentic.yaml\n#\n# AUTO-GENERATED — do not edit directly.\n# Gold entries only (no baseline) for agentic evaluation mode.\n# Source: tasks/*.yaml (single-definition format)\n# Run: pnpm generate-configs\n\n${agenticExpandedYaml}`, "utf-8");
-    console.log(`  ✓ tasks/.expanded.agentic.yaml (${agenticEntries.length} entries)`);
+    log.info(`  ✓ tasks/.expanded.agentic.yaml (${agenticEntries.length} entries)`);
     const taskFiles = ["file://tasks/.expanded.yaml"];
     const agenticTaskFiles = ["file://tasks/.expanded.agentic.yaml"];
     // Load prompt templates
     const prompts = loadPrompts(rootDir);
-    console.log(`  Loaded prompts: ${Object.keys(prompts).join(", ")}`);
+    log.debug("Prompt templates loaded", {
+        keys: Object.keys(prompts),
+        withDocsId: prompts.withDocs.id,
+        withoutDocsId: prompts.withoutDocs.id,
+        agenticId: prompts.agentic.id,
+    });
+    log.info(`  Loaded prompts: ${Object.keys(prompts).join(", ")}`);
     // Load optional documentation source configuration
     // Pre-resolved source wins over name-based lookup
     let source = options.resolvedSource;
     const sourceName = options.source;
     if (!source && sourceName) {
-        console.log(`\nLoading source: ${sourceName}`);
+        log.info(`\nLoading source: ${sourceName}`);
         try {
             source = loadSource(sourceName);
         }
         catch (err) {
             const msg = err instanceof Error ? err.message : String(err);
-            console.warn(`\n⚠ Failed to load source "${sourceName}": ${msg}`);
+            log.warn(`\n⚠ Failed to load source "${sourceName}": ${msg}`);
         }
     }
     if (source) {
-        console.log(`  Base URL: ${source.baseUrl}`);
-        console.log(`  Dataset: ${source.dataset}`);
+        log.info(`  Base URL: ${source.baseUrl}`);
+        log.info(`  Dataset: ${source.dataset}`);
         if (source.allowedOrigins?.length) {
-            console.log(`  Allowed origins: ${source.allowedOrigins.join(", ")}`);
+            log.info(`  Allowed origins: ${source.allowedOrigins.join(", ")}`);
         }
     }
-    console.log("\nGenerating configs...");
-    writeConfig(rootDir, "promptfooconfig.yaml", generateBaselineConfig(models, taskFiles, prompts), `# promptfooconfig.yaml\n#\n# AUTO-GENERATED from config/models.yaml — do not edit directly.\n# Run: pnpm generate-configs\n`);
-    writeConfig(rootDir, "promptfooconfig.observed.yaml", generateObservedConfig(models, taskFiles, prompts), `# promptfooconfig.observed.yaml\n#\n# AUTO-GENERATED from config/models.yaml — do not edit directly.\n# Run: pnpm generate-configs\n`);
-    writeConfig(rootDir, "promptfooconfig.agentic.yaml", generateAgenticConfig(models, agenticTaskFiles, prompts, source, options.searchMode, options.allowedOrigins), `# promptfooconfig.agentic.yaml\n#\n# AUTO-GENERATED from config/models.yaml — do not edit directly.\n# Run: pnpm generate-configs\n`);
-    console.log("\nDone! Configs are ready.");
+    log.info("\nGenerating configs...");
+    writeConfig(rootDir, "promptfooconfig.yaml", generateBaselineConfig(models, taskFiles, prompts), `# promptfooconfig.yaml\n#\n# AUTO-GENERATED from config/models.yaml — do not edit directly.\n# Run: pnpm generate-configs\n`, log);
+    writeConfig(rootDir, "promptfooconfig.observed.yaml", generateObservedConfig(models, taskFiles, prompts), `# promptfooconfig.observed.yaml\n#\n# AUTO-GENERATED from config/models.yaml — do not edit directly.\n# Run: pnpm generate-configs\n`, log);
+    writeConfig(rootDir, "promptfooconfig.agentic.yaml", generateAgenticConfig(models, agenticTaskFiles, prompts, source, options.searchMode, options.allowedOrigins), `# promptfooconfig.agentic.yaml\n#\n# AUTO-GENERATED from config/models.yaml — do not edit directly.\n# Run: pnpm generate-configs\n`, log);
+    log.info("\nDone! Configs are ready.");
     if (source) {
-        console.log(`  (using doc source: ${sourceName})`);
+        log.info(`  (using doc source: ${sourceName})`);
     }
 }
 // ---------------------------------------------------------------------------
 // File writing
 // ---------------------------------------------------------------------------
-function writeConfig(rootDir, filename, config, header) {
+function writeConfig(rootDir, filename, config, header, log) {
     const yamlStr = dump(config, {
         forceQuotes: false,
         lineWidth: 120,
@@ -391,5 +405,5 @@ function writeConfig(rootDir, filename, config, header) {
     const content = `${header}\n${yamlStr}`;
     const outPath = resolve(rootDir, filename);
     writeFileSync(outPath, content, "utf-8");
-    console.log(`  ✓ ${filename}`);
+    log.info(`  ✓ ${filename}`);
 }

package/dist/pipeline/grader-api.d.ts

@@ -12,6 +12,7 @@
  * Migrated from lib/grader-api.ts — no module-level side effects, no
  * process.exit(), accepts rootDir as parameter for file-based operations.
  */
+import type { Logger } from "../_vendor/ailf-core/index.d.ts";
 interface ProviderConfig {
     apiKey: string;
     baseUrl: string;
@@ -23,7 +24,7 @@ interface ProviderConfig {
  * Dispatches to the correct provider API based on the model prefix.
  * Returns a numeric score (0–100) or null if the call or parse fails.
  */
-export declare function gradeOnce(graderModel: string, responseText: string, rubricText: string): Promise<null | number>;
+export declare function gradeOnce(graderModel: string, responseText: string, rubricText: string, logger?: Logger): Promise<null | number>;
 /**
  * Load the grader model from `config/models.yaml`.
  * Returns both the model ID and human-readable label.

package/dist/pipeline/grader-api.js

@@ -15,6 +15,7 @@
 import { existsSync, readFileSync } from "fs";
 import { join } from "path";
 import { load } from "js-yaml";
+import { ConsoleLogger } from "../adapters/loggers/index.js";
 // ---------------------------------------------------------------------------
 // Public API
 // ---------------------------------------------------------------------------
@@ -24,7 +25,8 @@ import { load } from "js-yaml";
  * Dispatches to the correct provider API based on the model prefix.
  * Returns a numeric score (0–100) or null if the call or parse fails.
  */
-export async function gradeOnce(graderModel, responseText, rubricText) {
+export async function gradeOnce(graderModel, responseText, rubricText, logger) {
+    const log = logger ?? new ConsoleLogger();
     const config = resolveProvider(graderModel);
     const prompt = `You are evaluating an AI assistant's response. Grade the response according to the following rubric.
 
@@ -38,10 +40,10 @@ ${rubricText}
         const provider = graderModel.split(":")[0];
         let content;
         if (provider === "anthropic") {
-            content = await callAnthropic(config, prompt);
+            content = await callAnthropic(config, prompt, log);
         }
         else if (provider === "openai") {
-            content = await callOpenAI(config, prompt);
+            content = await callOpenAI(config, prompt, log);
         }
         else {
             // resolveProvider already throws for unknown providers, but just in case
@@ -51,12 +53,12 @@ ${rubricText}
             return null;
         const score = extractScore(content);
         if (score === null) {
-            console.error(`  ⚠ Could not parse grader response: ${content.slice(0, 100)}`);
+            log.error(`  ⚠ Could not parse grader response: ${content.slice(0, 100)}`);
         }
         return score;
     }
     catch (err) {
-        console.error(`  ⚠ Grader call failed: ${err instanceof Error ? err.message : String(err)}`);
+        log.error(`  ⚠ Grader call failed: ${err instanceof Error ? err.message : String(err)}`);
         return null;
     }
 }
@@ -152,7 +154,7 @@ export function resolveProvider(graderModel) {
 // ---------------------------------------------------------------------------
 // Provider-specific API calls
 // ---------------------------------------------------------------------------
-async function callAnthropic(config, prompt) {
+async function callAnthropic(config, prompt, log) {
     const response = await fetch(config.baseUrl, {
         body: JSON.stringify({
             max_tokens: 256,
@@ -169,14 +171,14 @@ async function callAnthropic(config, prompt) {
     });
     if (!response.ok) {
         const text = await response.text();
-        console.error(`  ⚠ Grader API error (Anthropic): ${response.status} ${text.slice(0, 200)}`);
+        log.error(`  ⚠ Grader API error (Anthropic): ${response.status} ${text.slice(0, 200)}`);
         return null;
     }
     const data = (await response.json());
     const textBlock = data.content?.find((c) => c.type === "text");
     return textBlock?.text ?? "";
 }
-async function callOpenAI(config, prompt) {
+async function callOpenAI(config, prompt, log) {
     const response = await fetch(config.baseUrl, {
         body: JSON.stringify({
             max_tokens: 256,
@@ -192,7 +194,7 @@ async function callOpenAI(config, prompt) {
     });
     if (!response.ok) {
         const text = await response.text();
-        console.error(`  ⚠ Grader API error (OpenAI): ${response.status} ${text.slice(0, 200)}`);
+        log.error(`  ⚠ Grader API error (OpenAI): ${response.status} ${text.slice(0, 200)}`);
         return null;
     }
     const data = (await response.json());

package/dist/pipeline/grader-compare-runner.d.ts

@@ -12,6 +12,7 @@
  *
  * @see docs/exec-plans/grader-reliability.md — Phase 3
  */
+import type { Logger } from "../_vendor/ailf-core/index.d.ts";
 import { type GraderComparison } from "./grader-comparison.js";
 export interface GraderCompareRunnerOptions {
     /** Candidate grader models to compare against the baseline */
@@ -21,6 +22,8 @@ export interface GraderCompareRunnerOptions {
     }[];
     /** Output format */
     format?: "json" | "table";
+    /** Logger instance (defaults to ConsoleLogger) */
+    logger?: Logger;
     /** Custom output path (default: results/latest/grader-comparison.json) */
     outputPath?: string;
     /** Path to eval results (default: results/latest/eval-results.json) */

package/dist/pipeline/grader-compare-runner.js

@@ -15,6 +15,7 @@
 import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
 import { join } from "path";
 import { load } from "js-yaml";
+import { ConsoleLogger } from "../adapters/loggers/index.js";
 import { compareGraders, } from "./grader-comparison.js";
 import { classifyCorrelation } from "./grader-validation.js";
 import { gradeOnce } from "./grader-api.js";
@@ -198,10 +199,11 @@ export function formatComparisonReport(result) {
  */
 export async function runGraderCompare(options) {
     const { rootDir, format = "table" } = options;
+    const log = options.logger ?? new ConsoleLogger();
     const resultsPath = options.resultsPath
         ? join(rootDir, options.resultsPath)
         : join(rootDir, "results", "latest", "eval-results.json");
-    console.log("=== Grader Comparison ===\n");
+    log.info("=== Grader Comparison ===\n");
     // Load config
     const { baseline, candidates } = loadConfig(rootDir, options.candidates);
     if (candidates.length === 0) {
@@ -215,32 +217,32 @@ export async function runGraderCompare(options) {
     const file = JSON.parse(readFileSync(resultsPath, "utf-8"));
     // Extract judgments
     const judgments = extractJudgments(file);
-    console.log(`  Baseline:     ${baseline.label} (${baseline.id})`);
-    console.log(`  Candidates:   ${candidates.map((c) => c.label).join(", ")}`);
-    console.log(`  Judgments:    ${judgments.length}`);
+    log.info(`  Baseline:     ${baseline.label} (${baseline.id})`);
+    log.info(`  Candidates:   ${candidates.map((c) => c.label).join(", ")}`);
+    log.info(`  Judgments:    ${judgments.length}`);
     if (judgments.length === 0) {
         throw new Error("No gradable judgments found in results.");
     }
     const totalCalls = judgments.length * (1 + candidates.length);
     const estimatedCost = totalCalls * 0.005;
-    console.log(`  API calls:    ${totalCalls} (${judgments.length} × ${1 + candidates.length} models)`);
-    console.log(`  Est. cost:    ~$${estimatedCost.toFixed(2)}`);
-    console.log();
+    log.info(`  API calls:    ${totalCalls} (${judgments.length} × ${1 + candidates.length} models)`);
+    log.info(`  Est. cost:    ~$${estimatedCost.toFixed(2)}`);
+    log.info("");
     // Grade with baseline
-    console.log(`  Grading with baseline: ${baseline.label}...`);
-    const baselineScores = await gradeJudgments(judgments, baseline.id);
+    log.info(`  Grading with baseline: ${baseline.label}...`);
+    const baselineScores = await gradeJudgments(judgments, baseline.id, log);
     // Grade with each candidate
     const candidateScoreSets = [];
     for (const candidate of candidates) {
-        console.log(`  Grading with candidate: ${candidate.label}...`);
-        const scores = await gradeJudgments(judgments, candidate.id);
+        log.info(`  Grading with candidate: ${candidate.label}...`);
+        const scores = await gradeJudgments(judgments, candidate.id, log);
         candidateScoreSets.push({
             label: candidate.label,
             modelId: candidate.id,
             scores,
         });
     }
-    console.log();
+    log.info("");
     // Compare
     const baselineScoreSet = {
         label: baseline.label,
@@ -250,10 +252,10 @@ export async function runGraderCompare(options) {
     const result = compareGraders(baselineScoreSet, candidateScoreSets);
     // Output
     if (format === "table") {
-        console.log(formatComparisonReport(result));
+        log.info(formatComparisonReport(result));
     }
     else {
-        console.log(JSON.stringify(result, null, 2));
+        log.info(JSON.stringify(result, null, 2));
     }
     // Write output
     const outPath = options.outputPath ??
@@ -261,7 +263,7 @@ export async function runGraderCompare(options) {
     const outDir = join(outPath, "..");
     mkdirSync(outDir, { recursive: true });
     writeFileSync(outPath, JSON.stringify(result, null, 2));
-    console.log(`\n  📄 Results written to ${outPath}`);
+    log.info(`\n  📄 Results written to ${outPath}`);
     return result;
 }
 // ---------------------------------------------------------------------------
@@ -271,7 +273,7 @@ export async function runGraderCompare(options) {
  * Grade a set of judgments with a specific grader model.
  * Returns GraderScore[] with one score per judgment.
  */
-async function gradeJudgments(judgments, graderModel) {
+async function gradeJudgments(judgments, graderModel, log) {
     const scores = [];
     let completed = 0;
     let failed = 0;
@@ -280,7 +282,7 @@ async function gradeJudgments(judgments, graderModel) {
         completed++;
         if (completed % 10 === 0 || completed === judgments.length) {
             const pct = Math.round((completed / judgments.length) * 100);
-            process.stdout.write(`\r    Progress: ${completed}/${judgments.length} (${pct}%)`);
+            log.info(`    Progress: ${completed}/${judgments.length} (${pct}%)`);
         }
         if (score === null) {
             failed++;
@@ -293,9 +295,9 @@ async function gradeJudgments(judgments, graderModel) {
             taskId: judgment.description,
         });
     }
-    console.log(); // newline after progress
+    log.info(""); // newline after progress
     if (failed > 0) {
-        console.log(`    ⚠ ${failed} grading calls failed (excluded)`);
+        log.warn(`    ⚠ ${failed} grading calls failed (excluded)`);
     }
     return scores;
 }

package/dist/pipeline/grader-consistency-runner.d.ts

@@ -14,11 +14,14 @@
  *
  * @see docs/exec-plans/grader-reliability.md — Phase 1
  */
+import { type Logger } from "../_vendor/ailf-core/index.d.ts";
 import type { RawPromptfooFile } from "./calculate-scores.js";
 import { type GraderConsistency } from "./grader-consistency.js";
 import type { DimensionName } from "./types.js";
 /** Options for the grader consistency runner. */
 export interface GraderConsistencyRunnerOptions {
+    /** Logger for structured output. Falls back to ConsoleLogger if omitted. */
+    logger?: Logger;
     /** Number of additional grading replications (default: 5) */
     replications: number;
     /** Path to eval-results.json */