@sanity/ailf 2.1.0 → 2.3.0

package/dist/pipeline/compiler/mode-handlers/agent-harness/sandbox.js

@@ -4,14 +4,26 @@
  * Builds Promptfoo beforeEach/afterEach hooks for provisioning and
  * tearing down sandbox working directories.
  */
+import { resolve } from "path";
 // ---------------------------------------------------------------------------
 // Sandbox configuration
 // ---------------------------------------------------------------------------
-export function buildSandboxConfig(task) {
+/**
+ * Build sandbox configuration from a task definition.
+ *
+ * Fixture paths are resolved to absolute at compile time using callerCwd
+ * (the directory the pipeline was invoked from), because promptfoo runs
+ * with cwd set to packages/eval/ — not the monorepo root where apps/ lives.
+ */
+export function buildSandboxConfig(task, callerCwd) {
+    const cwd = callerCwd ?? process.cwd();
     return {
         type: task.sandbox?.type ?? "tempdir",
         image: task.sandbox?.image,
-        fixtures: task.fixtures ?? [],
+        fixtures: (task.fixtures ?? []).map((f) => {
+            const stripped = f.startsWith("file://") ? f.slice(7) : f;
+            return resolve(cwd, stripped);
+        }),
         limits: task.sandbox?.limits
             ? {
                 cpus: task.sandbox.limits.cpus,
@@ -39,23 +51,41 @@ export function buildLifecycleExtensions(task, sandboxConfig) {
     return extensions;
 }
 export function buildBeforeEachHook(taskId, config) {
-    return (`// beforeEach: provision sandbox for ${taskId}\n` +
-        `async function({ vars }) {\n` +
-        `  const { mkdirSync, writeFileSync } = require('fs');\n` +
-        `  const { tmpdir } = require('os');\n` +
+    // Promptfoo extension hooks receive (hookName, context).
+    // beforeEach context is { test } — vars live at context.test.vars.
+    // Must return context for mutations to persist.
+    //
+    // The sandbox directory is created by the YAML writer at config-gen time
+    // (deterministic path in results/latest/sandbox-{taskId}/) so it exists
+    // before the provider is initialized. This hook copies fixtures into it.
+    //
+    // @see https://www.promptfoo.dev/docs/configuration/reference/ — extensions
+    return (`// beforeEach: copy fixtures into sandbox for ${taskId}\n` +
+        `async function(context) {\n` +
+        `  const { cpSync, existsSync, mkdirSync } = require('fs');\n` +
         `  const { resolve } = require('path');\n` +
-        `  const id = 'ailf-${taskId}-' + require('crypto').randomUUID().slice(0, 8);\n` +
-        `  const workDir = resolve(tmpdir(), id);\n` +
+        `  const workDir = context.test.vars?.__workingDir;\n` +
+        `  if (!workDir) return context;\n` +
         `  mkdirSync(workDir, { recursive: true });\n` +
-        `  vars.__workingDir = workDir;\n` +
-        `  vars.__sandboxId = id;\n` +
-        `  // Fixture list: ${JSON.stringify(config.fixtures)}\n` +
+        `  // Copy fixtures into sandbox\n` +
+        `  const fixtures = ${JSON.stringify(config.fixtures)};\n` +
+        `  for (const fixture of fixtures) {\n` +
+        `    const src = resolve(process.cwd(), fixture);\n` +
+        `    if (existsSync(src)) {\n` +
+        `      cpSync(src, workDir, { recursive: true });\n` +
+        `    }\n` +
+        `  }\n` +
+        `  return context;\n` +
         `}`);
 }
 export function buildAfterEachHook(taskId) {
+    // Promptfoo extension hooks receive (hookName, context).
+    // afterEach context is { test, result } — vars live at context.test.vars.
+    // @see https://www.promptfoo.dev/docs/configuration/reference/ — extensions
     return (`// afterEach: collect artifacts + teardown for ${taskId}\n` +
-        `async function({ vars }) {\n` +
+        `async function(context) {\n` +
         `  const { rmSync, readdirSync, existsSync } = require('fs');\n` +
+        `  const vars = context.test.vars || {};\n` +
         `  const workDir = vars.__workingDir;\n` +
         `  if (workDir && existsSync(workDir)) {\n` +
         `    try {\n` +

package/dist/pipeline/compiler/mode-handlers/agent-harness/types.d.ts

@@ -2,6 +2,7 @@
  * Shared types for the agent harness mode handler.
  */
 import type { PromptfooPrompt, PromptfooProvider, PromptfooTestCase } from "../../promptfoo-compiler.js";
+import type { RubricConfig } from "../../rubric-resolution.js";
 import type { SandboxType } from "../../sandbox/sandbox-strategy.js";
 /** Options for compiling an agent harness task */
 export interface AgentHarnessCompileOptions {
@@ -9,6 +10,8 @@ export interface AgentHarnessCompileOptions {
     graderProvider?: string;
     /** Root directory for fixture resolution */
     rootDir?: string;
+    /** Rubric config (templates, weights, profiles) — loaded from rubrics config */
+    rubricConfig?: RubricConfig;
 }
 /** Result of compiling a single agent harness task */
 export interface AgentHarnessCompileResult {

package/dist/pipeline/compiler/mode-handlers/literacy/assertions.js

@@ -4,6 +4,7 @@
  * Handles rubric template resolution, doc-coverage auto-generation,
  * and baseline assertion filtering.
  */
+import { resolveTemplatedAssertion } from "../../rubric-resolution.js";
 // ---------------------------------------------------------------------------
 // Assertion resolution
 // ---------------------------------------------------------------------------
@@ -37,33 +38,6 @@ export function resolveAssertions(task, options, warnings) {
     return assertions;
 }
 // ---------------------------------------------------------------------------
-// Rubric template resolution
-// ---------------------------------------------------------------------------
-function resolveTemplatedAssertion(a, rubricConfig, graderProvider, warnings) {
-    if (!rubricConfig) {
-        warnings.push(`No rubric config — template "${a.template}" cannot be resolved`);
-        return null;
-    }
-    const template = rubricConfig.templates[a.template];
-    if (!template) {
-        warnings.push(`Unknown rubric template: "${a.template}"`);
-        return null;
-    }
-    const scaleText = template.scale.map((s) => `- ${s}`).join("\n");
-    const criteriaText = a.criteria.map((c) => `- ${c}`).join("\n");
-    const rubricValue = `${template.header}\n${scaleText}\n\n` +
-        `${template.criteria_label ?? "Check for:"}\n${criteriaText}\n\n` +
-        `Return ONLY a JSON object: {"score": <number>, "reason": "<explanation>"}`;
-    return {
-        type: "llm-rubric",
-        value: rubricValue,
-        ...(graderProvider ? { provider: graderProvider } : {}),
-        ...(template.dimension
-            ? { metadata: { dimension: template.dimension, maxScore: 100 } }
-            : {}),
-    };
-}
-// ---------------------------------------------------------------------------
 // Doc-coverage assertion
 // ---------------------------------------------------------------------------
 function buildDocCoverageAssertion(rubricConfig, graderProvider) {

package/dist/pipeline/compiler/mode-handlers/literacy/types.d.ts

@@ -2,6 +2,8 @@
  * Shared types for the literacy mode handler.
  */
 import type { PromptfooPrompt, PromptfooProvider, PromptfooTestCase } from "../../promptfoo-compiler.js";
+export type { RubricConfig } from "../../rubric-resolution.js";
+import type { RubricConfig } from "../../rubric-resolution.js";
 /** Options for compiling a literacy task */
 export interface LiteracyCompileOptions {
     /** Grader provider for LLM-graded assertions */
@@ -19,15 +21,6 @@ export interface LiteracyCompileOptions {
     /** Rubric config (templates, weights, profiles) — loaded from rubrics config */
     rubricConfig?: RubricConfig;
 }
-/** Minimal rubric config needed by the handler */
-export interface RubricConfig {
-    templates: Record<string, {
-        dimension?: string;
-        header: string;
-        scale: string[];
-        criteria_label?: string;
-    }>;
-}
 /** Result of compiling a single literacy task */
 export interface LiteracyCompileResult {
     /** Promptfoo provider configs */

package/dist/pipeline/compiler/rubric-resolution.d.ts

@@ -0,0 +1,40 @@
+/**
+ * Shared rubric template resolution for all evaluation modes.
+ *
+ * Resolves templated LLM-rubric assertions (those with `template` + `criteria`
+ * fields) into fully assembled Promptfoo assertions with rubric text and
+ * dimension metadata.
+ *
+ * Used by both literacy and agent-harness compilers. Extracted from
+ * literacy/assertions.ts to fix the compilation bug where agent-harness
+ * tasks with templated rubrics produced empty rubric text (DOC-2029).
+ *
+ * @see docs/design-docs/mode-agnostic-scoring.md
+ * @see config/rubrics.ts — template definitions
+ */
+import type { PromptfooAssertion } from "./assertion-mapper.js";
+/** Minimal rubric config needed for template resolution */
+export interface RubricConfig {
+    templates: Record<string, {
+        criteria_label?: string;
+        dimension?: string;
+        header: string;
+        scale: string[];
+    }>;
+}
+/**
+ * Resolve a templated LLM-rubric assertion into a fully assembled
+ * Promptfoo assertion with rubric text and dimension metadata.
+ *
+ * A "templated" assertion has `template` (referencing a key in rubrics.ts)
+ * and `criteria` (task-specific bullet points). The template provides the
+ * scoring header, scale, and dimension metadata. The criteria are appended
+ * to create the final rubric prompt.
+ *
+ * Returns null (with a warning) if the template can't be resolved.
+ */
+export declare function resolveTemplatedAssertion(assertion: {
+    criteria: string[];
+    template: string;
+    type: string;
+}, rubricConfig: RubricConfig | undefined, graderProvider: string | undefined, warnings: string[]): PromptfooAssertion | null;

package/dist/pipeline/compiler/rubric-resolution.js

@@ -0,0 +1,52 @@
+/**
+ * Shared rubric template resolution for all evaluation modes.
+ *
+ * Resolves templated LLM-rubric assertions (those with `template` + `criteria`
+ * fields) into fully assembled Promptfoo assertions with rubric text and
+ * dimension metadata.
+ *
+ * Used by both literacy and agent-harness compilers. Extracted from
+ * literacy/assertions.ts to fix the compilation bug where agent-harness
+ * tasks with templated rubrics produced empty rubric text (DOC-2029).
+ *
+ * @see docs/design-docs/mode-agnostic-scoring.md
+ * @see config/rubrics.ts — template definitions
+ */
+// ---------------------------------------------------------------------------
+// Template resolution
+// ---------------------------------------------------------------------------
+/**
+ * Resolve a templated LLM-rubric assertion into a fully assembled
+ * Promptfoo assertion with rubric text and dimension metadata.
+ *
+ * A "templated" assertion has `template` (referencing a key in rubrics.ts)
+ * and `criteria` (task-specific bullet points). The template provides the
+ * scoring header, scale, and dimension metadata. The criteria are appended
+ * to create the final rubric prompt.
+ *
+ * Returns null (with a warning) if the template can't be resolved.
+ */
+export function resolveTemplatedAssertion(assertion, rubricConfig, graderProvider, warnings) {
+    if (!rubricConfig) {
+        warnings.push(`No rubric config — template "${assertion.template}" cannot be resolved`);
+        return null;
+    }
+    const template = rubricConfig.templates[assertion.template];
+    if (!template) {
+        warnings.push(`Unknown rubric template: "${assertion.template}"`);
+        return null;
+    }
+    const scaleText = template.scale.map((s) => `- ${s}`).join("\n");
+    const criteriaText = assertion.criteria.map((c) => `- ${c}`).join("\n");
+    const rubricValue = `${template.header}\n${scaleText}\n\n` +
+        `${template.criteria_label ?? "Check for:"}\n${criteriaText}\n\n` +
+        `Return ONLY a JSON object: {"score": <number>, "reason": "<explanation>"}`;
+    return {
+        type: "llm-rubric",
+        value: rubricValue,
+        ...(graderProvider ? { provider: graderProvider } : {}),
+        ...(template.dimension
+            ? { metadata: { dimension: template.dimension, maxScore: 100 } }
+            : {}),
+    };
+}

package/dist/pipeline/compiler/scoring-bridge.js

@@ -41,19 +41,25 @@ import { classifyRubric, parseRubricScore } from "../../_vendor/ailf-core/index.
 export function scoreTestGroup(tests, profile, taskId) {
     let totalCost = 0;
     // Step 1: Convert all ComponentResults into AssertionScore[] (0–1 scale)
+    //
+    // Two assertion types contribute to scoring:
+    //   - llm-rubric: dimension from metadata, score from grader (0–100 → [0,1])
+    //   - javascript: mapped to "assertion-pass-rate" dimension (pass=1, fail=0)
+    //
+    // Other types (cost, trajectory, contains, etc.) are metadata or guards —
+    // they don't produce dimension scores.
     const assertionScores = [];
     for (const test of tests) {
         totalCost += test.cost;
         for (const comp of test.gradingResult.componentResults) {
-            if (comp.assertion?.type !== "llm-rubric")
-                continue;
-            const converted = componentToAssertionScore(comp);
+            const converted = componentToScore(comp);
             if (converted)
                 assertionScores.push(converted);
         }
     }
     // Step 2: Aggregate into DimensionScores (0–1 scale)
     const dimensionLabels = {
+        "assertion-pass-rate": "Assertion Pass Rate",
         "code-correctness": "Code Correctness",
         "doc-coverage": "Doc Coverage",
         "task-completion": "Task Completion",
@@ -86,12 +92,34 @@ export function scoreTestGroup(tests, profile, taskId) {
 // Conversion helpers
 // ---------------------------------------------------------------------------
 /**
- * Convert a single Promptfoo ComponentResult into the scoring engine's
- * AssertionScore format.
+ * Route a ComponentResult to the appropriate scoring conversion.
  *
- * Returns null if the component doesn't map to a known dimension.
+ * Dispatches by assertion type:
+ *   - llm-rubric  → dimension from metadata, grader score (0–100 → [0,1])
+ *   - javascript   → "assertion-pass-rate" dimension, binary (pass=1, fail=0)
+ *   - everything else → null (not a scoring-relevant assertion)
+ *
+ * This replaces the previous llm-rubric-only filter that caused agent-harness
+ * javascript assertions to be invisible to the scoring engine (DOC-2029).
+ */
+function componentToScore(comp) {
+    const type = comp.assertion?.type;
+    if (type === "llm-rubric") {
+        return llmRubricToScore(comp);
+    }
+    if (type === "javascript") {
+        return javascriptAssertionToScore(comp);
+    }
+    // Other types (cost, trajectory, contains, etc.) don't produce scores
+    return null;
+}
+/**
+ * Convert an LLM-rubric ComponentResult into an AssertionScore.
+ *
+ * The dimension comes from metadata (set during rubric template resolution).
+ * Returns null if the component doesn't map to any dimension.
  */
-function componentToAssertionScore(comp) {
+function llmRubricToScore(comp) {
     const dim = classifyRubric(comp);
     if (!dim)
         return null;
@@ -108,6 +136,30 @@ function componentToAssertionScore(comp) {
         weight: 1.0,
     };
 }
+/**
+ * Convert a javascript assertion ComponentResult into an AssertionScore.
+ *
+ * Javascript assertions (fileExists, fileContains, commandSucceeds, etc.)
+ * produce binary pass/fail results. They map to the "assertion-pass-rate"
+ * dimension — the fraction of structural assertions that passed.
+ *
+ * Zero-weight assertions (like URL extraction) are excluded from scoring.
+ */
+function javascriptAssertionToScore(comp) {
+    // Skip zero-weight assertions (diagnostic-only, e.g., URL extraction)
+    const weight = comp.assertion?.weight;
+    if (weight === 0)
+        return null;
+    return {
+        assertionType: "javascript",
+        dimension: "assertion-pass-rate",
+        latencyMs: 0,
+        pass: comp.pass,
+        reason: comp.reason ?? "",
+        score: comp.pass ? 1.0 : 0.0,
+        weight: 1.0,
+    };
+}
 /** Convert kebab-case dimension key to camelCase (e.g., "task-completion" → "taskCompletion") */
 function kebabToCamel(kebab) {
     return kebab.replace(/-([a-z])/g, (_, c) => c.toUpperCase());

package/dist/pipeline/provenance.js

@@ -59,6 +59,12 @@ export function buildProvenance(input) {
         evalFingerprint: input.evalFingerprint,
         hasLineage: Boolean(lineage),
     });
+    // Non-literacy modes (agent-harness, mcp-server, etc.) don't use the
+    // config/models.ts model matrix — listing those models would be misleading.
+    // Only include them for literacy mode where they're the actual eval targets.
+    const evaluatedModels = input.mode === "literacy"
+        ? models.models.map((m) => ({ id: m.id, label: m.label }))
+        : [];
     return {
         areas: input.areas,
         autoScope: input.autoScope,
@@ -68,7 +74,7 @@ export function buildProvenance(input) {
         graderModel: models.grader.id,
         lineage,
         mode: input.mode,
-        models: models.models.map((m) => ({ id: m.id, label: m.label })),
+        models: evaluatedModels,
         promptfooUrl: input.promptfooUrl,
         promptfooUrls: input.promptfooUrls,
         source: {

package/dist/pipeline/validate.d.ts

@@ -51,10 +51,11 @@ export declare function validateReferenceSolutions(rootDir: string): ValidationI
  */
 export declare function validateRubricsYaml(rootDir: string): ValidationIssue[];
 /**
- * Check that tasks/*.yaml files exist, parse, and conform to the Zod schema.
- * Validates both the new single-definition format (with `id`) and the legacy
- * paired format. Uses `TaskFileSchema` from schemas.ts for structural
- * validation, plus cross-entry checks (duplicate IDs, docs path consistency).
+ * Check that task definition files exist.
+ *
+ * Tasks live as `*.task.ts` files in mode subdirectories (e.g.
+ * `tasks/literacy/groq.task.ts`). Legacy YAML task files are no longer
+ * used. Warns only if no task files are found at all.
  */
 export declare function validateTaskFiles(rootDir: string): ValidationIssue[];
 /**

package/dist/pipeline/validate.js

@@ -9,10 +9,9 @@
  */
 import fs from "fs";
 import path from "path";
-import { load } from "js-yaml";
 import { tryLoadConfigFile } from "./compiler/config-loader.js";
 import { resolveMappings } from "./resolve-mappings.js";
-import { FeatureRegistrySchema, formatZodErrors, RubricConfigSchema, TaskFileSchema, ThresholdConfigSchema, } from "./schemas.js";
+import { FeatureRegistrySchema, formatZodErrors, RubricConfigSchema, ThresholdConfigSchema, } from "./schemas.js";
 // ---------------------------------------------------------------------------
 // Helpers
 // ---------------------------------------------------------------------------
@@ -248,10 +247,11 @@ export function validateRubricsYaml(rootDir) {
     return issues;
 }
 /**
- * Check that tasks/*.yaml files exist, parse, and conform to the Zod schema.
- * Validates both the new single-definition format (with `id`) and the legacy
- * paired format. Uses `TaskFileSchema` from schemas.ts for structural
- * validation, plus cross-entry checks (duplicate IDs, docs path consistency).
+ * Check that task definition files exist.
+ *
+ * Tasks live as `*.task.ts` files in mode subdirectories (e.g.
+ * `tasks/literacy/groq.task.ts`). Legacy YAML task files are no longer
+ * used. Warns only if no task files are found at all.
  */
 export function validateTaskFiles(rootDir) {
     const source = "validateTaskFiles";
@@ -261,70 +261,9 @@ export function validateTaskFiles(rootDir) {
         issues.push(warning(source, "tasks/ directory not found (using Content Lake tasks?)", tasksDir));
         return issues;
     }
-    const yamlFiles = fs
-        .readdirSync(tasksDir)
-        .filter((f) => (f.endsWith(".yaml") || f.endsWith(".yml")) && !f.startsWith("."));
-    if (yamlFiles.length === 0) {
-        issues.push(warning(source, "No task YAML files found in tasks/ (using Content Lake tasks?)", tasksDir));
-        return issues;
-    }
-    const allIds = new Map(); // id → source file
-    const templateKeys = loadTemplateKeys(rootDir);
-    for (const file of yamlFiles) {
-        const filePath = path.join(tasksDir, file);
-        // Step 1: Parse YAML
-        const result = parseYamlFile(filePath, source);
-        if (!result.ok) {
-            issues.push(result.issue);
-            continue;
-        }
-        const { data } = result;
-        if (!Array.isArray(data)) {
-            issues.push(error(source, `${file} did not parse to an array of tasks`, filePath));
-            continue;
-        }
-        // Step 2: Validate each entry with Zod schema
-        const zodResult = TaskFileSchema.safeParse(data);
-        if (!zodResult.success) {
-            const lines = formatZodErrors(zodResult.error);
-            for (const line of lines) {
-                issues.push(error(source, `${file}: ${line.trim()}`, filePath));
-            }
-            continue;
-        }
-        // Step 3: Cross-entry validation (duplicate IDs, docs path consistency)
-        for (const entry of zodResult.data) {
-            if ("id" in entry && typeof entry.id === "string") {
-                // Check for duplicate IDs across all files
-                if (allIds.has(entry.id)) {
-                    issues.push(error(source, `${file}: duplicate id '${entry.id}' (also in ${allIds.get(entry.id)})`, filePath));
-                }
-                else {
-                    allIds.set(entry.id, file);
-                }
-                // Check docs path matches task id
-                const vars = entry.vars;
-                if (vars.docs && typeof vars.docs === "string") {
-                    const expectedPath = `file://contexts/canonical/${entry.id}.md`;
-                    if (vars.docs !== expectedPath) {
-                        issues.push(warning(source, `${file}: id is '${entry.id}' but docs path is '${vars.docs}' (expected '${expectedPath}')`, filePath));
-                    }
-                }
-                // Check that llm-rubric template references exist in config/rubrics
-                const asserts = entry.assert;
-                if (Array.isArray(asserts) && templateKeys.size > 0) {
-                    for (const a of asserts) {
-                        const assertion = a;
-                        if (assertion.type === "llm-rubric" &&
-                            typeof assertion.template === "string") {
-                            if (!templateKeys.has(assertion.template)) {
-                                issues.push(error(source, `${file}: task '${entry.id}' references unknown rubric template '${assertion.template}' (available: ${[...templateKeys].join(", ")})`, filePath));
-                            }
-                        }
-                    }
-                }
-            }
-        }
+    const taskAreas = collectTaskAreas(tasksDir);
+    if (taskAreas.size === 0) {
+        issues.push(warning(source, "No task files found in tasks/ (using Content Lake tasks?)", tasksDir));
     }
     return issues;
 }
@@ -355,15 +294,10 @@ export function validateThresholdsYaml(rootDir) {
     // Cross-reference: warn if an area override references an area with no task file
     if (zodResult.data.areas) {
         const tasksDir = path.join(rootDir, "tasks");
-        if (fs.existsSync(tasksDir)) {
-            const taskFiles = new Set(fs
-                .readdirSync(tasksDir)
-                .filter((f) => /\.(yaml|yml|task\.ts|task\.js)$/.test(f))
-                .map((f) => f.replace(/\.(yaml|yml|task\.ts|task\.js)$/, "")));
-            for (const areaName of Object.keys(zodResult.data.areas)) {
-                if (!taskFiles.has(areaName)) {
-                    issues.push(warning(source, `config/thresholds: area override '${areaName}' has no matching tasks/${areaName}`, loaded.filePath));
-                }
+        const taskAreas = collectTaskAreas(tasksDir);
+        for (const areaName of Object.keys(zodResult.data.areas)) {
+            if (!taskAreas.has(areaName)) {
+                issues.push(warning(source, `config/thresholds: area override '${areaName}' has no matching task file`, loaded.filePath));
             }
         }
     }
@@ -378,44 +312,31 @@ function error(source, message, filePath) {
     };
 }
 /**
- * Load the set of valid rubric template keys from config/rubrics.
- * Returns an empty set if the file is missing or invalid.
+ * Collect task area names from all subdirectories of `tasksDir`.
+ *
+ * Task files live in mode subdirectories (e.g. `tasks/literacy/groq.task.ts`).
+ * Returns a set of basenames without the `.task.ts`/`.task.js` extension.
  */
-function loadTemplateKeys(rootDir) {
-    const loaded = tryLoadConfigFile("rubrics", rootDir);
-    if (!loaded)
+function collectTaskAreas(tasksDir) {
+    if (!fs.existsSync(tasksDir))
         return new Set();
-    try {
-        const templates = loaded.data?.templates;
-        if (templates && typeof templates === "object") {
-            return new Set(Object.keys(templates));
+    const areas = new Set();
+    const taskFilePattern = /\.task\.(ts|js)$/;
+    for (const entry of fs.readdirSync(tasksDir, { withFileTypes: true })) {
+        if (entry.isDirectory()) {
+            const subdir = path.join(tasksDir, entry.name);
+            for (const file of fs.readdirSync(subdir)) {
+                if (taskFilePattern.test(file)) {
+                    areas.add(file.replace(taskFilePattern, ""));
+                }
+            }
+        }
+        // Also check top-level task files for backwards compatibility
+        if (entry.isFile() && taskFilePattern.test(entry.name)) {
+            areas.add(entry.name.replace(taskFilePattern, ""));
         }
     }
-    catch {
-        // Ignore — structural errors are caught by validateRubricsYaml
-    }
-    return new Set();
-}
-/** Safely parse a YAML file, returning the parsed value or a validation issue. */
-function parseYamlFile(filePath, source) {
-    if (!fs.existsSync(filePath)) {
-        return {
-            issue: error(source, `File not found: ${filePath}`, filePath),
-            ok: false,
-        };
-    }
-    try {
-        const raw = fs.readFileSync(filePath, "utf-8");
-        const data = load(raw);
-        return { data, ok: true };
-    }
-    catch (err) {
-        const message = err instanceof Error ? err.message : "Unknown YAML parse error";
-        return {
-            issue: error(source, `Failed to parse YAML: ${message}`, filePath),
-            ok: false,
-        };
-    }
+    return areas;
 }
 // ---------------------------------------------------------------------------
 // Main entry point

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sanity/ailf",
-  "version": "2.1.0",
+  "version": "2.3.0",
   "private": false,
   "publishConfig": {
     "access": "public"
@@ -46,6 +46,7 @@
     "zod": "^4.3.6"
   },
   "devDependencies": {
+    "@anthropic-ai/claude-agent-sdk": "^0.2.105",
     "@types/js-yaml": "^4.0.9",
     "@types/node": "^22.13.1",
     "tsx": "^4.19.2",