@sanity/ailf 2.2.0 → 2.3.0

package/dist/pipeline/compiler/__tests__/agent-harness-handler.test.js

@@ -232,6 +232,70 @@ describe("compileAgentHarnessTask — assertions", () => {
         }), { graderProvider: "openai:chat:gpt-5" });
         assert.equal(result.tests[0].assert?.[0]?.provider, "openai:chat:gpt-5");
     });
+    it("resolves templated llm-rubric with rubric text and dimension metadata", () => {
+        const rubricConfig = {
+            templates: {
+                "agent-output": {
+                    dimension: "agent-output",
+                    header: "Score the agent's final output from 0 to 100:",
+                    scale: ["0: Failed", "50: Partial", "100: Complete"],
+                    criteria_label: "Check for:",
+                },
+            },
+        };
+        const result = compileAgentHarnessTask(makeTask({
+            assertions: [
+                {
+                    type: "llm-rubric",
+                    template: "agent-output",
+                    criteria: ["File created", "Correct content"],
+                },
+            ],
+        }), { rubricConfig, graderProvider: "anthropic:messages:claude-opus-4-5" });
+        const assertion = result.tests[0].assert?.[0];
+        assert.ok(assertion, "should produce an assertion");
+        assert.equal(assertion.type, "llm-rubric");
+        // Rubric text should be fully rendered (not empty)
+        assert.ok(assertion.value.includes("Score the agent"), "should contain rendered rubric header");
+        assert.ok(assertion.value.includes("File created"), "should contain task-specific criteria");
+        // Dimension metadata should be attached
+        const metadata = assertion.metadata;
+        assert.ok(metadata, "should have metadata");
+        assert.equal(metadata.dimension, "agent-output");
+        assert.equal(metadata.maxScore, 100);
+        // Grader provider should be set
+        assert.equal(assertion.provider, "anthropic:messages:claude-opus-4-5");
+    });
+    it("warns when rubric template is unknown", () => {
+        const rubricConfig = { templates: {} };
+        const result = compileAgentHarnessTask(makeTask({
+            assertions: [
+                {
+                    type: "llm-rubric",
+                    template: "nonexistent-template",
+                    criteria: ["Something"],
+                },
+            ],
+        }), { rubricConfig });
+        // Unknown template produces a warning and no assertion
+        assert.ok(result.warnings.some((w) => w.includes("nonexistent-template")), "should warn about unknown template");
+        // The assertion should be null (filtered out)
+        assert.equal(result.tests[0].assert?.length ?? 0, 0, "should not produce an assertion for unknown template");
+    });
+    it("warns when rubricConfig is not provided for templated assertion", () => {
+        const result = compileAgentHarnessTask(makeTask({
+            assertions: [
+                {
+                    type: "llm-rubric",
+                    template: "agent-output",
+                    criteria: ["Something"],
+                },
+            ],
+        })
+        // No rubricConfig in options
+        );
+        assert.ok(result.warnings.some((w) => w.includes("No rubric config")), "should warn about missing rubric config");
+    });
 });
 // ---------------------------------------------------------------------------
 // compileAgentHarnessTask — lifecycle extensions

package/dist/pipeline/compiler/mode-handlers/agent-harness/assertions.d.ts

@@ -5,6 +5,12 @@
  * command-succeeds, diff-matches) as well as standard pass-through
  * assertion types.
  *
+ * Templated LLM-rubric assertions (those with `template` + `criteria`)
+ * are resolved via the shared rubric-resolution module, producing fully
+ * assembled rubric text and dimension metadata. This is critical for
+ * scoring — without it, the grader receives empty rubrics and the
+ * scoring pipeline has no dimension data to work with (DOC-2029).
+ *
  * Agent-specific assertions use file-based references to the assertions
  * runtime module (dist/agent-harness/assertions-runtime.js) because
  * promptfoo's inline `type: javascript` assertions run in a restricted

package/dist/pipeline/compiler/mode-handlers/agent-harness/assertions.js

@@ -5,6 +5,12 @@
  * command-succeeds, diff-matches) as well as standard pass-through
  * assertion types.
  *
+ * Templated LLM-rubric assertions (those with `template` + `criteria`)
+ * are resolved via the shared rubric-resolution module, producing fully
+ * assembled rubric text and dimension metadata. This is critical for
+ * scoring — without it, the grader receives empty rubrics and the
+ * scoring pipeline has no dimension data to work with (DOC-2029).
+ *
  * Agent-specific assertions use file-based references to the assertions
  * runtime module (dist/agent-harness/assertions-runtime.js) because
  * promptfoo's inline `type: javascript` assertions run in a restricted
@@ -14,6 +20,7 @@
  * @see https://www.promptfoo.dev/docs/configuration/expected-outputs/javascript/
  * @see src/agent-harness/assertions-runtime.ts — runtime implementations
  */
+import { resolveTemplatedAssertion } from "../../rubric-resolution.js";
 /** Base path for the file-based assertion runtime module */
 const RUNTIME = "file://dist/agent-harness/assertions-runtime.js";
 // ---------------------------------------------------------------------------
@@ -44,6 +51,13 @@ export function mapAgentAssertion(assertion, options, warnings) {
                     : {}),
             };
         case "llm-rubric":
+            // Templated assertions (template + criteria) need full resolution
+            // to produce rubric text and dimension metadata for scoring.
+            if ("template" in assertion && "criteria" in assertion) {
+                const resolved = resolveTemplatedAssertion(assertion, options?.rubricConfig, options?.graderProvider, warnings);
+                return resolved;
+            }
+            // Non-templated llm-rubric (inline value) — pass through
             return {
                 type: "llm-rubric",
                 ...("value" in assertion ? { value: assertion.value } : {}),

package/dist/pipeline/compiler/mode-handlers/agent-harness/index.js

@@ -28,6 +28,7 @@ export const handler = {
         const result = compileAgentHarnessTask(task, {
             graderProvider: ctx.graderProvider,
             rootDir: ctx.rootDir,
+            rubricConfig: ctx.rubricConfig,
         });
         return {
             providers: result.providers,

package/dist/pipeline/compiler/mode-handlers/agent-harness/types.d.ts

@@ -2,6 +2,7 @@
  * Shared types for the agent harness mode handler.
  */
 import type { PromptfooPrompt, PromptfooProvider, PromptfooTestCase } from "../../promptfoo-compiler.js";
+import type { RubricConfig } from "../../rubric-resolution.js";
 import type { SandboxType } from "../../sandbox/sandbox-strategy.js";
 /** Options for compiling an agent harness task */
 export interface AgentHarnessCompileOptions {
@@ -9,6 +10,8 @@ export interface AgentHarnessCompileOptions {
     graderProvider?: string;
     /** Root directory for fixture resolution */
     rootDir?: string;
+    /** Rubric config (templates, weights, profiles) — loaded from rubrics config */
+    rubricConfig?: RubricConfig;
 }
 /** Result of compiling a single agent harness task */
 export interface AgentHarnessCompileResult {

package/dist/pipeline/compiler/mode-handlers/literacy/assertions.js

@@ -4,6 +4,7 @@
  * Handles rubric template resolution, doc-coverage auto-generation,
  * and baseline assertion filtering.
  */
+import { resolveTemplatedAssertion } from "../../rubric-resolution.js";
 // ---------------------------------------------------------------------------
 // Assertion resolution
 // ---------------------------------------------------------------------------
@@ -37,33 +38,6 @@ export function resolveAssertions(task, options, warnings) {
     return assertions;
 }
 // ---------------------------------------------------------------------------
-// Rubric template resolution
-// ---------------------------------------------------------------------------
-function resolveTemplatedAssertion(a, rubricConfig, graderProvider, warnings) {
-    if (!rubricConfig) {
-        warnings.push(`No rubric config — template "${a.template}" cannot be resolved`);
-        return null;
-    }
-    const template = rubricConfig.templates[a.template];
-    if (!template) {
-        warnings.push(`Unknown rubric template: "${a.template}"`);
-        return null;
-    }
-    const scaleText = template.scale.map((s) => `- ${s}`).join("\n");
-    const criteriaText = a.criteria.map((c) => `- ${c}`).join("\n");
-    const rubricValue = `${template.header}\n${scaleText}\n\n` +
-        `${template.criteria_label ?? "Check for:"}\n${criteriaText}\n\n` +
-        `Return ONLY a JSON object: {"score": <number>, "reason": "<explanation>"}`;
-    return {
-        type: "llm-rubric",
-        value: rubricValue,
-        ...(graderProvider ? { provider: graderProvider } : {}),
-        ...(template.dimension
-            ? { metadata: { dimension: template.dimension, maxScore: 100 } }
-            : {}),
-    };
-}
-// ---------------------------------------------------------------------------
 // Doc-coverage assertion
 // ---------------------------------------------------------------------------
 function buildDocCoverageAssertion(rubricConfig, graderProvider) {

package/dist/pipeline/compiler/mode-handlers/literacy/types.d.ts

@@ -2,6 +2,8 @@
  * Shared types for the literacy mode handler.
  */
 import type { PromptfooPrompt, PromptfooProvider, PromptfooTestCase } from "../../promptfoo-compiler.js";
+export type { RubricConfig } from "../../rubric-resolution.js";
+import type { RubricConfig } from "../../rubric-resolution.js";
 /** Options for compiling a literacy task */
 export interface LiteracyCompileOptions {
     /** Grader provider for LLM-graded assertions */
@@ -19,15 +21,6 @@ export interface LiteracyCompileOptions {
     /** Rubric config (templates, weights, profiles) — loaded from rubrics config */
     rubricConfig?: RubricConfig;
 }
-/** Minimal rubric config needed by the handler */
-export interface RubricConfig {
-    templates: Record<string, {
-        dimension?: string;
-        header: string;
-        scale: string[];
-        criteria_label?: string;
-    }>;
-}
 /** Result of compiling a single literacy task */
 export interface LiteracyCompileResult {
     /** Promptfoo provider configs */

package/dist/pipeline/compiler/rubric-resolution.d.ts

@@ -0,0 +1,40 @@
+/**
+ * Shared rubric template resolution for all evaluation modes.
+ *
+ * Resolves templated LLM-rubric assertions (those with `template` + `criteria`
+ * fields) into fully assembled Promptfoo assertions with rubric text and
+ * dimension metadata.
+ *
+ * Used by both literacy and agent-harness compilers. Extracted from
+ * literacy/assertions.ts to fix the compilation bug where agent-harness
+ * tasks with templated rubrics produced empty rubric text (DOC-2029).
+ *
+ * @see docs/design-docs/mode-agnostic-scoring.md
+ * @see config/rubrics.ts — template definitions
+ */
+import type { PromptfooAssertion } from "./assertion-mapper.js";
+/** Minimal rubric config needed for template resolution */
+export interface RubricConfig {
+    templates: Record<string, {
+        criteria_label?: string;
+        dimension?: string;
+        header: string;
+        scale: string[];
+    }>;
+}
+/**
+ * Resolve a templated LLM-rubric assertion into a fully assembled
+ * Promptfoo assertion with rubric text and dimension metadata.
+ *
+ * A "templated" assertion has `template` (referencing a key in rubrics.ts)
+ * and `criteria` (task-specific bullet points). The template provides the
+ * scoring header, scale, and dimension metadata. The criteria are appended
+ * to create the final rubric prompt.
+ *
+ * Returns null (with a warning) if the template can't be resolved.
+ */
+export declare function resolveTemplatedAssertion(assertion: {
+    criteria: string[];
+    template: string;
+    type: string;
+}, rubricConfig: RubricConfig | undefined, graderProvider: string | undefined, warnings: string[]): PromptfooAssertion | null;

package/dist/pipeline/compiler/rubric-resolution.js

@@ -0,0 +1,52 @@
+/**
+ * Shared rubric template resolution for all evaluation modes.
+ *
+ * Resolves templated LLM-rubric assertions (those with `template` + `criteria`
+ * fields) into fully assembled Promptfoo assertions with rubric text and
+ * dimension metadata.
+ *
+ * Used by both literacy and agent-harness compilers. Extracted from
+ * literacy/assertions.ts to fix the compilation bug where agent-harness
+ * tasks with templated rubrics produced empty rubric text (DOC-2029).
+ *
+ * @see docs/design-docs/mode-agnostic-scoring.md
+ * @see config/rubrics.ts — template definitions
+ */
+// ---------------------------------------------------------------------------
+// Template resolution
+// ---------------------------------------------------------------------------
+/**
+ * Resolve a templated LLM-rubric assertion into a fully assembled
+ * Promptfoo assertion with rubric text and dimension metadata.
+ *
+ * A "templated" assertion has `template` (referencing a key in rubrics.ts)
+ * and `criteria` (task-specific bullet points). The template provides the
+ * scoring header, scale, and dimension metadata. The criteria are appended
+ * to create the final rubric prompt.
+ *
+ * Returns null (with a warning) if the template can't be resolved.
+ */
+export function resolveTemplatedAssertion(assertion, rubricConfig, graderProvider, warnings) {
+    if (!rubricConfig) {
+        warnings.push(`No rubric config — template "${assertion.template}" cannot be resolved`);
+        return null;
+    }
+    const template = rubricConfig.templates[assertion.template];
+    if (!template) {
+        warnings.push(`Unknown rubric template: "${assertion.template}"`);
+        return null;
+    }
+    const scaleText = template.scale.map((s) => `- ${s}`).join("\n");
+    const criteriaText = assertion.criteria.map((c) => `- ${c}`).join("\n");
+    const rubricValue = `${template.header}\n${scaleText}\n\n` +
+        `${template.criteria_label ?? "Check for:"}\n${criteriaText}\n\n` +
+        `Return ONLY a JSON object: {"score": <number>, "reason": "<explanation>"}`;
+    return {
+        type: "llm-rubric",
+        value: rubricValue,
+        ...(graderProvider ? { provider: graderProvider } : {}),
+        ...(template.dimension
+            ? { metadata: { dimension: template.dimension, maxScore: 100 } }
+            : {}),
+    };
+}

package/dist/pipeline/compiler/scoring-bridge.js

@@ -41,19 +41,25 @@ import { classifyRubric, parseRubricScore } from "../../_vendor/ailf-core/index.
 export function scoreTestGroup(tests, profile, taskId) {
     let totalCost = 0;
     // Step 1: Convert all ComponentResults into AssertionScore[] (0–1 scale)
+    //
+    // Two assertion types contribute to scoring:
+    //   - llm-rubric: dimension from metadata, score from grader (0–100 → [0,1])
+    //   - javascript: mapped to "assertion-pass-rate" dimension (pass=1, fail=0)
+    //
+    // Other types (cost, trajectory, contains, etc.) are metadata or guards —
+    // they don't produce dimension scores.
     const assertionScores = [];
     for (const test of tests) {
         totalCost += test.cost;
         for (const comp of test.gradingResult.componentResults) {
-            if (comp.assertion?.type !== "llm-rubric")
-                continue;
-            const converted = componentToAssertionScore(comp);
+            const converted = componentToScore(comp);
             if (converted)
                 assertionScores.push(converted);
         }
     }
     // Step 2: Aggregate into DimensionScores (0–1 scale)
     const dimensionLabels = {
+        "assertion-pass-rate": "Assertion Pass Rate",
         "code-correctness": "Code Correctness",
         "doc-coverage": "Doc Coverage",
         "task-completion": "Task Completion",
@@ -86,12 +92,34 @@ export function scoreTestGroup(tests, profile, taskId) {
 // Conversion helpers
 // ---------------------------------------------------------------------------
 /**
- * Convert a single Promptfoo ComponentResult into the scoring engine's
- * AssertionScore format.
+ * Route a ComponentResult to the appropriate scoring conversion.
  *
- * Returns null if the component doesn't map to a known dimension.
+ * Dispatches by assertion type:
+ *   - llm-rubric  → dimension from metadata, grader score (0–100 → [0,1])
+ *   - javascript   → "assertion-pass-rate" dimension, binary (pass=1, fail=0)
+ *   - everything else → null (not a scoring-relevant assertion)
+ *
+ * This replaces the previous llm-rubric-only filter that caused agent-harness
+ * javascript assertions to be invisible to the scoring engine (DOC-2029).
+ */
+function componentToScore(comp) {
+    const type = comp.assertion?.type;
+    if (type === "llm-rubric") {
+        return llmRubricToScore(comp);
+    }
+    if (type === "javascript") {
+        return javascriptAssertionToScore(comp);
+    }
+    // Other types (cost, trajectory, contains, etc.) don't produce scores
+    return null;
+}
+/**
+ * Convert an LLM-rubric ComponentResult into an AssertionScore.
+ *
+ * The dimension comes from metadata (set during rubric template resolution).
+ * Returns null if the component doesn't map to any dimension.
  */
-function componentToAssertionScore(comp) {
+function llmRubricToScore(comp) {
     const dim = classifyRubric(comp);
     if (!dim)
         return null;
@@ -108,6 +136,30 @@ function componentToAssertionScore(comp) {
         weight: 1.0,
     };
 }
+/**
+ * Convert a javascript assertion ComponentResult into an AssertionScore.
+ *
+ * Javascript assertions (fileExists, fileContains, commandSucceeds, etc.)
+ * produce binary pass/fail results. They map to the "assertion-pass-rate"
+ * dimension — the fraction of structural assertions that passed.
+ *
+ * Zero-weight assertions (like URL extraction) are excluded from scoring.
+ */
+function javascriptAssertionToScore(comp) {
+    // Skip zero-weight assertions (diagnostic-only, e.g., URL extraction)
+    const weight = comp.assertion?.weight;
+    if (weight === 0)
+        return null;
+    return {
+        assertionType: "javascript",
+        dimension: "assertion-pass-rate",
+        latencyMs: 0,
+        pass: comp.pass,
+        reason: comp.reason ?? "",
+        score: comp.pass ? 1.0 : 0.0,
+        weight: 1.0,
+    };
+}
 /** Convert kebab-case dimension key to camelCase (e.g., "task-completion" → "taskCompletion") */
 function kebabToCamel(kebab) {
     return kebab.replace(/-([a-z])/g, (_, c) => c.toUpperCase());

package/dist/pipeline/provenance.js

@@ -59,6 +59,12 @@ export function buildProvenance(input) {
         evalFingerprint: input.evalFingerprint,
         hasLineage: Boolean(lineage),
     });
+    // Non-literacy modes (agent-harness, mcp-server, etc.) don't use the
+    // config/models.ts model matrix — listing those models would be misleading.
+    // Only include them for literacy mode where they're the actual eval targets.
+    const evaluatedModels = input.mode === "literacy"
+        ? models.models.map((m) => ({ id: m.id, label: m.label }))
+        : [];
     return {
         areas: input.areas,
         autoScope: input.autoScope,
@@ -68,7 +74,7 @@ export function buildProvenance(input) {
         graderModel: models.grader.id,
         lineage,
         mode: input.mode,
-        models: models.models.map((m) => ({ id: m.id, label: m.label })),
+        models: evaluatedModels,
         promptfooUrl: input.promptfooUrl,
         promptfooUrls: input.promptfooUrls,
         source: {

package/dist/pipeline/validate.d.ts

@@ -51,10 +51,11 @@ export declare function validateReferenceSolutions(rootDir: string): ValidationI
  */
 export declare function validateRubricsYaml(rootDir: string): ValidationIssue[];
 /**
- * Check that tasks/*.yaml files exist, parse, and conform to the Zod schema.
- * Validates both the new single-definition format (with `id`) and the legacy
- * paired format. Uses `TaskFileSchema` from schemas.ts for structural
- * validation, plus cross-entry checks (duplicate IDs, docs path consistency).
+ * Check that task definition files exist.
+ *
+ * Tasks live as `*.task.ts` files in mode subdirectories (e.g.
+ * `tasks/literacy/groq.task.ts`). Legacy YAML task files are no longer
+ * used. Warns only if no task files are found at all.
  */
 export declare function validateTaskFiles(rootDir: string): ValidationIssue[];
 /**

package/dist/pipeline/validate.js

@@ -9,10 +9,9 @@
  */
 import fs from "fs";
 import path from "path";
-import { load } from "js-yaml";
 import { tryLoadConfigFile } from "./compiler/config-loader.js";
 import { resolveMappings } from "./resolve-mappings.js";
-import { FeatureRegistrySchema, formatZodErrors, RubricConfigSchema, TaskFileSchema, ThresholdConfigSchema, } from "./schemas.js";
+import { FeatureRegistrySchema, formatZodErrors, RubricConfigSchema, ThresholdConfigSchema, } from "./schemas.js";
 // ---------------------------------------------------------------------------
 // Helpers
 // ---------------------------------------------------------------------------
@@ -248,10 +247,11 @@ export function validateRubricsYaml(rootDir) {
     return issues;
 }
 /**
- * Check that tasks/*.yaml files exist, parse, and conform to the Zod schema.
- * Validates both the new single-definition format (with `id`) and the legacy
- * paired format. Uses `TaskFileSchema` from schemas.ts for structural
- * validation, plus cross-entry checks (duplicate IDs, docs path consistency).
+ * Check that task definition files exist.
+ *
+ * Tasks live as `*.task.ts` files in mode subdirectories (e.g.
+ * `tasks/literacy/groq.task.ts`). Legacy YAML task files are no longer
+ * used. Warns only if no task files are found at all.
  */
 export function validateTaskFiles(rootDir) {
     const source = "validateTaskFiles";
@@ -261,70 +261,9 @@ export function validateTaskFiles(rootDir) {
         issues.push(warning(source, "tasks/ directory not found (using Content Lake tasks?)", tasksDir));
         return issues;
     }
-    const yamlFiles = fs
-        .readdirSync(tasksDir)
-        .filter((f) => (f.endsWith(".yaml") || f.endsWith(".yml")) && !f.startsWith("."));
-    if (yamlFiles.length === 0) {
-        issues.push(warning(source, "No task YAML files found in tasks/ (using Content Lake tasks?)", tasksDir));
-        return issues;
-    }
-    const allIds = new Map(); // id → source file
-    const templateKeys = loadTemplateKeys(rootDir);
-    for (const file of yamlFiles) {
-        const filePath = path.join(tasksDir, file);
-        // Step 1: Parse YAML
-        const result = parseYamlFile(filePath, source);
-        if (!result.ok) {
-            issues.push(result.issue);
-            continue;
-        }
-        const { data } = result;
-        if (!Array.isArray(data)) {
-            issues.push(error(source, `${file} did not parse to an array of tasks`, filePath));
-            continue;
-        }
-        // Step 2: Validate each entry with Zod schema
-        const zodResult = TaskFileSchema.safeParse(data);
-        if (!zodResult.success) {
-            const lines = formatZodErrors(zodResult.error);
-            for (const line of lines) {
-                issues.push(error(source, `${file}: ${line.trim()}`, filePath));
-            }
-            continue;
-        }
-        // Step 3: Cross-entry validation (duplicate IDs, docs path consistency)
-        for (const entry of zodResult.data) {
-            if ("id" in entry && typeof entry.id === "string") {
-                // Check for duplicate IDs across all files
-                if (allIds.has(entry.id)) {
-                    issues.push(error(source, `${file}: duplicate id '${entry.id}' (also in ${allIds.get(entry.id)})`, filePath));
-                }
-                else {
-                    allIds.set(entry.id, file);
-                }
-                // Check docs path matches task id
-                const vars = entry.vars;
-                if (vars.docs && typeof vars.docs === "string") {
-                    const expectedPath = `file://contexts/canonical/${entry.id}.md`;
-                    if (vars.docs !== expectedPath) {
-                        issues.push(warning(source, `${file}: id is '${entry.id}' but docs path is '${vars.docs}' (expected '${expectedPath}')`, filePath));
-                    }
-                }
-                // Check that llm-rubric template references exist in config/rubrics
-                const asserts = entry.assert;
-                if (Array.isArray(asserts) && templateKeys.size > 0) {
-                    for (const a of asserts) {
-                        const assertion = a;
-                        if (assertion.type === "llm-rubric" &&
-                            typeof assertion.template === "string") {
-                            if (!templateKeys.has(assertion.template)) {
-                                issues.push(error(source, `${file}: task '${entry.id}' references unknown rubric template '${assertion.template}' (available: ${[...templateKeys].join(", ")})`, filePath));
-                            }
-                        }
-                    }
-                }
-            }
-        }
+    const taskAreas = collectTaskAreas(tasksDir);
+    if (taskAreas.size === 0) {
+        issues.push(warning(source, "No task files found in tasks/ (using Content Lake tasks?)", tasksDir));
     }
     return issues;
 }
@@ -355,15 +294,10 @@ export function validateThresholdsYaml(rootDir) {
     // Cross-reference: warn if an area override references an area with no task file
     if (zodResult.data.areas) {
         const tasksDir = path.join(rootDir, "tasks");
-        if (fs.existsSync(tasksDir)) {
-            const taskFiles = new Set(fs
-                .readdirSync(tasksDir)
-                .filter((f) => /\.(yaml|yml|task\.ts|task\.js)$/.test(f))
-                .map((f) => f.replace(/\.(yaml|yml|task\.ts|task\.js)$/, "")));
-            for (const areaName of Object.keys(zodResult.data.areas)) {
-                if (!taskFiles.has(areaName)) {
-                    issues.push(warning(source, `config/thresholds: area override '${areaName}' has no matching tasks/${areaName}`, loaded.filePath));
-                }
+        const taskAreas = collectTaskAreas(tasksDir);
+        for (const areaName of Object.keys(zodResult.data.areas)) {
+            if (!taskAreas.has(areaName)) {
+                issues.push(warning(source, `config/thresholds: area override '${areaName}' has no matching task file`, loaded.filePath));
             }
         }
     }
@@ -378,44 +312,31 @@ function error(source, message, filePath) {
     };
 }
 /**
- * Load the set of valid rubric template keys from config/rubrics.
- * Returns an empty set if the file is missing or invalid.
+ * Collect task area names from all subdirectories of `tasksDir`.
+ *
+ * Task files live in mode subdirectories (e.g. `tasks/literacy/groq.task.ts`).
+ * Returns a set of basenames without the `.task.ts`/`.task.js` extension.
  */
-function loadTemplateKeys(rootDir) {
-    const loaded = tryLoadConfigFile("rubrics", rootDir);
-    if (!loaded)
+function collectTaskAreas(tasksDir) {
+    if (!fs.existsSync(tasksDir))
         return new Set();
-    try {
-        const templates = loaded.data?.templates;
-        if (templates && typeof templates === "object") {
-            return new Set(Object.keys(templates));
+    const areas = new Set();
+    const taskFilePattern = /\.task\.(ts|js)$/;
+    for (const entry of fs.readdirSync(tasksDir, { withFileTypes: true })) {
+        if (entry.isDirectory()) {
+            const subdir = path.join(tasksDir, entry.name);
+            for (const file of fs.readdirSync(subdir)) {
+                if (taskFilePattern.test(file)) {
+                    areas.add(file.replace(taskFilePattern, ""));
+                }
+            }
+        }
+        // Also check top-level task files for backwards compatibility
+        if (entry.isFile() && taskFilePattern.test(entry.name)) {
+            areas.add(entry.name.replace(taskFilePattern, ""));
         }
     }
-    catch {
-        // Ignore — structural errors are caught by validateRubricsYaml
-    }
-    return new Set();
-}
-/** Safely parse a YAML file, returning the parsed value or a validation issue. */
-function parseYamlFile(filePath, source) {
-    if (!fs.existsSync(filePath)) {
-        return {
-            issue: error(source, `File not found: ${filePath}`, filePath),
-            ok: false,
-        };
-    }
-    try {
-        const raw = fs.readFileSync(filePath, "utf-8");
-        const data = load(raw);
-        return { data, ok: true };
-    }
-    catch (err) {
-        const message = err instanceof Error ? err.message : "Unknown YAML parse error";
-        return {
-            issue: error(source, `Failed to parse YAML: ${message}`, filePath),
-            ok: false,
-        };
-    }
+    return areas;
 }
 // ---------------------------------------------------------------------------
 // Main entry point

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sanity/ailf",
-  "version": "2.2.0",
+  "version": "2.3.0",
   "private": false,
   "publishConfig": {
     "access": "public"