@sanity/ailf 2.2.0 → 2.3.1

package/dist/pipeline/validate.d.ts

@@ -51,10 +51,11 @@ export declare function validateReferenceSolutions(rootDir: string): ValidationI
  */
 export declare function validateRubricsYaml(rootDir: string): ValidationIssue[];
 /**
- * Check that tasks/*.yaml files exist, parse, and conform to the Zod schema.
- * Validates both the new single-definition format (with `id`) and the legacy
- * paired format. Uses `TaskFileSchema` from schemas.ts for structural
- * validation, plus cross-entry checks (duplicate IDs, docs path consistency).
+ * Check that task definition files exist.
+ *
+ * Tasks live as `*.task.ts` files in mode subdirectories (e.g.
+ * `tasks/literacy/groq.task.ts`). Legacy YAML task files are no longer
+ * used. Warns only if no task files are found at all.
  */
 export declare function validateTaskFiles(rootDir: string): ValidationIssue[];
 /**

package/dist/pipeline/validate.js

@@ -9,10 +9,9 @@
  */
 import fs from "fs";
 import path from "path";
-import { load } from "js-yaml";
 import { tryLoadConfigFile } from "./compiler/config-loader.js";
 import { resolveMappings } from "./resolve-mappings.js";
-import { FeatureRegistrySchema, formatZodErrors, RubricConfigSchema, TaskFileSchema, ThresholdConfigSchema, } from "./schemas.js";
+import { FeatureRegistrySchema, formatZodErrors, RubricConfigSchema, ThresholdConfigSchema, } from "./schemas.js";
 // ---------------------------------------------------------------------------
 // Helpers
 // ---------------------------------------------------------------------------
@@ -248,10 +247,11 @@ export function validateRubricsYaml(rootDir) {
     return issues;
 }
 /**
- * Check that tasks/*.yaml files exist, parse, and conform to the Zod schema.
- * Validates both the new single-definition format (with `id`) and the legacy
- * paired format. Uses `TaskFileSchema` from schemas.ts for structural
- * validation, plus cross-entry checks (duplicate IDs, docs path consistency).
+ * Check that task definition files exist.
+ *
+ * Tasks live as `*.task.ts` files in mode subdirectories (e.g.
+ * `tasks/literacy/groq.task.ts`). Legacy YAML task files are no longer
+ * used. Warns only if no task files are found at all.
  */
 export function validateTaskFiles(rootDir) {
     const source = "validateTaskFiles";
@@ -261,70 +261,9 @@ export function validateTaskFiles(rootDir) {
         issues.push(warning(source, "tasks/ directory not found (using Content Lake tasks?)", tasksDir));
         return issues;
     }
-    const yamlFiles = fs
-        .readdirSync(tasksDir)
-        .filter((f) => (f.endsWith(".yaml") || f.endsWith(".yml")) && !f.startsWith("."));
-    if (yamlFiles.length === 0) {
-        issues.push(warning(source, "No task YAML files found in tasks/ (using Content Lake tasks?)", tasksDir));
-        return issues;
-    }
-    const allIds = new Map(); // id → source file
-    const templateKeys = loadTemplateKeys(rootDir);
-    for (const file of yamlFiles) {
-        const filePath = path.join(tasksDir, file);
-        // Step 1: Parse YAML
-        const result = parseYamlFile(filePath, source);
-        if (!result.ok) {
-            issues.push(result.issue);
-            continue;
-        }
-        const { data } = result;
-        if (!Array.isArray(data)) {
-            issues.push(error(source, `${file} did not parse to an array of tasks`, filePath));
-            continue;
-        }
-        // Step 2: Validate each entry with Zod schema
-        const zodResult = TaskFileSchema.safeParse(data);
-        if (!zodResult.success) {
-            const lines = formatZodErrors(zodResult.error);
-            for (const line of lines) {
-                issues.push(error(source, `${file}: ${line.trim()}`, filePath));
-            }
-            continue;
-        }
-        // Step 3: Cross-entry validation (duplicate IDs, docs path consistency)
-        for (const entry of zodResult.data) {
-            if ("id" in entry && typeof entry.id === "string") {
-                // Check for duplicate IDs across all files
-                if (allIds.has(entry.id)) {
-                    issues.push(error(source, `${file}: duplicate id '${entry.id}' (also in ${allIds.get(entry.id)})`, filePath));
-                }
-                else {
-                    allIds.set(entry.id, file);
-                }
-                // Check docs path matches task id
-                const vars = entry.vars;
-                if (vars.docs && typeof vars.docs === "string") {
-                    const expectedPath = `file://contexts/canonical/${entry.id}.md`;
-                    if (vars.docs !== expectedPath) {
-                        issues.push(warning(source, `${file}: id is '${entry.id}' but docs path is '${vars.docs}' (expected '${expectedPath}')`, filePath));
-                    }
-                }
-                // Check that llm-rubric template references exist in config/rubrics
-                const asserts = entry.assert;
-                if (Array.isArray(asserts) && templateKeys.size > 0) {
-                    for (const a of asserts) {
-                        const assertion = a;
-                        if (assertion.type === "llm-rubric" &&
-                            typeof assertion.template === "string") {
-                            if (!templateKeys.has(assertion.template)) {
-                                issues.push(error(source, `${file}: task '${entry.id}' references unknown rubric template '${assertion.template}' (available: ${[...templateKeys].join(", ")})`, filePath));
-                            }
-                        }
-                    }
-                }
-            }
-        }
+    const taskAreas = collectTaskAreas(tasksDir);
+    if (taskAreas.size === 0) {
+        issues.push(warning(source, "No task files found in tasks/ (using Content Lake tasks?)", tasksDir));
     }
     return issues;
 }
@@ -355,15 +294,10 @@ export function validateThresholdsYaml(rootDir) {
     // Cross-reference: warn if an area override references an area with no task file
     if (zodResult.data.areas) {
         const tasksDir = path.join(rootDir, "tasks");
-        if (fs.existsSync(tasksDir)) {
-            const taskFiles = new Set(fs
-                .readdirSync(tasksDir)
-                .filter((f) => /\.(yaml|yml|task\.ts|task\.js)$/.test(f))
-                .map((f) => f.replace(/\.(yaml|yml|task\.ts|task\.js)$/, "")));
-            for (const areaName of Object.keys(zodResult.data.areas)) {
-                if (!taskFiles.has(areaName)) {
-                    issues.push(warning(source, `config/thresholds: area override '${areaName}' has no matching tasks/${areaName}`, loaded.filePath));
-                }
+        const taskAreas = collectTaskAreas(tasksDir);
+        for (const areaName of Object.keys(zodResult.data.areas)) {
+            if (!taskAreas.has(areaName)) {
+                issues.push(warning(source, `config/thresholds: area override '${areaName}' has no matching task file`, loaded.filePath));
             }
         }
     }
@@ -378,44 +312,31 @@ function error(source, message, filePath) {
     };
 }
 /**
- * Load the set of valid rubric template keys from config/rubrics.
- * Returns an empty set if the file is missing or invalid.
+ * Collect task area names from all subdirectories of `tasksDir`.
+ *
+ * Task files live in mode subdirectories (e.g. `tasks/literacy/groq.task.ts`).
+ * Returns a set of basenames without the `.task.ts`/`.task.js` extension.
  */
-function loadTemplateKeys(rootDir) {
-    const loaded = tryLoadConfigFile("rubrics", rootDir);
-    if (!loaded)
+function collectTaskAreas(tasksDir) {
+    if (!fs.existsSync(tasksDir))
         return new Set();
-    try {
-        const templates = loaded.data?.templates;
-        if (templates && typeof templates === "object") {
-            return new Set(Object.keys(templates));
+    const areas = new Set();
+    const taskFilePattern = /\.task\.(ts|js)$/;
+    for (const entry of fs.readdirSync(tasksDir, { withFileTypes: true })) {
+        if (entry.isDirectory()) {
+            const subdir = path.join(tasksDir, entry.name);
+            for (const file of fs.readdirSync(subdir)) {
+                if (taskFilePattern.test(file)) {
+                    areas.add(file.replace(taskFilePattern, ""));
+                }
+            }
+        }
+        // Also check top-level task files for backwards compatibility
+        if (entry.isFile() && taskFilePattern.test(entry.name)) {
+            areas.add(entry.name.replace(taskFilePattern, ""));
         }
     }
-    catch {
-        // Ignore — structural errors are caught by validateRubricsYaml
-    }
-    return new Set();
-}
-/** Safely parse a YAML file, returning the parsed value or a validation issue. */
-function parseYamlFile(filePath, source) {
-    if (!fs.existsSync(filePath)) {
-        return {
-            issue: error(source, `File not found: ${filePath}`, filePath),
-            ok: false,
-        };
-    }
-    try {
-        const raw = fs.readFileSync(filePath, "utf-8");
-        const data = load(raw);
-        return { data, ok: true };
-    }
-    catch (err) {
-        const message = err instanceof Error ? err.message : "Unknown YAML parse error";
-        return {
-            issue: error(source, `Failed to parse YAML: ${message}`, filePath),
-            ok: false,
-        };
-    }
+    return areas;
 }
 // ---------------------------------------------------------------------------
 // Main entry point

package/dist/webhook/eval-request-handler.js

@@ -173,6 +173,10 @@ async function dispatchGitHubEval(repo, payload, config) {
             projectId: payload.projectId,
             publish: true,
             source: "production",
+            // Studio-initiated evals always use Content Lake as the task source.
+            // Without this, the pipeline only loads filesystem .task.ts files and
+            // Studio-owned tasks are invisible.
+            taskMode: "content-lake",
             // Release-scoped fields
             ...(hasPerspective ? { perspective: payload.perspective } : {}),
             // Task-scoped fields

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sanity/ailf",
-  "version": "2.2.0",
+  "version": "2.3.1",
   "private": false,
   "publishConfig": {
     "access": "public"