@sanity/ailf 0.1.23 → 0.1.24

package/dist/orchestration/steps/run-eval-step.js

@@ -47,7 +47,14 @@ export class RunEvalStep {
             }
             : undefined;
         const tasks = await ctx.taskSource.loadTasks(filter);
-        const taskIds = tasks.map((t) => t.id);
+        // Only check context files for tasks that have canonical docs.
+        // Tasks without canonical docs are skipped by FetchDocsStep (they
+        // have no docs to fetch), so no context file is written for them.
+        // The generated Promptfoo config still includes their "without-docs"
+        // variant (testing model knowledge alone), which doesn't need a
+        // context file.
+        const tasksWithDocs = tasks.filter((t) => t.canonicalDocs.length > 0);
+        const taskIds = tasksWithDocs.map((t) => t.id);
         const contextIssues = checkCanonicalContextsExist(rootDir, taskIds);
         const contextErrors = contextIssues.filter((i) => i.severity === "error");
         if (contextErrors.length > 0) {

package/dist/pipeline/expand-tasks.js

@@ -198,8 +198,15 @@ export function expandTask(task, rubricConfig, mode = "baseline") {
 /**
  * Convert a TaskDefinition (from @sanity/ailf-core) to the local
  * SingleTaskDefinition format used by expandTask().
+ *
+ * When a task has no canonical docs, the `docs` var is set to empty string
+ * instead of a file path. This prevents Promptfoo from trying to read a
+ * context file that was never created by FetchDocsStep (which skips tasks
+ * without canonical docs). The gold entry still runs — it just tests model
+ * knowledge alone, same as the baseline variant.
  */
 function taskDefinitionToSingle(task) {
+    const hasDocs = task.canonicalDocs.length > 0;
     return {
         assert: task.assertions.map((a) => ({ ...a })),
         baseline: task.baseline,
@@ -208,7 +215,7 @@ function taskDefinitionToSingle(task) {
         featureArea: task.featureArea,
         id: task.id,
         vars: {
-            docs: `file://contexts/canonical/${task.id}.md`,
+            docs: hasDocs ? `file://contexts/canonical/${task.id}.md` : "",
             task: task.taskPrompt,
             ...task.extraVars,
         },

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sanity/ailf",
-  "version": "0.1.23",
+  "version": "0.1.24",
   "private": false,
   "publishConfig": {
     "access": "restricted"