@sanity/ailf 0.5.0 → 2.0.0

package/dist/adapters/task-sources/content-lake-task-source.js

@@ -1,10 +1,11 @@
 /**
  * Adapter: Load task definitions from the Sanity Content Lake.
  *
- * Fetches ailf.task documents via GROQ and maps them to the canonical
- * TaskDefinition type. This adapter is the Content Lake counterpart of
- * YamlTaskSource — both produce identical TaskDefinition[] for the same
- * logical tasks. The pipeline never knows which adapter loaded the tasks.
+ * Fetches ailf.task documents via GROQ and maps them to
+ * GeneralizedTaskDefinition (LiteracyTaskDefinition variant). This
+ * adapter is the Content Lake counterpart of YamlTaskSource — both
+ * produce identical GeneralizedTaskDefinition[] for the same logical
+ * tasks. The pipeline never knows which adapter loaded the tasks.
  *
  * Wired in the composition root when --task-source content-lake is set.
  *
@@ -16,11 +17,11 @@
 // ---------------------------------------------------------------------------
 /**
  * GROQ query that projects ailf.task documents into a shape suitable for
- * mapping to TaskDefinition.
+ * mapping to LiteracyTaskDefinition.
  *
  * Key projections:
- * - featureArea reference → dereferenced areaId string
- * - canonicalDocs[] → dereferenced article slugs with reason
+ * - area reference → dereferenced areaId string
+ * - contextDocs[] → dereferenced article slugs with reason
  * - referenceSolution → title (for identification, not full content)
  *
  * Filter parameters:
@@ -29,7 +30,7 @@
  */
 const TASKS_QUERY = /* groq */ `
 *[_type == "ailf.task"
-  && (!defined($areas) || featureArea->areaId.current in $areas)
+  && (!defined($areas) || area->areaId.current in $areas)
   && (!defined($taskIds) || id.current in $taskIds)
   && (
     // Status-based filtering (unified — replaces execution.enabled)
@@ -40,13 +41,13 @@ const TASKS_QUERY = /* groq */ `
     || (defined($taskIds) && status != "archived")
   )
   && (!defined($tags) || count((tags)[@ in $tags]) > 0)
-] | order(featureArea->areaId.current asc, id.current asc) {
+] | order(area->areaId.current asc, id.current asc) {
   "taskId": id.current,
-  description,
-  "featureAreaId": featureArea->areaId.current,
-  taskPrompt,
+  title,
+  "areaId": area->areaId.current,
+  promptText,
   docCoverage,
-  "canonicalDocs": canonicalDocs[] {
+  "contextDocs": contextDocs[] {
     refType,
     "slug": doc->slug.current,
     "docRefId": doc->_id,
@@ -56,7 +57,7 @@ const TASKS_QUERY = /* groq */ `
     perspective,
     reason
   },
-  assert,
+  assertions,
   rawAssert,
   baseline,
   tags,
@@ -79,7 +80,7 @@ export class ContentLakeTaskSource {
         }
         const definitions = [];
         for (const entry of raw) {
-            const mapped = mapToTaskDefinition(entry);
+            const mapped = mapToLiteracyTask(entry);
             if (!mapped)
                 continue;
             definitions.push(mapped);
@@ -105,35 +106,29 @@ function buildGroqParams(filter) {
     };
 }
 // ---------------------------------------------------------------------------
-// Mapping: Content Lake → TaskDefinition
+// Mapping: Content Lake → LiteracyTaskDefinition
 // ---------------------------------------------------------------------------
 /**
- * Map a Content Lake ailf.task document to the canonical TaskDefinition.
+ * Map a Content Lake ailf.task document directly to a LiteracyTaskDefinition.
  *
  * Returns null if the document is missing required fields (taskId,
- * description, featureAreaId, taskPrompt). These are required by the
+ * title, areaId, promptText). These are required by the
  * Studio schema, but defensive coding handles edge cases (drafts,
  * partially-created documents, etc.).
  */
-function mapToTaskDefinition(raw) {
+function mapToLiteracyTask(raw) {
     // Required fields — skip malformed documents
-    if (!raw.taskId ||
-        !raw.description ||
-        !raw.featureAreaId ||
-        !raw.taskPrompt) {
+    if (!raw.taskId || !raw.title || !raw.areaId || !raw.promptText) {
         return null;
     }
-    const canonicalDocs = (raw.canonicalDocs ?? [])
+    const docs = (raw.contextDocs ?? [])
         .map(mapCanonicalDocRef)
         .filter((d) => d !== null);
-    const assertions = mapAssertions(raw.assert ?? []);
+    const assertions = mapAssertions(raw.assertions ?? []);
     // Append raw pass-through assertions (escape hatch for arbitrary Promptfoo
     // assertion types that aren't in the curated list). These bypass template
     // resolution and flow directly into the expanded Promptfoo test case as
-    // value-based assertions. In baseline mode, buildBaselineAsserts() with
-    // "full" (the default) copies all assertions as-is, so rawAssert
-    // entries only run in the gold variant — consistent with how regular
-    // value-based assertions like `contains` or `regex` behave.
+    // value-based assertions.
     const rawAssertions = (raw.rawAssert ?? [])
         .filter((a) => !!a.type)
         .map((a) => ({
@@ -141,7 +136,10 @@ function mapToTaskDefinition(raw) {
         ...(a.value !== undefined ? { value: a.value } : {}),
         ...(a.threshold !== undefined ? { threshold: a.threshold } : {}),
     }));
-    const allAssertions = [...assertions, ...rawAssertions];
+    const allAssertions = [
+        ...assertions,
+        ...rawAssertions,
+    ];
     const baseline = raw.baseline
         ? {
             ...(raw.baseline.enabled !== undefined
@@ -155,25 +153,23 @@ function mapToTaskDefinition(raw) {
         }
         : undefined;
     return {
+        mode: "literacy",
+        id: raw.taskId,
+        title: raw.title,
+        area: raw.areaId,
+        prompt: { text: raw.promptText },
+        context: { docs },
         assertions: allAssertions,
-        canonicalDocs,
-        description: raw.description,
         docCoverage: raw.docCoverage ?? false,
-        featureArea: raw.featureAreaId,
-        id: raw.taskId,
         // Reference solution path is not meaningful for CL tasks — the solution
         // content lives in the Content Lake document, not on the filesystem.
-        // Downstream consumers that need the solution content will need to fetch
-        // it via the referenceSolution reference. For now, use empty string
-        // (same as tasks without a reference solution).
         referenceSolution: "",
-        taskPrompt: raw.taskPrompt,
         ...(baseline ? { baseline } : {}),
         ...(raw.tags?.length ? { tags: raw.tags } : {}),
     };
 }
 /**
- * Map a Content Lake canonical doc entry to the polymorphic CanonicalDocRef.
+ * Map a Content Lake context doc entry to the polymorphic CanonicalDocRef.
  *
  * Uses `refType` to determine which value field to read. Falls back to
  * slug-based resolution for backward compatibility (documents created

package/dist/adapters/task-sources/index.d.ts

@@ -1,7 +1,8 @@
 export { CompositeTaskSource } from "./composite-task-source.js";
 export { ContentLakeTaskSource } from "./content-lake-task-source.js";
-export { parseRepoConfig, parseRepoTaskFile, RepoConfigSchema, RepoTaskSchema, CURATED_ASSERTION_TYPES, RUBRIC_TEMPLATE_NAMES, type RepoConfig, type RepoTask, } from "./repo-schemas.js";
+export { CanonicalTaskFileSchema, CanonicalTaskSchema, CURATED_ASSERTION_TYPES, detectLegacyFieldNames, parseCanonicalTaskFile, parseRepoConfig, RepoConfigSchema, RUBRIC_TEMPLATE_NAMES, type CanonicalTask, type CuratedAssertionType, type RepoConfig, type RubricTemplateName, } from "./repo-schemas.js";
 export { RepoTaskSource } from "./repo-task-source.js";
 export { detectTriggerContext, resolveTrigger, type ResolvedTrigger, type TriggerContext, } from "./repo-trigger.js";
-export { formatValidationResult, validateRepoTasks, type ValidationMessage, type ValidationResult, } from "./repo-validation.js";
+export { formatValidationResult, validateCanonicalTasks, type ValidationMessage, type ValidationResult, } from "./repo-validation.js";
+export { discoverTsTaskFiles, loadAllTsTaskFiles, loadTsTaskFile, } from "./task-file-loader.js";
 export { YamlTaskSource } from "./yaml-task-source.js";

package/dist/adapters/task-sources/index.js

@@ -1,7 +1,8 @@
 export { CompositeTaskSource } from "./composite-task-source.js";
 export { ContentLakeTaskSource } from "./content-lake-task-source.js";
-export { parseRepoConfig, parseRepoTaskFile, RepoConfigSchema, RepoTaskSchema, CURATED_ASSERTION_TYPES, RUBRIC_TEMPLATE_NAMES, } from "./repo-schemas.js";
+export { CanonicalTaskFileSchema, CanonicalTaskSchema, CURATED_ASSERTION_TYPES, detectLegacyFieldNames, parseCanonicalTaskFile, parseRepoConfig, RepoConfigSchema, RUBRIC_TEMPLATE_NAMES, } from "./repo-schemas.js";
 export { RepoTaskSource } from "./repo-task-source.js";
 export { detectTriggerContext, resolveTrigger, } from "./repo-trigger.js";
-export { formatValidationResult, validateRepoTasks, } from "./repo-validation.js";
+export { formatValidationResult, validateCanonicalTasks, } from "./repo-validation.js";
+export { discoverTsTaskFiles, loadAllTsTaskFiles, loadTsTaskFile, } from "./task-file-loader.js";
 export { YamlTaskSource } from "./yaml-task-source.js";

package/dist/adapters/task-sources/repo-schemas.d.ts

@@ -1,29 +1,231 @@
 /**
- * repo-schemas.ts — Re-exports task schemas + defines config schemas.
+ * repo-schemas.ts — Canonical Zod schemas for task and config validation.
  *
- * Task schemas (RepoTaskSchema, assertions, etc.) are the single source
- * of truth in @sanity/ailf-tasks. This file re-exports them so existing
- * importers within the eval package don't need to change their paths.
+ * Task schemas validate .ailf/tasks/*.yaml and .task.ts files against the
+ * canonical GeneralizedTaskDefinition shape. Field names match the internal
+ * domain model: `area` (not featureArea), `assertions` (not assert),
+ * `context.docs` (not canonicalDocs), `prompt.text` (not vars.task).
  *
- * Config schemas (RepoConfigSchema, trigger config, etc.) remain here
- * because they are specific to the eval pipeline and not needed by
- * external tools that only validate task YAML.
+ * Previously this file re-exported from @sanity/ailf-tasks. That package
+ * has been eliminated — all schema logic now lives here.
  *
- * @see packages/tasks/src/schemas.ts — task schema source of truth
+ * Config schemas (RepoConfigSchema, trigger config) are eval-pipeline-
+ * specific and remain here unchanged.
+ *
+ * @see packages/core/src/types/generalized-task.ts — canonical TypeScript types
  * @see docs/exec-plans/tasks-as-content/phase-4-repo-based-tasks.md
  */
-import { RepoTaskFileSchema as _Schema } from "../../_vendor/ailf-tasks/index.d.ts";
 import { z } from "zod";
-export { CURATED_ASSERTION_TYPES, RepoTaskFileSchema, RepoTaskSchema, RUBRIC_TEMPLATE_NAMES, type CuratedAssertionType, type RepoTask, type RubricTemplateName, } from "../../_vendor/ailf-tasks/index.d.ts";
-export { loadTaskDir, parseTaskFile } from "../../_vendor/ailf-tasks/index.d.ts";
 /**
- * Parse and validate a repo task file's content. Returns typed tasks or throws
- * with a user-friendly Zod error message.
+ * The set of assertion types allowed in task files.
+ *
+ * This is a curated subset of Promptfoo assertion types — we expose only the
+ * types that are stable, well-documented, and useful for external authors.
+ */
+export declare const CURATED_ASSERTION_TYPES: readonly ["llm-rubric", "contains", "contains-any", "contains-all", "not-contains", "icontains", "icontains-any", "regex", "javascript", "similar", "cost", "latency"];
+export type CuratedAssertionType = (typeof CURATED_ASSERTION_TYPES)[number];
+/**
+ * Valid rubric template names — must match keys in config/rubrics.yaml.
+ */
+export declare const RUBRIC_TEMPLATE_NAMES: readonly ["task-completion", "code-correctness", "doc-coverage"];
+export type RubricTemplateName = (typeof RUBRIC_TEMPLATE_NAMES)[number];
+/**
+ * Zod schema for a single task definition using canonical field names.
+ *
+ * Uses .passthrough() to allow mode-specific fields (serverConfig, sandbox,
+ * handler, etc.) without listing every possible field. Mode-specific
+ * validation is deferred to the pipeline's mode handlers.
+ */
+export declare const CanonicalTaskSchema: z.ZodObject<{
+    id: z.ZodString;
+    mode: z.ZodDefault<z.ZodString>;
+    title: z.ZodString;
+    description: z.ZodOptional<z.ZodString>;
+    area: z.ZodOptional<z.ZodString>;
+    difficulty: z.ZodOptional<z.ZodEnum<{
+        basic: "basic";
+        intermediate: "intermediate";
+        advanced: "advanced";
+    }>>;
+    status: z.ZodDefault<z.ZodOptional<z.ZodEnum<{
+        active: "active";
+        draft: "draft";
+        paused: "paused";
+        archived: "archived";
+    }>>>;
+    tags: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    prompt: z.ZodOptional<z.ZodObject<{
+        template: z.ZodOptional<z.ZodString>;
+        text: z.ZodOptional<z.ZodString>;
+        systemMessage: z.ZodOptional<z.ZodString>;
+        vars: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+    }, z.core.$strip>>;
+    context: z.ZodOptional<z.ZodObject<{
+        docs: z.ZodOptional<z.ZodArray<z.ZodUnion<readonly [z.ZodObject<{
+            id: z.ZodString;
+            reason: z.ZodDefault<z.ZodOptional<z.ZodString>>;
+            slug: z.ZodOptional<z.ZodString>;
+            path: z.ZodOptional<z.ZodString>;
+        }, z.core.$strip>, z.ZodObject<{
+            slug: z.ZodString;
+            reason: z.ZodDefault<z.ZodOptional<z.ZodString>>;
+        }, z.core.$strip>, z.ZodObject<{
+            path: z.ZodString;
+            reason: z.ZodDefault<z.ZodOptional<z.ZodString>>;
+        }, z.core.$strip>, z.ZodObject<{
+            perspective: z.ZodString;
+            reason: z.ZodDefault<z.ZodOptional<z.ZodString>>;
+        }, z.core.$strip>]>>>;
+        fixtures: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    }, z.core.$strip>>;
+    assertions: z.ZodOptional<z.ZodArray<z.ZodUnion<readonly [z.ZodObject<{
+        type: z.ZodLiteral<"llm-rubric">;
+        template: z.ZodEnum<{
+            "task-completion": "task-completion";
+            "code-correctness": "code-correctness";
+            "doc-coverage": "doc-coverage";
+        }>;
+        criteria: z.ZodArray<z.ZodString>;
+        weight: z.ZodOptional<z.ZodNumber>;
+    }, z.core.$strip>, z.ZodObject<{
+        type: z.ZodEnum<{
+            "llm-rubric": "llm-rubric";
+            contains: "contains";
+            "contains-any": "contains-any";
+            "contains-all": "contains-all";
+            "not-contains": "not-contains";
+            icontains: "icontains";
+            "icontains-any": "icontains-any";
+            regex: "regex";
+            javascript: "javascript";
+            similar: "similar";
+            cost: "cost";
+            latency: "latency";
+        }>;
+        value: z.ZodOptional<z.ZodUnknown>;
+        threshold: z.ZodOptional<z.ZodNumber>;
+        weight: z.ZodOptional<z.ZodNumber>;
+    }, z.core.$loose>]>>>;
+    referenceSolution: z.ZodOptional<z.ZodString>;
+    docCoverage: z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
+    baseline: z.ZodOptional<z.ZodObject<{
+        enabled: z.ZodOptional<z.ZodBoolean>;
+        rubric: z.ZodOptional<z.ZodEnum<{
+            full: "full";
+            abbreviated: "abbreviated";
+            none: "none";
+        }>>;
+    }, z.core.$strip>>;
+    rubric: z.ZodOptional<z.ZodUnknown>;
+    providers: z.ZodOptional<z.ZodArray<z.ZodUnknown>>;
+    options: z.ZodOptional<z.ZodUnknown>;
+    metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+}, z.core.$loose>;
+export type CanonicalTask = z.infer<typeof CanonicalTaskSchema>;
+/**
+ * Schema for an array of canonical tasks — what a single .ailf/tasks/*.yaml
+ * file contains. Each file must define at least one task.
+ */
+export declare const CanonicalTaskFileSchema: z.ZodArray<z.ZodObject<{
+    id: z.ZodString;
+    mode: z.ZodDefault<z.ZodString>;
+    title: z.ZodString;
+    description: z.ZodOptional<z.ZodString>;
+    area: z.ZodOptional<z.ZodString>;
+    difficulty: z.ZodOptional<z.ZodEnum<{
+        basic: "basic";
+        intermediate: "intermediate";
+        advanced: "advanced";
+    }>>;
+    status: z.ZodDefault<z.ZodOptional<z.ZodEnum<{
+        active: "active";
+        draft: "draft";
+        paused: "paused";
+        archived: "archived";
+    }>>>;
+    tags: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    prompt: z.ZodOptional<z.ZodObject<{
+        template: z.ZodOptional<z.ZodString>;
+        text: z.ZodOptional<z.ZodString>;
+        systemMessage: z.ZodOptional<z.ZodString>;
+        vars: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+    }, z.core.$strip>>;
+    context: z.ZodOptional<z.ZodObject<{
+        docs: z.ZodOptional<z.ZodArray<z.ZodUnion<readonly [z.ZodObject<{
+            id: z.ZodString;
+            reason: z.ZodDefault<z.ZodOptional<z.ZodString>>;
+            slug: z.ZodOptional<z.ZodString>;
+            path: z.ZodOptional<z.ZodString>;
+        }, z.core.$strip>, z.ZodObject<{
+            slug: z.ZodString;
+            reason: z.ZodDefault<z.ZodOptional<z.ZodString>>;
+        }, z.core.$strip>, z.ZodObject<{
+            path: z.ZodString;
+            reason: z.ZodDefault<z.ZodOptional<z.ZodString>>;
+        }, z.core.$strip>, z.ZodObject<{
+            perspective: z.ZodString;
+            reason: z.ZodDefault<z.ZodOptional<z.ZodString>>;
+        }, z.core.$strip>]>>>;
+        fixtures: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    }, z.core.$strip>>;
+    assertions: z.ZodOptional<z.ZodArray<z.ZodUnion<readonly [z.ZodObject<{
+        type: z.ZodLiteral<"llm-rubric">;
+        template: z.ZodEnum<{
+            "task-completion": "task-completion";
+            "code-correctness": "code-correctness";
+            "doc-coverage": "doc-coverage";
+        }>;
+        criteria: z.ZodArray<z.ZodString>;
+        weight: z.ZodOptional<z.ZodNumber>;
+    }, z.core.$strip>, z.ZodObject<{
+        type: z.ZodEnum<{
+            "llm-rubric": "llm-rubric";
+            contains: "contains";
+            "contains-any": "contains-any";
+            "contains-all": "contains-all";
+            "not-contains": "not-contains";
+            icontains: "icontains";
+            "icontains-any": "icontains-any";
+            regex: "regex";
+            javascript: "javascript";
+            similar: "similar";
+            cost: "cost";
+            latency: "latency";
+        }>;
+        value: z.ZodOptional<z.ZodUnknown>;
+        threshold: z.ZodOptional<z.ZodNumber>;
+        weight: z.ZodOptional<z.ZodNumber>;
+    }, z.core.$loose>]>>>;
+    referenceSolution: z.ZodOptional<z.ZodString>;
+    docCoverage: z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
+    baseline: z.ZodOptional<z.ZodObject<{
+        enabled: z.ZodOptional<z.ZodBoolean>;
+        rubric: z.ZodOptional<z.ZodEnum<{
+            full: "full";
+            abbreviated: "abbreviated";
+            none: "none";
+        }>>;
+    }, z.core.$strip>>;
+    rubric: z.ZodOptional<z.ZodUnknown>;
+    providers: z.ZodOptional<z.ZodArray<z.ZodUnknown>>;
+    options: z.ZodOptional<z.ZodUnknown>;
+    metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+}, z.core.$loose>>;
+/**
+ * Parse and validate a task file's content against the canonical schema.
+ * Returns typed tasks or throws with a user-friendly Zod error message.
+ *
+ * Accepts pre-parsed YAML data (unknown), not a raw string.
+ */
+export declare function parseCanonicalTaskFile(raw: unknown, filename: string): CanonicalTask[];
+/**
+ * Detect legacy field names in raw task data and return helpful messages.
  *
- * NOTE: This accepts pre-parsed YAML data (unknown), not a raw string.
- * For raw YAML strings, use `parseTaskFile()` from @sanity/ailf-tasks.
+ * Runs BEFORE Zod parsing to catch the most common migration mistake —
+ * using old field names from @sanity/ailf-tasks instead of the canonical
+ * GeneralizedTaskDefinition shape.
  */
-export declare function parseRepoTaskFile(raw: unknown, filename: string): z.infer<typeof _Schema>;
+export declare function detectLegacyFieldNames(raw: unknown, filename: string): string[];
 /**
  * Zod schema for .ailf/config.yaml — controls documentation source,
  * report destination, and trigger behavior for evaluations from an