@sanity/ailf 0.5.0 → 1.0.0

package/dist/sinks/loader.js

@@ -7,11 +7,10 @@
  *
  * @see docs/design-docs/report-store/sink-architecture.md
  */
-import { existsSync, readFileSync } from "fs";
 import { dirname, resolve } from "path";
 import { fileURLToPath } from "url";
-import { load } from "js-yaml";
 import { interpolate } from "../interpolate.js";
+import { tryLoadConfigFile } from "../pipeline/compiler/config-loader.js";
 import { BigQuerySink } from "./bigquery/index.js";
 import { SinksFileSchema } from "./schema.js";
 import { SlackSink } from "./slack/index.js";
@@ -20,7 +19,7 @@ import { WebhookSink } from "./webhook/index.js";
 // Paths
 // ---------------------------------------------------------------------------
 const __dirname = dirname(fileURLToPath(import.meta.url));
-const SINKS_CONFIG_PATH = resolve(__dirname, "..", "..", "config", "sinks.yaml");
+const PACKAGE_ROOT = resolve(__dirname, "..", "..");
 // ---------------------------------------------------------------------------
 // Public API
 // ---------------------------------------------------------------------------
@@ -32,13 +31,12 @@ const SINKS_CONFIG_PATH = resolve(__dirname, "..", "..", "config", "sinks.yaml")
  * - The config file fails to parse or validate
  */
 export function loadSinks() {
-    if (!existsSync(SINKS_CONFIG_PATH)) {
+    const loaded = tryLoadConfigFile("sinks", PACKAGE_ROOT);
+    if (!loaded) {
         return [];
     }
     try {
-        const raw = readFileSync(SINKS_CONFIG_PATH, "utf-8");
-        const parsed = load(raw);
-        const interpolated = interpolate(parsed);
+        const interpolated = interpolate(loaded.data);
         // When all sinks are commented out, js-yaml returns { sinks: null }.
         // Normalize null → undefined so Zod's .default([]) can apply.
         const normalized = interpolated && typeof interpolated === "object"

package/dist/sources.d.ts

@@ -1,7 +1,7 @@
 /**
  * Sources.ts
  *
- * Loads and validates documentation source configurations from config/sources.yaml.
+ * Loads and validates documentation source configurations from config/sources.
  * Each source defines where to find documentation for evaluation — a Sanity
  * dataset, a branch deploy URL, a local dev server, etc.
  *
@@ -9,7 +9,7 @@
  *   - Baseline/Observed: uses projectId + dataset (GROQ queries to Sanity CMS)
  *   - Agentic: uses baseUrl (agent browses the live site, llms.txt derived automatically)
  *
- * Environment variables in config/sources.yaml are resolved via ${{ VAR | default }}
+ * Environment variables in config/sources are resolved via ${{ VAR | default }}
  * interpolation at load time.
  */
 import type { Logger } from "./_vendor/ailf-core/index.d.ts";
@@ -22,7 +22,7 @@ export type DocSource = ResolvedSourceConfig;
  * Fully resolved documentation source configuration.
  *
  * After all CLI flags, environment variables, URL classifications, and
- * config/sources.yaml defaults are merged, the system produces a single
+ * config/sources defaults are merged, the system produces a single
  * ResolvedSourceConfig that drives the entire pipeline.
  *
  * All ${{ }} expressions have been expanded by the time this is constructed.
@@ -40,7 +40,7 @@ export interface ResolvedSourceConfig {
     headers?: Record<string, string>;
     /** URL to the llms.txt file — always derived as baseUrl + "/llms.txt" */
     llmsTxt: string;
-    /** Source name (the key from config/sources.yaml, or "env-override") */
+    /** Source name (the key from config/sources, or "env-override") */
     name: string;
     /** Sanity release perspective ID (optional — for evaluating content releases) */
     perspective?: string;
@@ -84,11 +84,11 @@ export interface SourceOverrides {
  *
  * Resolution order:
  *   1. If DOC_BASE_URL env var is set, build an ad-hoc source from env vars
- *   2. If config/sources.yaml exists, load and interpolate the named source
+ *   2. If config/sources exists, load and interpolate the named source
  *   3. Fall back to the built-in default (sanity.io production)
  *
- * @param name - Source name from config/sources.yaml. If omitted, uses DOC_SOURCE
- *               env var or the first source defined in config/sources.yaml.
+ * @param name - Source name from config/sources. If omitted, uses DOC_SOURCE
+ *               env var or the first source defined in config/sources.
  */
 export declare function loadSource(name?: string, overrides?: SourceOverrides, logger?: Logger): ResolvedSourceConfig;
 /**

package/dist/sources.js

@@ -1,7 +1,7 @@
 /**
  * Sources.ts
  *
- * Loads and validates documentation source configurations from config/sources.yaml.
+ * Loads and validates documentation source configurations from config/sources.
  * Each source defines where to find documentation for evaluation — a Sanity
  * dataset, a branch deploy URL, a local dev server, etc.
  *
@@ -9,22 +9,21 @@
  *   - Baseline/Observed: uses projectId + dataset (GROQ queries to Sanity CMS)
  *   - Agentic: uses baseUrl (agent browses the live site, llms.txt derived automatically)
  *
- * Environment variables in config/sources.yaml are resolved via ${{ VAR | default }}
+ * Environment variables in config/sources are resolved via ${{ VAR | default }}
  * interpolation at load time.
  */
-import { readFileSync } from "fs";
 import { dirname, resolve } from "path";
 import { fileURLToPath } from "url";
-import { load } from "js-yaml";
 import { ConsoleLogger } from "./adapters/loggers/index.js";
 import { interpolate } from "./interpolate.js";
+import { tryLoadConfigFile } from "./pipeline/compiler/config-loader.js";
 // ---------------------------------------------------------------------------
 // Paths
 // ---------------------------------------------------------------------------
 const __dirname = dirname(fileURLToPath(import.meta.url));
-const SOURCES_PATH = resolve(__dirname, "..", "config", "sources.yaml");
+const PACKAGE_ROOT = resolve(__dirname, "..");
 // ---------------------------------------------------------------------------
-// Default source — used when no config/sources.yaml exists or no --source flag
+// Default source — used when no config/sources exists or no --source flag
 // ---------------------------------------------------------------------------
 const DEFAULT_SOURCE = {
     baseUrl: "https://www.sanity.io/docs",
@@ -67,11 +66,11 @@ export function isAllowedOrigin(url, origins) {
  *
  * Resolution order:
  *   1. If DOC_BASE_URL env var is set, build an ad-hoc source from env vars
- *   2. If config/sources.yaml exists, load and interpolate the named source
+ *   2. If config/sources exists, load and interpolate the named source
  *   3. Fall back to the built-in default (sanity.io production)
  *
- * @param name - Source name from config/sources.yaml. If omitted, uses DOC_SOURCE
- *               env var or the first source defined in config/sources.yaml.
+ * @param name - Source name from config/sources. If omitted, uses DOC_SOURCE
+ *               env var or the first source defined in config/sources.
  */
 export function loadSource(name, overrides, logger) {
     const log = logger ?? new ConsoleLogger();
@@ -105,32 +104,31 @@ export function loadSource(name, overrides, logger) {
             urls: overrides?.directUrls ?? parseDirectUrlsEnv(),
         };
     }
-    // Priority 2: Load from config/sources.yaml
+    // Priority 2: Load from config/sources (TS, JS, YAML, or JSON)
     let rawFile;
-    try {
-        const content = readFileSync(SOURCES_PATH, "utf-8");
-        rawFile = load(content);
+    const loaded = tryLoadConfigFile("sources", PACKAGE_ROOT);
+    if (loaded) {
+        rawFile = loaded.data;
     }
-    catch {
-        // No config/sources.yaml — use built-in default
-        log.debug("No config/sources.yaml found, falling back to built-in default", {
-            path: SOURCES_PATH,
+    else {
+        // No config file found — use built-in default
+        log.debug("No config/sources found, falling back to built-in default", {
             defaultName: DEFAULT_SOURCE.name,
             defaultBaseUrl: DEFAULT_SOURCE.baseUrl,
         });
-        console.log("  No config/sources.yaml found, using built-in default (sanity.io production)");
+        console.log("  No config/sources found, using built-in default (sanity.io production)");
         return DEFAULT_SOURCE;
     }
     if (!rawFile?.sources || Object.keys(rawFile.sources).length === 0) {
-        log.debug("config/sources.yaml is empty, falling back to built-in default");
-        console.log("  config/sources.yaml is empty, using built-in default");
+        log.debug("config/sources is empty, falling back to built-in default");
+        console.log("  config/sources is empty, using built-in default");
         return DEFAULT_SOURCE;
     }
     // Resolve which source to use
     const sourceName = 
     // oxlint-disable-next-line @typescript-eslint/prefer-nullish-coalescing -- empty strings should fall through to next candidate
     name || process.env.DOC_SOURCE || Object.keys(rawFile.sources)[0];
-    log.debug("Resolving source from config/sources.yaml", {
+    log.debug("Resolving source from config/sources", {
         requestedName: name,
         resolvedName: sourceName,
         availableSources: Object.keys(rawFile.sources),
@@ -140,7 +138,7 @@ export function loadSource(name, overrides, logger) {
         const available = Object.keys(rawFile.sources).join(", ");
         throw new Error('Source "' +
             sourceName +
-            '" not found in config/sources.yaml. Available: ' +
+            '" not found in config/sources. Available: ' +
             available);
     }
     // Interpolate ${{ VAR }} expressions
@@ -151,7 +149,7 @@ export function loadSource(name, overrides, logger) {
     // An interpolation like ${{ DOC_ALLOWED_ORIGIN | }} resolves to "",
     // Which would produce [""] — matching nothing and silently blocking all origins.
     const filteredOrigins = resolved.allowedOrigins?.filter(Boolean);
-    log.debug("Source resolved from config/sources.yaml", {
+    log.debug("Source resolved from config/sources", {
         name: sourceName,
         baseUrl: resolved.baseUrl,
         dataset: resolved.dataset,
@@ -208,7 +206,7 @@ function extractHostname(url) {
     }
 }
 /**
- * Merge headers from config/sources.yaml with DOC_HEADERS env var.
+ * Merge headers from config/sources with DOC_HEADERS env var.
  * Env var headers take precedence over source-defined headers.
  * Returns undefined if both are empty.
  */

package/dist/webhook/dispatch.js

@@ -15,6 +15,7 @@
  * @see .github/workflows/external-eval.yml
  * @see docs/design-docs/cross-repo-eval.md
  */
+import { LiteracyVariant } from "../pipeline/normalize-mode.js";
 // ---------------------------------------------------------------------------
 // Constants
 // ---------------------------------------------------------------------------
@@ -43,7 +44,7 @@ export async function dispatchEvaluation(request, options) {
                 ? request.documentSlug.split(",").filter(Boolean)
                 : undefined,
             compare: true,
-            mode: request.mode ?? "baseline",
+            mode: request.mode ?? LiteracyVariant.STANDARD,
             publish: true,
             source: "production",
             tasks: request.taskIds.length > 0 ? request.taskIds : undefined,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sanity/ailf",
-  "version": "0.5.0",
+  "version": "1.0.0",
   "private": false,
   "publishConfig": {
     "access": "restricted"
@@ -31,6 +31,7 @@
     "commander": "^14.0.3",
     "dotenv": "^16.4.7",
     "dotenv-cli": "^11.0.0",
+    "jiti": "^2.6.1",
     "js-yaml": "^4.1.0",
     "promptfoo": "^0.120.24",
     "zod": "^4.3.6"
@@ -41,8 +42,8 @@
     "tsx": "^4.19.2",
     "typescript": "^5.7.3",
     "@sanity/ailf-core": "0.1.0",
-    "@sanity/ailf-tasks": "0.1.4",
-    "@sanity/ailf-shared": "0.1.0"
+    "@sanity/ailf-shared": "0.1.0",
+    "@sanity/ailf-tasks": "0.1.4"
   },
   "scripts": {
     "build": "tsc && tsx scripts/bundle-workspace-deps.ts",
@@ -63,6 +64,8 @@
     "pipeline": "tsx src/cli.ts pipeline",
     "validate": "tsx src/cli.ts validate",
     "test": "tsx --test src/__tests__/*.test.ts",
+    "test:e2e": "AILF_E2E=1 tsx --test src/__tests__/e2e/*.e2e.test.ts",
+    "test:all": "AILF_E2E=1 tsx --test src/__tests__/*.test.ts src/pipeline/compiler/__tests__/*.test.ts src/__tests__/e2e/*.e2e.test.ts",
     "pr-comment": "tsx src/cli.ts pr-comment",
     "coverage-audit": "tsx src/cli.ts coverage-audit",
     "readiness-report": "tsx src/cli.ts readiness-report",

package/tasks/knowledge-probe/define-type-api.task.ts

@@ -0,0 +1,55 @@
+/**
+ * Knowledge probe: Sanity defineType API.
+ *
+ * Tests whether models know the current typed schema API
+ * vs the legacy untyped format.
+ */
+import { defineTask } from "@sanity/ailf-core"
+
+export default defineTask({
+  mode: "knowledge-probe",
+  id: "kp-define-type-api",
+  title: "What is Sanity's defineType API?",
+  description:
+    "Explain how to define document schemas in Sanity using the defineType, " +
+    "defineField, and defineArrayMember helper functions.",
+  area: "studio",
+  difficulty: "basic",
+  tags: ["knowledge-probe", "studio", "schema"],
+  probeStrategy: "breadth-first",
+  prompt: {
+    text:
+      "Explain Sanity's schema definition API:\n\n" +
+      "1. What is `defineType` and how do you use it?\n" +
+      "2. What are `defineField` and `defineArrayMember`?\n" +
+      "3. Why were these typed helpers introduced? What did they replace?\n" +
+      "4. Show a complete example of a document schema with various field types\n" +
+      "5. How do you add validation rules using the typed API?",
+    vars: {
+      task:
+        "Explain Sanity's defineType/defineField schema API with examples, " +
+        "motivation, and validation rules.",
+    },
+  },
+  assertions: [
+    { type: "contains", value: "defineType" },
+    { type: "contains", value: "defineField" },
+    {
+      type: "llm-rubric",
+      value:
+        "The response should accurately explain the typed schema helpers. " +
+        "Check that the code examples use the current API, not the legacy " +
+        "untyped format. Penalize if the response uses the old pattern " +
+        "without mentioning defineType.",
+      weight: 0.5,
+    },
+    {
+      type: "llm-rubric",
+      value:
+        "Evaluate currency: does the response know about defineType " +
+        "(introduced in Sanity v3)? Does it mention TypeScript type " +
+        "inference benefits?",
+      weight: 0.5,
+    },
+  ],
+})

package/tasks/knowledge-probe/groq-projections.task.ts

@@ -0,0 +1,59 @@
+/**
+ * Knowledge probe: GROQ projection syntax.
+ *
+ * Tests deep knowledge of Sanity's query language without
+ * providing any documentation context.
+ */
+import { defineTask } from "@sanity/ailf-core"
+
+export default defineTask({
+  mode: "knowledge-probe",
+  id: "kp-groq-projections",
+  title: "Explain GROQ Projection Syntax",
+  description:
+    "Explain GROQ's projection syntax in detail, including: " +
+    "object projections ({}), array projections, spread operator (...), " +
+    "computed field names, the dereference operator (->), and " +
+    "conditional projections using select().",
+  area: "groq",
+  difficulty: "intermediate",
+  tags: ["knowledge-probe", "groq", "syntax"],
+  probeStrategy: "depth-first",
+  prompt: {
+    text:
+      "Explain GROQ's projection syntax in Sanity. Cover these topics:\n\n" +
+      "1. Basic object projections with `{}`\n" +
+      "2. Nested projections and the spread operator `...`\n" +
+      "3. Computed field names\n" +
+      "4. The dereference operator `->` for following references\n" +
+      "5. Array slicing with `[0..5]` and `[0...5]`\n" +
+      "6. Conditional projections using `select()`\n\n" +
+      "Provide working code examples for each.",
+    vars: {
+      task:
+        "Explain GROQ projection syntax with working code examples " +
+        "covering projections, spread, dereference, slicing, and select().",
+    },
+  },
+  assertions: [
+    { type: "contains", value: "->" },
+    { type: "contains", value: "select(" },
+    {
+      type: "llm-rubric",
+      value:
+        "The response should demonstrate accurate knowledge of GROQ " +
+        "projection syntax with working code examples. Check that the " +
+        "dereference operator, spread syntax, and select() are correctly " +
+        "explained with valid GROQ code.",
+      weight: 0.6,
+    },
+    {
+      type: "llm-rubric",
+      value:
+        "Evaluate whether the response reflects current GROQ syntax " +
+        "(post-2023). Check for deprecated patterns or outdated " +
+        "recommendations.",
+      weight: 0.4,
+    },
+  ],
+})

package/tasks/literacy/frameworks.task.ts

@@ -0,0 +1,128 @@
+import { defineTask } from "@sanity/ailf-core"
+
+export default [
+  defineTask({
+    id: "remix-integration",
+    mode: "literacy",
+    title: "Frameworks - Remix integration with data fetching",
+    description: "Frameworks - Remix integration with data fetching",
+    area: "frameworks",
+    docCoverage: true,
+    context: {
+      docs: [
+        {
+          slug: "displaying-content-in-a-react-router-front-end",
+          reason: "React Router front-end content display guide",
+        },
+        {
+          slug: "visual-editing-with-react-router",
+          reason: "React Router / Remix integration with visual editing",
+        },
+        {
+          slug: "functions-js-client",
+          reason: "Configuring @sanity/client for data fetching",
+        },
+      ],
+    },
+    referenceSolution: "reference-solutions/frameworks/remix.tsx",
+    prompt: {
+      vars: {
+        task: `Integrate Sanity into a Remix application:
+
+1. Set up the Sanity client
+2. Create a loader that fetches blog posts using GROQ
+3. Build a route component that renders the fetched posts
+4. Handle loading and error states properly
+
+Provide all necessary files for a working Remix + Sanity integration.`,
+        docs: "file://contexts/canonical/remix-integration.md",
+      },
+    },
+    assertions: [
+      {
+        type: "llm-rubric",
+        template: "task-completion",
+        criteria: [
+          "Sanity client configuration",
+          "Remix loader function with GROQ query",
+          "Route component using useLoaderData",
+          "Proper typing",
+        ],
+      },
+      {
+        type: "llm-rubric",
+        template: "code-correctness",
+        criteria: [
+          "Modern Remix patterns (v2 conventions)",
+          "Proper loader/component separation",
+          "Valid GROQ queries",
+          "No deprecated APIs",
+        ],
+      },
+      {
+        type: "contains-any",
+        value: ["useLoaderData", "loader"],
+        weight: 1,
+      },
+    ],
+  }),
+
+  defineTask({
+    id: "nuxt-integration",
+    mode: "literacy",
+    title: "Frameworks - Nuxt 4 integration",
+    description: "Frameworks - Nuxt 4 integration",
+    area: "frameworks",
+    docCoverage: true,
+    context: {
+      docs: [
+        {
+          slug: "displaying-content-in-nuxt-js",
+          reason: "Nuxt.js front-end content display guide",
+        },
+        {
+          slug: "visual-editing-with-nuxt",
+          reason: "Nuxt visual editing integration",
+        },
+      ],
+    },
+    referenceSolution: "reference-solutions/frameworks/nuxt.ts",
+    prompt: {
+      vars: {
+        task: `Integrate Sanity into a Nuxt 4 application:
+
+1. Install and configure the @nuxtjs/sanity module
+2. Create a page that fetches and displays blog posts
+3. Use Nuxt composables for data fetching
+
+Provide all necessary configuration and component code.`,
+        docs: "file://contexts/canonical/nuxt-integration.md",
+      },
+    },
+    assertions: [
+      {
+        type: "llm-rubric",
+        template: "task-completion",
+        criteria: [
+          "@nuxtjs/sanity module setup in nuxt.config.ts",
+          "Page component using Nuxt data fetching composables",
+          "Sanity GROQ query",
+        ],
+      },
+      {
+        type: "llm-rubric",
+        template: "code-correctness",
+        criteria: [
+          "Nuxt 3 module configuration syntax",
+          "Uses useSanityQuery or equivalent composable",
+          "Proper Nuxt 3 patterns (not Nuxt 2)",
+        ],
+      },
+      {
+        type: "contains-any",
+        value: ["@nuxtjs/sanity", "useSanityQuery", "sanity:"],
+        weight: 1,
+      },
+    ],
+  }),
+]

package/tasks/literacy/functions.task.ts

@@ -0,0 +1,69 @@
+import { defineTask } from "@sanity/ailf-core"
+
+export default [
+  defineTask({
+    id: "functions-webhook",
+    mode: "literacy",
+    title: "Functions - Webhook on document publish",
+    description: "Functions - Webhook on document publish",
+    area: "functions",
+    docCoverage: true,
+    context: {
+      docs: [
+        {
+          slug: "functions-introduction",
+          reason: "Functions overview — deployment, triggers, lifecycle",
+        },
+        {
+          slug: "webhooks",
+          reason: "GROQ-powered webhooks — configuration and filtering",
+        },
+        {
+          slug: "function-wrapper",
+          reason: "Function handler reference — event shape, exports",
+        },
+        {
+          slug: "functions-cheatsheet",
+          reason: "Quick reference for common function patterns",
+        },
+      ],
+    },
+    referenceSolution: "reference-solutions/functions/publish-webhook.ts",
+    prompt: {
+      vars: {
+        task: `Deploy a Sanity function that triggers when a document is published
+and sends a webhook notification to an external endpoint.
+
+Requirements:
+1. The function should trigger on document publish events
+2. Filter to only fire for "post" document types
+3. Send a POST request to an external URL with the document data
+4. Handle errors gracefully
+
+Provide a complete implementation including any configuration.`,
+        docs: "file://contexts/canonical/functions-webhook.md",
+      },
+    },
+    assertions: [
+      {
+        type: "llm-rubric",
+        template: "task-completion",
+        criteria: [
+          "Event-driven function definition",
+          "Document type filtering (post type only)",
+          "HTTP webhook call with document payload",
+          "Basic error handling",
+        ],
+      },
+      {
+        type: "llm-rubric",
+        template: "code-correctness",
+        criteria: [
+          "No deprecated API patterns",
+          "Proper error handling",
+          "Idiomatic function handler structure",
+        ],
+      },
+    ],
+  }),
+]