@sanity/ailf 1.0.0 → 2.0.1

package/dist/sinks/slack-sink.d.ts

@@ -0,0 +1,27 @@
+/**
+ * sinks/slack-sink.ts
+ *
+ * Slack notification sink — posts formatted score change messages to
+ * configured Slack channels via incoming webhooks.
+ *
+ * By default, only posts when regressions are detected (avoids notification
+ * fatigue). Set `alwaysPost: true` to receive all reports.
+ *
+ * @see docs/design-docs/report-store/sink-architecture.md
+ * @see docs/design-docs/report-store/notifications.md
+ */
+import type { Report, SinkHealthStatus, SinkResult } from "../pipeline/types.js";
+import type { ReportSink } from "./types.js";
+export interface SlackSinkOptions {
+    /** Post all reports, not just regressions (default: false — only regressions) */
+    alwaysPost?: boolean;
+}
+export declare class SlackSink implements ReportSink {
+    private readonly webhookUrl;
+    private readonly channel?;
+    private readonly options;
+    readonly name = "slack";
+    constructor(webhookUrl: string, channel?: string | undefined, options?: SlackSinkOptions);
+    healthCheck(): Promise<SinkHealthStatus>;
+    publish(report: Report): Promise<SinkResult>;
+}

package/dist/sinks/slack-sink.js

@@ -0,0 +1,78 @@
+/**
+ * sinks/slack-sink.ts
+ *
+ * Slack notification sink — posts formatted score change messages to
+ * configured Slack channels via incoming webhooks.
+ *
+ * By default, only posts when regressions are detected (avoids notification
+ * fatigue). Set `alwaysPost: true` to receive all reports.
+ *
+ * @see docs/design-docs/report-store/sink-architecture.md
+ * @see docs/design-docs/report-store/notifications.md
+ */
+import { formatRegressionAlert, formatScoreSummary, } from "./format-slack.js";
+export class SlackSink {
+    webhookUrl;
+    channel;
+    options;
+    name = "slack";
+    constructor(webhookUrl, channel, options = {}) {
+        this.webhookUrl = webhookUrl;
+        this.channel = channel;
+        this.options = options;
+    }
+    healthCheck() {
+        try {
+            const url = new URL(this.webhookUrl);
+            if (url.protocol !== "https:") {
+                return Promise.resolve({
+                    healthy: false,
+                    reason: `Webhook URL must use HTTPS, got ${url.protocol}`,
+                });
+            }
+            if (!this.webhookUrl.startsWith("https://hooks.slack.com/")) {
+                // Non-standard URL — warn but don't fail (could be a proxy)
+                return Promise.resolve({ healthy: true });
+            }
+            return Promise.resolve({ healthy: true });
+        }
+        catch {
+            return Promise.resolve({
+                healthy: false,
+                reason: `Invalid webhook URL: ${this.webhookUrl}`,
+            });
+        }
+    }
+    async publish(report) {
+        const hasRegressions = report.comparison !== undefined && report.comparison.regressed.length > 0;
+        let message;
+        if (hasRegressions) {
+            message = formatRegressionAlert(report);
+        }
+        else if (this.options.alwaysPost) {
+            message = formatScoreSummary(report);
+        }
+        else {
+            return { reason: "No regressions detected", status: "skipped" };
+        }
+        const body = { ...message };
+        if (this.channel) {
+            body.channel = this.channel;
+        }
+        const response = await fetch(this.webhookUrl, {
+            body: JSON.stringify(body),
+            headers: { "Content-Type": "application/json" },
+            method: "POST",
+        });
+        if (!response.ok) {
+            return {
+                error: `Slack webhook returned HTTP ${response.status}`,
+                status: "failed",
+            };
+        }
+        return {
+            detail: hasRegressions ? "regression alert" : "score summary",
+            status: "success",
+        };
+    }
+}

package/dist/sinks/types.d.ts

@@ -54,6 +54,6 @@ export interface ReportSink {
  * - The payload's maxSeverity matches an enabled routing rule
  * - The payload has regressions and the sink's `regression` rule is enabled
  *
- * @see docs/exec-plans/scenario-matrix-implementation/phase-5-readiness-thresholds.md
+ * @see docs/archive/exec-plans/scenario-matrix-implementation/phase-5-readiness-thresholds.md
  */
 export declare function shouldDeliver(routing: SinkRouting | undefined, payload: SinkPayload): boolean;

package/dist/sinks/types.js

@@ -21,7 +21,7 @@
  * - The payload's maxSeverity matches an enabled routing rule
  * - The payload has regressions and the sink's `regression` rule is enabled
  *
- * @see docs/exec-plans/scenario-matrix-implementation/phase-5-readiness-thresholds.md
+ * @see docs/archive/exec-plans/scenario-matrix-implementation/phase-5-readiness-thresholds.md
  */
 export function shouldDeliver(routing, payload) {
     // No routing config = deliver everything (backward compatible)

package/dist/sinks/webhook-sink.d.ts

@@ -0,0 +1,19 @@
+/**
+ * sinks/webhook-sink.ts
+ *
+ * Generic HTTP webhook sink — POSTs the full report JSON to any endpoint.
+ * This is the universal adapter for integrations that don't have a
+ * dedicated sink (Airbyte, Zapier, custom services, etc.).
+ *
+ * @see docs/design-docs/report-store/sink-architecture.md
+ */
+import type { Report, SinkHealthStatus, SinkResult } from "../pipeline/types.js";
+import type { ReportSink } from "./types.js";
+export declare class WebhookSink implements ReportSink {
+    private readonly url;
+    private readonly headers;
+    readonly name: string;
+    constructor(url: string, headers?: Record<string, string>);
+    healthCheck(): Promise<SinkHealthStatus>;
+    publish(report: Report): Promise<SinkResult>;
+}

package/dist/sinks/webhook-sink.js

@@ -0,0 +1,50 @@
+/**
+ * sinks/webhook-sink.ts
+ *
+ * Generic HTTP webhook sink — POSTs the full report JSON to any endpoint.
+ * This is the universal adapter for integrations that don't have a
+ * dedicated sink (Airbyte, Zapier, custom services, etc.).
+ *
+ * @see docs/design-docs/report-store/sink-architecture.md
+ */
+export class WebhookSink {
+    url;
+    headers;
+    name;
+    constructor(url, headers = {}) {
+        this.url = url;
+        this.headers = headers;
+        try {
+            this.name = `webhook:${new URL(url).hostname}`;
+        }
+        catch {
+            this.name = "webhook:invalid-url";
+        }
+    }
+    healthCheck() {
+        try {
+            new URL(this.url);
+            return Promise.resolve({ healthy: true });
+        }
+        catch {
+            return Promise.resolve({
+                healthy: false,
+                reason: `Invalid URL: ${this.url}`,
+            });
+        }
+    }
+    async publish(report) {
+        const response = await fetch(this.url, {
+            body: JSON.stringify(report),
+            headers: { "Content-Type": "application/json", ...this.headers },
+            method: "POST",
+        });
+        if (!response.ok) {
+            return {
+                error: `HTTP ${response.status}: ${response.statusText}`,
+                status: "failed",
+            };
+        }
+        return { detail: `HTTP ${response.status}`, status: "success" };
+    }
+}

package/dist/tasks/knowledge-probe/define-type-api.task.ts

@@ -0,0 +1,66 @@
+/**
+ * Knowledge probe: Sanity defineType API.
+ *
+ * Tests whether models know the current typed schema API
+ * vs the legacy untyped format.
+ *
+ * Knowledge probes test the model's built-in knowledge WITHOUT providing documentation
+ * context (no `context.docs`). Contrast with "literacy" tasks which inject docs.
+ */
+import { defineTask } from "../../_vendor/ailf-core/index.js"
+
+export default defineTask({
+  // "knowledge-probe" mode: no docs injected — measures what the model already knows
+  mode: "knowledge-probe",
+  id: "kp-define-type-api",
+  title: "What is Sanity's defineType API?",
+  description:
+    "Explain how to define document schemas in Sanity using the defineType, " +
+    "defineField, and defineArrayMember helper functions.",
+  // Used for score aggregation in reports and --area CLI filtering
+  area: "studio",
+  // Metadata for reporting; does not affect evaluation behavior
+  difficulty: "basic",
+  // Freeform labels for --tag CLI filtering
+  tags: ["knowledge-probe", "studio", "schema"],
+  // Controls how the probe explores knowledge: "breadth-first" covers many topics, "depth-first" drills deep
+  probeStrategy: "breadth-first",
+  prompt: {
+    // Direct prompt text sent to the model (knowledge probes use text, literacy tasks use vars.task with a template)
+    text:
+      "Explain Sanity's schema definition API:\n\n" +
+      "1. What is `defineType` and how do you use it?\n" +
+      "2. What are `defineField` and `defineArrayMember`?\n" +
+      "3. Why were these typed helpers introduced? What did they replace?\n" +
+      "4. Show a complete example of a document schema with various field types\n" +
+      "5. How do you add validation rules using the typed API?",
+    vars: {
+      task:
+        "Explain Sanity's defineType/defineField schema API with examples, " +
+        "motivation, and validation rules.",
+    },
+  },
+  assertions: [
+    { type: "contains", value: "defineType" },
+    { type: "contains", value: "defineField" },
+    // Inline llm-rubric (value is the rubric text itself, unlike template+criteria in literacy tasks)
+    {
+      type: "llm-rubric",
+      value:
+        "The response should accurately explain the typed schema helpers. " +
+        "Check that the code examples use the current API, not the legacy " +
+        "untyped format. Penalize if the response uses the old pattern " +
+        "without mentioning defineType.",
+      // weight: relative weight in the overall score (these two rubrics split evenly at 0.5 each)
+      weight: 0.5,
+    },
+    {
+      type: "llm-rubric",
+      value:
+        "Evaluate currency: does the response know about defineType " +
+        "(introduced in Sanity v3)? Does it mention TypeScript type " +
+        "inference benefits?",
+      weight: 0.5,
+    },
+  ],
+})

package/dist/tasks/knowledge-probe/groq-projections.task.ts

@@ -0,0 +1,62 @@
+/**
+ * Knowledge probe: GROQ projection syntax.
+ *
+ * Tests deep knowledge of Sanity's query language without
+ * providing any documentation context.
+ *
+ * See define-type-api.task.ts for detailed explanations of knowledge-probe properties.
+ */
+import { defineTask } from "../../_vendor/ailf-core/index.js"
+
+export default defineTask({
+  mode: "knowledge-probe",
+  id: "kp-groq-projections",
+  title: "Explain GROQ Projection Syntax",
+  description:
+    "Explain GROQ's projection syntax in detail, including: " +
+    "object projections ({}), array projections, spread operator (...), " +
+    "computed field names, the dereference operator (->), and " +
+    "conditional projections using select().",
+  area: "groq",
+  difficulty: "intermediate",
+  tags: ["knowledge-probe", "groq", "syntax"],
+  // "depth-first" drills deep into one topic (vs "breadth-first" which covers many shallowly)
+  probeStrategy: "depth-first",
+  prompt: {
+    text:
+      "Explain GROQ's projection syntax in Sanity. Cover these topics:\n\n" +
+      "1. Basic object projections with `{}`\n" +
+      "2. Nested projections and the spread operator `...`\n" +
+      "3. Computed field names\n" +
+      "4. The dereference operator `->` for following references\n" +
+      "5. Array slicing with `[0..5]` and `[0...5]`\n" +
+      "6. Conditional projections using `select()`\n\n" +
+      "Provide working code examples for each.",
+    vars: {
+      task:
+        "Explain GROQ projection syntax with working code examples " +
+        "covering projections, spread, dereference, slicing, and select().",
+    },
+  },
+  assertions: [
+    { type: "contains", value: "->" },
+    { type: "contains", value: "select(" },
+    {
+      type: "llm-rubric",
+      value:
+        "The response should demonstrate accurate knowledge of GROQ " +
+        "projection syntax with working code examples. Check that the " +
+        "dereference operator, spread syntax, and select() are correctly " +
+        "explained with valid GROQ code.",
+      weight: 0.6,
+    },
+    {
+      type: "llm-rubric",
+      value:
+        "Evaluate whether the response reflects current GROQ syntax " +
+        "(post-2023). Check for deprecated patterns or outdated " +
+        "recommendations.",
+      weight: 0.4,
+    },
+  ],
+})

package/dist/tasks/literacy/content-lake.task.ts

@@ -0,0 +1,181 @@
+import { defineTask } from "../../_vendor/ailf-core/index.js"
+
+export default [
+  defineTask({
+    id: "content-lake-mutations",
+    mode: "literacy",
+    title: "Content Lake - CRUD operations with @sanity/client",
+    description: "Content Lake - CRUD operations with @sanity/client",
+    // Used for score aggregation in reports and --area CLI filtering
+    area: "content-lake",
+    // Auto-generates an additional rubric checking if the response uses the provided docs
+    docCoverage: true,
+    context: {
+      // Sanity doc pages fetched and injected into the prompt as context; slug resolves via content lake
+      docs: [
+        {
+          slug: "js-client-mutations",
+          reason: "Creating and updating documents with @sanity/client",
+        },
+        {
+          slug: "js-client-transactions",
+          reason: "Atomic transactions — multiple mutations in one request",
+        },
+        {
+          slug: "js-client-deleting",
+          reason: "Deleting documents with @sanity/client",
+        },
+        {
+          slug: "http-patches",
+          reason: "Patch operations — set, unset, inc, insert, diffMatchPatch",
+        },
+      ],
+    },
+    // Path (relative to eval package root) to a gold-standard implementation the grader compares against
+    referenceSolution: "reference-solutions/content-lake/mutations.ts",
+    prompt: {
+      vars: {
+        // The instruction the model under evaluation sees (interpolated into the prompt template)
+        task: `Implement a content management service using @sanity/client that
+performs CRUD operations on Sanity documents.
+
+Requirements:
+1. Create a new document with typed fields
+2. Patch an existing document (set fields, unset fields, increment a counter)
+3. Delete a document by ID
+4. Use a transaction to atomically create multiple documents
+5. Include proper TypeScript types
+
+Provide a complete, reusable implementation.`,
+        // file:// URI resolved at runtime to a generated canonical context file (from npx @sanity/ailf fetch-docs)
+        docs: "file://contexts/canonical/content-lake-mutations.md",
+      },
+    },
+    // Grading criteria applied to the model's response; each assertion produces a pass/fail contributing to the task score
+    assertions: [
+      // llm-rubric with template: the grader model evaluates the response against these criteria
+      {
+        type: "llm-rubric",
+        template: "task-completion",
+        criteria: [
+          "Document creation with client.create() or client.createOrReplace()",
+          "Patch operations with set/unset/inc",
+          "Document deletion with client.delete()",
+          "Transaction with multiple mutations",
+        ],
+      },
+      {
+        type: "llm-rubric",
+        template: "code-correctness",
+        criteria: [
+          "Correct @sanity/client API usage",
+          "Proper patch chaining syntax",
+          "Transaction commit pattern",
+          "TypeScript types for document shapes",
+        ],
+      },
+      // Deterministic string-match assertions — fast, cheap, no grader model needed
+      { type: "contains", value: "createClient", weight: 1 },
+      // contains-any: passes if ANY string in the array appears in the response
+      {
+        type: "contains-any",
+        value: [".create(", ".createOrReplace(", ".createIfNotExists("],
+        weight: 1,
+      },
+      {
+        type: "contains-any",
+        value: [".patch(", ".set("],
+        weight: 1,
+      },
+      {
+        type: "contains-any",
+        value: [".transaction(", "transaction"],
+        weight: 1,
+      },
+      {
+        type: "contains-any",
+        value: [".delete(", "client.delete"],
+        weight: 1,
+      },
+    ],
+  }),
+
+  defineTask({
+    id: "content-lake-realtime",
+    mode: "literacy",
+    title: "Content Lake - Real-time listeners with @sanity/client",
+    description: "Content Lake - Real-time listeners with @sanity/client",
+    area: "content-lake",
+    docCoverage: true,
+    context: {
+      docs: [
+        {
+          slug: "js-client-realtime",
+          reason: "Listening to content updates with @sanity/client",
+        },
+        {
+          slug: "realtime-updates",
+          reason: "Real-time updates — listener protocol, event types",
+        },
+        {
+          slug: "js-client-getting-started",
+          reason: "Client setup — configuration, tokens, API version",
+        },
+        {
+          slug: "js-client-querying",
+          reason:
+            "Querying content — needed for initial data fetch before listening",
+        },
+      ],
+    },
+    referenceSolution: "reference-solutions/content-lake/realtime.ts",
+    prompt: {
+      vars: {
+        task: `Implement real-time content synchronization using Sanity's listener API.
+
+Requirements:
+1. Set up a listener that watches for changes to documents of a specific type
+2. Handle mutation events (create, update, delete)
+3. Apply changes to a local in-memory cache
+4. Handle reconnection and error scenarios
+5. Provide a way to unsubscribe/clean up the listener
+
+Provide a complete implementation with TypeScript types.`,
+        docs: "file://contexts/canonical/content-lake-realtime.md",
+      },
+    },
+    assertions: [
+      {
+        type: "llm-rubric",
+        template: "task-completion",
+        criteria: [
+          "Listener setup with GROQ filter for document type",
+          "Mutation event handling (create, update, delete differentiation)",
+          "Local cache synchronization",
+          "Cleanup/unsubscribe mechanism",
+          "Error or reconnection handling",
+        ],
+      },
+      {
+        type: "llm-rubric",
+        template: "code-correctness",
+        criteria: [
+          "Correct client.listen() API usage",
+          "Proper GROQ filter syntax in listener",
+          "Subscription cleanup pattern (unsubscribe)",
+          "No deprecated listener APIs",
+        ],
+      },
+      {
+        type: "contains-any",
+        value: [".listen(", "client.listen"],
+        weight: 1,
+      },
+      {
+        type: "contains-any",
+        value: ["subscribe", ".subscribe("],
+        weight: 1,
+      },
+    ],
+  }),
+]

package/dist/tasks/literacy/frameworks.task.ts

@@ -0,0 +1,129 @@
+import { defineTask } from "../../_vendor/ailf-core/index.js"
+
+// See content-lake.task.ts for detailed explanations of common task properties
+export default [
+  defineTask({
+    id: "remix-integration",
+    mode: "literacy",
+    title: "Frameworks - Remix integration with data fetching",
+    description: "Frameworks - Remix integration with data fetching",
+    area: "frameworks",
+    docCoverage: true,
+    context: {
+      docs: [
+        {
+          slug: "displaying-content-in-a-react-router-front-end",
+          reason: "React Router front-end content display guide",
+        },
+        {
+          slug: "visual-editing-with-react-router",
+          reason: "React Router / Remix integration with visual editing",
+        },
+        {
+          slug: "functions-js-client",
+          reason: "Configuring @sanity/client for data fetching",
+        },
+      ],
+    },
+    referenceSolution: "reference-solutions/frameworks/remix.tsx",
+    prompt: {
+      vars: {
+        task: `Integrate Sanity into a Remix application:
+
+1. Set up the Sanity client
+2. Create a loader that fetches blog posts using GROQ
+3. Build a route component that renders the fetched posts
+4. Handle loading and error states properly
+
+Provide all necessary files for a working Remix + Sanity integration.`,
+        docs: "file://contexts/canonical/remix-integration.md",
+      },
+    },
+    assertions: [
+      {
+        type: "llm-rubric",
+        template: "task-completion",
+        criteria: [
+          "Sanity client configuration",
+          "Remix loader function with GROQ query",
+          "Route component using useLoaderData",
+          "Proper typing",
+        ],
+      },
+      {
+        type: "llm-rubric",
+        template: "code-correctness",
+        criteria: [
+          "Modern Remix patterns (v2 conventions)",
+          "Proper loader/component separation",
+          "Valid GROQ queries",
+          "No deprecated APIs",
+        ],
+      },
+      {
+        type: "contains-any",
+        value: ["useLoaderData", "loader"],
+        weight: 1,
+      },
+    ],
+  }),
+
+  defineTask({
+    id: "nuxt-integration",
+    mode: "literacy",
+    title: "Frameworks - Nuxt 4 integration",
+    description: "Frameworks - Nuxt 4 integration",
+    area: "frameworks",
+    docCoverage: true,
+    context: {
+      docs: [
+        {
+          slug: "displaying-content-in-nuxt-js",
+          reason: "Nuxt.js front-end content display guide",
+        },
+        {
+          slug: "visual-editing-with-nuxt",
+          reason: "Nuxt visual editing integration",
+        },
+      ],
+    },
+    referenceSolution: "reference-solutions/frameworks/nuxt.ts",
+    prompt: {
+      vars: {
+        task: `Integrate Sanity into a Nuxt 4 application:
+
+1. Install and configure the @nuxtjs/sanity module
+2. Create a page that fetches and displays blog posts
+3. Use Nuxt composables for data fetching
+
+Provide all necessary configuration and component code.`,
+        docs: "file://contexts/canonical/nuxt-integration.md",
+      },
+    },
+    assertions: [
+      {
+        type: "llm-rubric",
+        template: "task-completion",
+        criteria: [
+          "@nuxtjs/sanity module setup in nuxt.config.ts",
+          "Page component using Nuxt data fetching composables",
+          "Sanity GROQ query",
+        ],
+      },
+      {
+        type: "llm-rubric",
+        template: "code-correctness",
+        criteria: [
+          "Nuxt 3 module configuration syntax",
+          "Uses useSanityQuery or equivalent composable",
+          "Proper Nuxt 3 patterns (not Nuxt 2)",
+        ],
+      },
+      {
+        type: "contains-any",
+        value: ["@nuxtjs/sanity", "useSanityQuery", "sanity:"],
+        weight: 1,
+      },
+    ],
+  }),
+]