@sanity/ailf 0.1.0

package/dist/webhook/handler.js

@@ -0,0 +1,203 @@
+/**
+ * webhook/handler.ts
+ *
+ * Platform-agnostic webhook handler for Sanity content change events.
+ *
+ * Receives Sanity webhook payloads, determines which evaluation areas
+ * are affected by the document change, debounces rapid edits, enforces
+ * daily budget limits, and dispatches scoped evaluations via GitHub Actions.
+ *
+ * This handler is stateless between requests (debounce and budget state
+ * is held in-memory by the enclosing WebhookHandler instance). It can be
+ * mounted in any HTTP framework: Express, Hono, Cloudflare Workers, etc.
+ *
+ * Flow:
+ * 1. Receive Sanity webhook payload
+ * 2. Extract document slug from payload
+ * 3. Look up affected areas via reverse mapping
+ * 4. If no areas affected → ignore (untracked document)
+ * 5. Check daily budget → rate-limit if exceeded
+ * 6. Push slug into debounce window
+ * 7. When debounce window closes → dispatch scoped eval via GitHub Actions
+ *
+ * @see docs/design-docs/report-store/visibility-workflows.md
+ */
+import { assessImpact, buildReverseMapping, } from "../pipeline/reverse-mapping.js";
+import { createBudgetTracker } from "./budget.js";
+import { createDebouncer } from "./debounce.js";
+import { dispatchEvaluation } from "./dispatch.js";
+// ---------------------------------------------------------------------------
+// Defaults
+// ---------------------------------------------------------------------------
+const DEFAULT_DEBOUNCE_MS = 300_000; // 5 minutes
+const DEFAULT_DAILY_BUDGET = 20;
+const DEFAULT_REPO = "sanity-labs/ai-literacy-framework";
+// ---------------------------------------------------------------------------
+// WebhookHandler
+// ---------------------------------------------------------------------------
+/**
+ * A stateful webhook handler that manages debouncing, budgeting,
+ * and evaluation dispatch.
+ *
+ * Create one instance per process and call `handle()` for each
+ * incoming webhook payload.
+ *
+ * ```ts
+ * const handler = new WebhookHandler({
+ *   githubToken: process.env.GITHUB_TOKEN!,
+ *   rootDir: "/path/to/packages/eval",
+ * })
+ *
+ * // In your HTTP handler:
+ * app.post("/webhook", async (req) => {
+ *   const result = await handler.handle(req.body)
+ *   return Response.json(result)
+ * })
+ * ```
+ */
+export class WebhookHandler {
+    budget;
+    config;
+    debouncer;
+    /** Recent dispatch results (for diagnostics) */
+    recentDispatches = [];
+    reverseMapping;
+    constructor(config) {
+        this.config = {
+            dailyBudget: config.dailyBudget ?? DEFAULT_DAILY_BUDGET,
+            debounceMs: config.debounceMs ?? DEFAULT_DEBOUNCE_MS,
+            githubRepo: config.githubRepo ?? DEFAULT_REPO,
+            githubToken: config.githubToken,
+            rootDir: config.rootDir ?? process.cwd(),
+            webhookSecret: config.webhookSecret,
+        };
+        this.reverseMapping = buildReverseMapping(this.config.rootDir);
+        this.budget = createBudgetTracker(this.config.dailyBudget);
+        this.debouncer = createDebouncer(this.config.debounceMs, this.onDebounceFlush.bind(this));
+    }
+    /**
+     * Get handler diagnostics (for health check endpoints).
+     */
+    diagnostics() {
+        const budgetState = this.budget.currentState();
+        return {
+            budget: {
+                count: budgetState.count,
+                limit: budgetState.limit,
+                remaining: this.budget.remaining(),
+            },
+            pendingSlugs: this.debouncer.pending(),
+            recentDispatches: this.recentDispatches.slice(-10),
+            trackedSlugs: this.reverseMapping.size,
+        };
+    }
+    /**
+     * Handle an incoming Sanity webhook payload.
+     *
+     * This is the main entry point — call once per webhook request.
+     * Returns a result indicating what happened (dispatched, debounced,
+     * rate-limited, or ignored).
+     */
+    handle(payload) {
+        // Extract the document slug from the payload
+        const slug = extractSlug(payload);
+        if (!slug) {
+            return {
+                reason: "No document slug in payload",
+                status: "ignored",
+            };
+        }
+        // Look up affected areas
+        const impact = assessImpact([slug], this.reverseMapping);
+        if (impact.areas.length === 0) {
+            return {
+                reason: `Document "${slug}" is not tracked by any evaluation task`,
+                status: "ignored",
+            };
+        }
+        // Check daily budget
+        if (!this.budget.canDispatch()) {
+            const state = this.budget.currentState();
+            return {
+                dailyBudget: state.limit,
+                dailyCount: state.count,
+                status: "rate-limited",
+            };
+        }
+        // Push into debounce window
+        this.debouncer.push(slug);
+        const expiresAt = new Date(Date.now() + this.config.debounceMs).toISOString();
+        return {
+            pendingSlugs: [...this.debouncer.state().slugs],
+            status: "debounced",
+            windowExpiresAt: expiresAt,
+        };
+    }
+    /**
+     * Force-flush the debounce window (for graceful shutdown).
+     */
+    async shutdown() {
+        await this.debouncer.flush();
+    }
+    // -------------------------------------------------------------------------
+    // Private
+    // -------------------------------------------------------------------------
+    /**
+     * Called when the debounce window closes — dispatches a scoped evaluation.
+     */
+    async onDebounceFlush(slugs) {
+        // Re-check budget (may have been exhausted during debounce window)
+        if (!this.budget.canDispatch()) {
+            console.warn(`  ⚠️  Budget exhausted during debounce window. Skipping dispatch for: ${slugs.join(", ")}`);
+            return;
+        }
+        const impact = assessImpact(slugs, this.reverseMapping);
+        if (impact.areas.length === 0)
+            return;
+        console.log(`  📤 Dispatching evaluation for areas [${impact.areas.join(", ")}] ` +
+            `triggered by document changes: ${slugs.join(", ")}`);
+        const result = await dispatchEvaluation({
+            areas: impact.areas,
+            documentSlug: slugs.join(","),
+            taskIds: impact.taskIds,
+        }, {
+            githubToken: this.config.githubToken,
+            repo: this.config.githubRepo,
+        });
+        // Record dispatch
+        this.budget.record();
+        this.recentDispatches.push({
+            areas: impact.areas,
+            result,
+            timestamp: new Date().toISOString(),
+        });
+        // Keep only last 50 dispatches
+        if (this.recentDispatches.length > 50) {
+            this.recentDispatches.splice(0, this.recentDispatches.length - 50);
+        }
+        if (result.ok) {
+            console.log(`  ✅ Dispatch accepted (HTTP ${result.httpStatus})`);
+        }
+        else {
+            console.warn(`  ⚠️  Dispatch failed: ${result.error}`);
+        }
+    }
+}
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+/**
+ * Extract the document slug from a Sanity webhook payload.
+ *
+ * Sanity webhooks include the document in `result` — the slug is at
+ * `result.slug.current` for article documents.
+ */
+function extractSlug(payload) {
+    // Direct slug from projected document
+    if (payload.result?.slug?.current) {
+        return payload.result.slug.current;
+    }
+    // Fallback: extract from _id (e.g., "article-groq-introduction")
+    // This handles cases where the projection doesn't include slug
+    return undefined;
+}

package/dist/webhook/index.d.ts

@@ -0,0 +1,17 @@
+/**
+ * webhook/index.ts
+ *
+ * Barrel exports for the webhook event-driven trigger system.
+ *
+ * @see docs/design-docs/report-store/visibility-workflows.md
+ */
+export { handleEvalRequest } from "./eval-request-handler.js";
+export type { EvalRequestHandlerConfig, EvalRequestPayload, EvalRequestResult, } from "./eval-request-handler.js";
+export { createBudgetTracker } from "./budget.js";
+export type { BudgetState, BudgetTracker } from "./budget.js";
+export { createDebouncer } from "./debounce.js";
+export type { Debouncer, DebounceCallback, DebounceState } from "./debounce.js";
+export { dispatchEvaluation } from "./dispatch.js";
+export type { DispatchOptions, DispatchResult } from "./dispatch.js";
+export { WebhookHandler } from "./handler.js";
+export type { DispatchRequest, SanityWebhookPayload, WebhookHandlerConfig, WebhookResult, } from "./types.js";

package/dist/webhook/index.js

@@ -0,0 +1,12 @@
+/**
+ * webhook/index.ts
+ *
+ * Barrel exports for the webhook event-driven trigger system.
+ *
+ * @see docs/design-docs/report-store/visibility-workflows.md
+ */
+export { handleEvalRequest } from "./eval-request-handler.js";
+export { createBudgetTracker } from "./budget.js";
+export { createDebouncer } from "./debounce.js";
+export { dispatchEvaluation } from "./dispatch.js";
+export { WebhookHandler } from "./handler.js";

package/dist/webhook/types.d.ts

@@ -0,0 +1,109 @@
+/**
+ * webhook/types.ts
+ *
+ * Types for the Sanity Content Lake webhook payload and the
+ * event-driven evaluation trigger system.
+ *
+ * @see https://www.sanity.io/docs/webhooks
+ * @see docs/design-docs/report-store/visibility-workflows.md
+ */
+/** Evaluation dispatch request (sent to GitHub Actions) */
+export interface DispatchRequest {
+    /** Feature areas to evaluate */
+    areas: string[];
+    /** The document that triggered this evaluation */
+    documentId?: string;
+    /** Document slug */
+    documentSlug?: string;
+    /** Evaluation mode */
+    mode?: string;
+    /** Specific task IDs to evaluate */
+    taskIds: string[];
+}
+/**
+ * Sanity webhook payload shape.
+ *
+ * When a GROQ-powered webhook fires, Sanity POSTs a JSON body with
+ * the matching document and operation metadata.
+ */
+export interface SanityWebhookPayload {
+    /** The ID of the dataset */
+    dataset?: string;
+    /** The IDs of the documents that triggered the webhook */
+    ids: {
+        /** The created document ID (for create operations) */
+        created?: string;
+        /** The deleted document ID (for delete operations) */
+        deleted?: string;
+        /** The updated document ID (for update operations) */
+        updated?: string;
+    };
+    /** The operation that triggered the webhook */
+    operation: "create" | "delete" | "update";
+    /** The ID of the project */
+    projectId?: string;
+    /** The projected document data (shape depends on webhook GROQ projection) */
+    result?: {
+        _id: string;
+        _type: string;
+        slug?: {
+            current: string;
+        };
+        [key: string]: unknown;
+    };
+}
+/** Configuration for the webhook handler */
+export interface WebhookHandlerConfig {
+    /**
+     * Budget: maximum evaluations per day.
+     * When exceeded, webhooks are acknowledged but not dispatched.
+     * Default: 20
+     */
+    dailyBudget?: number;
+    /**
+     * Debounce window in milliseconds.
+     * Rapid edits within this window are coalesced into a single evaluation.
+     * Default: 300_000 (5 minutes)
+     */
+    debounceMs?: number;
+    /**
+     * The GitHub repository to dispatch to (owner/repo format).
+     * Default: "sanity-labs/ai-literacy-framework"
+     */
+    githubRepo?: string;
+    /**
+     * GitHub personal access token for dispatching repository_dispatch events.
+     * Required for triggering evaluations.
+     */
+    githubToken: string;
+    /**
+     * Path to the eval package root (for building reverse mappings).
+     * Default: process.cwd()
+     */
+    rootDir?: string;
+    /**
+     * Sanity webhook secret for verifying payload authenticity.
+     * If provided, the handler validates the X-Sanity-Webhook-Signature header.
+     */
+    webhookSecret?: string;
+}
+/** Result of handling a webhook */
+export type WebhookResult = {
+    status: "acknowledged";
+    reason: string;
+} | {
+    status: "debounced";
+    pendingSlugs: string[];
+    windowExpiresAt: string;
+} | {
+    status: "dispatched";
+    areas: string[];
+    taskIds: string[];
+} | {
+    status: "ignored";
+    reason: string;
+} | {
+    status: "rate-limited";
+    dailyCount: number;
+    dailyBudget: number;
+};

package/dist/webhook/types.js

@@ -0,0 +1,10 @@
+/**
+ * webhook/types.ts
+ *
+ * Types for the Sanity Content Lake webhook payload and the
+ * event-driven evaluation trigger system.
+ *
+ * @see https://www.sanity.io/docs/webhooks
+ * @see docs/design-docs/report-store/visibility-workflows.md
+ */
+export {};

package/package.json

@@ -0,0 +1,72 @@
+{
+  "name": "@sanity/ailf",
+  "version": "0.1.0",
+  "private": false,
+  "publishConfig": {
+    "access": "restricted"
+  },
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/sanity-labs/ai-literacy-framework.git",
+    "directory": "packages/eval"
+  },
+  "description": "AI Literacy Framework - Evaluation tool for Sanity documentation",
+  "type": "module",
+  "bin": {
+    "ailf": "./bin/ailf.js"
+  },
+  "files": [
+    "bin",
+    "dist",
+    "config",
+    "canonical",
+    "tasks"
+  ],
+  "scripts": {
+    "build": "tsc && tsx scripts/bundle-workspace-deps.ts",
+    "generate-configs": "tsx src/cli.ts generate-configs",
+    "fetch-docs": "tsx src/cli.ts fetch-docs",
+    "measure-retrieval": "tsx src/cli.ts measure-retrieval",
+    "eval": "tsx src/cli.ts eval",
+    "compare": "tsx src/cli.ts compare",
+    "grader-consistency": "tsx src/cli.ts grader consistency",
+    "grader-validate": "tsx src/cli.ts grader validate",
+    "grader-compare": "tsx src/cli.ts grader compare",
+    "grader-sensitivity": "tsx src/cli.ts grader sensitivity",
+    "calculate-scores": "tsx src/cli.ts calculate-scores",
+    "agent-report": "tsx src/cli.ts agent-report",
+    "share": "dotenv -e ../../.env -o -- promptfoo share",
+    "view": "dotenv -e ../../.env -o -- promptfoo view",
+    "cli": "tsx src/cli.ts",
+    "pipeline": "tsx src/cli.ts pipeline",
+    "validate": "tsx src/cli.ts validate",
+    "test": "tsx --test src/__tests__/*.test.ts",
+    "pr-comment": "tsx src/cli.ts pr-comment",
+    "coverage-audit": "tsx src/cli.ts coverage-audit",
+    "readiness-report": "tsx src/cli.ts readiness-report",
+    "discovery-report": "tsx src/cli.ts discovery-report",
+    "webhook-server": "tsx src/cli.ts webhook-server",
+    "weekly-digest": "tsx src/cli.ts weekly-digest"
+  },
+  "dependencies": {
+    "@google-cloud/bigquery": "^8.1.1",
+    "@inquirer/prompts": "^8.3.0",
+    "@portabletext/markdown": "^1.0.0",
+    "@sanity/client": "^7.3.0",
+    "commander": "^14.0.3",
+    "dotenv": "^16.4.7",
+    "dotenv-cli": "^11.0.0",
+    "js-yaml": "^4.1.0",
+    "promptfoo": "^0.120.24",
+    "zod": "^4.3.6"
+  },
+  "devDependencies": {
+    "@sanity/ailf-core": "workspace:*",
+    "@sanity/ailf-shared": "workspace:*",
+    "@types/js-yaml": "^4.0.9",
+    "@types/node": "^22.13.1",
+    "tsx": "^4.19.2",
+    "typescript": "^5.7.3"
+  }
+}

package/tasks/.expanded.agentic.yaml

@@ -0,0 +1,51 @@
+# .expanded.agentic.yaml
+#
+# AUTO-GENERATED — do not edit directly.
+# Gold entries only (no baseline) for agentic evaluation mode.
+# Source: tasks/*.yaml (single-definition format)
+# Run: pnpm generate-configs
+
+- assert:
+    - type: llm-rubric
+      value: |-
+        Score task completion from 0 to 100:
+        - 0: Couldn't attempt — missing critical information
+        - 20: Attempted but fundamentally wrong approach
+        - 50: Partial implementation — major functional gaps
+        - 80: Mostly complete — minor issues or missing edge cases
+        - 100: Fully functional code — works as expected
+
+        Must demonstrate:
+        - Configures a GROQ-powered webhook
+        - Webhook triggers on content changes
+        - Includes agent integration concepts
+
+        Return ONLY a JSON object: {"score": <number>, "reason": "<explanation>"}
+      metadata:
+        dimension: task-completion
+        maxScore: 100
+    - type: contains-any
+      value:
+        - webhook
+        - GROQ
+      weight: 1
+    - type: llm-rubric
+      value: |-
+        Score documentation coverage from 0 to 100:
+        - 0: Had to hallucinate/guess most implementation details
+        - 30: Significant gaps — filled with assumptions
+        - 50: Some gaps — inferred from partial information
+        - 80: Minor gaps — almost everything was documented
+        - 100: Complete coverage — all necessary info was in docs
+
+        Return ONLY a JSON object: {"score": <number>, "reason": "<explanation>"}
+      metadata:
+        dimension: doc-coverage
+        maxScore: 100
+  description: Test - Perspective ref expansion (gold)
+  vars:
+    docs: file://contexts/canonical/perspective-ref-test.md
+    task: |
+      Build a webhook handler that integrates with an AI agent pipeline.
+      Configure a GROQ-powered webhook that triggers when blog posts are
+      published and sends a payload to an agent endpoint.

package/tasks/.expanded.yaml

@@ -0,0 +1,66 @@
+# .expanded.yaml
+#
+# AUTO-GENERATED — do not edit directly.
+# Source: tasks/*.yaml (single-definition format)
+# Run: pnpm generate-configs
+
+- assert:
+    - type: llm-rubric
+      value: |-
+        Score task completion from 0 to 100:
+        - 0: Couldn't attempt — missing critical information
+        - 20: Attempted but fundamentally wrong approach
+        - 50: Partial implementation — major functional gaps
+        - 80: Mostly complete — minor issues or missing edge cases
+        - 100: Fully functional code — works as expected
+
+        Must demonstrate:
+        - Configures a GROQ-powered webhook
+        - Webhook triggers on content changes
+        - Includes agent integration concepts
+
+        Return ONLY a JSON object: {"score": <number>, "reason": "<explanation>"}
+      metadata:
+        dimension: task-completion
+        maxScore: 100
+    - type: contains-any
+      value:
+        - webhook
+        - GROQ
+      weight: 1
+    - type: llm-rubric
+      value: |-
+        Score documentation coverage from 0 to 100:
+        - 0: Had to hallucinate/guess most implementation details
+        - 30: Significant gaps — filled with assumptions
+        - 50: Some gaps — inferred from partial information
+        - 80: Minor gaps — almost everything was documented
+        - 100: Complete coverage — all necessary info was in docs
+
+        Return ONLY a JSON object: {"score": <number>, "reason": "<explanation>"}
+      metadata:
+        dimension: doc-coverage
+        maxScore: 100
+  description: Test - Perspective ref expansion (gold)
+  prompts:
+    - with-docs
+  vars:
+    docs: file://contexts/canonical/perspective-ref-test.md
+    task: |
+      Build a webhook handler that integrates with an AI agent pipeline.
+      Configure a GROQ-powered webhook that triggers when blog posts are
+      published and sends a payload to an agent endpoint.
+- description: Test - Perspective ref expansion (baseline)
+  prompts:
+    - without-docs
+  vars:
+    docs: ''
+    task: |
+      Build a webhook handler that integrates with an AI agent pipeline.
+      Configure a GROQ-powered webhook that triggers when blog posts are
+      published and sends a payload to an agent endpoint.
+  assert:
+    - type: llm-rubric
+      value: |-
+        Score task completion from 0 to 100 (same criteria as above).
+        Return ONLY a JSON object: {"score": <number>, "reason": "<explanation>"}

package/tasks/frameworks.yaml

@@ -0,0 +1,98 @@
+# tasks/frameworks.yaml
+#
+# Other Framework Integrations — Remix, Nuxt, etc.
+#
+# Each task is defined once. The pipeline auto-generates gold (with docs)
+# and baseline (without docs) variants from each definition.
+
+# ============================================================
+# TASK: Remix Integration
+# ============================================================
+- id: remix-integration
+  description: "Frameworks - Remix integration with data fetching"
+  doc_coverage: true
+  canonical_docs:
+    - slug: displaying-content-in-a-react-router-front-end
+      reason: "React Router front-end content display guide"
+    - slug: visual-editing-with-react-router
+      reason: "React Router / Remix integration with visual editing"
+    - slug: functions-js-client
+      reason: "Configuring @sanity/client for data fetching"
+  reference_solution: reference-solutions/frameworks/remix.tsx
+  vars:
+    task: |
+      Integrate Sanity into a Remix application:
+
+      1. Set up the Sanity client
+      2. Create a loader that fetches blog posts using GROQ
+      3. Build a route component that renders the fetched posts
+      4. Handle loading and error states properly
+
+      Provide all necessary files for a working Remix + Sanity integration.
+    docs: file://contexts/canonical/remix-integration.md
+  assert:
+    - type: llm-rubric
+      template: task-completion
+      criteria:
+        - Sanity client configuration
+        - Remix loader function with GROQ query
+        - Route component using useLoaderData
+        - Proper typing
+
+    - type: llm-rubric
+      template: code-correctness
+      criteria:
+        - Modern Remix patterns (v2 conventions)
+        - Proper loader/component separation
+        - Valid GROQ queries
+        - No deprecated APIs
+
+    - type: contains-any
+      value:
+        - "useLoaderData"
+        - "loader"
+      weight: 1
+
+# ============================================================
+# TASK: Nuxt 3 Integration
+# ============================================================
+- id: nuxt-integration
+  description: "Frameworks - Nuxt 4 integration"
+  doc_coverage: true
+  canonical_docs:
+    - slug: displaying-content-in-nuxt-js
+      reason: "Nuxt.js front-end content display guide"
+    - slug: visual-editing-with-nuxt
+      reason: "Nuxt visual editing integration"
+  reference_solution: reference-solutions/frameworks/nuxt.ts
+  vars:
+    task: |
+      Integrate Sanity into a Nuxt 4 application:
+
+      1. Install and configure the @nuxtjs/sanity module
+      2. Create a page that fetches and displays blog posts
+      3. Use Nuxt composables for data fetching
+
+      Provide all necessary configuration and component code.
+    docs: file://contexts/canonical/nuxt-integration.md
+  assert:
+    - type: llm-rubric
+      template: task-completion
+      criteria:
+        - "@nuxtjs/sanity module setup in nuxt.config.ts"
+        - Page component using Nuxt data fetching composables
+        - Sanity GROQ query
+
+    - type: llm-rubric
+      template: code-correctness
+      criteria:
+        - Nuxt 3 module configuration syntax
+        - Uses useSanityQuery or equivalent composable
+        - Proper Nuxt 3 patterns (not Nuxt 2)
+
+    - type: contains-any
+      value:
+        - "@nuxtjs/sanity"
+        - "useSanityQuery"
+        - "sanity:"
+      weight: 1