@sanity/ailf 0.1.0

package/dist/webhook/debounce.js

@@ -0,0 +1,76 @@
+/**
+ * webhook/debounce.ts
+ *
+ * Time-window debouncing for content change webhooks.
+ *
+ * When multiple documents are edited in rapid succession (e.g., an editor
+ * updating several related articles), we coalesce them into a single
+ * evaluation dispatch. This prevents wasting evaluation budget on
+ * intermediate states.
+ *
+ * Strategy: accumulate changed slugs in a time window (default: 5 minutes).
+ * When the window expires, dispatch a single evaluation for all accumulated
+ * areas. New edits during the window reset the timer ("trailing edge").
+ *
+ * @see docs/design-docs/report-store/visibility-workflows.md — Debouncing
+ */
+/**
+ * Create a debouncer with a configurable time window.
+ *
+ * @param windowMs - Debounce window in milliseconds (default: 300_000 = 5 min)
+ * @param onFlush - Callback invoked when the window closes
+ */
+export function createDebouncer(windowMs, onFlush) {
+    const state = {
+        slugs: new Set(),
+        timer: null,
+        windowStart: 0,
+    };
+    function resetTimer() {
+        if (state.timer !== null) {
+            clearTimeout(state.timer);
+        }
+        state.timer = setTimeout(() => {
+            void doFlush();
+        }, windowMs);
+    }
+    async function doFlush() {
+        if (state.slugs.size === 0)
+            return;
+        const slugs = [...state.slugs];
+        // Reset state before callback (so new pushes start a fresh window)
+        state.slugs.clear();
+        if (state.timer !== null) {
+            clearTimeout(state.timer);
+            state.timer = null;
+        }
+        state.windowStart = 0;
+        await onFlush(slugs);
+    }
+    return {
+        cancel() {
+            if (state.timer !== null) {
+                clearTimeout(state.timer);
+                state.timer = null;
+            }
+            state.slugs.clear();
+            state.windowStart = 0;
+        },
+        async flush() {
+            await doFlush();
+        },
+        pending() {
+            return state.slugs.size;
+        },
+        push(slug) {
+            if (state.slugs.size === 0) {
+                state.windowStart = Date.now();
+            }
+            state.slugs.add(slug);
+            resetTimer();
+        },
+        state() {
+            return { ...state, slugs: new Set(state.slugs) };
+        },
+    };
+}

package/dist/webhook/dispatch.d.ts

@@ -0,0 +1,45 @@
+/**
+ * webhook/dispatch.ts
+ *
+ * Triggers AI Literacy Framework evaluations via GitHub Actions
+ * repository_dispatch events.
+ *
+ * When a content webhook determines that a document change affects
+ * tracked evaluation areas, this module dispatches a scoped pipeline
+ * run to the AILF repository.
+ *
+ * The dispatch uses the `external-eval` event type. The client_payload
+ * conforms to the PipelineRequestSchema so the receiving workflow can
+ * pass it directly to the CLI via `--config` without field translation.
+ *
+ * @see .github/workflows/external-eval.yml
+ * @see docs/design-docs/cross-repo-eval.md
+ */
+import type { DispatchRequest } from "./types.js";
+export interface DispatchOptions {
+    /** GitHub personal access token with repo scope */
+    githubToken: string;
+    /** Target repository (owner/repo). Default: sanity-labs/ai-literacy-framework */
+    repo?: string;
+}
+export interface DispatchResult {
+    /** Error message on failure */
+    error?: string;
+    /** The HTTP status code from GitHub API */
+    httpStatus: number;
+    /** Whether the dispatch was accepted */
+    ok: boolean;
+}
+/**
+ * Dispatch an evaluation via GitHub Actions repository_dispatch.
+ *
+ * Sends a `repository_dispatch` event with the `external-eval` event type.
+ * The payload includes area and task scoping so the pipeline evaluates
+ * only what's needed.
+ *
+ * GitHub API response:
+ * - 204: Dispatch accepted
+ * - 404: Repository not found or token lacks access
+ * - 422: Validation error
+ */
+export declare function dispatchEvaluation(request: DispatchRequest, options: DispatchOptions): Promise<DispatchResult>;

package/dist/webhook/dispatch.js

@@ -0,0 +1,84 @@
+/**
+ * webhook/dispatch.ts
+ *
+ * Triggers AI Literacy Framework evaluations via GitHub Actions
+ * repository_dispatch events.
+ *
+ * When a content webhook determines that a document change affects
+ * tracked evaluation areas, this module dispatches a scoped pipeline
+ * run to the AILF repository.
+ *
+ * The dispatch uses the `external-eval` event type. The client_payload
+ * conforms to the PipelineRequestSchema so the receiving workflow can
+ * pass it directly to the CLI via `--config` without field translation.
+ *
+ * @see .github/workflows/external-eval.yml
+ * @see docs/design-docs/cross-repo-eval.md
+ */
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+const DEFAULT_REPO = "sanity-labs/ai-literacy-framework";
+const GITHUB_API = "https://api.github.com";
+/**
+ * Dispatch an evaluation via GitHub Actions repository_dispatch.
+ *
+ * Sends a `repository_dispatch` event with the `external-eval` event type.
+ * The payload includes area and task scoping so the pipeline evaluates
+ * only what's needed.
+ *
+ * GitHub API response:
+ * - 204: Dispatch accepted
+ * - 404: Repository not found or token lacks access
+ * - 422: Validation error
+ */
+export async function dispatchEvaluation(request, options) {
+    const repo = options.repo ?? DEFAULT_REPO;
+    const url = `${GITHUB_API}/repos/${repo}/dispatches`;
+    const payload = {
+        client_payload: {
+            areas: request.areas,
+            caller_repo: "ailf-webhook-handler",
+            changedDocs: request.documentSlug
+                ? request.documentSlug.split(",").filter(Boolean)
+                : undefined,
+            compare: true,
+            mode: request.mode ?? "baseline",
+            publish: true,
+            source: "production",
+            tasks: request.taskIds.length > 0 ? request.taskIds : undefined,
+        },
+        event_type: "external-eval",
+    };
+    try {
+        const response = await fetch(url, {
+            body: JSON.stringify(payload),
+            headers: {
+                Accept: "application/vnd.github+json",
+                Authorization: `Bearer ${options.githubToken}`,
+                "X-GitHub-Api-Version": "2022-11-28",
+            },
+            method: "POST",
+        });
+        if (response.status === 204) {
+            return { httpStatus: 204, ok: true };
+        }
+        // Try to extract error message
+        let error;
+        try {
+            const body = (await response.json());
+            error = body.message ?? `HTTP ${response.status}`;
+        }
+        catch {
+            error = `HTTP ${response.status}: ${response.statusText}`;
+        }
+        return { error, httpStatus: response.status, ok: false };
+    }
+    catch (err) {
+        return {
+            error: err instanceof Error ? err.message : String(err),
+            httpStatus: 0,
+            ok: false,
+        };
+    }
+}

package/dist/webhook/eval-request-handler.d.ts

@@ -0,0 +1,87 @@
+/**
+ * webhook/eval-request-handler.ts
+ *
+ * Server-side handler for `ailf.evalRequest` documents from the Sanity
+ * Content Lake. This is the counterpart to the Studio's "Request Evaluation"
+ * action — when a content editor creates an eval request document via the
+ * Studio UI, a Sanity webhook fires and calls this handler.
+ *
+ * Designed to run in any HTTP environment: Cloudflare Workers, Vercel
+ * functions, Express, Hono, etc.
+ *
+ * Flow:
+ * 1. Receive eval request payload (from Sanity webhook projection)
+ * 2. Validate: must be `ailf.evalRequest` type, `pending` status,
+ *    non-empty `perspective`
+ * 3. Dispatch a full evaluation to GitHub Actions via `repository_dispatch`
+ *    with `external-eval` event type and release-scoped client payload
+ * 4. On success: PATCH the eval request document → `status: "dispatched"`
+ * 5. On failure: PATCH the eval request document → `status: "failed"` + error
+ * 6. Return a structured result
+ *
+ * @see packages/studio/src/actions/dispatch.ts — Studio-side dispatch
+ * @see .github/workflows/external-eval.yml — receiving workflow
+ * @see docs/design-docs/report-store/visibility-workflows.md
+ */
+/** Projected shape of an `ailf.evalRequest` document from a Sanity webhook. */
+export interface EvalRequestPayload {
+    /** The Sanity document _id */
+    _id: string;
+    /** The Sanity document _type (should be "ailf.evalRequest") */
+    _type: string;
+    /** Sanity dataset */
+    dataset: string;
+    /** Error message (only if status is "failed") */
+    error?: string;
+    /** Evaluation mode */
+    mode: string;
+    /** Content release perspective ID */
+    perspective: string;
+    /** Sanity project ID */
+    projectId: string;
+    /** ISO datetime of when the request was created */
+    requestedAt: string;
+    /** User ID who requested */
+    requestedBy?: string;
+    /** Publish tag */
+    tag?: string;
+    /** Request status */
+    status: string;
+}
+/** Configuration for the eval request handler. */
+export interface EvalRequestHandlerConfig {
+    /** GitHub PAT with repo or actions:write scope */
+    githubToken: string;
+    /** Target AILF repo (default: "sanity-labs/ai-literacy-framework") */
+    githubRepo?: string;
+    /** Sanity API token for updating the eval request document status */
+    sanityToken: string;
+}
+/** Result of handling an eval request. */
+export interface EvalRequestResult {
+    /** Whether the dispatch succeeded */
+    ok: boolean;
+    /** The eval request document ID */
+    requestId: string;
+    /** Error message on failure */
+    error?: string;
+}
+/**
+ * Handle an incoming `ailf.evalRequest` document from a Sanity webhook.
+ *
+ * Validates the payload, dispatches a GitHub Actions evaluation, and
+ * updates the eval request document status in the Content Lake.
+ *
+ * ```ts
+ * // In a Cloudflare Worker or Vercel function:
+ * export default async function handler(req: Request) {
+ *   const payload = await req.json()
+ *   const result = await handleEvalRequest(payload, {
+ *     githubToken: env.GITHUB_TOKEN,
+ *     sanityToken: env.SANITY_API_TOKEN,
+ *   })
+ *   return Response.json(result, { status: result.ok ? 200 : 400 })
+ * }
+ * ```
+ */
+export declare function handleEvalRequest(payload: EvalRequestPayload, config: EvalRequestHandlerConfig): Promise<EvalRequestResult>;

package/dist/webhook/eval-request-handler.js

@@ -0,0 +1,181 @@
+/**
+ * webhook/eval-request-handler.ts
+ *
+ * Server-side handler for `ailf.evalRequest` documents from the Sanity
+ * Content Lake. This is the counterpart to the Studio's "Request Evaluation"
+ * action — when a content editor creates an eval request document via the
+ * Studio UI, a Sanity webhook fires and calls this handler.
+ *
+ * Designed to run in any HTTP environment: Cloudflare Workers, Vercel
+ * functions, Express, Hono, etc.
+ *
+ * Flow:
+ * 1. Receive eval request payload (from Sanity webhook projection)
+ * 2. Validate: must be `ailf.evalRequest` type, `pending` status,
+ *    non-empty `perspective`
+ * 3. Dispatch a full evaluation to GitHub Actions via `repository_dispatch`
+ *    with `external-eval` event type and release-scoped client payload
+ * 4. On success: PATCH the eval request document → `status: "dispatched"`
+ * 5. On failure: PATCH the eval request document → `status: "failed"` + error
+ * 6. Return a structured result
+ *
+ * @see packages/studio/src/actions/dispatch.ts — Studio-side dispatch
+ * @see .github/workflows/external-eval.yml — receiving workflow
+ * @see docs/design-docs/report-store/visibility-workflows.md
+ */
+import { createClient } from "@sanity/client";
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+const DEFAULT_REPO = "sanity-labs/ai-literacy-framework";
+const GITHUB_API = "https://api.github.com";
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+/**
+ * Handle an incoming `ailf.evalRequest` document from a Sanity webhook.
+ *
+ * Validates the payload, dispatches a GitHub Actions evaluation, and
+ * updates the eval request document status in the Content Lake.
+ *
+ * ```ts
+ * // In a Cloudflare Worker or Vercel function:
+ * export default async function handler(req: Request) {
+ *   const payload = await req.json()
+ *   const result = await handleEvalRequest(payload, {
+ *     githubToken: env.GITHUB_TOKEN,
+ *     sanityToken: env.SANITY_API_TOKEN,
+ *   })
+ *   return Response.json(result, { status: result.ok ? 200 : 400 })
+ * }
+ * ```
+ */
+export async function handleEvalRequest(payload, config) {
+    const requestId = payload._id ?? "unknown";
+    // -------------------------------------------------------------------------
+    // 1. Validate payload
+    // -------------------------------------------------------------------------
+    if (payload._type !== "ailf.evalRequest") {
+        return {
+            error: `Unexpected document type: "${payload._type}" (expected "ailf.evalRequest")`,
+            ok: false,
+            requestId,
+        };
+    }
+    if (payload.status !== "pending") {
+        return {
+            error: `Eval request is not pending (status: "${payload.status}")`,
+            ok: false,
+            requestId,
+        };
+    }
+    if (!payload.perspective) {
+        return {
+            error: "Missing required field: perspective",
+            ok: false,
+            requestId,
+        };
+    }
+    // -------------------------------------------------------------------------
+    // 2. Dispatch evaluation via GitHub Actions
+    // -------------------------------------------------------------------------
+    const repo = config.githubRepo ?? DEFAULT_REPO;
+    const dispatchResult = await dispatchGitHubEval(repo, payload, config);
+    // -------------------------------------------------------------------------
+    // 3. Update eval request document status
+    // -------------------------------------------------------------------------
+    const client = createClient({
+        apiVersion: "2026-03-11",
+        dataset: payload.dataset,
+        projectId: payload.projectId,
+        token: config.sanityToken,
+        useCdn: false,
+    });
+    if (dispatchResult.ok) {
+        try {
+            await client
+                .patch(payload._id)
+                .set({
+                dispatchedAt: new Date().toISOString(),
+                status: "dispatched",
+            })
+                .commit();
+        }
+        catch (err) {
+            // Dispatch succeeded but status update failed — log and still return ok
+            console.warn(`  ⚠️  Dispatch succeeded but failed to update document status: ${err instanceof Error ? err.message : String(err)}`);
+        }
+        return { ok: true, requestId };
+    }
+    // Dispatch failed — mark the document as failed
+    try {
+        await client
+            .patch(payload._id)
+            .set({
+            error: dispatchResult.error ?? "Unknown dispatch error",
+            status: "failed",
+        })
+            .commit();
+    }
+    catch (err) {
+        console.warn(`  ⚠️  Failed to update document with error status: ${err instanceof Error ? err.message : String(err)}`);
+    }
+    return {
+        error: dispatchResult.error,
+        ok: false,
+        requestId,
+    };
+}
+/**
+ * Dispatch a release-scoped evaluation via GitHub Actions repository_dispatch.
+ *
+ * Uses the `external-eval` event type with a client_payload conforming to
+ * PipelineRequestSchema. The workflow passes it directly to the CLI via
+ * `--config` without field translation.
+ */
+async function dispatchGitHubEval(repo, payload, config) {
+    const url = `${GITHUB_API}/repos/${repo}/dispatches`;
+    const body = {
+        client_payload: {
+            caller_repo: "sanity-io/www-sanity-io",
+            dataset: payload.dataset,
+            mode: payload.mode,
+            perspective: payload.perspective,
+            projectId: payload.projectId,
+            publish: true,
+            ...(payload.tag ? { publishTag: payload.tag } : {}),
+            source: "production",
+        },
+        event_type: "external-eval",
+    };
+    try {
+        const response = await fetch(url, {
+            body: JSON.stringify(body),
+            headers: {
+                Accept: "application/vnd.github+json",
+                Authorization: `Bearer ${config.githubToken}`,
+                "X-GitHub-Api-Version": "2022-11-28",
+            },
+            method: "POST",
+        });
+        if (response.status === 204) {
+            return { ok: true };
+        }
+        // Try to extract error message from GitHub response
+        let error;
+        try {
+            const responseBody = (await response.json());
+            error = responseBody.message ?? `HTTP ${response.status}`;
+        }
+        catch {
+            error = `HTTP ${response.status}: ${response.statusText}`;
+        }
+        return { error, ok: false };
+    }
+    catch (err) {
+        return {
+            error: err instanceof Error ? err.message : String(err),
+            ok: false,
+        };
+    }
+}

package/dist/webhook/handler.d.ts

@@ -0,0 +1,88 @@
+/**
+ * webhook/handler.ts
+ *
+ * Platform-agnostic webhook handler for Sanity content change events.
+ *
+ * Receives Sanity webhook payloads, determines which evaluation areas
+ * are affected by the document change, debounces rapid edits, enforces
+ * daily budget limits, and dispatches scoped evaluations via GitHub Actions.
+ *
+ * This handler is stateless between requests (debounce and budget state
+ * is held in-memory by the enclosing WebhookHandler instance). It can be
+ * mounted in any HTTP framework: Express, Hono, Cloudflare Workers, etc.
+ *
+ * Flow:
+ * 1. Receive Sanity webhook payload
+ * 2. Extract document slug from payload
+ * 3. Look up affected areas via reverse mapping
+ * 4. If no areas affected → ignore (untracked document)
+ * 5. Check daily budget → rate-limit if exceeded
+ * 6. Push slug into debounce window
+ * 7. When debounce window closes → dispatch scoped eval via GitHub Actions
+ *
+ * @see docs/design-docs/report-store/visibility-workflows.md
+ */
+import { type DispatchResult } from "./dispatch.js";
+import type { SanityWebhookPayload, WebhookHandlerConfig, WebhookResult } from "./types.js";
+/**
+ * A stateful webhook handler that manages debouncing, budgeting,
+ * and evaluation dispatch.
+ *
+ * Create one instance per process and call `handle()` for each
+ * incoming webhook payload.
+ *
+ * ```ts
+ * const handler = new WebhookHandler({
+ *   githubToken: process.env.GITHUB_TOKEN!,
+ *   rootDir: "/path/to/packages/eval",
+ * })
+ *
+ * // In your HTTP handler:
+ * app.post("/webhook", async (req) => {
+ *   const result = await handler.handle(req.body)
+ *   return Response.json(result)
+ * })
+ * ```
+ */
+export declare class WebhookHandler {
+    private readonly budget;
+    private readonly config;
+    private readonly debouncer;
+    /** Recent dispatch results (for diagnostics) */
+    private readonly recentDispatches;
+    private readonly reverseMapping;
+    constructor(config: WebhookHandlerConfig);
+    /**
+     * Get handler diagnostics (for health check endpoints).
+     */
+    diagnostics(): {
+        budget: {
+            count: number;
+            limit: number;
+            remaining: number;
+        };
+        pendingSlugs: number;
+        recentDispatches: {
+            areas: string[];
+            result: DispatchResult;
+            timestamp: string;
+        }[];
+        trackedSlugs: number;
+    };
+    /**
+     * Handle an incoming Sanity webhook payload.
+     *
+     * This is the main entry point — call once per webhook request.
+     * Returns a result indicating what happened (dispatched, debounced,
+     * rate-limited, or ignored).
+     */
+    handle(payload: SanityWebhookPayload): WebhookResult;
+    /**
+     * Force-flush the debounce window (for graceful shutdown).
+     */
+    shutdown(): Promise<void>;
+    /**
+     * Called when the debounce window closes — dispatches a scoped evaluation.
+     */
+    private onDebounceFlush;
+}