@sanity/ailf 0.4.1 → 1.0.0

package/config/features.ts

@@ -0,0 +1,23 @@
+/**
+ * features.ts — Product feature registry for documentation coverage auditing.
+ *
+ * Default features are provided by the sanity-literacy preset registered
+ * in the composition root. This file exists as an override point — any
+ * features defined here take precedence over preset-provided features
+ * during coverage auditing.
+ *
+ * To track custom features, define them here:
+ *
+ *   export default defineFeatures({
+ *     features: [
+ *       { id: "my-feature", name: "My Feature", sections: ["api"], ... },
+ *     ],
+ *   })
+ *
+ * @see packages/eval/src/pipeline/compiler/presets/sanity-literacy.ts
+ * @see docs/exec-plans/scenario-matrix-implementation/phase-3-gap-analysis.md
+ */
+
+import { defineFeatures } from "@sanity/ailf-core"
+
+export default defineFeatures({ features: [] })

package/config/models.ts

@@ -0,0 +1,83 @@
+/**
+ * models.ts — Central model registry for AILF evaluations.
+ *
+ * Define all models to test here. Each eval mode (baseline, observed, agentic)
+ * reads this config and generates the appropriate provider entries.
+ *
+ * @see docs/exec-plans/architecture-overhaul/phase-1-ts-config-loading.md
+ */
+
+import { defineModels } from "@sanity/ailf-core"
+
+export default defineModels({
+  models: [
+    // ── Anthropic ──────────────────────────────────────────────
+    {
+      id: "anthropic:messages:claude-opus-4-6",
+      label: "Claude Opus 4.6",
+      config: { temperature: 0.2, max_tokens: 4096 },
+      modes: ["baseline", "observed", "agentic-naive", "agentic-optimized"],
+    },
+
+    // ── Google ─────────────────────────────────────────────────
+    // {
+    //   id: "google:gemini-2.5-pro",
+    //   label: "Gemini 2.5 Pro",
+    //   config: { temperature: 0.2, max_tokens: 4096 },
+    //   modes: ["baseline", "observed", "agentic-naive", "agentic-optimized"],
+    // },
+
+    // ── OpenAI ─────────────────────────────────────────────────
+    {
+      id: "openai:chat:gpt-5.2",
+      label: "GPT 5.2",
+      config: { temperature: 0.2, max_tokens: 4096 },
+      modes: ["baseline", "observed", "agentic-naive", "agentic-optimized"],
+    },
+    {
+      id: "openai:chat:gpt-5.4",
+      label: "GPT 5.4",
+      config: {
+        reasoning_effort: "medium",
+        max_output_tokens: 4096,
+        maxRetries: 1,
+      },
+      modes: ["baseline", "observed", "agentic-naive", "agentic-optimized"],
+    },
+
+    // ── Disabled models (uncomment to enable) ──────────────────
+    // { id: "anthropic:claude-sonnet-4-20250514", label: "Claude Sonnet 4",
+    //   config: { temperature: 0.2, max_tokens: 4096 }, modes: ["baseline"] },
+    // { id: "anthropic:claude-3.5-sonnet-20241022", label: "Claude 3.5 Sonnet",
+    //   config: { temperature: 0.2, max_tokens: 4096 },
+    //   modes: ["baseline", "agentic-naive", "agentic-optimized"] },
+    // { id: "google:gemini-2.0-flash", label: "Gemini 2.0 Flash",
+    //   config: { temperature: 0.2, max_tokens: 4096 }, modes: ["baseline"] },
+    // { id: "openrouter:deepseek/deepseek-r1", label: "DeepSeek R1",
+    //   config: { temperature: 0.2, max_tokens: 4096 }, modes: ["baseline"] },
+  ],
+
+  // ── Grading Model ──────────────────────────────────────────
+  // Which model scores the responses. Separate from the models being tested.
+  grader: {
+    id: "anthropic:messages:claude-opus-4-5-20251101",
+    label: "Claude Opus 4.5 (grader)",
+  },
+
+  // ── Evaluation Options ─────────────────────────────────────
+  maxConcurrency: 32, // max parallel API calls — benchmarked in DOC-1896
+
+  // ── Default Config ─────────────────────────────────────────
+  // Applied to all models unless overridden per-model.
+  defaults: {
+    temperature: 0.2,
+    max_tokens: 4096,
+    maxToolRounds: 5, // for agentic modes
+    observerOptions: {
+      maxPreviewBytes: 2048,
+      captureResponsePreview: true,
+      includePatterns: ["sanity.io", "sanity.dev", "cdn.sanity.io"],
+      sensitiveHeaders: ["authorization", "cookie", "x-api-key"],
+    },
+  },
+})

package/config/prompts.ts

@@ -0,0 +1,16 @@
+/**
+ * prompts.ts — User-override prompt templates.
+ *
+ * Canonical literacy prompt templates now live in the literacy mode handler:
+ *   src/pipeline/compiler/mode-handlers/literacy-handler.ts
+ *
+ * Each mode handler owns its own prompts via getPrompts(). This file exists
+ * for user-level overrides only. Add entries here to replace handler-owned
+ * defaults for specific prompt IDs.
+ *
+ * @see packages/eval/src/pipeline/compiler/mode-handlers/literacy-handler.ts
+ */
+
+import { definePrompts } from "@sanity/ailf-core"
+
+export default definePrompts([])

package/config/rubrics.ts

@@ -0,0 +1,225 @@
+/**
+ * rubrics.ts — Centralized rubric templates for LLM grading assertions.
+ *
+ * Tasks reference these templates by key and provide only their unique
+ * criteria bullet points. The pipeline assembles the full rubric text
+ * at expansion time.
+ *
+ * @see docs/design-docs/structured-dimensions.md
+ * @see docs/design-docs/uniform-dimension-scoring.md
+ */
+
+import { defineRubrics } from "@sanity/ailf-core"
+
+export default defineRubrics({
+  templates: {
+    // ── Core literacy dimensions ────────────────────────────
+    "task-completion": {
+      dimension: "task-completion",
+      header: "Score task completion from 0 to 100:",
+      scale: [
+        "0: Couldn't attempt — missing critical information",
+        "20: Attempted but fundamentally wrong approach",
+        "50: Partial implementation — major functional gaps",
+        "80: Mostly complete — minor issues or missing edge cases",
+        "100: Fully functional code — works as expected",
+      ],
+      criteria_label: "Must demonstrate:",
+    },
+    "code-correctness": {
+      dimension: "code-correctness",
+      header: "Score code correctness from 0 to 100:",
+      scale: [
+        "0: Broken code, syntax errors, or deprecated APIs",
+        "30: Works but uses anti-patterns or inefficient approaches",
+        "50: Works but not idiomatic",
+        "80: Follows most best practices",
+        "100: Follows all best practices, idiomatic implementation",
+      ],
+      criteria_label: "Check for:",
+    },
+    "doc-coverage": {
+      dimension: "doc-coverage",
+      header: "Score documentation coverage from 0 to 100:",
+      scale: [
+        "0: Had to hallucinate/guess most implementation details",
+        "30: Significant gaps — filled with assumptions",
+        "50: Some gaps — inferred from partial information",
+        "80: Minor gaps — almost everything was documented",
+        "100: Complete coverage — all necessary info was in docs",
+      ],
+    },
+
+    // ── MCP server dimensions ───────────────────────────────
+    "mcp-input-validation": {
+      dimension: "input-validation",
+      header: "Score MCP tool input correctness from 0 to 100:",
+      scale: [
+        "0: Completely wrong tool inputs — missing or invalid parameters",
+        "25: Attempted correct inputs but with significant errors",
+        "50: Partially correct — some required fields present, others wrong",
+        "75: Mostly correct — minor issues with optional parameters",
+        "100: Perfect tool inputs — all parameters correct and well-formed",
+      ],
+      criteria_label: "Evaluate:",
+    },
+    "mcp-output-correctness": {
+      dimension: "output-correctness",
+      header: "Score MCP tool output handling from 0 to 100:",
+      scale: [
+        "0: Completely failed to use tool outputs",
+        "25: Used outputs but misinterpreted the data",
+        "50: Partially correct interpretation of tool outputs",
+        "75: Mostly correct — minor misuse of returned data",
+        "100: Perfect output handling — correctly interpreted all tool responses",
+      ],
+      criteria_label: "Check for:",
+    },
+    "mcp-error-handling": {
+      dimension: "error-handling",
+      header: "Score MCP error handling from 0 to 100:",
+      scale: [
+        "0: Crashed or ignored errors entirely",
+        "25: Acknowledged error but took wrong corrective action",
+        "50: Partial error handling — recovered from some errors",
+        "75: Good error handling — recovered gracefully with minor issues",
+        "100: Excellent — handled all errors appropriately with clear messaging",
+      ],
+      criteria_label: "Evaluate:",
+    },
+    "mcp-security": {
+      dimension: "security",
+      header: "Score MCP security compliance from 0 to 100:",
+      scale: [
+        "0: Attempted unauthorized operations or leaked sensitive data",
+        "25: Made questionable tool calls outside the expected scope",
+        "50: Stayed within scope but with minor security concerns",
+        "75: Good security — no unauthorized access, minor concerns",
+        "100: Perfect security — only used authorized tools with safe inputs",
+      ],
+      criteria_label: "Check for:",
+    },
+
+    // ── Knowledge probe dimensions ──────────────────────────
+    "factual-correctness": {
+      dimension: "factual-correctness",
+      header: "Score factual correctness from 0 to 100:",
+      scale: [
+        "0: Completely wrong — fabricated facts, hallucinated APIs, incorrect syntax",
+        "25: Mostly wrong — some real concepts but significant factual errors",
+        "50: Mixed — some correct facts alongside notable inaccuracies",
+        "75: Mostly correct — minor inaccuracies or imprecisions",
+        "100: Fully correct — all statements are accurate and verifiable",
+      ],
+      criteria_label: "Verify:",
+    },
+    completeness: {
+      dimension: "completeness",
+      header: "Score knowledge completeness from 0 to 100:",
+      scale: [
+        "0: Superficial — only knows the name, no substantive knowledge",
+        "25: Minimal — knows basic concepts but misses most important details",
+        "50: Partial — covers some key aspects but has significant gaps",
+        "75: Good coverage — covers most key aspects with minor gaps",
+        "100: Comprehensive — thorough coverage of all important aspects",
+      ],
+      criteria_label: "Check coverage of:",
+    },
+    currency: {
+      dimension: "currency",
+      header: "Score knowledge currency (up-to-dateness) from 0 to 100:",
+      scale: [
+        "0: Severely outdated — references deprecated APIs or removed features",
+        "25: Mostly outdated — aware of old version but not recent changes",
+        "50: Partially current — knows some recent changes but misses others",
+        "75: Mostly current — knows recent API but minor gaps on latest features",
+        "100: Fully current — references latest APIs, patterns, and best practices",
+      ],
+      criteria_label: "Check for:",
+    },
+
+    // ── Agent harness dimensions ────────────────────────────
+    "process-quality": {
+      dimension: "process-quality",
+      header:
+        "Score the agent's process quality from 0 to 100 (advisory — does not gate pass/fail):",
+      scale: [
+        "0: Chaotic process — random tool calls, no planning, no error handling",
+        "25: Poor process — some structure but significant inefficiencies",
+        "50: Adequate process — gets the job done but not efficiently",
+        "75: Good process — reads before writing, handles errors, incremental changes",
+        "100: Excellent process — optimal tool usage, clear planning, graceful recovery",
+      ],
+      criteria_label: "Evaluate:",
+    },
+    "agent-output": {
+      dimension: "agent-output",
+      header: "Score the agent's final output from 0 to 100:",
+      scale: [
+        "0: No useful output produced — task completely failed",
+        "25: Partial output — attempted but with fundamental errors",
+        "50: Usable output — works but with significant issues",
+        "75: Good output — mostly correct with minor issues",
+        "100: Excellent output — fully correct, clean, and complete",
+      ],
+      criteria_label: "Check for:",
+    },
+    "agent-tool-usage": {
+      dimension: "tool-usage",
+      header: "Score the agent's tool usage from 0 to 100:",
+      scale: [
+        "0: Completely wrong tool usage — called wrong tools or with invalid inputs",
+        "25: Poor tool usage — correct tools but wrong parameters or sequencing",
+        "50: Adequate — correct tools and basic parameters, some inefficiency",
+        "75: Good — efficient tool usage with proper error handling",
+        "100: Excellent — optimal tool selection, correct inputs, minimal redundancy",
+      ],
+      criteria_label: "Evaluate:",
+    },
+  },
+
+  // ── Named scoring profiles ────────────────────────────────
+  profiles: {
+    default: {
+      "task-completion": 0.5,
+      "code-correctness": 0.25,
+      "doc-coverage": 0.25,
+    },
+    "output-only": {
+      "task-completion": 0.6,
+      "code-correctness": 0.4,
+    },
+    "mcp-behavior": {
+      "input-validation": 0.25,
+      "output-correctness": 0.35,
+      "error-handling": 0.25,
+      security: 0.15,
+    },
+    "knowledge-probe": {
+      "factual-correctness": 0.45,
+      completeness: 0.35,
+      currency: 0.2,
+    },
+    "agent-harness": {
+      "agent-output": 0.45,
+      "tool-usage": 0.4,
+      "process-quality": 0.15,
+    },
+  },
+
+  // ── Mode-to-profile bindings ──────────────────────────────
+  // Literacy mode uses variant sub-keys because 'baseline' and 'agentic'
+  // are scoring variants within the same canonical mode, not separate modes.
+  "mode-profiles": {
+    "literacy": {
+      baseline: { gold: "default", baseline: "output-only" },
+      agentic: { gold: "default" },
+    },
+    "mcp-server": { gold: "mcp-behavior" },
+    "knowledge-probe": { gold: "knowledge-probe" },
+    "agent-harness": { gold: "agent-harness" },
+  },
+
+  footer:
+    'Return ONLY a JSON object: {"score": <number>, "reason": "<explanation>"}',
+})

package/config/schedules.ts

@@ -0,0 +1,47 @@
+/**
+ * schedules.ts — Scheduled evaluation configuration.
+ *
+ * Each schedule defines a recurring pipeline run with its own source,
+ * mode, and delivery preferences. The GitHub Actions cron workflow reads
+ * this config to determine which evaluations to run and when.
+ *
+ * Cron expressions use UTC timezone (GitHub Actions standard).
+ *
+ * @see docs/design-docs/report-store/implementation.md — Phase 5
+ */
+
+import { defineSchedules, env } from "@sanity/ailf-core"
+
+export default defineSchedules({
+  schedules: [
+    // Daily baseline — track score trends against production docs
+    {
+      name: "daily-baseline",
+      cron: "0 2 * * *", // 2:00 AM UTC, every day
+      mode: "baseline",
+      source: "production",
+      publish: true,
+      compare: true,
+      enabled: true,
+    },
+
+    // Weekly full decomposition — complete floor/ceiling/actual report
+    {
+      name: "weekly-full",
+      cron: "0 3 * * 0", // 3:00 AM UTC, every Sunday
+      mode: "full",
+      source: "production",
+      publish: true,
+      compare: true,
+      enabled: true,
+    },
+  ],
+
+  // Digest — aggregates reports into periodic summaries
+  digest: {
+    enabled: true,
+    cron: "0 9 * * 1", // 9:00 AM UTC, every Monday
+    lookbackDays: 7,
+    slackWebhookUrl: env("SLACK_WEBHOOK_URL", ""),
+  },
+})

package/config/sinks.ts

@@ -0,0 +1,37 @@
+/**
+ * sinks.ts — Report delivery sink configuration.
+ *
+ * Sinks receive published evaluation reports and deliver them to external
+ * systems (BigQuery, Slack, GitHub, webhooks, etc.).
+ *
+ * Sinks are fire-and-forget: a sink failure is logged but never blocks
+ * the pipeline. The Sanity Content Lake is the system of record.
+ *
+ * Sinks activate only when their required environment variables are present.
+ * A developer running locally with no env vars gets zero sinks.
+ *
+ * @see docs/design-docs/report-store/sink-architecture.md
+ */
+
+import { defineSinks } from "@sanity/ailf-core"
+
+export default defineSinks({
+  sinks: [
+    // All sinks are currently disabled (commented out in YAML).
+    // Uncomment and configure as needed:
+    // BigQuery — disabled; Airbyte ELT is the primary delivery mechanism
+    // { type: "bigquery", enabled: false, project: env("BIGQUERY_PROJECT", "data-platform-302218"),
+    //   dataset: env("BIGQUERY_DATASET", "ailf"), credentials: env("GOOGLE_APPLICATION_CREDENTIALS", "") },
+    // Slack — regression alerts to configured channels
+    // { type: "slack", enabled: true, webhookUrl: env("SLACK_WEBHOOK_URL", ""),
+    //   channel: "#docs-ai-literacy",
+    //   routing: { critical: "#docs-alerts", warning: "#docs-team",
+    //              regression: "#docs-team", digest: "#docs-weekly" } },
+    // GitHub PR comments
+    // { type: "github-comment", enabled: false, token: env("GITHUB_TOKEN", "") },
+    // Webhook — generic HTTP relay
+    // { type: "webhook", enabled: false, url: env("AILF_WEBHOOK_URL", ""),
+    //   headers: { Authorization: `Bearer ${env("AILF_WEBHOOK_TOKEN", "")}` },
+    //   routing: { critical: true } },
+  ],
+})

package/config/sources.ts

@@ -0,0 +1,21 @@
+/**
+ * sources.ts — Documentation source definitions for AILF evaluations.
+ *
+ * Default sources (production, branch, local) are provided by the
+ * sanity-literacy preset registered in the composition root. This file
+ * exists as an override point — any sources defined here are merged with
+ * (and take precedence over) preset-provided sources.
+ *
+ * To add a custom source, define it here:
+ *
+ *   export default defineSources([
+ *     { name: "my-docs", baseUrl: "https://docs.example.com", ... },
+ *   ])
+ *
+ * @see packages/eval/src/pipeline/compiler/presets/sanity-literacy.ts
+ * @see docs/exec-plans/dynamic-doc-sources.md
+ */
+
+import { defineSources } from "@sanity/ailf-core"
+
+export default defineSources([])

package/config/thresholds.ts

@@ -0,0 +1,61 @@
+/**
+ * thresholds.ts — Quality thresholds for readiness gates and regression alerts.
+ *
+ * Used by:
+ * - `npx @sanity/ailf pipeline --readiness` (launch readiness checklist)
+ * - `npx @sanity/ailf pipeline --publish` (severity-aware sink routing)
+ * - `npx @sanity/ailf pipeline --compare` (regression alerting)
+ *
+ * @see docs/exec-plans/scenario-matrix-implementation/phase-5-readiness-thresholds.md
+ */
+
+import { defineThresholds } from "@sanity/ailf-core"
+
+export default defineThresholds({
+  // Global defaults (apply to all areas unless overridden)
+  defaults: {
+    composite: 50,
+    dimensions: {
+      "task-completion": 40,
+      "code-correctness": 30,
+      "doc-coverage": 30,
+    },
+    "doc-lift": 0, // docs must not hurt
+    ceiling: 40, // minimum ceiling score (doc quality floor)
+  },
+
+  // Per-area overrides (inherit from defaults, override specific values)
+  areas: {
+    groq: {
+      composite: 60, // GROQ is critical — higher bar
+      dimensions: {
+        "task-completion": 50,
+      },
+    },
+    // "visual-editing": {
+    //   composite: 45, // currently at 36, set achievable near-term target
+    // },
+  },
+
+  // Regression thresholds (for comparison reports)
+  regression: {
+    composite: -3, // alert if composite drops more than 3 points
+    "per-area": -5, // alert if any area drops more than 5 points
+    "per-dimension": -8, // alert if any dimension drops more than 8 points
+  },
+
+  // Severity classification
+  severity: {
+    critical: {
+      "composite-below": 30,
+      "negative-doc-lift": true,
+    },
+    warning: {
+      "composite-below": 50,
+      "regression-exceeds": -3,
+    },
+    info: {
+      "composite-below": 60,
+    },
+  },
+})