@sanity/ailf 1.0.0 → 2.0.1

package/canonical/reference-solutions/portable-text/custom-blocks.ts

@@ -0,0 +1,204 @@
+/**
+ * Reference Solution: Custom Block Types in Portable Text
+ *
+ * Demonstrates:
+ * - Defining custom block types with defineArrayMember
+ * - Adding "code" and "callout" blocks to a PT field
+ * - Rendering custom blocks with @portabletext/react
+ * - TypeScript types for custom block shapes
+ */
+
+// === Part 1: Schema Definition (schemas/post.ts) ===
+
+import {
+  defineType,
+  defineField,
+  defineArrayMember,
+} from "sanity"
+
+export const postType = defineType({
+  name: "post",
+  title: "Post",
+  type: "document",
+  fields: [
+    defineField({
+      name: "title",
+      title: "Title",
+      type: "string",
+    }),
+    defineField({
+      name: "body",
+      title: "Body",
+      type: "array",
+      of: [
+        // Standard block (paragraphs, headings, lists)
+        defineArrayMember({
+          type: "block",
+          styles: [
+            { title: "Normal", value: "normal" },
+            { title: "H2", value: "h2" },
+            { title: "H3", value: "h3" },
+            { title: "Quote", value: "blockquote" },
+          ],
+          marks: {
+            decorators: [
+              { title: "Bold", value: "strong" },
+              { title: "Italic", value: "em" },
+              { title: "Code", value: "code" },
+            ],
+          },
+        }),
+
+        // Custom: Code block with language selection
+        defineArrayMember({
+          name: "code",
+          title: "Code Block",
+          type: "object",
+          fields: [
+            defineField({
+              name: "language",
+              title: "Language",
+              type: "string",
+              options: {
+                list: [
+                  { title: "JavaScript", value: "javascript" },
+                  { title: "TypeScript", value: "typescript" },
+                  { title: "HTML", value: "html" },
+                  { title: "CSS", value: "css" },
+                  { title: "Shell", value: "bash" },
+                  { title: "JSON", value: "json" },
+                ],
+              },
+            }),
+            defineField({
+              name: "code",
+              title: "Code",
+              type: "text",
+              rows: 10,
+            }),
+            defineField({
+              name: "filename",
+              title: "Filename",
+              type: "string",
+            }),
+          ],
+          preview: {
+            select: { language: "language", code: "code" },
+            prepare({ language, code }) {
+              return {
+                title: `Code: ${language || "plain"}`,
+                subtitle: code ? code.slice(0, 50) + "…" : "",
+              }
+            },
+          },
+        }),
+
+        // Custom: Callout block with tone
+        defineArrayMember({
+          name: "callout",
+          title: "Callout",
+          type: "object",
+          fields: [
+            defineField({
+              name: "tone",
+              title: "Tone",
+              type: "string",
+              options: {
+                list: [
+                  { title: "Info", value: "info" },
+                  { title: "Warning", value: "warning" },
+                  { title: "Error", value: "error" },
+                  { title: "Tip", value: "tip" },
+                ],
+                layout: "radio",
+              },
+              initialValue: "info",
+            }),
+            defineField({
+              name: "text",
+              title: "Text",
+              type: "text",
+              rows: 3,
+            }),
+          ],
+          preview: {
+            select: { tone: "tone", text: "text" },
+            prepare({ tone, text }) {
+              const icons = { info: "ℹ️", warning: "⚠️", error: "🚨", tip: "💡" }
+              return {
+                title: `${icons[tone as keyof typeof icons] || ""} ${tone || "info"} callout`,
+                subtitle: text,
+              }
+            },
+          },
+        }),
+      ],
+    }),
+  ],
+})
+
+// === Part 2: Frontend Rendering (components/PortableTextBody.tsx) ===
+
+// import { PortableText, type PortableTextComponents } from "@portabletext/react"
+
+interface CodeBlockValue {
+  _type: "code"
+  language?: string
+  code: string
+  filename?: string
+}
+
+interface CalloutBlockValue {
+  _type: "callout"
+  tone: "info" | "warning" | "error" | "tip"
+  text: string
+}
+
+function CodeBlock({ value }: { value: CodeBlockValue }) {
+  return (
+    <figure>
+      {value.filename && (
+        <figcaption>{value.filename}</figcaption>
+      )}
+      <pre data-language={value.language}>
+        <code>{value.code}</code>
+      </pre>
+    </figure>
+  )
+}
+
+const toneStyles: Record<string, { background: string; border: string }> = {
+  info: { background: "#e8f4fd", border: "#2196f3" },
+  warning: { background: "#fff8e1", border: "#ff9800" },
+  error: { background: "#fde8e8", border: "#f44336" },
+  tip: { background: "#e8f5e9", border: "#4caf50" },
+}
+
+function CalloutBlock({ value }: { value: CalloutBlockValue }) {
+  const style = toneStyles[value.tone] || toneStyles.info
+
+  return (
+    <aside
+      role="note"
+      style={{
+        padding: "1rem",
+        borderLeft: `4px solid ${style.border}`,
+        background: style.background,
+        margin: "1.5rem 0",
+      }}
+    >
+      <p>{value.text}</p>
+    </aside>
+  )
+}
+
+// Register custom types in the PortableText component map
+export const components = {
+  types: {
+    code: CodeBlock,
+    callout: CalloutBlock,
+  },
+}
+
+// Usage:
+// <PortableText value={post.body} components={components} />

package/canonical/reference-solutions/portable-text/rendering.tsx

@@ -0,0 +1,163 @@
+/**
+ * Reference Solution: Portable Text Rendering in React
+ *
+ * Demonstrates:
+ * - Setting up @portabletext/react with custom components
+ * - Handling image blocks with @sanity/image-url
+ * - Custom marks for links (internal + external)
+ * - Code block rendering with language metadata
+ * - TypeScript types for component props
+ */
+
+import { PortableText, type PortableTextComponents } from "@portabletext/react"
+import imageUrlBuilder from "@sanity/image-url"
+import { createClient } from "@sanity/client"
+
+// === Sanity Client & Image Builder ===
+
+const client = createClient({
+  projectId: "your-project-id",
+  dataset: "production",
+  apiVersion: "2024-01-01",
+  useCdn: true,
+})
+
+const builder = imageUrlBuilder(client)
+
+function urlFor(source: SanityImageSource) {
+  return builder.image(source)
+}
+
+// === Types ===
+
+interface SanityImageSource {
+  _type: "image"
+  asset: {
+    _ref: string
+    _type: "reference"
+  }
+  hotspot?: { x: number; y: number; height: number; width: number }
+  crop?: { top: number; bottom: number; left: number; right: number }
+}
+
+interface ImageBlockValue {
+  _type: "image"
+  asset: { _ref: string; _type: "reference" }
+  hotspot?: SanityImageSource["hotspot"]
+  crop?: SanityImageSource["crop"]
+  alt?: string
+  caption?: string
+}
+
+interface CodeBlockValue {
+  _type: "code"
+  language?: string
+  code: string
+}
+
+interface LinkMarkValue {
+  _type: "link"
+  href: string
+  blank?: boolean
+}
+
+interface InternalLinkMarkValue {
+  _type: "internalLink"
+  reference: { _ref: string; _type: "reference" }
+  slug?: { current: string }
+}
+
+interface PortableTextBodyProps {
+  value: unknown[]
+  className?: string
+}
+
+// === Custom Components ===
+
+// Image block — renders Sanity images with hotspot/crop via the URL builder
+function ImageBlock({ value }: { value: ImageBlockValue }) {
+  if (!value?.asset) return null
+
+  const url = urlFor(value)
+    .width(800)
+    .auto("format")
+    .url()
+
+  return (
+    <figure>
+      <img
+        src={url}
+        alt={value.alt || ""}
+        loading="lazy"
+        style={{ maxWidth: "100%", height: "auto" }}
+      />
+      {value.caption && <figcaption>{value.caption}</figcaption>}
+    </figure>
+  )
+}
+
+// Code block — renders preformatted code with language annotation
+function CodeBlock({ value }: { value: CodeBlockValue }) {
+  return (
+    <pre data-language={value.language}>
+      <code>{value.code}</code>
+    </pre>
+  )
+}
+
+// External link mark — opens in new tab when blank is true
+function LinkMark({
+  value,
+  children,
+}: {
+  value?: LinkMarkValue
+  children: React.ReactNode
+}) {
+  const target = value?.blank ? "_blank" : undefined
+  const rel = value?.blank ? "noopener noreferrer" : undefined
+
+  return (
+    <a href={value?.href} target={target} rel={rel}>
+      {children}
+    </a>
+  )
+}
+
+// Internal link mark — resolves to a local route
+function InternalLinkMark({
+  value,
+  children,
+}: {
+  value?: InternalLinkMarkValue
+  children: React.ReactNode
+}) {
+  const slug = value?.slug?.current
+  const href = slug ? `/${slug}` : "#"
+
+  return <a href={href}>{children}</a>
+}
+
+// === Component Map ===
+
+const components: PortableTextComponents = {
+  types: {
+    image: ImageBlock,
+    code: CodeBlock,
+  },
+  marks: {
+    link: LinkMark,
+    internalLink: InternalLinkMark,
+  },
+}
+
+// === Main Component ===
+
+export function PortableTextBody({ value, className }: PortableTextBodyProps) {
+  if (!value) return null
+
+  return (
+    <div className={className}>
+      <PortableText value={value} components={components} />
+    </div>
+  )
+}

package/config/features.ts

@@ -15,7 +15,7 @@
  *   })
  *
  * @see packages/eval/src/pipeline/compiler/presets/sanity-literacy.ts
- * @see docs/exec-plans/scenario-matrix-implementation/phase-3-gap-analysis.md
+ * @see docs/archive/exec-plans/scenario-matrix-implementation/phase-3-gap-analysis.md
  */
 
 import { defineFeatures } from "@sanity/ailf-core"

package/config/models.ts

@@ -1,10 +1,14 @@
 /**
  * models.ts — Central model registry for AILF evaluations.
  *
- * Define all models to test here. Each eval mode (baseline, observed, agentic)
- * reads this config and generates the appropriate provider entries.
+ * Define all models to test here. Each model declares which evaluation
+ * modes it participates in (e.g., "literacy", "mcp-server") and
+ * optionally which variants within those modes.
  *
- * @see docs/exec-plans/architecture-overhaul/phase-1-ts-config-loading.md
+ * When a model enrolls in a mode without specifying variants, all
+ * variants defined by the mode base are included by default.
+ *
+ * @see docs/archive/exec-plans/architecture-overhaul/phase-1-ts-config-loading.md
  */
 
 import { defineModels } from "@sanity/ailf-core"
@@ -16,7 +20,9 @@ export default defineModels({
       id: "anthropic:messages:claude-opus-4-6",
       label: "Claude Opus 4.6",
       config: { temperature: 0.2, max_tokens: 4096 },
-      modes: ["baseline", "observed", "agentic-naive", "agentic-optimized"],
+      modes: ["literacy", "mcp-server", "knowledge-probe"],
+      // All literacy variants included by default (baseline, observed,
+      // agentic-naive, agentic-optimized)
     },
 
     // ── Google ─────────────────────────────────────────────────
@@ -24,7 +30,7 @@ export default defineModels({
     //   id: "google:gemini-2.5-pro",
     //   label: "Gemini 2.5 Pro",
     //   config: { temperature: 0.2, max_tokens: 4096 },
-    //   modes: ["baseline", "observed", "agentic-naive", "agentic-optimized"],
+    //   modes: ["literacy"],
     // },
 
     // ── OpenAI ─────────────────────────────────────────────────
@@ -32,29 +38,39 @@ export default defineModels({
       id: "openai:chat:gpt-5.2",
       label: "GPT 5.2",
       config: { temperature: 0.2, max_tokens: 4096 },
-      modes: ["baseline", "observed", "agentic-naive", "agentic-optimized"],
+      modes: ["literacy", "knowledge-probe"],
+      // All literacy variants included by default
     },
     {
-      id: "openai:chat:gpt-5.4",
+      id: "openai:responses:gpt-5.4",
       label: "GPT 5.4",
       config: {
         reasoning_effort: "medium",
         max_output_tokens: 4096,
         maxRetries: 1,
       },
-      modes: ["baseline", "observed", "agentic-naive", "agentic-optimized"],
+      timeoutMs: 600_000, // 10 min — reasoning model needs more headroom
+      modes: ["literacy", "mcp-server", "knowledge-probe"],
+      // All literacy variants included by default
     },
 
     // ── Disabled models (uncomment to enable) ──────────────────
     // { id: "anthropic:claude-sonnet-4-20250514", label: "Claude Sonnet 4",
-    //   config: { temperature: 0.2, max_tokens: 4096 }, modes: ["baseline"] },
+    //   config: { temperature: 0.2, max_tokens: 4096 },
+    //   modes: ["literacy"],
+    //   variants: { literacy: ["baseline"] } },
     // { id: "anthropic:claude-3.5-sonnet-20241022", label: "Claude 3.5 Sonnet",
     //   config: { temperature: 0.2, max_tokens: 4096 },
-    //   modes: ["baseline", "agentic-naive", "agentic-optimized"] },
+    //   modes: ["literacy"],
+    //   variants: { literacy: ["baseline", "agentic-naive", "agentic-optimized"] } },
     // { id: "google:gemini-2.0-flash", label: "Gemini 2.0 Flash",
-    //   config: { temperature: 0.2, max_tokens: 4096 }, modes: ["baseline"] },
+    //   config: { temperature: 0.2, max_tokens: 4096 },
+    //   modes: ["literacy"],
+    //   variants: { literacy: ["baseline"] } },
     // { id: "openrouter:deepseek/deepseek-r1", label: "DeepSeek R1",
-    //   config: { temperature: 0.2, max_tokens: 4096 }, modes: ["baseline"] },
+    //   config: { temperature: 0.2, max_tokens: 4096 },
+    //   modes: ["literacy"],
+    //   variants: { literacy: ["baseline"] } },
   ],
 
   // ── Grading Model ──────────────────────────────────────────
@@ -65,6 +81,7 @@ export default defineModels({
   },
 
   // ── Evaluation Options ─────────────────────────────────────
+  evalBudgetMs: 1_200_000, // 20 min per eval mode — outer kill switch
   maxConcurrency: 32, // max parallel API calls — benchmarked in DOC-1896
 
   // ── Default Config ─────────────────────────────────────────

package/config/sources.ts

@@ -13,7 +13,7 @@
  *   ])
  *
  * @see packages/eval/src/pipeline/compiler/presets/sanity-literacy.ts
- * @see docs/exec-plans/dynamic-doc-sources.md
+ * @see docs/archive/exec-plans/dynamic-doc-sources.md
  */
 
 import { defineSources } from "@sanity/ailf-core"

package/config/thresholds.ts

@@ -6,7 +6,7 @@
  * - `npx @sanity/ailf pipeline --publish` (severity-aware sink routing)
  * - `npx @sanity/ailf pipeline --compare` (regression alerting)
  *
- * @see docs/exec-plans/scenario-matrix-implementation/phase-5-readiness-thresholds.md
+ * @see docs/archive/exec-plans/scenario-matrix-implementation/phase-5-readiness-thresholds.md
  */
 
 import { defineThresholds } from "@sanity/ailf-core"

package/dist/_vendor/ailf-core/__tests__/comparison-formatters.test.d.ts

@@ -0,0 +1,10 @@
+/**
+ * comparison-formatters.test.ts
+ *
+ * Verifies that formatComparisonMarkdown() and formatComparisonTable()
+ * dynamically derive column headers from the dimension keys present
+ * in the report data, rather than hardcoding literacy-specific names.
+ *
+ * Run: npx tsx --test src/__tests__/comparison-formatters.test.ts
+ */
+export {};

package/dist/_vendor/ailf-core/__tests__/comparison-formatters.test.js

@@ -0,0 +1,185 @@
+/**
+ * comparison-formatters.test.ts
+ *
+ * Verifies that formatComparisonMarkdown() and formatComparisonTable()
+ * dynamically derive column headers from the dimension keys present
+ * in the report data, rather than hardcoding literacy-specific names.
+ *
+ * Run: npx tsx --test src/__tests__/comparison-formatters.test.ts
+ */
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import { formatComparisonMarkdown, formatComparisonTable, } from "../services/comparison-formatters.js";
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+/** Minimal ScoreSummary stub — only fields the formatters actually read */
+function stubSummary(avgScore) {
+    return {
+        belowCritical: [],
+        lowestArea: "area-a",
+        lowestScore: 40,
+        overall: {
+            avgCeilingScore: 80,
+            avgScore,
+            avgDocLift: 10,
+            avgDocQualityGap: 20,
+            avgFloorScore: 30,
+            negativeDocLiftCount: 0,
+        },
+        scores: [],
+        timestamp: "2026-04-05T00:00:00.000Z",
+    };
+}
+function makeReport(overrides) {
+    return {
+        areas: [
+            {
+                area: "area-a",
+                baseline: 60,
+                experiment: 65,
+                delta: 5,
+                change: "improved",
+                dimensions: overrides.areaDimensions,
+                ceilingDelta: 0,
+                docLiftDelta: 2,
+                floorDelta: 0,
+            },
+        ],
+        baseline: stubSummary(60),
+        experiment: stubSummary(65),
+        deltas: {
+            overall: 5,
+            perArea: { "area-a": 5 },
+            perDimension: overrides.perDimension,
+            docLift: 2,
+        },
+        generatedAt: "2026-04-05T00:00:00.000Z",
+        improved: ["area-a"],
+        regressed: [],
+        unchanged: [],
+        notEvaluated: [],
+        mismatched: { onlyInBaseline: [], onlyInExperiment: [] },
+        noiseThreshold: 2,
+        noiseThresholdEmpirical: false,
+    };
+}
+// ---------------------------------------------------------------------------
+// Tests — literacy dimensions (backward compatibility)
+// ---------------------------------------------------------------------------
+describe("formatComparisonMarkdown", () => {
+    it("renders literacy dimension columns dynamically", () => {
+        const report = makeReport({
+            areaDimensions: {
+                "task-completion": { baseline: 60, experiment: 65, delta: 5 },
+                "code-correctness": { baseline: 50, experiment: 55, delta: 5 },
+                "doc-coverage": { baseline: 40, experiment: 42, delta: 2 },
+            },
+            perDimension: {
+                "task-completion": 5,
+                "code-correctness": 5,
+                "doc-coverage": 2,
+            },
+        });
+        const md = formatComparisonMarkdown(report);
+        // Column headers should be title-cased from kebab-case
+        assert.ok(md.includes("Task Completion"), "should have Task Completion column header");
+        assert.ok(md.includes("Code Correctness"), "should have Code Correctness column header");
+        assert.ok(md.includes("Doc Coverage"), "should have Doc Coverage column header");
+        // Per-dimension averages section should also show dynamic labels
+        assert.ok(md.includes("| Task Completion |"), "dimension averages should include Task Completion");
+        assert.ok(md.includes("| Code Correctness |"), "dimension averages should include Code Correctness");
+        assert.ok(md.includes("| Doc Coverage |"), "dimension averages should include Doc Coverage");
+    });
+    it("renders MCP dimension columns dynamically", () => {
+        const report = makeReport({
+            areaDimensions: {
+                "input-validation": { baseline: 50, experiment: 60, delta: 10 },
+                "output-correctness": { baseline: 70, experiment: 75, delta: 5 },
+                "error-handling": { baseline: 40, experiment: 45, delta: 5 },
+                security: { baseline: 80, experiment: 82, delta: 2 },
+            },
+            perDimension: {
+                "input-validation": 10,
+                "output-correctness": 5,
+                "error-handling": 5,
+                security: 2,
+            },
+        });
+        const md = formatComparisonMarkdown(report);
+        // 4 MCP columns instead of 3 literacy columns
+        assert.ok(md.includes("Input Validation"), "should have Input Validation column");
+        assert.ok(md.includes("Output Correctness"), "should have Output Correctness column");
+        assert.ok(md.includes("Error Handling"), "should have Error Handling column");
+        assert.ok(md.includes("Security"), "should have Security column");
+        // Per-dimension averages
+        assert.ok(md.includes("| Input Validation |"), "dimension averages should include Input Validation");
+        assert.ok(md.includes("| Security |"), "dimension averages should include Security");
+    });
+});
+describe("formatComparisonTable", () => {
+    it("renders literacy dimension columns dynamically", () => {
+        const report = makeReport({
+            areaDimensions: {
+                "task-completion": { baseline: 60, experiment: 65, delta: 5 },
+                "code-correctness": { baseline: 50, experiment: 55, delta: 5 },
+                "doc-coverage": { baseline: 40, experiment: 42, delta: 2 },
+            },
+            perDimension: {
+                "task-completion": 5,
+                "code-correctness": 5,
+                "doc-coverage": 2,
+            },
+        });
+        const table = formatComparisonTable(report);
+        // Dimension averages section
+        assert.ok(table.includes("Task Completion:"), "should show Task Completion in dimension averages");
+        assert.ok(table.includes("Code Correctness:"), "should show Code Correctness in dimension averages");
+        assert.ok(table.includes("Doc Coverage:"), "should show Doc Coverage in dimension averages");
+        // Per-area table header
+        assert.ok(table.includes("Task Completion"), "per-area table should have Task Completion header");
+        assert.ok(table.includes("Code Correctness"), "per-area table should have Code Correctness header");
+        assert.ok(table.includes("Doc Coverage"), "per-area table should have Doc Coverage header");
+    });
+    it("renders MCP dimension columns dynamically", () => {
+        const report = makeReport({
+            areaDimensions: {
+                "input-validation": { baseline: 50, experiment: 60, delta: 10 },
+                "output-correctness": { baseline: 70, experiment: 75, delta: 5 },
+                "error-handling": { baseline: 40, experiment: 45, delta: 5 },
+                security: { baseline: 80, experiment: 82, delta: 2 },
+            },
+            perDimension: {
+                "input-validation": 10,
+                "output-correctness": 5,
+                "error-handling": 5,
+                security: 2,
+            },
+        });
+        const table = formatComparisonTable(report);
+        // 4 MCP columns in the per-area table
+        assert.ok(table.includes("Input Validation"), "should have Input Validation");
+        assert.ok(table.includes("Output Correctness"), "should have Output Correctness");
+        assert.ok(table.includes("Error Handling"), "should have Error Handling");
+        assert.ok(table.includes("Security"), "should have Security");
+        // Should NOT have literacy dimension headers
+        assert.ok(!table.includes("Task Completion"), "should not contain Task Completion");
+        assert.ok(!table.includes("Doc Coverage"), "should not contain Doc Coverage");
+    });
+    it("includes delta values for each dimension in the per-area rows", () => {
+        const report = makeReport({
+            areaDimensions: {
+                "input-validation": { baseline: 50, experiment: 60, delta: 10 },
+                "output-correctness": { baseline: 70, experiment: 75, delta: 5 },
+            },
+            perDimension: {
+                "input-validation": 10,
+                "output-correctness": 5,
+            },
+        });
+        const table = formatComparisonTable(report);
+        // The per-area row should include the delta values (+10 and +5)
+        assert.ok(table.includes("+10"), "should show +10 delta for area-a");
+        assert.ok(table.includes("+5"), "should show +5 delta for area-a");
+    });
+});