@sanity/ailf 0.1.0

package/dist/pipeline/probe.js

@@ -0,0 +1,390 @@
+/**
+ * pipeline/probe.ts
+ *
+ * Tier B probes for new features without evaluation tasks.
+ *
+ * Phase 4b of the Scenario Matrix implementation.
+ *
+ * When a content release adds documents for a genuinely new feature (no
+ * evaluation tasks exist), a probe provides a directional "usability"
+ * signal by analyzing the document content and, optionally, evaluating
+ * a generic implementation prompt.
+ *
+ * Probes are NOT scored evaluations — they answer "are these docs usable?"
+ * not "are these docs good enough?" The output is always labeled as
+ * directional and never displayed on the same scale as scored evaluations.
+ *
+ * @see docs/exec-plans/completed/scenario-matrix-implementation/phase-4-content-release-integration.md
+ */
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+/** Generic probe prompt template */
+export const PROBE_PROMPT = `Given the following documentation about a Sanity feature, write a TypeScript
+implementation that demonstrates the feature's core functionality.
+
+Use only APIs and patterns described in the documentation. If the documentation
+is insufficient to implement something, say so explicitly rather than guessing.
+
+Documentation:
+{{docs}}`;
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+/**
+ * Analyze a model's probe output against the source documentation.
+ *
+ * Extracts API names from both the documentation and the model output,
+ * identifies hallucinations (APIs used but not documented), and classifies
+ * the overall usability.
+ *
+ * This is a pure function — it doesn't call any LLM. The model output
+ * is provided as input (from a prior evaluation step or manual run).
+ *
+ * @param documentContent - The concatenated documentation text
+ * @param modelOutput - The model's response to the probe prompt
+ * @param documentSlugs - The slugs of the documents that were probed
+ * @returns Probe result with usability classification
+ */
+export function analyzeProbeOutput(documentContent, modelOutput, documentSlugs) {
+    const docApis = extractApiNames(documentContent);
+    const outputApis = extractApiNames(modelOutput);
+    // APIs in the output that aren't in the docs = hallucinations
+    const docApiSet = new Set(docApis.map((a) => a.toLowerCase()));
+    const hallucinatedApis = outputApis.filter((api) => !docApiSet.has(api.toLowerCase()));
+    // APIs in both docs and output
+    const apiNamesExtracted = outputApis.filter((api) => docApiSet.has(api.toLowerCase()));
+    const producedCode = hasCodeOutput(modelOutput);
+    const suggestedTasks = generateTaskSuggestions(documentContent, documentSlugs);
+    const usability = classifyUsability(producedCode, apiNamesExtracted.length, hallucinatedApis.length, docApis.length);
+    return {
+        apiNamesExtracted,
+        documentSlugs,
+        hallucinatedApis,
+        producedCode,
+        rawOutput: modelOutput,
+        suggestedTasks,
+        usability,
+    };
+}
+/**
+ * Format a probe result for console output.
+ */
+export function formatProbeConsole(probe) {
+    const lines = [];
+    lines.push("🔍 TIER B PROBE RESULT");
+    lines.push("");
+    lines.push(`  Documents: ${probe.documentSlugs.join(", ")}`);
+    lines.push(`  Usability: ${usabilityIcon(probe.usability)} ${probe.usability}`);
+    lines.push(`  Produced code: ${probe.producedCode ? "yes" : "no"}`);
+    lines.push("");
+    if (probe.apiNamesExtracted.length > 0) {
+        lines.push("  APIs correctly extracted from docs:");
+        for (const api of probe.apiNamesExtracted) {
+            lines.push(`    ✅ ${api}`);
+        }
+        lines.push("");
+    }
+    if (probe.hallucinatedApis.length > 0) {
+        lines.push("  Hallucinated APIs (not in docs):");
+        for (const api of probe.hallucinatedApis) {
+            lines.push(`    ❌ ${api}`);
+        }
+        lines.push("");
+    }
+    if (probe.suggestedTasks.length > 0) {
+        lines.push("  Suggested evaluation tasks:");
+        for (let i = 0; i < probe.suggestedTasks.length; i++) {
+            lines.push(`    ${i + 1}. ${probe.suggestedTasks[i]}`);
+        }
+        lines.push("");
+    }
+    return lines.join("\n");
+}
+// ---------------------------------------------------------------------------
+// Formatting
+// ---------------------------------------------------------------------------
+/**
+ * Format a probe result as markdown.
+ */
+export function formatProbeMarkdown(probe) {
+    const lines = [];
+    const featureName = inferFeatureName(probe.documentSlugs);
+    lines.push(`#### 🔍 New Feature Probe: ${featureName}`);
+    lines.push("");
+    lines.push("**Status:** No evaluation tasks exist for this feature. " +
+        "**Assessment:** Directional only — not a scored evaluation.");
+    lines.push("");
+    lines.push("**Documents Analyzed:**");
+    for (const slug of probe.documentSlugs) {
+        lines.push(`- \`${slug}\` (new)`);
+    }
+    lines.push("");
+    lines.push(`**Usability:** ${usabilityIcon(probe.usability)} ${probe.usability}`);
+    lines.push("");
+    if (probe.apiNamesExtracted.length > 0 || probe.hallucinatedApis.length > 0) {
+        lines.push("**Probe Findings:**");
+        for (const api of probe.apiNamesExtracted) {
+            lines.push(`- ✅ Correctly used \`${api}\``);
+        }
+        for (const api of probe.hallucinatedApis) {
+            lines.push(`- ❌ Hallucinated \`${api}\` (not in docs)`);
+        }
+        lines.push("");
+    }
+    if (probe.suggestedTasks.length > 0) {
+        lines.push("<details>", "<summary>Suggested evaluation tasks</summary>", "");
+        for (let i = 0; i < probe.suggestedTasks.length; i++) {
+            lines.push(`${i + 1}. ${probe.suggestedTasks[i]}`);
+        }
+        lines.push("", "</details>", "");
+    }
+    return lines.join("\n");
+}
+/**
+ * Generate task scaffolding suggestions from document content analysis.
+ *
+ * Extracts function names, configuration patterns, and common operations
+ * from code blocks in the documentation to suggest evaluation tasks.
+ */
+export function generateTaskSuggestions(documentContent, documentSlugs) {
+    const suggestions = [];
+    const featureName = inferFeatureName(documentSlugs);
+    // Extract patterns from code blocks
+    const codeBlocks = extractCodeBlocks(documentContent);
+    const functionNames = extractFunctionNamesFromCode(codeBlocks);
+    const configPatterns = detectConfigPatterns(documentContent);
+    const crudOperations = detectCrudOperations(documentContent);
+    // Generate suggestions based on detected patterns
+    if (configPatterns.length > 0) {
+        suggestions.push(`Configure ${featureName} in a Sanity Studio project (${configPatterns.join(", ")})`);
+    }
+    if (functionNames.length > 0) {
+        const topFns = functionNames.slice(0, 3).join(", ");
+        suggestions.push(`Implement core ${featureName} functionality using ${topFns}`);
+    }
+    if (crudOperations.length > 0) {
+        suggestions.push(`Perform ${crudOperations.join("/")} operations with ${featureName}`);
+    }
+    // Always suggest an integration task
+    if (suggestions.length > 0) {
+        suggestions.push(`Integrate ${featureName} with an existing Next.js application`);
+    }
+    return suggestions;
+}
+// ---------------------------------------------------------------------------
+// Internal helpers
+// ---------------------------------------------------------------------------
+/** Classify overall usability based on probe metrics */
+function classifyUsability(producedCode, correctApis, hallucinatedApis, totalDocApis) {
+    if (!producedCode)
+        return "not-usable";
+    const hallucinationRate = correctApis + hallucinatedApis > 0
+        ? hallucinatedApis / (correctApis + hallucinatedApis)
+        : 0;
+    const coverageRate = totalDocApis > 0 ? correctApis / totalDocApis : 0;
+    // Usable: produced code, low hallucination, reasonable API coverage
+    if (hallucinationRate < 0.2 && coverageRate > 0.5)
+        return "usable";
+    // Not usable: high hallucination or very low coverage
+    if (hallucinationRate > 0.5 || (coverageRate < 0.2 && totalDocApis > 2)) {
+        return "not-usable";
+    }
+    return "partially-usable";
+}
+/** Detect configuration patterns in documentation */
+function detectConfigPatterns(text) {
+    const patterns = [];
+    const lower = text.toLowerCase();
+    if (/sanity\.config|defineconfig/i.test(lower))
+        patterns.push("sanity.config");
+    if (/plugin|definePlugin/i.test(lower))
+        patterns.push("plugin setup");
+    if (/env|environment|api.?key/i.test(lower))
+        patterns.push("environment config");
+    return patterns;
+}
+/** Detect CRUD operation patterns in text */
+function detectCrudOperations(text) {
+    const ops = [];
+    const lower = text.toLowerCase();
+    if (/\bcreate\b/.test(lower))
+        ops.push("create");
+    if (/\bread\b|\bfetch\b|\bquery\b|\bget\b/.test(lower))
+        ops.push("read");
+    if (/\bupdate\b|\bpatch\b|\bmutate\b/.test(lower))
+        ops.push("update");
+    if (/\bdelete\b|\bremove\b/.test(lower))
+        ops.push("delete");
+    return ops;
+}
+/** Extract API-like names from text (function calls, imports, etc.) */
+function extractApiNames(text) {
+    const apis = new Set();
+    // Match function calls: functionName(
+    const fnCalls = text.match(/\b([a-zA-Z_]\w{2,})\s*\(/g);
+    if (fnCalls) {
+        for (const call of fnCalls) {
+            const name = call.replace(/\s*\($/, "");
+            // Filter out common JS keywords and control flow
+            if (!isCommonKeyword(name)) {
+                apis.add(name);
+            }
+        }
+    }
+    // Match import names: import { name } from
+    const imports = text.match(/import\s*\{([^}]+)\}/g);
+    if (imports) {
+        for (const imp of imports) {
+            const names = imp
+                .replace(/import\s*\{/, "")
+                .replace(/\}/, "")
+                .split(",")
+                .map((n) => n.trim())
+                .filter((n) => n.length > 0 && !n.includes(" as "));
+            for (const name of names) {
+                apis.add(name);
+            }
+        }
+    }
+    return [...apis].sort();
+}
+/** Extract code blocks from markdown content */
+function extractCodeBlocks(text) {
+    const blocks = [];
+    const regex = /```(?:typescript|ts|javascript|js|tsx|jsx)?\n([\s\S]*?)```/g;
+    let match;
+    while ((match = regex.exec(text)) !== null) {
+        blocks.push(match[1]);
+    }
+    return blocks;
+}
+/** Extract function/method names from code blocks */
+function extractFunctionNamesFromCode(codeBlocks) {
+    const names = new Set();
+    for (const block of codeBlocks) {
+        // Function declarations
+        const fnDecls = block.match(/(?:function|const|let|var)\s+(\w+)/g);
+        if (fnDecls) {
+            for (const decl of fnDecls) {
+                const name = decl.replace(/^(?:function|const|let|var)\s+/, "");
+                if (name.length > 2 && !isCommonKeyword(name)) {
+                    names.add(name);
+                }
+            }
+        }
+        // Method calls: obj.method(
+        const methods = block.match(/\.([a-zA-Z_]\w+)\s*\(/g);
+        if (methods) {
+            for (const method of methods) {
+                const name = method.replace(/^\./, "").replace(/\s*\($/, "");
+                if (name.length > 2) {
+                    names.add(name);
+                }
+            }
+        }
+    }
+    return [...names].sort();
+}
+/** Detect whether the model output contains code */
+function hasCodeOutput(output) {
+    // Check for code fences or obvious code patterns
+    if (/```/.test(output))
+        return true;
+    if (/import\s+\{/.test(output))
+        return true;
+    if (/(?:const|let|var|function)\s+\w+/.test(output))
+        return true;
+    return false;
+}
+/** Infer a human-readable feature name from document slugs */
+function inferFeatureName(slugs) {
+    if (slugs.length === 0)
+        return "Unknown Feature";
+    // Find common prefix
+    const parts = slugs[0].split("-");
+    if (parts.length > 0) {
+        // Capitalize and join first 2 parts
+        return parts
+            .slice(0, 2)
+            .map((p) => p.charAt(0).toUpperCase() + p.slice(1))
+            .join(" ");
+    }
+    return slugs[0];
+}
+/** Check if a name is a common JavaScript keyword */
+function isCommonKeyword(name) {
+    const keywords = new Set([
+        "async",
+        "await",
+        "break",
+        "case",
+        "catch",
+        "class",
+        "const",
+        "continue",
+        "debugger",
+        "default",
+        "delete",
+        "else",
+        "export",
+        "extends",
+        "finally",
+        "for",
+        "from",
+        "function",
+        "get",
+        "if",
+        "import",
+        "in",
+        "let",
+        "new",
+        "of",
+        "return",
+        "set",
+        "switch",
+        "throw",
+        "try",
+        "typeof",
+        "var",
+        "void",
+        "while",
+        "with",
+        "yield",
+        // Common global functions
+        "console",
+        "log",
+        "error",
+        "warn",
+        "require",
+        "module",
+        "exports",
+        "describe",
+        "it",
+        "test",
+        "expect",
+        "assert",
+        "JSON",
+        "Array",
+        "Object",
+        "Map",
+        "Set",
+        "Promise",
+        "String",
+        "Number",
+        "Boolean",
+        "Error",
+    ]);
+    return keywords.has(name);
+}
+/** Get usability icon */
+function usabilityIcon(usability) {
+    switch (usability) {
+        case "not-usable":
+            return "❌";
+        case "partially-usable":
+            return "🟡";
+        case "usable":
+            return "✅";
+    }
+}

package/dist/pipeline/provenance.d.ts

@@ -0,0 +1,47 @@
+/**
+ * pipeline/provenance.ts
+ *
+ * Builds ReportProvenance from data available during a pipeline run.
+ *
+ * Provenance captures what produced an evaluation report: which models,
+ * which source, which mode, what triggered it, git metadata, etc.
+ * Most of this data already flows through the pipeline — this module
+ * just captures what would otherwise be ephemeral.
+ *
+ * @see docs/design-docs/report-store/domain-model.md
+ * @see docs/design-docs/report-store/architecture.md — Provenance collection
+ */
+import type { ResolvedSourceConfig } from "../sources.js";
+import type { EvalMode, PromptfooUrlEntry, ReportProvenance } from "./types.js";
+export interface ProvenanceInput {
+    /** Feature areas that were evaluated */
+    areas: string[];
+    /** SHA-256 hash of the doc context files (from cache system) */
+    contextHash?: string;
+    /** Evaluation fingerprint for cross-environment cache lookup */
+    evalFingerprint?: string;
+    /** Evaluation mode */
+    mode: EvalMode;
+    /** @deprecated Use `promptfooUrls` — kept for backward compatibility */
+    promptfooUrl?: string;
+    /** Per-mode Promptfoo share URLs */
+    promptfooUrls?: PromptfooUrlEntry[];
+    /** Path to the package root (for reading models.yaml) */
+    rootDir: string;
+    /** Sanity document IDs targeted */
+    sanityDocumentIds?: string[];
+    /** Resolved documentation source */
+    source: ResolvedSourceConfig;
+    /** Specific task IDs evaluated (if scoped) */
+    taskIds?: string[];
+}
+/**
+ * Build a ReportProvenance object from pipeline context.
+ *
+ * Assembles provenance from:
+ * - Pipeline options (mode, source, areas, tasks)
+ * - config/models.yaml (model list, grader)
+ * - Environment variables (CI metadata, trigger detection)
+ * - Optional metadata (context hash, Promptfoo URL)
+ */
+export declare function buildProvenance(input: ProvenanceInput): ReportProvenance;

package/dist/pipeline/provenance.js

@@ -0,0 +1,146 @@
+/**
+ * pipeline/provenance.ts
+ *
+ * Builds ReportProvenance from data available during a pipeline run.
+ *
+ * Provenance captures what produced an evaluation report: which models,
+ * which source, which mode, what triggered it, git metadata, etc.
+ * Most of this data already flows through the pipeline — this module
+ * just captures what would otherwise be ephemeral.
+ *
+ * @see docs/design-docs/report-store/domain-model.md
+ * @see docs/design-docs/report-store/architecture.md — Provenance collection
+ */
+import { readFileSync } from "fs";
+import { resolve } from "path";
+import { load } from "js-yaml";
+/**
+ * Build a ReportProvenance object from pipeline context.
+ *
+ * Assembles provenance from:
+ * - Pipeline options (mode, source, areas, tasks)
+ * - config/models.yaml (model list, grader)
+ * - Environment variables (CI metadata, trigger detection)
+ * - Optional metadata (context hash, Promptfoo URL)
+ */
+export function buildProvenance(input) {
+    const models = loadModelsConfig(input.rootDir);
+    return {
+        areas: input.areas,
+        contextHash: input.contextHash,
+        evalFingerprint: input.evalFingerprint,
+        git: detectGitMetadata(),
+        graderModel: models.grader.id,
+        mode: input.mode,
+        models: models.models.map((m) => ({ id: m.id, label: m.label })),
+        promptfooUrl: input.promptfooUrl,
+        promptfooUrls: input.promptfooUrls,
+        source: {
+            baseUrl: input.source.baseUrl,
+            dataset: input.source.dataset,
+            name: input.source.name,
+            perspective: input.source.perspective,
+            projectId: input.source.projectId,
+        },
+        targetDocuments: input.sanityDocumentIds,
+        taskIds: input.taskIds,
+        trigger: detectTrigger(),
+    };
+}
+// ---------------------------------------------------------------------------
+// Trigger detection
+// ---------------------------------------------------------------------------
+/**
+ * Extract git metadata from GitHub Actions environment variables.
+ * Returns undefined when not running in CI.
+ */
+function detectGitMetadata() {
+    const repo = process.env.GITHUB_REPOSITORY;
+    if (!repo)
+        return undefined;
+    const sha = process.env.GITHUB_SHA ?? "unknown";
+    const ref = process.env.GITHUB_REF ?? "";
+    // Extract branch name from ref (refs/heads/main → main)
+    const branch = ref.startsWith("refs/heads/")
+        ? ref.slice("refs/heads/".length)
+        : ref.startsWith("refs/pull/")
+            ? `pr-${ref.split("/")[2]}`
+            : ref;
+    // Extract PR number from GITHUB_REF (refs/pull/123/merge)
+    const prMatch = ref.match(/^refs\/pull\/(\d+)\//);
+    const prNumber = prMatch ? parseInt(prMatch[1], 10) : undefined;
+    return { branch, prNumber, repo, sha };
+}
+// ---------------------------------------------------------------------------
+// Git metadata
+// ---------------------------------------------------------------------------
+/**
+ * Infer what triggered this evaluation from environment variables.
+ *
+ * Detection order:
+ * 1. AILF_TRIGGER_TYPE — explicit override (for custom integrations)
+ * 2. GITHUB_EVENT_NAME === "schedule" — cron-triggered
+ * 3. GITHUB_EVENT_NAME === "repository_dispatch" — cross-repo trigger
+ * 4. GITHUB_ACTIONS === "true" — CI-triggered
+ * 5. Default: manual
+ */
+function detectTrigger() {
+    const explicit = process.env.AILF_TRIGGER_TYPE;
+    if (explicit === "scheduled") {
+        return {
+            schedule: process.env.AILF_SCHEDULE ?? "unknown",
+            type: "scheduled",
+        };
+    }
+    if (explicit === "webhook") {
+        return {
+            documentId: process.env.AILF_WEBHOOK_DOCUMENT_ID,
+            source: process.env.AILF_WEBHOOK_SOURCE ?? "unknown",
+            type: "webhook",
+        };
+    }
+    // GitHub Actions context
+    const eventName = process.env.GITHUB_EVENT_NAME;
+    if (eventName === "schedule") {
+        return {
+            schedule: process.env.GITHUB_SCHEDULE ?? "unknown",
+            type: "scheduled",
+        };
+    }
+    if (eventName === "repository_dispatch") {
+        return {
+            callerRef: process.env.GITHUB_REF,
+            callerRepo: process.env.GITHUB_REPOSITORY_OWNER_ID ?? "unknown",
+            type: "cross-repo",
+        };
+    }
+    if (process.env.GITHUB_ACTIONS === "true") {
+        return {
+            runId: process.env.GITHUB_RUN_ID ?? "unknown",
+            type: "ci",
+            workflow: process.env.GITHUB_WORKFLOW ?? "unknown",
+        };
+    }
+    return { type: "manual" };
+}
+// ---------------------------------------------------------------------------
+// Model config loading
+// ---------------------------------------------------------------------------
+/**
+ * Load config/models.yaml to extract model list and grader info.
+ * Falls back to a minimal config if the file can't be read.
+ */
+function loadModelsConfig(rootDir) {
+    try {
+        const content = readFileSync(resolve(rootDir, "config", "models.yaml"), "utf-8");
+        return load(content);
+    }
+    catch {
+        console.warn("  ⚠️  Could not read config/models.yaml for provenance");
+        return {
+            defaults: {},
+            grader: { id: "unknown" },
+            models: [],
+        };
+    }
+}

package/dist/pipeline/readiness-report.d.ts

@@ -0,0 +1,87 @@
+/**
+ * pipeline/readiness-report.ts
+ *
+ * Pure computation module for launch readiness reports.
+ *
+ * Combines threshold evaluation, ceiling decomposition, and gap analysis
+ * into a single actionable readiness checklist for a given feature area.
+ *
+ * This module has NO side effects — no file I/O, no process.argv, no env vars.
+ * Callers (command handlers, orchestration steps) handle I/O and pass data in.
+ *
+ * Exports pure functions:
+ *   - generateReadinessReport()  — builds the structured report
+ *   - formatReadinessMarkdown()  — renders the report as markdown
+ *
+ * @see docs/exec-plans/completed/scenario-matrix-implementation/phase-5-readiness-thresholds.md
+ * @see docs/exec-plans/active/eliminate-lib-layer.md
+ */
+import type { ThresholdConfig } from "./schemas.js";
+import type { GapAnalysisReport, GapEstimate, ScoreSummary, ThresholdEvaluation, ThresholdViolation } from "./types.js";
+/** A single dimension's readiness check */
+export interface DimensionCheck {
+    /** Dimension display name */
+    dimension: string;
+    /** Whether the dimension meets its threshold */
+    pass: boolean;
+    /** Actual score */
+    score: number;
+    /** Required threshold */
+    threshold: number;
+}
+/** A historical baseline entry for trend tracking */
+export interface HistoryEntry {
+    /** The area's composite score at this point in time */
+    score: number;
+    /** Optional tag (e.g., "pre-groq-rewrite") */
+    tag?: string;
+    /** When this baseline was captured */
+    timestamp: string;
+}
+/** Structured readiness report — the output of generateReadinessReport() */
+export interface ReadinessReport {
+    /** The feature area being evaluated */
+    area: string;
+    /** Ceiling decomposition data for this area */
+    ceiling: {
+        ceilingScore: number;
+        docLift: number;
+        docQualityGap: number;
+        floorScore: number;
+    };
+    /** Per-dimension breakdown with threshold comparison */
+    dimensions: DimensionCheck[];
+    /** Gap analysis entries for this area (empty if no gap data) */
+    gaps: GapEstimate[];
+    /** Historical scores (empty unless --history was used) */
+    history: HistoryEntry[];
+    /** Whether the area passes all thresholds */
+    pass: boolean;
+    /** The area's composite score */
+    score: number;
+    /** The composite threshold for this area */
+    threshold: number;
+    /** Threshold evaluation result */
+    thresholdEvaluation: ThresholdEvaluation;
+    /** Violations specific to this area */
+    violations: ThresholdViolation[];
+}
+/**
+ * Format a readiness report as markdown.
+ *
+ * Pure function — takes a structured report and returns a markdown string.
+ */
+export declare function formatReadinessMarkdown(report: ReadinessReport): string;
+/**
+ * Generate a structured readiness report for a given feature area.
+ *
+ * This is a pure function — it takes all data as parameters and produces
+ * a structured report. No I/O.
+ */
+export declare function generateReadinessReport(opts: {
+    area: string;
+    gapAnalysis?: GapAnalysisReport;
+    history?: HistoryEntry[];
+    scoreSummary: ScoreSummary;
+    thresholdConfig: ThresholdConfig;
+}): ReadinessReport;