@sanity/ailf 2.0.1 → 2.1.0

package/dist/scripts/fetch-docs.js

@@ -1,472 +0,0 @@
-/**
- * Fetch-docs.ts
- *
- * Pulls documentation from the Sanity CMS and generates markdown context
- * files for use in Promptfoo evaluations. Always produces canonical contexts;
- * other outputs are opt-in:
- *
- *   1. Canonical contexts     — one file per evaluation task, containing
- *      only the manually-annotated "gold" documents for that task (always)
- *   2. Feature-area contexts  — one file per GROQ feature area query
- *      (opt-in via --include-feature-areas)
- *   3. Full corpus            — all articles in one file
- *      (opt-in via --include-corpus)
- */
-// oxlint-disable-next-line import/no-unassigned-import -- side-effect: loads .env into process.env
-import "dotenv/config";
-import { mkdirSync, writeFileSync } from "fs";
-import { dirname, join } from "path";
-import { fetchUrlContent, } from "../pipeline/fetch-url-content.js";
-import { resolveMappings, } from "../pipeline/resolve-mappings.js";
-import { createPerspectiveClient, createPublishedClient, getSanityClient, } from "../sanity/client.js";
-import { ALL_ARTICLES_QUERY, ALL_FEATURE_AREAS, ARTICLES_METADATA_BY_SLUGS_QUERY, ARTICLE_BY_ID_QUERY, ARTICLE_BY_SLUG_QUERY, ARTICLE_BY_SLUG_WITH_PERSPECTIVE_QUERY, FEATURE_AREA_QUERIES, } from "../sanity/queries.js";
-import { toMarkdown } from "../sanity/portable-text.js";
-import { loadSource } from "../sources.js";
-// ---------------------------------------------------------------------------
-// Helpers
-// ---------------------------------------------------------------------------
-const ROOT = join(dirname(new URL(import.meta.url).pathname), "..", "..");
-/**
- * Escape `{{` and `}}` so Promptfoo's Nunjucks template engine doesn't
- * try to interpret them as template variables (e.g. Vue `{{ post.title }}`
- * in Nuxt docs would break Nunjucks parsing).
- *
- * Uses a single-pass regex so replacement text isn't re-scanned.
- */
-function escapeNunjucks(text) {
-    return text.replace(/\{\{|\}\}/g, (match) => match === "{{" ? '{{ "{{" }}' : '{{ "}}" }}');
-}
-function estimateTokens(text) {
-    return Math.ceil(text.length / 4);
-}
-// ---------------------------------------------------------------------------
-// Perspective diffing
-// ---------------------------------------------------------------------------
-/**
- * Fetch a single article by its document ID.
- *
- * When a perspective is active, the document is fetched through the
- * perspective client — this is the natural behavior for Studio document
- * URLs that include a ?perspective= parameter.
- */
-async function fetchArticleById(id, source) {
-    const client = source.perspective
-        ? createPerspectiveClient(source.perspective, source)
-        : getSanityClient();
-    return client.fetch(ARTICLE_BY_ID_QUERY, { id });
-}
-async function fetchArticleBySlug(slug) {
-    const client = getSanityClient();
-    const doc = await client.fetch(ARTICLE_BY_SLUG_QUERY, {
-        slug,
-    });
-    if (!doc) {
-        console.warn(`    [warn] No article found for slug "${slug}"`);
-        return "";
-    }
-    return formatArticle(doc);
-}
-/**
- * Fetch a single article by slug through a perspective-enabled client.
- */
-async function fetchArticleBySlugWithPerspective(slug, source) {
-    if (!source.perspective) {
-        return fetchArticleBySlug(slug);
-    }
-    const client = createPerspectiveClient(source.perspective, source);
-    const doc = await client.fetch(ARTICLE_BY_SLUG_WITH_PERSPECTIVE_QUERY, { slug });
-    if (!doc) {
-        console.warn(`    [warn] No article found for slug "${slug}" in perspective "${source.perspective}"`);
-        return "";
-    }
-    return formatArticle(doc);
-}
-// ---------------------------------------------------------------------------
-// Document ID overlay
-// ---------------------------------------------------------------------------
-function formatArticle(doc) {
-    const sectionLabel = doc.section ? `Section: ${doc.section.title}\n` : "";
-    const desc = doc.description ? `${doc.description}\n\n` : "";
-    // Convert Portable Text to Markdown
-    const ptBlocks = doc.content ?? [];
-    const markdown = toMarkdown(ptBlocks);
-    return `## ${doc.title}\n\n${sectionLabel}${desc}${markdown}`;
-}
-async function generateCanonicalContexts(source) {
-    const mappings = resolveMappings(ROOT);
-    const canonicalDir = join(ROOT, "contexts", "canonical");
-    mkdirSync(canonicalDir, { recursive: true });
-    const hasPerspective = Boolean(source.perspective);
-    const hasDocumentIds = source.documentIds !== undefined && source.documentIds.length > 0;
-    const hasDirectUrls = source.urls.length > 0;
-    // Build descriptive header
-    const parts = [];
-    if (hasPerspective) {
-        parts.push(`perspective "${source.perspective}"`);
-    }
-    if (hasDocumentIds) {
-        parts.push(`${source.documentIds.length} document overlay(s)`);
-    }
-    if (hasDirectUrls) {
-        parts.push(`${source.urls.length} direct URL(s)`);
-    }
-    if (parts.length > 0) {
-        console.log(`\nGenerating canonical contexts with ${parts.join(" + ")}...\n`);
-    }
-    else {
-        console.log("\nGenerating canonical (gold-retrieval) contexts...\n");
-    }
-    // Collect all canonical slugs across all tasks for a single diff pass
-    const allSlugs = new Set();
-    for (const areaData of Object.values(mappings.feature_areas)) {
-        for (const task of areaData.tasks) {
-            for (const doc of task.canonical_docs) {
-                allSlugs.add(doc.slug);
-            }
-        }
-    }
-    // Fetch metadata for all canonical docs and write the document manifest.
-    // This captures _id, _rev, slug, title at evaluation time for traceability.
-    const metadataClient = hasPerspective
-        ? createPerspectiveClient(source.perspective, source)
-        : getSanityClient();
-    const allMetadata = await metadataClient.fetch(ARTICLES_METADATA_BY_SLUGS_QUERY, { slugs: [...allSlugs] });
-    const manifestPath = join(ROOT, "contexts", "document-manifest.json");
-    const manifest = allMetadata
-        .map((m) => ({ _id: m._id, _rev: m._rev, slug: m.slug, title: m.title }))
-        .sort((a, b) => a.slug.localeCompare(b.slug));
-    writeFileSync(manifestPath, JSON.stringify(manifest, null, 2));
-    console.log(`  📋 Document manifest: ${manifest.length} docs → contexts/document-manifest.json\n`);
-    // If a perspective is active, diff all canonical slugs at once
-    let releaseImpact;
-    if (hasPerspective) {
-        releaseImpact = await identifyAffectedDocs(source, [...allSlugs]);
-        const affected = releaseImpact.added.length +
-            releaseImpact.modified.length +
-            releaseImpact.removed.length;
-        console.log(`  Perspective diff: ${affected} of ${allSlugs.size} canonical docs affected`);
-        if (releaseImpact.added.length > 0) {
-            console.log(`    Added:    ${releaseImpact.added.join(", ")}`);
-        }
-        if (releaseImpact.modified.length > 0) {
-            console.log(`    Modified: ${releaseImpact.modified.join(", ")}`);
-        }
-        if (releaseImpact.removed.length > 0) {
-            console.log(`    Removed:  ${releaseImpact.removed.join(", ")}`);
-        }
-        console.log();
-    }
-    // If document IDs are specified, resolve them against the canonical set
-    let documentOverlay;
-    if (hasDocumentIds) {
-        console.log("  Resolving " +
-            source.documentIds.length +
-            " document ID(s) against canonical set...");
-        documentOverlay = await resolveDocumentOverlay(source.documentIds, allSlugs, source);
-        console.log("  Document overlay: " +
-            documentOverlay.replacements.size +
-            " replacement(s), " +
-            documentOverlay.appendedContent.length +
-            " appended");
-        console.log();
-    }
-    // If direct URLs are specified, fetch their content
-    const urlContent = [];
-    const urlFetchMetadata = [];
-    if (source.urls.length > 0) {
-        console.log(`  Fetching ${source.urls.length} direct URL(s)...`);
-        for (const url of source.urls) {
-            const result = await fetchUrlContent(url, source.headers);
-            urlFetchMetadata.push({
-                contentLength: result.content?.length,
-                error: result.error,
-                method: result.method,
-                status: result.status,
-                url: result.url,
-            });
-            if (result.content) {
-                urlContent.push(result.content);
-                const tokens = estimateTokens(result.content);
-                console.log(`    ✅ ${url} (via ${result.method}, ~${tokens} tokens)`);
-            }
-            else {
-                console.warn(`    ⚠️  ${url}: ${result.error}`);
-            }
-        }
-        console.log();
-    }
-    const affectedSlugs = new Set([
-        ...(releaseImpact?.added ?? []),
-        ...(releaseImpact?.modified ?? []),
-    ]);
-    const removedSlugs = new Set(releaseImpact?.removed ?? []);
-    for (const [area, areaData] of Object.entries(mappings.feature_areas)) {
-        console.log(`  Feature area: ${area}`);
-        for (const task of areaData.tasks) {
-            console.log(`    Task: ${task.id}`);
-            const docContents = [];
-            for (const doc of task.canonical_docs) {
-                if (removedSlugs.has(doc.slug)) {
-                    console.log(`      ⚠️ Skipping: ${doc.slug} (removed in perspective)`);
-                    continue;
-                }
-                // Check if this slug has a document overlay replacement
-                if (documentOverlay?.replacements.has(doc.slug)) {
-                    console.log(`      Fetching: ${doc.slug} (from document overlay)`);
-                    docContents.push(documentOverlay.replacements.get(doc.slug));
-                }
-                else if (affectedSlugs.has(doc.slug)) {
-                    console.log(`      Fetching: ${doc.slug} (from perspective)`);
-                    const content = await fetchArticleBySlugWithPerspective(doc.slug, source);
-                    if (content) {
-                        docContents.push(content);
-                    }
-                }
-                else {
-                    console.log(`      Fetching: ${doc.slug}`);
-                    const content = await fetchArticleBySlug(doc.slug);
-                    if (content) {
-                        docContents.push(content);
-                    }
-                }
-            }
-            // Append any extra documents from the overlay that didn't match
-            // Canonical slugs — these are added to every task's context
-            if (documentOverlay && documentOverlay.appendedContent.length > 0) {
-                docContents.push(...documentOverlay.appendedContent);
-                console.log(`      + ${documentOverlay.appendedContent.length} appended document(s) from overlay`);
-            }
-            // Append URL-fetched content (from --url classified as direct-url)
-            if (urlContent.length > 0) {
-                docContents.push(...urlContent);
-                console.log(`      + ${urlContent.length} URL-fetched document(s)`);
-            }
-            const combined = docContents.join("\n\n---\n\n");
-            const outPath = join(canonicalDir, `${task.id}.md`);
-            writeFileSync(outPath, escapeNunjucks(combined));
-            console.log(`      -> ${task.id}.md: ~${estimateTokens(combined)} tokens`);
-        }
-    }
-    // Write release impact file for downstream consumption (scoring, reports)
-    if (releaseImpact) {
-        const impactPath = join(ROOT, "contexts", "release-impact.json");
-        writeFileSync(impactPath, JSON.stringify(releaseImpact, null, 2));
-        console.log(`\n  📄 Release impact written to contexts/release-impact.json`);
-    }
-    // Write document overlay metadata for downstream consumption
-    if (documentOverlay) {
-        const overlayMeta = {
-            appendedCount: documentOverlay.appendedContent.length,
-            documentIds: source.documentIds,
-            replacedSlugs: [...documentOverlay.replacements.keys()],
-        };
-        const overlayPath = join(ROOT, "contexts", "document-overlay.json");
-        writeFileSync(overlayPath, JSON.stringify(overlayMeta, null, 2));
-        console.log("\n  📄 Document overlay written to contexts/document-overlay.json");
-    }
-    // Write URL fetch metadata for downstream consumption
-    if (urlFetchMetadata.length > 0) {
-        const fetched = urlFetchMetadata.filter((m) => m.method !== "failed");
-        const failures = urlFetchMetadata.filter((m) => m.method === "failed");
-        const meta = {
-            failures: failures.map((f) => ({ error: f.error, url: f.url })),
-            fetchedUrls: fetched,
-            totalFailed: failures.length,
-            totalFetched: fetched.length,
-        };
-        const urlFetchPath = join(ROOT, "contexts", "url-fetch.json");
-        writeFileSync(urlFetchPath, JSON.stringify(meta, null, 2));
-        console.log("\n  📄 URL fetch metadata written to contexts/url-fetch.json");
-    }
-}
-// ---------------------------------------------------------------------------
-// Feature-area contexts
-// ---------------------------------------------------------------------------
-async function generateFeatureAreaContexts(source) {
-    const client = getSanityClient(undefined, source);
-    const contextsDir = join(ROOT, "contexts");
-    mkdirSync(contextsDir, { recursive: true });
-    console.log("Generating feature-area contexts...\n");
-    for (const feature of ALL_FEATURE_AREAS) {
-        const query = FEATURE_AREA_QUERIES[feature];
-        const docs = await client.fetch(query);
-        if (docs.length === 0) {
-            console.warn(`  [warn] No articles found for "${feature}"`);
-            continue;
-        }
-        const combined = docs.map(formatArticle).join("\n\n---\n\n");
-        const outPath = join(contextsDir, `${feature}.md`);
-        writeFileSync(outPath, escapeNunjucks(combined));
-        console.log(`  ${feature}: ${docs.length} articles, ~${estimateTokens(combined)} tokens`);
-    }
-}
-// ---------------------------------------------------------------------------
-// Canonical (gold-retrieval) contexts
-// ---------------------------------------------------------------------------
-async function generateFullCorpus(source) {
-    const client = getSanityClient(undefined, source);
-    console.log("\nGenerating full corpus...");
-    const docs = await client.fetch(ALL_ARTICLES_QUERY);
-    const corpus = docs
-        .map((d) => {
-        const ptBlocks = d.content ?? [];
-        const markdown = toMarkdown(ptBlocks);
-        return (`## ${d.title}\n\n` +
-            // oxlint-disable-next-line @typescript-eslint/prefer-nullish-coalescing -- empty string title should fall back to "General"
-            `Section: ${d.section?.title || "General"}\n` +
-            `URL: ${source.baseUrl}/${d.slug}\n\n` +
-            markdown);
-    })
-        .join("\n\n---\n\n");
-    const outDir = join(ROOT, "contexts");
-    mkdirSync(outDir, { recursive: true });
-    writeFileSync(join(outDir, "full-corpus.md"), escapeNunjucks(corpus));
-    console.log(`  full-corpus.md: ${docs.length} articles, ~${estimateTokens(corpus)} tokens`);
-}
-/**
- * Compare canonical documents between the published dataset and a perspective
- * to identify which docs have been added, modified, or removed.
- *
- * The comparison uses _rev to detect modifications — if the _rev differs
- * between published and perspective, the document has been modified.
- */
-async function identifyAffectedDocs(source, canonicalSlugs) {
-    if (!source.perspective) {
-        return {
-            added: [],
-            modified: [],
-            removed: [],
-            unchanged: [...canonicalSlugs],
-        };
-    }
-    const publishedClient = createPublishedClient(source);
-    const perspectiveClient = createPerspectiveClient(source.perspective, source);
-    const [publishedMeta, perspectiveMeta] = await Promise.all([
-        publishedClient.fetch(ARTICLES_METADATA_BY_SLUGS_QUERY, {
-            slugs: canonicalSlugs,
-        }),
-        perspectiveClient.fetch(ARTICLES_METADATA_BY_SLUGS_QUERY, { slugs: canonicalSlugs }),
-    ]);
-    const publishedMap = new Map(publishedMeta.map((d) => [d.slug, d]));
-    const perspectiveMap = new Map(perspectiveMeta.map((d) => [d.slug, d]));
-    const added = [];
-    const modified = [];
-    const removed = [];
-    const unchanged = [];
-    for (const slug of canonicalSlugs) {
-        const pub = publishedMap.get(slug);
-        const persp = perspectiveMap.get(slug);
-        if (!pub && persp) {
-            added.push(slug);
-        }
-        else if (pub && !persp) {
-            removed.push(slug);
-        }
-        else if (pub && persp && pub._rev !== persp._rev) {
-            modified.push(slug);
-        }
-        else {
-            unchanged.push(slug);
-        }
-    }
-    return { added, modified, removed, unchanged };
-}
-// ---------------------------------------------------------------------------
-// Full corpus (optional — useful for retrieval experiments)
-// ---------------------------------------------------------------------------
-async function main() {
-    console.log("=== ai-literacy-framework — Documentation Fetcher ===\n");
-    const args = process.argv.slice(2);
-    const includeFeatureAreas = args.includes("--include-feature-areas");
-    const includeCorpus = args.includes("--include-corpus");
-    // Parse --source <name> argument
-    const sourceIdx = args.indexOf("--source");
-    const sourceName = sourceIdx !== -1 ? args[sourceIdx + 1] : undefined;
-    const source = loadSource(sourceName);
-    console.log(`  Source: ${sourceName ?? "default (production)"}`);
-    console.log(`  Base URL: ${source.baseUrl}`);
-    if (source.dataset) {
-        console.log(`  Dataset: ${source.dataset}`);
-    }
-    if (source.perspective) {
-        console.log(`  Perspective: ${source.perspective}`);
-    }
-    if (source.documentIds && source.documentIds.length > 0) {
-        console.log(`  Documents: ${source.documentIds.length} document ID(s)`);
-        for (const id of source.documentIds) {
-            console.log(`             ${id}`);
-        }
-    }
-    if (source.urls.length > 0) {
-        console.log(`  URLs:      ${source.urls.length} direct URL(s)`);
-        for (const u of source.urls) {
-            console.log(`             ${u}`);
-        }
-    }
-    console.log();
-    if (includeFeatureAreas) {
-        await generateFeatureAreaContexts(source);
-    }
-    await generateCanonicalContexts(source);
-    if (includeCorpus) {
-        await generateFullCorpus(source);
-    }
-    console.log("\nDone!");
-}
-// ---------------------------------------------------------------------------
-// Main
-// ---------------------------------------------------------------------------
-/**
- * Resolve document IDs into a DocumentOverlay that describes how they
- * interact with the canonical doc set.
- *
- * For each document ID:
- *   1. Fetch the document from Sanity (through perspective if active)
- *   2. Check if the fetched document's slug matches a canonical slug
- *   3. If it matches -> add to replacements (replaces the canonical fetch)
- *   4. If it doesn't match -> add to appendedContent (extra context)
- *
- * @param documentIds - Array of Sanity document IDs to resolve
- * @param canonicalSlugs - Set of all canonical doc slugs across all tasks
- * @param source - The resolved documentation source configuration
- */
-async function resolveDocumentOverlay(documentIds, canonicalSlugs, source) {
-    const overlay = {
-        appendedContent: [],
-        replacements: new Map(),
-    };
-    if (documentIds.length === 0) {
-        return overlay;
-    }
-    // Fetch all documents in parallel
-    const results = await Promise.all(documentIds.map(async (id) => {
-        const doc = await fetchArticleById(id, source);
-        return { doc, id };
-    }));
-    for (const { doc, id } of results) {
-        if (!doc) {
-            console.warn('    [warn] No article found for document ID "' + id + '"');
-            continue;
-        }
-        const content = formatArticle(doc);
-        if (!content) {
-            continue;
-        }
-        if (doc.slug && canonicalSlugs.has(doc.slug)) {
-            // This document matches a canonical slug — replace it
-            overlay.replacements.set(doc.slug, content);
-            console.log("    📄 Document " + id + ' → replaces canonical doc "' + doc.slug + '"');
-        }
-        else {
-            // This document doesn't match any canonical slug — append it
-            overlay.appendedContent.push(content);
-            const slugInfo = doc.slug ? ' (slug: "' + doc.slug + '")' : "";
-            console.log("    📄 Document " + id + " → appended as additional context" + slugInfo);
-        }
-    }
-    return overlay;
-}
-main().catch((err) => {
-    console.error("Fatal error:", err);
-    process.exit(1);
-});

package/dist/scripts/generate-configs.d.ts

@@ -1,66 +0,0 @@
-/**
- * Generate-configs.ts
- *
- * Reads config/models.yaml (the central model registry) and generates all
- * promptfoo config files with the correct provider entries.
- *
- * This keeps model definitions in one place — add a model to config/models.yaml
- * and run `pnpm generate-configs` to propagate it to all eval modes.
- *
- * Generated configs:
- *   - promptfooconfig.yaml           (baseline: with-docs vs without-docs)
- *   - promptfooconfig.observed.yaml  (instrumented HTTP recording)
- *   - promptfooconfig.agentic.yaml   (agentic tool-calling: naive vs optimized)
- *
- * Usage:
- *   pnpm generate-configs
- *   # or
- *   tsx src/scripts/generate-configs.ts
- */
-import type { ModelEntry } from "../pipeline/types.js";
-/** Auto-discover all task YAML files in the tasks/ directory. */
-export declare function discoverTaskFiles(rootDir: string): string[];
-interface LoadedPrompts {
-    agentic: {
-        id: string;
-        label: string;
-        raw: string;
-    };
-    withDocs: {
-        id: string;
-        label: string;
-        raw: string;
-    };
-    withoutDocs: {
-        id: string;
-        label: string;
-        raw: string;
-    };
-}
-/**
- * Extract the raw API model name from a promptfoo provider ID.
- *
- * Promptfoo IDs encode the provider + sub-protocol + model, e.g.:
- *   "openai:chat:gpt-5.2"       → "gpt-5.2"
- *   "anthropic:messages:claude-opus-4-6" → "claude-opus-4-6"
- *   "openai:gpt-4o"             → "gpt-4o"
- *   "google:gemini-2.5-pro"     → "gemini-2.5-pro"
- *
- * Falls back to stripping everything before the first colon for unknown
- * providers (e.g., "openrouter:deepseek/deepseek-r1" → "deepseek/deepseek-r1").
- */
-export declare function extractModelName(id: string): string;
-/**
- * Extract the LLM provider family from a promptfoo provider ID.
- *
- *   "openai:chat:gpt-5.2"                → "openai"
- *   "anthropic:messages:claude-opus-4-6"  → "anthropic"
- *   "google:gemini-2.5-pro"              → "google"
- */
-export declare function extractProvider(id: string): string;
-/** Load prompt templates from config/prompts.yaml. Throws if missing or malformed. */
-export declare function loadPrompts(rootDir: string): LoadedPrompts;
-/** Merge default config with model-specific config */
-export declare function mergeConfig(defaults: Record<string, unknown>, modelConfig?: Record<string, unknown>, overrides?: Record<string, unknown>): Record<string, unknown>;
-export declare function modelMatchesMode(model: ModelEntry, mode: string): boolean;
-export {};