@sanity/ailf 4.1.0 → 4.3.0

package/dist/pipeline/compiler/mode-handlers/literacy/compiler.js

@@ -6,10 +6,24 @@
  * - Baseline entry with without-docs prompt and empty docs
  * - Rubric assertions with structured dimension metadata
  */
+import { existsSync, readFileSync } from "node:fs";
+import { resolve } from "node:path";
 import { LiteracyVariant, } from "../../../normalize-mode.js";
 import { buildBaselineAssertions, resolveAssertions } from "./assertions.js";
 import { LITERACY_PROMPT_TEMPLATES } from "./prompts.js";
 import { validateLiteracyTask } from "./validation.js";
+/**
+ * Variable keys reserved by the AILF compilers. Authoring these via
+ * `prompt.vars` is rejected by `PromptVars` at compile time and by
+ * `TaskPromptSchema` at parse time; this constant exists to defend
+ * the literacy compiler at runtime against legacy-shape `*.task.ts`
+ * files that bypass both gates.
+ */
+const RESERVED_PROMPT_VAR_KEYS = [
+    "task",
+    "docs",
+    "__featureArea",
+];
 /**
  * Compile a literacy task into Promptfoo configuration.
  */
@@ -58,20 +72,55 @@ function buildPrompts(evalMode) {
 // ---------------------------------------------------------------------------
 function buildTestCases(task, evalMode, options, warnings) {
     const tests = [];
-    const promptText = task.prompt?.text ?? task.prompt?.template ?? "";
+    // W0193: type-erased read of prompt.vars so we can defensively detect
+    // reserved keys on legacy-shape `*.task.ts` files (the type narrow makes
+    // `task.prompt.vars.task` `never`, but TS task files bypass both the
+    // type and the parse-time schema). YAML/inline-task paths have already
+    // been migrated by `migratePromptShape` upstream.
+    const rawVars = (task.prompt?.vars ?? {});
+    const legacyTaskBody = typeof rawVars.task === "string" ? rawVars.task : undefined;
+    const promptText = task.prompt?.text ?? legacyTaskBody ?? task.prompt?.template ?? "";
     const contextDocs = task.context?.docs ?? [];
     const taskArea = task.area ?? "";
     const taskTitle = task.title;
-    const promptVars = task.prompt?.vars ?? {};
+    // Strip reserved keys from the vars spread so they cannot override the
+    // canonical assignments below. `safePromptVars` carries only freeform
+    // template extras.
+    const safePromptVars = {};
+    const presentReserved = [];
+    for (const [key, value] of Object.entries(rawVars)) {
+        if (RESERVED_PROMPT_VAR_KEYS.includes(key)) {
+            presentReserved.push(key);
+            continue;
+        }
+        safePromptVars[key] = value;
+    }
+    // Single deduplicated deprecation warning per task — even when several
+    // reserved keys are present.
+    if (presentReserved.length > 0) {
+        warnings.push(`Literacy task "${task.id}": deprecated prompt.vars keys ` +
+            `(${presentReserved.join(", ")}) — use prompt.text for the prompt ` +
+            `body and context.docs for documentation references. The compiler ` +
+            `migrated them in-memory, but the task source should be updated.`);
+    }
     const hasDocs = contextDocs.length > 0;
     const docsVar = hasDocs ? `file://contexts/canonical/${task.id}.md` : "";
-    const assertions = resolveAssertions(task, options, warnings);
-    // Gold entry — canonical docs injected
+    // When `graderContext` is "with-docs" and the task has canonical docs
+    // declared, read the resolved doc file so we can inject it into the
+    // assertion rubricPrompt as authoritative ground truth. The file is
+    // produced by `FetchDocsStep` (which runs before `GenerateConfigsStep`),
+    // so it should exist on disk by the time we're compiling.
+    const canonicalReference = options?.graderContext === "with-docs" && hasDocs
+        ? readGraderReference(task.id, options?.rootDir, warnings)
+        : undefined;
+    const assertions = resolveAssertions(task, options, warnings, canonicalReference);
+    // Gold entry — canonical docs injected. Spread freeform extras first so
+    // canonical keys (task / docs / __featureArea) cannot be overridden.
     const goldVars = {
+        ...safePromptVars,
         task: promptText,
         docs: docsVar,
         __featureArea: taskArea,
-        ...promptVars,
     };
     tests.push({
         description: `${taskTitle} (gold)`,
@@ -89,10 +138,10 @@ function buildTestCases(task, evalMode, options, warnings) {
             tests.push({
                 description: `${taskTitle} (baseline)`,
                 vars: {
+                    ...safePromptVars,
                     task: promptText,
                     docs: "",
                     __featureArea: taskArea,
-                    ...promptVars,
                 },
                 prompts: ["without-docs"],
                 ...(baselineAssertions.length > 0
@@ -103,3 +152,138 @@ function buildTestCases(task, evalMode, options, warnings) {
     }
     return tests;
 }
+// ---------------------------------------------------------------------------
+// Canonical doc resolution
+// ---------------------------------------------------------------------------
+/**
+ * Read the grader-context reference for a task, preferring the W0197
+ * symbol-reference index when one was emitted by the doc fetcher and
+ * falling back to the full canonical doc otherwise.
+ *
+ * Resolution is driven by the per-task manifest at
+ * `<rootDir>/contexts/canonical-symbols/manifest.json` (W0197). The
+ * manifest distinguishes three states:
+ *   1. Manifest missing → fetcher hadn't run with W0197. Silent fallback
+ *      to the full canonical doc; this is the backwards-compat path.
+ *   2. Manifest present, entry has `symbolCount > 0` → use the per-task
+ *      symbol-index file as the grader reference (W0197 path).
+ *   3. Manifest present, entry has `symbolCount === 0` → extraction ran
+ *      and produced nothing. Observable fallback (warning surfaced)
+ *      to the full canonical doc.
+ *
+ * Returns undefined and pushes a warning when the canonical doc itself
+ * is missing or unreadable, when rootDir is unset, or when a resolved
+ * path escapes rootDir (path-traversal guard mirroring
+ * `fixture-resolver.ts:resolveFileRef`).
+ */
+function readGraderReference(taskId, rootDir, warnings) {
+    if (!rootDir) {
+        warnings.push(`graderContext "with-docs" requires rootDir — canonical reference for "${taskId}" not injected`);
+        return undefined;
+    }
+    const manifestEntry = readSymbolIndexManifestEntry(taskId, rootDir, warnings);
+    if (manifestEntry !== "missing-manifest") {
+        if (manifestEntry === "path-traversal")
+            return undefined;
+        if (manifestEntry === null) {
+            // Manifest exists but task isn't listed — shouldn't happen for tasks
+            // the fetcher processed. Treat as silent fallback (matches "missing
+            // manifest" semantics for safety) but log a warning so an unexpected
+            // mismatch is observable.
+            warnings.push(`graderContext: task "${taskId}" not present in symbol-index manifest — falling back to full canonical doc injection`);
+        }
+        else if (manifestEntry.symbolCount === 0) {
+            warnings.push(`graderContext: symbol index empty for "${taskId}" — falling back to full canonical doc injection`);
+        }
+        else {
+            // Symbol-index path: read the per-task .md file.
+            const symbolsRelative = manifestEntry.path;
+            const symbolsAbsolute = resolveWithinRoot(rootDir, symbolsRelative);
+            if (symbolsAbsolute === null) {
+                warnings.push(`graderContext: path traversal blocked for task "${taskId}" — canonical reference not injected`);
+                return undefined;
+            }
+            if (existsSync(symbolsAbsolute)) {
+                const content = readFileSafe(symbolsAbsolute, symbolsRelative, warnings, taskId);
+                if (content && content.trim().length > 0)
+                    return content;
+            }
+            // Manifest claimed symbols but the file is missing/unreadable —
+            // surface as a warning and continue to full-doc fallback.
+            warnings.push(`graderContext: symbol index file at ${symbolsRelative} is missing or empty despite manifest entry — falling back to full canonical doc for "${taskId}"`);
+        }
+    }
+    // Fall back to the full canonical doc (W0196 path).
+    const canonicalRelative = `contexts/canonical/${taskId}.md`;
+    const canonicalAbsolute = resolveWithinRoot(rootDir, canonicalRelative);
+    if (canonicalAbsolute === null) {
+        warnings.push(`graderContext: path traversal blocked for task "${taskId}" — canonical reference not injected`);
+        return undefined;
+    }
+    if (!existsSync(canonicalAbsolute)) {
+        warnings.push(`graderContext "with-docs": canonical doc not found at ${canonicalRelative} — run fetch-docs first; reference not injected for "${taskId}"`);
+        return undefined;
+    }
+    return (readFileSafe(canonicalAbsolute, canonicalRelative, warnings, taskId) ??
+        undefined);
+}
+function readSymbolIndexManifestEntry(taskId, rootDir, warnings) {
+    const manifestRelative = "contexts/canonical-symbols/manifest.json";
+    const manifestAbsolute = resolveWithinRoot(rootDir, manifestRelative);
+    if (manifestAbsolute === null)
+        return "path-traversal";
+    if (!existsSync(manifestAbsolute))
+        return "missing-manifest";
+    const body = readFileSafe(manifestAbsolute, manifestRelative, warnings, taskId);
+    if (body === null)
+        return "missing-manifest";
+    let parsed;
+    try {
+        parsed = JSON.parse(body);
+    }
+    catch (err) {
+        const msg = err instanceof Error ? err.message : String(err);
+        warnings.push(`graderContext: symbol-index manifest at ${manifestRelative} is unparseable (${msg}) — falling back to full canonical doc for "${taskId}"`);
+        return "missing-manifest";
+    }
+    if (!parsed || typeof parsed !== "object")
+        return null;
+    const entries = parsed.entries;
+    if (!Array.isArray(entries))
+        return null;
+    for (const e of entries) {
+        if (!e || typeof e !== "object")
+            continue;
+        const entry = e;
+        if (typeof entry.taskId === "string" &&
+            typeof entry.path === "string" &&
+            typeof entry.symbolCount === "number" &&
+            entry.taskId === taskId) {
+            return {
+                taskId: entry.taskId,
+                path: entry.path,
+                symbolCount: entry.symbolCount,
+            };
+        }
+    }
+    return null;
+}
+function resolveWithinRoot(rootDir, relativePath) {
+    const absolutePath = resolve(rootDir, relativePath);
+    const normalizedBase = resolve(rootDir) + "/";
+    if (!absolutePath.startsWith(normalizedBase) &&
+        absolutePath !== resolve(rootDir)) {
+        return null;
+    }
+    return absolutePath;
+}
+function readFileSafe(absolutePath, relativePath, warnings, taskId) {
+    try {
+        return readFileSync(absolutePath, "utf-8");
+    }
+    catch (err) {
+        const msg = err instanceof Error ? err.message : String(err);
+        warnings.push(`graderContext: failed to read ${relativePath}: ${msg} — reference not injected for "${taskId}"`);
+        return null;
+    }
+}

package/dist/pipeline/compiler/mode-handlers/literacy/index.js

@@ -25,6 +25,8 @@ export const handler = {
             rootDir: ctx.rootDir,
             models: ctx.models,
             rubricConfig: ctx.rubricConfig,
+            graderContext: ctx.graderContext,
+            preflightContext: ctx.preflightContext,
             evalMode: ctx
                 .evalMode,
         });

package/dist/pipeline/compiler/mode-handlers/literacy/types.d.ts

@@ -2,8 +2,8 @@
  * Shared types for the literacy mode handler.
  */
 import type { PromptfooPrompt, PromptfooProvider, PromptfooTestCase } from "../../promptfoo-compiler.js";
-export type { RubricResolutionInput } from "../../rubric-resolution.js";
-import type { RubricResolutionInput } from "../../rubric-resolution.js";
+export type { PreflightRubricContext, RubricResolutionInput, } from "../../rubric-resolution.js";
+import type { PreflightRubricContext, RubricResolutionInput } from "../../rubric-resolution.js";
 /** Options for compiling a literacy task */
 export interface LiteracyCompileOptions {
     /** Grader provider for LLM-graded assertions */
@@ -20,6 +20,21 @@ export interface LiteracyCompileOptions {
     }[];
     /** Rubric config (templates, weights, profiles) — loaded from rubrics config */
     rubricConfig?: RubricResolutionInput;
+    /**
+     * Grader context policy. When `"with-docs"` and the task declares
+     * `context.docs`, the canonical doc content is read from
+     * `<rootDir>/contexts/canonical/<task.id>.md` and injected into the
+     * Promptfoo `rubricPrompt`.
+     */
+    graderContext?: "rubric-only" | "with-docs";
+    /**
+     * W0198 Phase 6 — when supplied, the `code-correctness` rubric is
+     * prefixed with a "DETERMINISTIC PREFLIGHT" instruction telling the
+     * grader to treat the deterministic lane's existence verdicts as
+     * ground truth. Sourced from the package-surface manifest at compile
+     * time; absence collapses cleanly to the pre-W0198 rubric.
+     */
+    preflightContext?: PreflightRubricContext;
 }
 /** Result of compiling a single literacy task */
 export interface LiteracyCompileResult {

package/dist/pipeline/compiler/rubric-resolution.d.ts

@@ -10,9 +10,12 @@
  * tasks with templated rubrics produced empty rubric text (DOC-2029).
  *
  * @see docs/design-docs/mode-agnostic-scoring.md
+ * @see docs/design-docs/two-stage-grader-symbol-preflight.md — W0198 Phase 6
  * @see config/rubrics.ts — template definitions
  */
+import type { PreflightRubricContext } from "../../_vendor/ailf-core/index.d.ts";
 import type { PromptfooAssertion } from "./assertion-mapper.js";
+export type { PreflightRubricContext } from "../../_vendor/ailf-core/index.d.ts";
 /** Minimal rubric config needed for template resolution */
 export interface RubricResolutionInput {
     templates: Record<string, {
@@ -31,10 +34,23 @@ export interface RubricResolutionInput {
  * scoring header, scale, and dimension metadata. The criteria are appended
  * to create the final rubric prompt.
  *
+ * When `canonicalReference` is supplied, it is emitted on the assertion's
+ * `rubricPrompt` field — Promptfoo's per-assertion grader-prompt override —
+ * wrapped in a ground-truth framing so the grader treats the supplied
+ * content as authoritative. The rubric `value` itself is unchanged. Without
+ * it, the grader falls back on training priors and may hallucinate against
+ * neighboring API surfaces.
+ *
  * Returns null (with a warning) if the template can't be resolved.
  */
 export declare function resolveTemplatedAssertion(assertion: {
     criteria: string[];
     template: string;
     type: string;
-}, rubricConfig: RubricResolutionInput | undefined, graderProvider: string | undefined, warnings: string[]): PromptfooAssertion | null;
+}, rubricConfig: RubricResolutionInput | undefined, graderProvider: string | undefined, warnings: string[], canonicalReference?: string, preflightContext?: PreflightRubricContext): PromptfooAssertion | null;
+/**
+ * Build the W0198 Phase 6 preflight preface for a `code-correctness`
+ * rubric. Returned with a trailing newline so it composes cleanly with
+ * the existing rubric body.
+ */
+export declare function buildPreflightSection(context: PreflightRubricContext): string;

package/dist/pipeline/compiler/rubric-resolution.js

@@ -10,6 +10,7 @@
  * tasks with templated rubrics produced empty rubric text (DOC-2029).
  *
  * @see docs/design-docs/mode-agnostic-scoring.md
+ * @see docs/design-docs/two-stage-grader-symbol-preflight.md — W0198 Phase 6
  * @see config/rubrics.ts — template definitions
  */
 // ---------------------------------------------------------------------------
@@ -24,9 +25,16 @@
  * scoring header, scale, and dimension metadata. The criteria are appended
  * to create the final rubric prompt.
  *
+ * When `canonicalReference` is supplied, it is emitted on the assertion's
+ * `rubricPrompt` field — Promptfoo's per-assertion grader-prompt override —
+ * wrapped in a ground-truth framing so the grader treats the supplied
+ * content as authoritative. The rubric `value` itself is unchanged. Without
+ * it, the grader falls back on training priors and may hallucinate against
+ * neighboring API surfaces.
+ *
  * Returns null (with a warning) if the template can't be resolved.
  */
-export function resolveTemplatedAssertion(assertion, rubricConfig, graderProvider, warnings) {
+export function resolveTemplatedAssertion(assertion, rubricConfig, graderProvider, warnings, canonicalReference, preflightContext) {
     if (!rubricConfig) {
         warnings.push(`No rubric config — template "${assertion.template}" cannot be resolved`);
         return null;
@@ -38,15 +46,83 @@ export function resolveTemplatedAssertion(assertion, rubricConfig, graderProvide
     }
     const scaleText = template.scale.map((s) => `- ${s}`).join("\n");
     const criteriaText = assertion.criteria.map((c) => `- ${c}`).join("\n");
-    const rubricValue = `${template.header}\n${scaleText}\n\n` +
+    // W0198 Phase 6 — when the deterministic preflight lane is wired and this
+    // rubric scores `code-correctness`, prefix a system instruction so the
+    // grader does not re-judge symbol existence. The lane separation is the
+    // whole reason W0198 exists; if both lanes weigh existence, the rubric's
+    // hallucinations sneak back into a dimension we want deterministic.
+    const preflightSection = preflightContext && template.dimension === "code-correctness"
+        ? buildPreflightSection(preflightContext)
+        : "";
+    const rubricValue = preflightSection +
+        `${template.header}\n${scaleText}\n\n` +
         `${template.criteria_label ?? "Check for:"}\n${criteriaText}\n\n` +
         `Return ONLY a JSON object: {"score": <number>, "reason": "<explanation>"}`;
+    const rubricPrompt = canonicalReference
+        ? buildRubricPromptWithReference(rubricValue, canonicalReference)
+        : undefined;
     return {
         type: "llm-rubric",
         value: rubricValue,
+        ...(rubricPrompt ? { rubricPrompt } : {}),
         ...(graderProvider ? { provider: graderProvider } : {}),
         ...(template.dimension
             ? { metadata: { dimension: template.dimension, maxScore: 100 } }
             : {}),
     };
 }
+/**
+ * Build the W0198 Phase 6 preflight preface for a `code-correctness`
+ * rubric. Returned with a trailing newline so it composes cleanly with
+ * the existing rubric body.
+ */
+export function buildPreflightSection(context) {
+    const packageList = context.packages.length > 0
+        ? context.packages.join(", ")
+        : "(no packages in manifest)";
+    return [
+        "DETERMINISTIC PREFLIGHT — A separate lane checks whether each named",
+        "binding the candidate imports (e.g. `import { foo } from 'pkg'`) actually",
+        "exports from its source package, against the actual `.d.ts` of the",
+        "installed version. Default and namespace imports (`import x from 'pkg'`,",
+        "`import * as x from 'pkg'`) are NOT preflight-checked and fall through to",
+        "you. Treat any existence verdict the preflight returns as ground truth —",
+        "do NOT include symbol-existence concerns in your `code-correctness`",
+        "judgment. Confine your code-correctness score to idiomatic usage, code",
+        "organization, type safety, and completeness against the task.",
+        "",
+        `In-scope packages (preflight rules on named bindings imported from`,
+        `these): ${packageList}.`,
+        "",
+        "",
+    ].join("\n");
+}
+/**
+ * Build the full rubric-prompt string sent to the grader when canonical
+ * reference content is available. The framing explicitly tells the grader
+ * to treat the reference as authoritative ground truth so it cannot fall
+ * back on training priors when checking symbol existence or API shape.
+ *
+ * Promptfoo interpolates `{{output}}` and `{{rubric}}` at grade time —
+ * `{{output}}` is the candidate response; `{{rubric}}` is the assertion's
+ * `value` field (the rubric text built above).
+ */
+function buildRubricPromptWithReference(rubric, reference) {
+    return [
+        "You are grading a candidate response against a rubric.",
+        "",
+        "AUTHORITATIVE REFERENCE — treat this as ground truth. If a symbol, function,",
+        "API, or pattern appears here, it exists; do not contradict it from prior",
+        "knowledge. If something is absent here, do not assume it exists.",
+        "",
+        "--- BEGIN REFERENCE ---",
+        reference,
+        "--- END REFERENCE ---",
+        "",
+        "RUBRIC:",
+        rubric,
+        "",
+        "CANDIDATE RESPONSE:",
+        "{{output}}",
+    ].join("\n");
+}

package/dist/pipeline/compiler/scoring-bridge.d.ts

@@ -16,11 +16,19 @@
  * engine works in [0, 1]; this module handles the conversion at
  * boundaries.
  *
+ * W0198 Phase 5 — when a `preflightForTest` callback is provided and
+ * returns a `SymbolPreflightReport`, the bridge synthesizes one extra
+ * `AssertionScore` per test in the `code-correctness` dimension. The
+ * deterministic preflight and the LLM rubric merge through D0010's
+ * weighted dimension aggregation; the relative share is set by
+ * `preflightWeight` in `[0, 1]`.
+ *
  * @see packages/core/src/services/scoring-engine.ts — the 4-tier engine
  * @see packages/eval/src/pipeline/calculate-scores.ts — the consumer
  * @see docs/archive/exec-plans/architecture-overhaul/phase-7-migrate-literacy.md
+ * @see docs/design-docs/two-stage-grader-symbol-preflight.md — W0198 design
  */
-import { type DimensionScore } from "../../_vendor/ailf-core/index.d.ts";
+import { type AssertionScore, type DimensionScore, type SymbolPreflightReport } from "../../_vendor/ailf-core/index.d.ts";
 import type { TestResult } from "../../_vendor/ailf-core/index.d.ts";
 /** Result of scoring a group of tests via the 4-tier engine */
 export interface BridgedScoreResult {
@@ -33,6 +41,24 @@ export interface BridgedScoreResult {
     /** Raw DimensionScore objects from the engine (0–1 scale) */
     rawDimensions: DimensionScore[];
 }
+/**
+ * W0198 Phase 5 options. When `preflightForTest` is omitted (or returns
+ * `undefined`) the bridge stays purely rubric-driven — identical to the
+ * pre-W0198 behavior. When a report is returned, one `code-correctness`
+ * preflight assertion is synthesized per test and merged with the rubric
+ * via weighted-mean.
+ */
+export interface ScoreTestGroupOptions {
+    /** Look up the preflight report attached to a particular test, if any. */
+    preflightForTest?: (test: TestResult) => SymbolPreflightReport | undefined;
+    /**
+     * Preflight's share of the `code-correctness` dimension, in `[0, 1]`.
+     * The complementary share belongs to the LLM rubric. Defaults to
+     * `DEFAULT_PREFLIGHT_CODE_CORRECTNESS_WEIGHT` (0.4). Ignored when
+     * `preflightForTest` is omitted.
+     */
+    preflightWeight?: number;
+}
 /**
  * Score a group of test results using the 4-tier scoring engine.
  *
@@ -44,6 +70,27 @@ export interface BridgedScoreResult {
  * @param profile  Weight profile mapping kebab-case dimension names to weights
  *                 (e.g., `{ "task-completion": 0.4, "code-correctness": 0.35, "doc-coverage": 0.25 }`)
  * @param taskId   Optional task identifier for traceability in TaskScore output
+ * @param options  Optional W0198 Phase 5 preflight integration
  * @returns        Dimensions (0–100) and composite (0–100), matching legacy output format
  */
-export declare function scoreTestGroup(tests: TestResult[], profile: Record<string, number>, taskId?: string): BridgedScoreResult;
+export declare function scoreTestGroup(tests: TestResult[], profile: Record<string, number>, taskId?: string, options?: ScoreTestGroupOptions): BridgedScoreResult;
+/**
+ * Synthesize a `code-correctness` AssertionScore from a W0198 symbol-
+ * preflight report.
+ *
+ * The score is `1 - min(1, total / cap)`. With the default
+ * `{ perMissing: 20, cap: 60 }` config: 0 missing → 1.0, 1 missing → 0.667,
+ * 2 missing → 0.333, ≥3 missing → 0.0.
+ *
+ * Edge cases for the deduction config:
+ * - `cap === 0` (measurement-only config): score is 1.0 — divide-by-zero
+ *   would NaN otherwise.
+ * - `cap < 0` (misconfigured): score is 1.0 and the merge silently
+ *   collapses to rubric-only on this dimension. The Phase 3 Zod schema
+ *   gates against this upstream so it should never reach here, but the
+ *   guard preserves the never-deduct invariant if it does.
+ *
+ * `unresolved` findings never deduct (the preflight's never-deduct rule):
+ * they're not part of `total`, so they fall through to the LLM rubric.
+ */
+export declare function preflightToScore(report: SymbolPreflightReport, weight: number): AssertionScore;