@sanity/ailf 4.2.0 → 4.3.1

package/dist/_vendor/ailf-core/schemas/symbol-preflight-report.d.ts

@@ -0,0 +1,51 @@
+/**
+ * @sanity/ailf-core — SymbolPreflightReport schema
+ *
+ * The trust-boundary parser for the W0198 deterministic-lane artifact.
+ * Used when reading a previously-emitted preflight report back from
+ * disk / GCS so a downstream step (re-grading, comparison, dashboard
+ * rendering) can trust its shape.
+ *
+ * The schema asserts `satisfies z.ZodType<SymbolPreflightReport>`
+ * against the domain type in
+ * `packages/core/src/types/symbol-preflight-report.ts` (D0045 / W0187),
+ * so any drift between the two is a build error.
+ */
+import { z } from "zod";
+export declare const SymbolPreflightReportSchema: z.ZodObject<{
+    candidate: z.ZodObject<{
+        taskId: z.ZodString;
+        testIndex: z.ZodNumber;
+    }, z.core.$strip>;
+    findings: z.ZodArray<z.ZodDiscriminatedUnion<[z.ZodObject<{
+        result: z.ZodLiteral<"exists">;
+        pkg: z.ZodString;
+        version: z.ZodString;
+        binding: z.ZodString;
+        source: z.ZodEnum<{
+            types: "types";
+            runtime: "runtime";
+        }>;
+    }, z.core.$strip>, z.ZodObject<{
+        result: z.ZodLiteral<"missing">;
+        pkg: z.ZodString;
+        version: z.ZodString;
+        binding: z.ZodString;
+    }, z.core.$strip>, z.ZodObject<{
+        result: z.ZodLiteral<"unresolved">;
+        pkg: z.ZodString;
+        binding: z.ZodString;
+        reason: z.ZodEnum<{
+            "package-not-installed": "package-not-installed";
+            "types-entry-missing": "types-entry-missing";
+            "parse-failed": "parse-failed";
+            "reexport-hop-unfollowed": "reexport-hop-unfollowed";
+        }>;
+    }, z.core.$strip>], "result">>;
+    deduction: z.ZodObject<{
+        perMissing: z.ZodNumber;
+        cap: z.ZodNumber;
+        total: z.ZodNumber;
+    }, z.core.$strip>;
+}, z.core.$strip>;
+export type { SymbolPreflightReport } from "../types/symbol-preflight-report.js";

package/dist/_vendor/ailf-core/schemas/symbol-preflight-report.js

@@ -0,0 +1,57 @@
+/**
+ * @sanity/ailf-core — SymbolPreflightReport schema
+ *
+ * The trust-boundary parser for the W0198 deterministic-lane artifact.
+ * Used when reading a previously-emitted preflight report back from
+ * disk / GCS so a downstream step (re-grading, comparison, dashboard
+ * rendering) can trust its shape.
+ *
+ * The schema asserts `satisfies z.ZodType<SymbolPreflightReport>`
+ * against the domain type in
+ * `packages/core/src/types/symbol-preflight-report.ts` (D0045 / W0187),
+ * so any drift between the two is a build error.
+ */
+import { z } from "zod";
+const ExistsFindingSchema = z.object({
+    result: z.literal("exists"),
+    pkg: z.string().min(1),
+    version: z.string().min(1),
+    binding: z.string().min(1),
+    source: z.enum(["types", "runtime"]),
+});
+const MissingFindingSchema = z.object({
+    result: z.literal("missing"),
+    pkg: z.string().min(1),
+    version: z.string().min(1),
+    binding: z.string().min(1),
+});
+const UnresolvedReasonSchema = z.enum([
+    "package-not-installed",
+    "types-entry-missing",
+    "parse-failed",
+    "reexport-hop-unfollowed",
+]);
+const UnresolvedFindingSchema = z.object({
+    result: z.literal("unresolved"),
+    pkg: z.string().min(1),
+    binding: z.string().min(1),
+    reason: UnresolvedReasonSchema,
+});
+const FindingSchema = z.discriminatedUnion("result", [
+    ExistsFindingSchema,
+    MissingFindingSchema,
+    UnresolvedFindingSchema,
+]);
+const DeductionSchema = z.object({
+    perMissing: z.number().nonnegative(),
+    cap: z.number().nonnegative(),
+    total: z.number().nonnegative(),
+});
+export const SymbolPreflightReportSchema = z.object({
+    candidate: z.object({
+        taskId: z.string().min(1),
+        testIndex: z.number().int().nonnegative(),
+    }),
+    findings: z.array(FindingSchema),
+    deduction: DeductionSchema,
+});

package/dist/_vendor/ailf-core/types/index.d.ts

@@ -11,6 +11,7 @@
  */
 import type { DocumentRef as _DocumentRef, EvalMode, RunContext } from "../../ailf-shared/index.d.ts";
 import type { ArtifactType } from "../artifact-registry.js";
+import type { SymbolPreflightReport } from "./symbol-preflight-report.js";
 import type { AssociationValues, RunId } from "./branded-ids.js";
 export type { ActualScoreEntry, ComponentResult, TestResult, UrlMetadata, } from "./scoring-input.js";
 export type { DocumentRef, RunContext, RunTrigger } from "../../ailf-shared/index.d.ts";
@@ -26,6 +27,9 @@ export type { EvalTrace, ToolCallCategory, ToolCallRecord, TraceEvent, TraceSpan
 export type { CompareRequest, CompareScoreSummaryRef, LookupDocRequest, ValidateConfigRequest, ValidateSourceRequest, ValidateTaskAssertionDef, ValidateTaskCanonicalDocRef, ValidateTaskDef, ValidateTaskRequest, } from "./api-requests.js";
 export type { AilfEvalWorkflow, AilfEvalWorkflowJob, AilfEvalWorkflowStep, RepoAgenticConfig, RepoArtifactsConfig, RepoConfig, RepoExecutionConfig, RepoOutputConfig, RepoOwnerConfig, RepoPublishConfig, RepoReportStoreConfig, RepoSourceConfig, RepoTaskSourceConfig, RepoTriggersConfig, ScheduleTriggerConfig, TriggerConfig, TriggerMode, } from "./repo-config.js";
 export type { PipelineRequest, PipelineRequestCallback, PipelineRequestCallerExecutor, PipelineRequestCallerGit, PipelineRequestCallerOwner, PipelineRequestDebug, PipelineRequestTaskSource, } from "./pipeline-request.js";
+export type { PackageSurfaceConfig, PackageSurfaceEntry, } from "./package-surface.js";
+export type { SymbolPreflightDeduction, SymbolPreflightFinding, SymbolPreflightReport, SymbolPreflightUnresolvedReason, } from "./symbol-preflight-report.js";
+export { DEFAULT_PREFLIGHT_CODE_CORRECTNESS_WEIGHT, type PreflightRubricContext, type PreflightScoringConfig, } from "./preflight-scoring.js";
 export type { ArtifactId, AssociationAxis, AssociationValues, Brand, EntryKey, Err, FixtureId, IdValidationError, NewReportId, Ok, ProviderId, PromptId, Result, ResultId, RubricId, RunFingerprint, RunId, SuiteId, TaskId, TaskSlug, TraceId, } from "./branded-ids.js";
 export { err, fixtureId, generateRunId, ok, providerId, resultId, runId, suiteId, taskId, traceId, } from "./branded-ids.js";
 export type { AgentHarnessTaskDefinition, ContentLakeAuthorableMode, ContentLakeAuthorableTask, CustomTaskDefinition, GeneralizedAssertionDefinition, GeneralizedDocRef, GeneralizedTaskDefinition, GeneralizedTemplatedAssertion, GeneralizedValueAssertion, IdDocRef, KnowledgeProbeTaskDefinition, LiteracyTaskDefinition, MCPServerTaskDefinition, PromptVars, PathDocRef, PerspectiveDocRef, ReservedPromptVarKey, RubricRef, SlugDocRef, TaskCommonFields, TaskDifficulty, TaskOptions, TaskProviderConfig, TaskStatus, } from "./generalized-task.js";
@@ -593,6 +597,14 @@ export interface PipelineState {
      * Consumed by the orchestrator for the enriched PipelineResult.
      */
     testSummary?: TestSummary;
+    /**
+     * W0198 deterministic-lane reports keyed by `${run}/${mode}/${task}/${model}`.
+     * Set by RunEvalStep after `emitSymbolPreflight`, consumed by the
+     * scoring step (Phase 5) so the per-test merge does not need to read
+     * the artifact back from disk. The map is intentionally small — one
+     * entry per (task, model) per mode.
+     */
+    preflightReports?: Map<string, SymbolPreflightReport>;
 }
 /**
  * Release auto-scope metadata — which tasks are affected by a content

package/dist/_vendor/ailf-core/types/index.js

@@ -16,6 +16,7 @@ export { InMemoryPluginRegistry } from "./plugin-registry.js";
 // version is used internally by LiteracyModeConfig. If consumers need
 // the mode-specific version, they import from "./eval-mode-config.js".
 export { evalModeType } from "./eval-mode-config.js";
+export { DEFAULT_PREFLIGHT_CODE_CORRECTNESS_WEIGHT, } from "./preflight-scoring.js";
 export { err, fixtureId, generateRunId, ok, providerId, resultId, runId, suiteId, taskId, traceId, } from "./branded-ids.js";
 // ---------------------------------------------------------------------------
 // Comparison (Approach 2: structured comparison output)

package/dist/_vendor/ailf-core/types/package-surface.d.ts

@@ -0,0 +1,36 @@
+/**
+ * Package surface manifest types — framework-level pin of which package
+ * surfaces the symbol-resolution preflight (W0198) reads against.
+ *
+ * Tasks reference packages by name; version metadata lives here, not on
+ * the task. A semver-major bump is an editorial event (regenerate cached
+ * surfaces, run the historical comparison set, ride a release) — patch
+ * and minor releases within a pinned major flow silently because semver
+ * disallows the export removals that would change a deduction outcome.
+ *
+ * @see docs/design-docs/two-stage-grader-symbol-preflight.md
+ */
+/**
+ * One in-scope package, pinned to a semver-major range.
+ *
+ * - `pkg` — npm package name (e.g. `"@sanity/sdk-react"`).
+ * - `semverPin` — semver range pinned to a single major (e.g. `"^2.0.0"`).
+ *   The W0198 resolver answers existence questions against this major;
+ *   minor and patch upgrades within the major do not require a manifest
+ *   bump because semver disallows export removal.
+ */
+export interface PackageSurfaceEntry {
+    pkg: string;
+    semverPin: string;
+}
+/**
+ * Framework-level package-surface manifest.
+ *
+ * Authored once in `packages/eval/config/package-surface.ts` via
+ * `definePackageSurface(...)`. Per-task pin overrides remain a deliberate
+ * future extension point; the shape leaves room for them but the initial
+ * slice does not implement them.
+ */
+export interface PackageSurfaceConfig {
+    packages: PackageSurfaceEntry[];
+}

package/dist/_vendor/ailf-core/types/package-surface.js

@@ -0,0 +1,13 @@
+/**
+ * Package surface manifest types — framework-level pin of which package
+ * surfaces the symbol-resolution preflight (W0198) reads against.
+ *
+ * Tasks reference packages by name; version metadata lives here, not on
+ * the task. A semver-major bump is an editorial event (regenerate cached
+ * surfaces, run the historical comparison set, ride a release) — patch
+ * and minor releases within a pinned major flow silently because semver
+ * disallows the export removals that would change a deduction outcome.
+ *
+ * @see docs/design-docs/two-stage-grader-symbol-preflight.md
+ */
+export {};

package/dist/_vendor/ailf-core/types/pipeline-request.d.ts

@@ -85,6 +85,7 @@ export interface PipelineRequest {
     debug?: PipelineRequestDebug | boolean;
     executor?: PipelineRequestCallerExecutor;
     gapAnalysis?: boolean;
+    graderContext?: "rubric-only" | "with-docs";
     graderReplications?: number;
     headers?: Record<string, string>;
     inlineTasks?: Record<string, unknown>[];

package/dist/_vendor/ailf-core/types/preflight-scoring.d.ts

@@ -0,0 +1,52 @@
+/**
+ * Preflight-scoring config — knob for how heavily the W0198 symbol-existence
+ * preflight contributes to the `code-correctness` dimension.
+ *
+ * The deterministic preflight (`SymbolPreflightReport`) and the LLM rubric
+ * both feed into `code-correctness` per D0010. This config sets the relative
+ * share of the two lanes: `codeCorrectnessWeight: 0.4` means preflight is
+ * 40% of the dimension, rubric 60%.
+ *
+ * Authored in `packages/eval/config/preflight-scoring.ts` via
+ * `definePreflightScoring()`. Loaded lazily by the scoring step; absence
+ * means "use the default" rather than "disable" — disabling the preflight
+ * entirely is a separate axis (no manifest, no resolver wired).
+ *
+ * @see docs/design-docs/two-stage-grader-symbol-preflight.md
+ */
+export interface PreflightScoringConfig {
+    /**
+     * Preflight's relative share of the `code-correctness` dimension, in
+     * `[0, 1]`. The complementary `1 - codeCorrectnessWeight` is the LLM
+     * rubric's share. Default `0.4`.
+     */
+    codeCorrectnessWeight: number;
+}
+/** Default preflight share of `code-correctness` when no config file is authored. */
+export declare const DEFAULT_PREFLIGHT_CODE_CORRECTNESS_WEIGHT = 0.4;
+/**
+ * W0198 Phase 6 — describes the deterministic symbol-preflight lane to the
+ * grader. When supplied to the literacy compiler, the rubric for the
+ * `code-correctness` dimension is prefixed with a system instruction telling
+ * the grader to treat the preflight's existence findings as ground truth and
+ * confine its code-correctness judgment to non-existence concerns (idiomatic
+ * usage, code organization, type safety, completeness).
+ *
+ * The actual per-test findings live on the deterministic lane (Phase 4
+ * artifact + Phase 5 score merge) and never enter the grader prompt — the
+ * lanes are kept structurally independent so the rubric never sees, and
+ * therefore cannot relitigate, an existence verdict.
+ *
+ * Lives in `@sanity/ailf-core` so the `CompilationContext` port and the
+ * eval-package compiler can both reference the same shape without creating
+ * a cross-package import cycle.
+ */
+export interface PreflightRubricContext {
+    /**
+     * Names of the packages currently in the framework-level package-surface
+     * manifest. The grader is shown this list so it knows the boundary of the
+     * deterministic lane (named imports from these packages are existence-
+     * checked; everything else falls through to the rubric).
+     */
+    readonly packages: readonly string[];
+}

package/dist/_vendor/ailf-core/types/preflight-scoring.js

@@ -0,0 +1,18 @@
+/**
+ * Preflight-scoring config — knob for how heavily the W0198 symbol-existence
+ * preflight contributes to the `code-correctness` dimension.
+ *
+ * The deterministic preflight (`SymbolPreflightReport`) and the LLM rubric
+ * both feed into `code-correctness` per D0010. This config sets the relative
+ * share of the two lanes: `codeCorrectnessWeight: 0.4` means preflight is
+ * 40% of the dimension, rubric 60%.
+ *
+ * Authored in `packages/eval/config/preflight-scoring.ts` via
+ * `definePreflightScoring()`. Loaded lazily by the scoring step; absence
+ * means "use the default" rather than "disable" — disabling the preflight
+ * entirely is a separate axis (no manifest, no resolver wired).
+ *
+ * @see docs/design-docs/two-stage-grader-symbol-preflight.md
+ */
+/** Default preflight share of `code-correctness` when no config file is authored. */
+export const DEFAULT_PREFLIGHT_CODE_CORRECTNESS_WEIGHT = 0.4;

package/dist/_vendor/ailf-core/types/repo-config.d.ts

@@ -51,6 +51,19 @@ export interface RepoExecutionConfig {
     gapAnalysis?: boolean;
     graderReplications?: number;
 }
+/** Grader configuration. */
+export interface RepoGraderConfig {
+    /**
+     * Grader context policy.
+     *
+     * - `"rubric-only"` — grader sees only the rubric template + criteria +
+     *   candidate response.
+     * - `"with-docs"` — canonical reference content for each task is injected
+     *   into the assertion's `rubricPrompt` so the grader has authoritative
+     *   ground truth.
+     */
+    context?: "rubric-only" | "with-docs";
+}
 /** Where the pipeline writes results (replaces `--output-dir`). */
 export interface RepoOutputConfig {
     dir?: string;
@@ -95,6 +108,7 @@ export interface RepoConfig {
     agentic?: RepoAgenticConfig;
     artifacts?: RepoArtifactsConfig;
     execution?: RepoExecutionConfig;
+    grader?: RepoGraderConfig;
     output?: RepoOutputConfig;
     owner?: RepoOwnerConfig;
     publish?: RepoPublishConfig;

package/dist/_vendor/ailf-core/types/symbol-preflight-report.d.ts

@@ -0,0 +1,66 @@
+/**
+ * Symbol-preflight report — the deterministic half of the W0198 two-stage
+ * grader. Built per candidate output by `computePreflight()` in
+ * `packages/eval/src/pipeline/preflight/`; consumed by the scoring bridge
+ * (Phase 5) and emitted as an artifact (Phase 4) so reviewers can audit
+ * which surface a deduction was computed against.
+ *
+ * The shape is a deliberate three-way discriminated union — `exists`,
+ * `missing`, `unresolved` — not a binary plus a `confident` flag. Treating
+ * the lanes as structurally distinct is load-bearing for measurement
+ * integrity: collapsing `unresolved` into `missing` would re-introduce
+ * hallucination in deterministic disguise, which is the whole failure
+ * mode this tier exists to fix.
+ *
+ * Score impact, by lane:
+ * - `exists`     — no deduction; LLM rubric is told the binding exists
+ *                  and instructed not to litigate its existence.
+ * - `missing`    — deterministic deduction in `code-correctness`; LLM
+ *                  does not re-judge the existence question.
+ * - `unresolved` — no deduction; grading for that binding falls through
+ *                  to the W0196 / W0197 LLM-rubric path.
+ *
+ * @see docs/design-docs/two-stage-grader-symbol-preflight.md
+ */
+/**
+ * Reason an `unresolved` finding was emitted. Each value names a specific
+ * resolver-side limit; collapsing these into a single bucket would lose
+ * the measurement-quality signal CI surfaces back to the framework's
+ * authors.
+ */
+export type SymbolPreflightUnresolvedReason = "package-not-installed" | "types-entry-missing" | "parse-failed" | "reexport-hop-unfollowed";
+export type SymbolPreflightFinding = {
+    readonly result: "exists";
+    readonly pkg: string;
+    readonly version: string;
+    readonly binding: string;
+    readonly source: "types" | "runtime";
+} | {
+    readonly result: "missing";
+    readonly pkg: string;
+    readonly version: string;
+    readonly binding: string;
+} | {
+    readonly result: "unresolved";
+    readonly pkg: string;
+    readonly binding: string;
+    readonly reason: SymbolPreflightUnresolvedReason;
+};
+/**
+ * Deduction roll-up — `total = min(missing_count * perMissing, cap)`.
+ * The score the rubric merges in is `1 - total / cap`, computed in the
+ * scoring bridge (Phase 5) so this report stays a pure data artifact.
+ */
+export interface SymbolPreflightDeduction {
+    readonly perMissing: number;
+    readonly cap: number;
+    readonly total: number;
+}
+export interface SymbolPreflightReport {
+    readonly candidate: {
+        readonly taskId: string;
+        readonly testIndex: number;
+    };
+    readonly findings: readonly SymbolPreflightFinding[];
+    readonly deduction: SymbolPreflightDeduction;
+}

package/dist/_vendor/ailf-core/types/symbol-preflight-report.js

@@ -0,0 +1,25 @@
+/**
+ * Symbol-preflight report — the deterministic half of the W0198 two-stage
+ * grader. Built per candidate output by `computePreflight()` in
+ * `packages/eval/src/pipeline/preflight/`; consumed by the scoring bridge
+ * (Phase 5) and emitted as an artifact (Phase 4) so reviewers can audit
+ * which surface a deduction was computed against.
+ *
+ * The shape is a deliberate three-way discriminated union — `exists`,
+ * `missing`, `unresolved` — not a binary plus a `confident` flag. Treating
+ * the lanes as structurally distinct is load-bearing for measurement
+ * integrity: collapsing `unresolved` into `missing` would re-introduce
+ * hallucination in deterministic disguise, which is the whole failure
+ * mode this tier exists to fix.
+ *
+ * Score impact, by lane:
+ * - `exists`     — no deduction; LLM rubric is told the binding exists
+ *                  and instructed not to litigate its existence.
+ * - `missing`    — deterministic deduction in `code-correctness`; LLM
+ *                  does not re-judge the existence question.
+ * - `unresolved` — no deduction; grading for that binding falls through
+ *                  to the W0196 / W0197 LLM-rubric path.
+ *
+ * @see docs/design-docs/two-stage-grader-symbol-preflight.md
+ */
+export {};

package/dist/adapters/api-client/build-request.d.ts

@@ -57,6 +57,7 @@ export interface RemoteConfigSlice {
     datasetOverride?: string;
     projectIdOverride?: string;
     perspectiveOverride?: string;
+    graderContext?: "rubric-only" | "with-docs";
     graderReplications?: number;
     gapAnalysisEnabled?: boolean;
     noRemoteCache?: boolean;

package/dist/adapters/api-client/build-request.js

@@ -124,6 +124,9 @@ export async function buildRemoteRequest(options) {
     if (config.perspectiveOverride)
         raw.perspective = config.perspectiveOverride;
     // Advanced
+    if (config.graderContext) {
+        raw.graderContext = config.graderContext;
+    }
     if (config.graderReplications) {
         raw.graderReplications = config.graderReplications;
     }

package/dist/adapters/config-sources/file-config-adapter.js

@@ -125,6 +125,7 @@ function mapEvalConfigToResolvedConfig(config, rootDir) {
         noCache: config.noCache ?? false,
         noRemoteCache: config.noRemoteCache ?? false,
         graderReplications: config.execution?.graderReplications,
+        graderContext: config.grader?.context,
         urls: config.urls,
         headers: config.agentic?.headers,
         allowedOrigins: config.agentic?.allowedOrigins,

package/dist/adapters/doc-fetchers/sanity-doc-fetcher.d.ts

@@ -42,6 +42,10 @@ export declare class SanityDocFetcher implements DocFetcher {
      * `typesReference` renderer to inline typedoc JSON. Returns `null`
      * on any HTTP/network failure rather than throwing — the renderer
      * surfaces a placeholder so the rest of the context still renders.
+     *
+     * Wrapped in a 30s `AbortSignal.timeout` so a slow CDN can't hang the
+     * eval pipeline indefinitely. Timeouts surface as a single `null`
+     * return like any other fetch failure.
      */
     private fetchAttachmentBody;
     /**