@sanity/ailf 0.4.1 → 1.0.0

package/dist/_vendor/ailf-core/types/index.d.ts

@@ -9,12 +9,23 @@
  * Ports & Adapters migration (Phase 0c). The original file is now a
  * re-export barrel that preserves backward compatibility.
  */
-import type { ConcreteEvalMode as _ConcreteEvalMode, DocumentRef as _DocumentRef, EvalMode as _EvalMode } from "../../ailf-shared/index.d.ts";
+import type { DocumentRef as _DocumentRef, EvalMode as _EvalMode } from "../../ailf-shared/index.d.ts";
 export type { ActualScoreEntry, ComponentResult, TestResult, UrlMetadata, } from "./scoring-input.js";
 export type { DocumentRef } from "../../ailf-shared/index.d.ts";
+export type { StoredBaseline, StoredReport, StoredRun, StoredTaskResult, StoredTrace, SchemaVersioned, } from "./storage-schema.js";
+export { CURRENT_SCHEMA_VERSION, isSchemaVersioned, migrateDocument, } from "./storage-schema.js";
+export type { AssertionRegistration, FixtureResolverRegistration, ModeRegistration, PluginManifest, PluginRegistry, PresetDefinition, ReportSinkRegistration, RubricTemplateRegistration, } from "./plugin-registry.js";
+export { InMemoryPluginRegistry } from "./plugin-registry.js";
+export type { AgentHarnessConfig, AgentHarnessModeConfig, CustomModeConfig, EvalModeConfig, EvalModeType, KnowledgeBaseRef, KnowledgeProbeModeConfig, LiteracyModeConfig, MCPServerConfig, MCPServerModeConfig, ProbeStrategy, SandboxConfig, ToolDef, } from "./eval-mode-config.js";
+export { evalModeType } from "./eval-mode-config.js";
+export type { DependencyEdge, ResolvedFixture, TaskGraph, TaskNode, } from "./task-graph.js";
+export type { VariableDeclaration, VariableEnvelope, VariableProvenance, VariableSource, } from "./variable-envelope.js";
+export type { EvalTrace, ToolCallCategory, ToolCallRecord, TraceEvent, TraceSpan, TraceTokenUsage, } from "./trace.js";
+export type { ArtifactId, Brand, Err, FixtureId, IdValidationError, NewReportId, Ok, ProviderId, PromptId, Result, ResultId, RubricId, RunFingerprint, RunId, SuiteId, TaskId, TaskSlug, TraceId, } from "./branded-ids.js";
+export { err, fixtureId, ok, providerId, resultId, runId, suiteId, taskId, traceId, } from "./branded-ids.js";
+export type { AgentHarnessTaskDefinition, CustomTaskDefinition, GeneralizedAssertionDefinition, GeneralizedDocRef, GeneralizedTaskDefinition, GeneralizedTemplatedAssertion, GeneralizedValueAssertion, IdDocRef, KnowledgeProbeTaskDefinition, LiteracyTaskDefinition, MCPServerTaskDefinition, PathDocRef, PerspectiveDocRef, RubricRef, SlugDocRef, TaskCommonFields, TaskDifficulty, TaskOptions, TaskProviderConfig, TaskStatus, } from "./generalized-task.js";
 type DocumentRef = _DocumentRef;
 type EvalMode = _EvalMode;
-type ConcreteEvalMode = _ConcreteEvalMode;
 /** Aggregated retrieval metrics for a feature area */
 export interface AreaRetrievalMetrics {
     area: string;
@@ -76,8 +87,7 @@ export interface DebugOptions {
     /** Random sample of N tests */
     sample?: number;
 }
-export type { DimensionName, EvalMode, ConcreteEvalMode, } from "../../ailf-shared/index.d.ts";
-export { FULL_MODE_SUBMODES } from "../../ailf-shared/index.d.ts";
+export type { EvalMode } from "../../ailf-shared/index.d.ts";
 /** A classified failure mode with confidence level */
 export interface FailureMode {
     /** How confident we are in this classification */
@@ -263,21 +273,11 @@ export interface GraderReliability {
         totalJudgments: number;
         /** Recommended noise threshold for comparisons (2× max dimension σ) */
         recommendedThreshold: number;
-        /** Per-dimension consistency */
-        perDimension: {
-            taskCompletion: {
-                avgStdDev: number;
-                maxStdDev: number;
-            };
-            codeCorrectness: {
-                avgStdDev: number;
-                maxStdDev: number;
-            };
-            docCoverage: {
-                avgStdDev: number;
-                maxStdDev: number;
-            };
-        };
+        /** Per-dimension consistency (keyed by dimension name) */
+        perDimension: Record<string, {
+            avgStdDev: number;
+            maxStdDev: number;
+        }>;
     };
     /** Grader model used for this evaluation */
     graderModel: string;
@@ -289,21 +289,11 @@ export interface GraderReliability {
         avgSeparation: number;
         /** Total paired comparisons analyzed */
         totalPairs: number;
-        /** Per-dimension sensitivity */
-        perDimension: {
-            taskCompletion: {
-                concordanceRate: number;
-                avgSeparation: number;
-            };
-            codeCorrectness: {
-                concordanceRate: number;
-                avgSeparation: number;
-            };
-            docCoverage: {
-                concordanceRate: number;
-                avgSeparation: number;
-            };
-        };
+        /** Per-dimension sensitivity (keyed by dimension name) */
+        perDimension: Record<string, {
+            concordanceRate: number;
+            avgSeparation: number;
+        }>;
     };
     /** Criterion validity (from human reference grades) — Phase 2 */
     validity?: {
@@ -313,12 +303,8 @@ export interface GraderReliability {
         correlation: number;
         /** Systematic bias (positive = grader scores higher than humans) */
         bias: number;
-        /** Per-dimension correlation with human grades */
-        perDimension: {
-            taskCompletion: number;
-            codeCorrectness: number;
-            docCoverage: number;
-        };
+        /** Per-dimension correlation with human grades (keyed by dimension name) */
+        perDimension: Record<string, number>;
         /** Number of human-graded reference samples */
         sampleSize: number;
         /** Whether the grader passes the MAE threshold */
@@ -420,7 +406,7 @@ export interface PipelineOptions {
 }
 /** A Promptfoo share URL tagged with the evaluation mode that produced it. */
 export interface PromptfooUrlEntry {
-    mode: ConcreteEvalMode;
+    mode: string;
     url: string;
 }
 /**
@@ -694,14 +680,18 @@ export interface ScoreSummary {
      */
     documentManifest?: DocumentRef[];
     /**
-     * Which evaluation modes contributed data to this summary.
+     * Which evaluation variant contributed data to this summary.
+     *
+     * For literacy mode this is a variant name:
      * - `'full'`: both baseline and agentic data present
      * - `'baseline'`: floor + ceiling only (no agentic data)
      * - `'agentic'`: actual only (no floor/ceiling data)
      * - `'observed'`: observed mode data
+     *
+     * For non-literacy modes this is the canonical mode name.
      * Absent in legacy summaries — treat as `'baseline'` for backward compat.
      */
-    evaluationMode?: EvalMode;
+    evaluationMode?: string;
     /** Failure mode analysis (Phase 3a) — diagnostic breakdown of why scores are low */
     failureModes?: FailureModeReport;
     /**
@@ -870,24 +860,12 @@ export interface AreaDelta {
     costDelta?: number;
     /** Overall score delta (experiment − baseline) */
     delta: number;
-    /** Per-dimension deltas */
-    dimensions: {
-        taskCompletion: {
-            baseline: number;
-            experiment: number;
-            delta: number;
-        };
-        codeCorrectness: {
-            baseline: number;
-            experiment: number;
-            delta: number;
-        };
-        docCoverage: {
-            baseline: number;
-            experiment: number;
-            delta: number;
-        };
-    };
+    /** Per-dimension deltas (keyed by dimension name, e.g. 'taskCompletion') */
+    dimensions: Record<string, {
+        baseline: number;
+        experiment: number;
+        delta: number;
+    }>;
     /** Doc Lift delta */
     docLiftDelta: number;
     /** Experiment total score */
@@ -958,11 +936,7 @@ export interface ComparisonReport {
         /** Per-area total score deltas */
         perArea: Record<string, number>;
         /** Per-dimension average deltas (across all areas) */
-        perDimension: {
-            taskCompletion: number;
-            codeCorrectness: number;
-            docCoverage: number;
-        };
+        perDimension: Record<string, number>;
         /** Doc Lift average delta */
         docLift: number;
         /** Cost delta (if both runs have cost data) */
@@ -1018,21 +992,11 @@ export interface ConfidenceAnnotation {
  * Matches the shape produced by pipeline/grader-consistency.ts.
  */
 export interface GraderConsistencyData {
-    /** Per-dimension consistency metrics */
-    perDimension: {
-        codeCorrectness: {
-            avgStdDev: number;
-            maxStdDev: number;
-        };
-        docCoverage: {
-            avgStdDev: number;
-            maxStdDev: number;
-        };
-        taskCompletion: {
-            avgStdDev: number;
-            maxStdDev: number;
-        };
-    };
+    /** Per-dimension consistency metrics (keyed by dimension name) */
+    perDimension: Record<string, {
+        avgStdDev: number;
+        maxStdDev: number;
+    }>;
     /** Recommended noise threshold for comparisons (2× max dimension σ) */
     recommendedThreshold: number;
 }

package/dist/_vendor/ailf-core/types/index.js

@@ -9,7 +9,14 @@
  * Ports & Adapters migration (Phase 0c). The original file is now a
  * re-export barrel that preserves backward compatibility.
  */
-export { FULL_MODE_SUBMODES } from "../../ailf-shared/index.js";
+export { CURRENT_SCHEMA_VERSION, isSchemaVersioned, migrateDocument, } from "./storage-schema.js";
+export { InMemoryPluginRegistry } from "./plugin-registry.js";
+// Note: DocSourceConfig is NOT re-exported here — it conflicts with the
+// existing DocSourceConfig in ports/doc-fetcher.ts. The eval-mode-config
+// version is used internally by LiteracyModeConfig. If consumers need
+// the mode-specific version, they import from "./eval-mode-config.js".
+export { evalModeType } from "./eval-mode-config.js";
+export { err, fixtureId, ok, providerId, resultId, runId, suiteId, taskId, traceId, } from "./branded-ids.js";
 // ---------------------------------------------------------------------------
 // Comparison (Approach 2: structured comparison output)
 // ---------------------------------------------------------------------------

package/dist/_vendor/ailf-core/types/plugin-registry.d.ts

@@ -0,0 +1,202 @@
+/**
+ * Plugin registry — typed extension points for AILF evaluation capabilities.
+ *
+ * Twelve extension points: evaluation modes, providers, assertions,
+ * rubric templates, fixture resolvers, report sinks, dashboard renderers,
+ * prompt templates, scoring profiles, doc fetcher factory, source defs,
+ * and feature defs.
+ *
+ * Presets bundle multiple extensions into a single installable unit.
+ *
+ * @see docs/design-docs/architecture-overhaul/extensibility-plugins.md
+ */
+import type { PromptTemplate } from "../ports/mode-handler.js";
+import type { DocFetcher } from "../ports/doc-fetcher.js";
+import type { SourceEntry } from "../config-helpers.js";
+import type { FeatureRegistry } from "../schemas/pipeline.js";
+/** A registered evaluation mode handler */
+export interface ModeRegistration {
+    /** Unique mode identifier (e.g., "api-contract") */
+    id: string;
+    /** Human-readable label */
+    label: string;
+    /** Valid provider pattern regexes */
+    validProviderPatterns: string[];
+    /** Rubric template IDs available for this mode */
+    rubricTemplateIds: string[];
+    /** Compile function module path (loaded at runtime) */
+    handlerModule: string;
+}
+/** A registered assertion type */
+export interface AssertionRegistration {
+    /** Assertion type name (e.g., "api-contract-match") */
+    type: string;
+    /** Human-readable label */
+    label: string;
+    /** Which modes this assertion is compatible with */
+    compatibleModes: string[];
+    /** Assertion handler module path */
+    handlerModule: string;
+}
+/** A registered rubric template */
+export interface RubricTemplateRegistration {
+    /** Template ID (e.g., "api-accuracy") */
+    id: string;
+    /** Scoring dimension this template contributes to */
+    dimension: string;
+    /** Scale header text */
+    header: string;
+    /** Scale entries */
+    scale: string[];
+    /** Criteria label */
+    criteriaLabel?: string;
+}
+/** A registered fixture resolver */
+export interface FixtureResolverRegistration {
+    /** URI scheme this resolver handles (e.g., "graphql://") */
+    scheme: string;
+    /** Resolver module path */
+    handlerModule: string;
+}
+/** A registered report sink */
+export interface ReportSinkRegistration {
+    /** Sink identifier (e.g., "bigquery", "slack") */
+    id: string;
+    /** Sink module path */
+    handlerModule: string;
+}
+/** Plugin manifest describing a single plugin */
+export interface PluginManifest {
+    /** Plugin name (npm package style) */
+    name: string;
+    /** Semver version */
+    version: string;
+    /** Human-readable description */
+    description?: string;
+    /** Plugin API version this plugin targets */
+    pluginApiVersion: number;
+    /** Minimum AILF version required */
+    minAILFVersion?: string;
+    /** Dependencies on other plugins */
+    requires?: string[];
+}
+/** A preset bundles multiple extensions into an installable unit */
+export interface PresetDefinition {
+    /** Preset name */
+    name: string;
+    /** Plugin manifest */
+    manifest: PluginManifest;
+    /** Evaluation modes to register */
+    modes?: ModeRegistration[];
+    /** Assertion types to register */
+    assertions?: AssertionRegistration[];
+    /** Rubric templates to register */
+    rubricTemplates?: RubricTemplateRegistration[];
+    /** Fixture resolvers to register */
+    fixtureResolvers?: FixtureResolverRegistration[];
+    /** Report sinks to register */
+    reportSinks?: ReportSinkRegistration[];
+    /** Prompt templates keyed by template name (e.g. "with-docs", "agentic") */
+    promptTemplates?: Record<string, PromptTemplate>;
+    /** Scoring profiles mapping profile name to dimension-weight pairs */
+    scoringProfiles?: Record<string, Record<string, number>>;
+    /** Factory function that creates a DocFetcher instance */
+    docFetcher?: () => DocFetcher;
+    /** Documentation source definitions (production, branch, local, etc.) */
+    sourceDefs?: SourceEntry[];
+    /** Product feature registry for coverage tracking */
+    featureDefs?: FeatureRegistry;
+}
+/**
+ * PluginRegistry — central registry for all AILF extensions.
+ *
+ * Plugins register their capabilities here. The pipeline queries the
+ * registry to discover available modes, assertions, templates, etc.
+ */
+export interface PluginRegistry {
+    /** Register a complete preset (bundles multiple extensions) */
+    registerPreset(preset: PresetDefinition): void;
+    /** Register a single evaluation mode */
+    registerMode(mode: ModeRegistration): void;
+    /** Register a single assertion type */
+    registerAssertion(assertion: AssertionRegistration): void;
+    /** Register a rubric template */
+    registerRubricTemplate(template: RubricTemplateRegistration): void;
+    /** Register a fixture resolver */
+    registerFixtureResolver(resolver: FixtureResolverRegistration): void;
+    /** Register a report sink */
+    registerReportSink(sink: ReportSinkRegistration): void;
+    /** Get all registered modes */
+    getModes(): ModeRegistration[];
+    /** Get a mode by ID */
+    getMode(id: string): ModeRegistration | undefined;
+    /** Get all registered assertion types */
+    getAssertions(): AssertionRegistration[];
+    /** Get all registered rubric templates */
+    getRubricTemplates(): RubricTemplateRegistration[];
+    /** Get all registered fixture resolvers */
+    getFixtureResolvers(): FixtureResolverRegistration[];
+    /** Get all registered report sinks */
+    getReportSinks(): ReportSinkRegistration[];
+    /** Register prompt templates (merged with existing) */
+    registerPromptTemplates(templates: Record<string, PromptTemplate>): void;
+    /** Get all registered prompt templates */
+    getPromptTemplates(): Record<string, PromptTemplate>;
+    /** Register scoring profiles (merged with existing) */
+    registerScoringProfiles(profiles: Record<string, Record<string, number>>): void;
+    /** Get all registered scoring profiles */
+    getScoringProfiles(): Record<string, Record<string, number>>;
+    /** Register a doc fetcher factory (last-write-wins) */
+    registerDocFetcherFactory(factory: () => DocFetcher): void;
+    /** Get the registered doc fetcher factory, if any */
+    getDocFetcherFactory(): (() => DocFetcher) | undefined;
+    /** Register source definitions (concatenated with existing) */
+    registerSourceDefs(sources: SourceEntry[]): void;
+    /** Get all registered source definitions */
+    getSourceDefs(): SourceEntry[];
+    /** Register a feature registry (last-write-wins) */
+    registerFeatureDefs(features: FeatureRegistry): void;
+    /** Get the registered feature registry, if any */
+    getFeatureDefs(): FeatureRegistry | undefined;
+    /** Get all registered presets */
+    getPresets(): PresetDefinition[];
+}
+/**
+ * In-memory plugin registry implementation.
+ */
+export declare class InMemoryPluginRegistry implements PluginRegistry {
+    private readonly modes;
+    private readonly assertions_;
+    private readonly rubricTemplates_;
+    private readonly fixtureResolvers_;
+    private readonly reportSinks_;
+    private readonly presets_;
+    private promptTemplates_;
+    private scoringProfiles_;
+    private docFetcherFactory_;
+    private sourceDefs_;
+    private featureDefs_;
+    registerPreset(preset: PresetDefinition): void;
+    registerMode(mode: ModeRegistration): void;
+    registerAssertion(assertion: AssertionRegistration): void;
+    registerRubricTemplate(template: RubricTemplateRegistration): void;
+    registerFixtureResolver(resolver: FixtureResolverRegistration): void;
+    registerReportSink(sink: ReportSinkRegistration): void;
+    getModes(): ModeRegistration[];
+    getMode(id: string): ModeRegistration | undefined;
+    getAssertions(): AssertionRegistration[];
+    getRubricTemplates(): RubricTemplateRegistration[];
+    getFixtureResolvers(): FixtureResolverRegistration[];
+    getReportSinks(): ReportSinkRegistration[];
+    getPresets(): PresetDefinition[];
+    registerPromptTemplates(templates: Record<string, PromptTemplate>): void;
+    getPromptTemplates(): Record<string, PromptTemplate>;
+    registerScoringProfiles(profiles: Record<string, Record<string, number>>): void;
+    getScoringProfiles(): Record<string, Record<string, number>>;
+    registerDocFetcherFactory(factory: () => DocFetcher): void;
+    getDocFetcherFactory(): (() => DocFetcher) | undefined;
+    registerSourceDefs(sources: SourceEntry[]): void;
+    getSourceDefs(): SourceEntry[];
+    registerFeatureDefs(features: FeatureRegistry): void;
+    getFeatureDefs(): FeatureRegistry | undefined;
+}

package/dist/_vendor/ailf-core/types/plugin-registry.js

@@ -0,0 +1,132 @@
+/**
+ * Plugin registry — typed extension points for AILF evaluation capabilities.
+ *
+ * Twelve extension points: evaluation modes, providers, assertions,
+ * rubric templates, fixture resolvers, report sinks, dashboard renderers,
+ * prompt templates, scoring profiles, doc fetcher factory, source defs,
+ * and feature defs.
+ *
+ * Presets bundle multiple extensions into a single installable unit.
+ *
+ * @see docs/design-docs/architecture-overhaul/extensibility-plugins.md
+ */
+/**
+ * In-memory plugin registry implementation.
+ */
+export class InMemoryPluginRegistry {
+    modes = new Map();
+    assertions_ = new Map();
+    rubricTemplates_ = new Map();
+    fixtureResolvers_ = new Map();
+    reportSinks_ = new Map();
+    presets_ = new Map();
+    promptTemplates_ = {};
+    scoringProfiles_ = {};
+    docFetcherFactory_;
+    sourceDefs_ = [];
+    featureDefs_;
+    registerPreset(preset) {
+        this.presets_.set(preset.name, preset);
+        if (preset.modes) {
+            for (const mode of preset.modes)
+                this.registerMode(mode);
+        }
+        if (preset.assertions) {
+            for (const a of preset.assertions)
+                this.registerAssertion(a);
+        }
+        if (preset.rubricTemplates) {
+            for (const t of preset.rubricTemplates)
+                this.registerRubricTemplate(t);
+        }
+        if (preset.fixtureResolvers) {
+            for (const r of preset.fixtureResolvers)
+                this.registerFixtureResolver(r);
+        }
+        if (preset.reportSinks) {
+            for (const s of preset.reportSinks)
+                this.registerReportSink(s);
+        }
+        if (preset.promptTemplates) {
+            this.registerPromptTemplates(preset.promptTemplates);
+        }
+        if (preset.scoringProfiles) {
+            this.registerScoringProfiles(preset.scoringProfiles);
+        }
+        if (preset.docFetcher) {
+            this.registerDocFetcherFactory(preset.docFetcher);
+        }
+        if (preset.sourceDefs) {
+            this.registerSourceDefs(preset.sourceDefs);
+        }
+        if (preset.featureDefs) {
+            this.registerFeatureDefs(preset.featureDefs);
+        }
+    }
+    registerMode(mode) {
+        this.modes.set(mode.id, mode);
+    }
+    registerAssertion(assertion) {
+        this.assertions_.set(assertion.type, assertion);
+    }
+    registerRubricTemplate(template) {
+        this.rubricTemplates_.set(template.id, template);
+    }
+    registerFixtureResolver(resolver) {
+        this.fixtureResolvers_.set(resolver.scheme, resolver);
+    }
+    registerReportSink(sink) {
+        this.reportSinks_.set(sink.id, sink);
+    }
+    getModes() {
+        return [...this.modes.values()];
+    }
+    getMode(id) {
+        return this.modes.get(id);
+    }
+    getAssertions() {
+        return [...this.assertions_.values()];
+    }
+    getRubricTemplates() {
+        return [...this.rubricTemplates_.values()];
+    }
+    getFixtureResolvers() {
+        return [...this.fixtureResolvers_.values()];
+    }
+    getReportSinks() {
+        return [...this.reportSinks_.values()];
+    }
+    getPresets() {
+        return [...this.presets_.values()];
+    }
+    registerPromptTemplates(templates) {
+        Object.assign(this.promptTemplates_, templates);
+    }
+    getPromptTemplates() {
+        return this.promptTemplates_;
+    }
+    registerScoringProfiles(profiles) {
+        Object.assign(this.scoringProfiles_, profiles);
+    }
+    getScoringProfiles() {
+        return this.scoringProfiles_;
+    }
+    registerDocFetcherFactory(factory) {
+        this.docFetcherFactory_ = factory;
+    }
+    getDocFetcherFactory() {
+        return this.docFetcherFactory_;
+    }
+    registerSourceDefs(sources) {
+        this.sourceDefs_ = [...this.sourceDefs_, ...sources];
+    }
+    getSourceDefs() {
+        return this.sourceDefs_;
+    }
+    registerFeatureDefs(features) {
+        this.featureDefs_ = features;
+    }
+    getFeatureDefs() {
+        return this.featureDefs_;
+    }
+}