@sanity/ailf 0.5.0 → 2.0.0

package/dist/_vendor/ailf-core/types/index.d.ts

@@ -9,12 +9,23 @@
  * Ports & Adapters migration (Phase 0c). The original file is now a
  * re-export barrel that preserves backward compatibility.
  */
-import type { ConcreteEvalMode as _ConcreteEvalMode, DocumentRef as _DocumentRef, EvalMode as _EvalMode } from "../../ailf-shared/index.d.ts";
+import type { DocumentRef as _DocumentRef, EvalMode as _EvalMode } from "../../ailf-shared/index.d.ts";
 export type { ActualScoreEntry, ComponentResult, TestResult, UrlMetadata, } from "./scoring-input.js";
 export type { DocumentRef } from "../../ailf-shared/index.d.ts";
+export type { StoredBaseline, StoredReport, StoredRun, StoredTaskResult, StoredTrace, SchemaVersioned, } from "./storage-schema.js";
+export { CURRENT_SCHEMA_VERSION, isSchemaVersioned, migrateDocument, } from "./storage-schema.js";
+export type { AssertionRegistration, FixtureResolverRegistration, ModeBase, ModeRegistration, PluginManifest, PluginRegistry, PresetDefinition, ReportSinkRegistration, RubricTemplateRegistration, } from "./plugin-registry.js";
+export { InMemoryPluginRegistry } from "./plugin-registry.js";
+export type { AgentHarnessConfig, AgentHarnessModeConfig, CustomModeConfig, EvalModeConfig, EvalModeType, KnowledgeBaseRef, KnowledgeProbeModeConfig, LiteracyModeConfig, MCPServerConfig, MCPServerModeConfig, ProbeStrategy, SandboxConfig, ToolDef, } from "./eval-mode-config.js";
+export { evalModeType } from "./eval-mode-config.js";
+export type { DependencyEdge, ResolvedFixture, TaskGraph, TaskNode, } from "./task-graph.js";
+export type { VariableDeclaration, VariableEnvelope, VariableProvenance, VariableSource, } from "./variable-envelope.js";
+export type { EvalTrace, ToolCallCategory, ToolCallRecord, TraceEvent, TraceSpan, TraceTokenUsage, } from "./trace.js";
+export type { ArtifactId, Brand, Err, FixtureId, IdValidationError, NewReportId, Ok, ProviderId, PromptId, Result, ResultId, RubricId, RunFingerprint, RunId, SuiteId, TaskId, TaskSlug, TraceId, } from "./branded-ids.js";
+export { err, fixtureId, ok, providerId, resultId, runId, suiteId, taskId, traceId, } from "./branded-ids.js";
+export type { AgentHarnessTaskDefinition, CustomTaskDefinition, GeneralizedAssertionDefinition, GeneralizedDocRef, GeneralizedTaskDefinition, GeneralizedTemplatedAssertion, GeneralizedValueAssertion, IdDocRef, KnowledgeProbeTaskDefinition, LiteracyTaskDefinition, MCPServerTaskDefinition, PathDocRef, PerspectiveDocRef, RubricRef, SlugDocRef, TaskCommonFields, TaskDifficulty, TaskOptions, TaskProviderConfig, TaskStatus, } from "./generalized-task.js";
 type DocumentRef = _DocumentRef;
 type EvalMode = _EvalMode;
-type ConcreteEvalMode = _ConcreteEvalMode;
 /** Aggregated retrieval metrics for a feature area */
 export interface AreaRetrievalMetrics {
     area: string;
@@ -76,8 +87,7 @@ export interface DebugOptions {
     /** Random sample of N tests */
     sample?: number;
 }
-export type { DimensionName, EvalMode, ConcreteEvalMode, } from "../../ailf-shared/index.d.ts";
-export { FULL_MODE_SUBMODES } from "../../ailf-shared/index.d.ts";
+export type { EvalMode } from "../../ailf-shared/index.d.ts";
 /** A classified failure mode with confidence level */
 export interface FailureMode {
     /** How confident we are in this classification */
@@ -263,21 +273,11 @@ export interface GraderReliability {
         totalJudgments: number;
         /** Recommended noise threshold for comparisons (2× max dimension σ) */
         recommendedThreshold: number;
-        /** Per-dimension consistency */
-        perDimension: {
-            taskCompletion: {
-                avgStdDev: number;
-                maxStdDev: number;
-            };
-            codeCorrectness: {
-                avgStdDev: number;
-                maxStdDev: number;
-            };
-            docCoverage: {
-                avgStdDev: number;
-                maxStdDev: number;
-            };
-        };
+        /** Per-dimension consistency (keyed by dimension name) */
+        perDimension: Record<string, {
+            avgStdDev: number;
+            maxStdDev: number;
+        }>;
     };
     /** Grader model used for this evaluation */
     graderModel: string;
@@ -289,21 +289,11 @@ export interface GraderReliability {
         avgSeparation: number;
         /** Total paired comparisons analyzed */
         totalPairs: number;
-        /** Per-dimension sensitivity */
-        perDimension: {
-            taskCompletion: {
-                concordanceRate: number;
-                avgSeparation: number;
-            };
-            codeCorrectness: {
-                concordanceRate: number;
-                avgSeparation: number;
-            };
-            docCoverage: {
-                concordanceRate: number;
-                avgSeparation: number;
-            };
-        };
+        /** Per-dimension sensitivity (keyed by dimension name) */
+        perDimension: Record<string, {
+            concordanceRate: number;
+            avgSeparation: number;
+        }>;
     };
     /** Criterion validity (from human reference grades) — Phase 2 */
     validity?: {
@@ -313,12 +303,8 @@ export interface GraderReliability {
         correlation: number;
         /** Systematic bias (positive = grader scores higher than humans) */
         bias: number;
-        /** Per-dimension correlation with human grades */
-        perDimension: {
-            taskCompletion: number;
-            codeCorrectness: number;
-            docCoverage: number;
-        };
+        /** Per-dimension correlation with human grades (keyed by dimension name) */
+        perDimension: Record<string, number>;
         /** Number of human-graded reference samples */
         sampleSize: number;
         /** Whether the grader passes the MAE threshold */
@@ -420,7 +406,7 @@ export interface PipelineOptions {
 }
 /** A Promptfoo share URL tagged with the evaluation mode that produced it. */
 export interface PromptfooUrlEntry {
-    mode: ConcreteEvalMode;
+    mode: string;
     url: string;
 }
 /**
@@ -589,8 +575,6 @@ export interface ProductFeature {
     sections: string[];
     /** Coverage status */
     status: "covered" | "out-of-scope" | "planned" | "uncovered";
-    /** Number of evaluation tasks (if covered) */
-    taskCount?: number;
 }
 /** Full classification of a content release for evaluation */
 export interface ReleaseClassification {
@@ -694,14 +678,18 @@ export interface ScoreSummary {
      */
     documentManifest?: DocumentRef[];
     /**
-     * Which evaluation modes contributed data to this summary.
+     * Which evaluation variant contributed data to this summary.
+     *
+     * For literacy mode this is a variant name:
      * - `'full'`: both baseline and agentic data present
      * - `'baseline'`: floor + ceiling only (no agentic data)
      * - `'agentic'`: actual only (no floor/ceiling data)
      * - `'observed'`: observed mode data
+     *
+     * For non-literacy modes this is the canonical mode name.
      * Absent in legacy summaries — treat as `'baseline'` for backward compat.
      */
-    evaluationMode?: EvalMode;
+    evaluationMode?: string;
     /** Failure mode analysis (Phase 3a) — diagnostic breakdown of why scores are low */
     failureModes?: FailureModeReport;
     /**
@@ -870,24 +858,12 @@ export interface AreaDelta {
     costDelta?: number;
     /** Overall score delta (experiment − baseline) */
     delta: number;
-    /** Per-dimension deltas */
-    dimensions: {
-        taskCompletion: {
-            baseline: number;
-            experiment: number;
-            delta: number;
-        };
-        codeCorrectness: {
-            baseline: number;
-            experiment: number;
-            delta: number;
-        };
-        docCoverage: {
-            baseline: number;
-            experiment: number;
-            delta: number;
-        };
-    };
+    /** Per-dimension deltas (keyed by dimension name, e.g. 'taskCompletion') */
+    dimensions: Record<string, {
+        baseline: number;
+        experiment: number;
+        delta: number;
+    }>;
     /** Doc Lift delta */
     docLiftDelta: number;
     /** Experiment total score */
@@ -958,11 +934,7 @@ export interface ComparisonReport {
         /** Per-area total score deltas */
         perArea: Record<string, number>;
         /** Per-dimension average deltas (across all areas) */
-        perDimension: {
-            taskCompletion: number;
-            codeCorrectness: number;
-            docCoverage: number;
-        };
+        perDimension: Record<string, number>;
         /** Doc Lift average delta */
         docLift: number;
         /** Cost delta (if both runs have cost data) */
@@ -1018,21 +990,11 @@ export interface ConfidenceAnnotation {
  * Matches the shape produced by pipeline/grader-consistency.ts.
  */
 export interface GraderConsistencyData {
-    /** Per-dimension consistency metrics */
-    perDimension: {
-        codeCorrectness: {
-            avgStdDev: number;
-            maxStdDev: number;
-        };
-        docCoverage: {
-            avgStdDev: number;
-            maxStdDev: number;
-        };
-        taskCompletion: {
-            avgStdDev: number;
-            maxStdDev: number;
-        };
-    };
+    /** Per-dimension consistency metrics (keyed by dimension name) */
+    perDimension: Record<string, {
+        avgStdDev: number;
+        maxStdDev: number;
+    }>;
     /** Recommended noise threshold for comparisons (2× max dimension σ) */
     recommendedThreshold: number;
 }

package/dist/_vendor/ailf-core/types/index.js

@@ -9,7 +9,14 @@
  * Ports & Adapters migration (Phase 0c). The original file is now a
  * re-export barrel that preserves backward compatibility.
  */
-export { FULL_MODE_SUBMODES } from "../../ailf-shared/index.js";
+export { CURRENT_SCHEMA_VERSION, isSchemaVersioned, migrateDocument, } from "./storage-schema.js";
+export { InMemoryPluginRegistry } from "./plugin-registry.js";
+// Note: DocSourceConfig is NOT re-exported here — it conflicts with the
+// existing DocSourceConfig in ports/doc-fetcher.ts. The eval-mode-config
+// version is used internally by LiteracyModeConfig. If consumers need
+// the mode-specific version, they import from "./eval-mode-config.js".
+export { evalModeType } from "./eval-mode-config.js";
+export { err, fixtureId, ok, providerId, resultId, runId, suiteId, taskId, traceId, } from "./branded-ids.js";
 // ---------------------------------------------------------------------------
 // Comparison (Approach 2: structured comparison output)
 // ---------------------------------------------------------------------------

package/dist/_vendor/ailf-core/types/plugin-registry.d.ts

@@ -0,0 +1,257 @@
+/**
+ * Plugin registry — typed extension points for AILF evaluation capabilities.
+ *
+ * Three-tier architecture:
+ * - **Mode bases** define evaluation methodology (rubrics, scoring, prompts)
+ * - **Domain presets** target a mode base and add domain config (sources,
+ *   features, doc fetcher)
+ * - **Framework assertions** are generic evaluation primitives available to
+ *   all modes
+ *
+ * @see docs/design-docs/architecture-overhaul/extensibility-plugins.md
+ */
+import type { PromptTemplate } from "../ports/mode-handler.js";
+import type { DocFetcher } from "../ports/doc-fetcher.js";
+import type { SourceEntry } from "../config-helpers.js";
+import type { FeatureRegistry } from "../schemas/pipeline.js";
+/** A registered evaluation mode handler */
+export interface ModeRegistration {
+    /** Unique mode identifier (e.g., "api-contract") */
+    id: string;
+    /** Human-readable label */
+    label: string;
+    /** Valid provider pattern regexes */
+    validProviderPatterns: string[];
+    /** Rubric template IDs available for this mode */
+    rubricTemplateIds: string[];
+    /** Compile function module path (loaded at runtime) */
+    handlerModule: string;
+}
+/** A registered assertion type */
+export interface AssertionRegistration {
+    /** Assertion type name (e.g., "api-contract-match") */
+    type: string;
+    /** Human-readable label */
+    label: string;
+    /**
+     * Which modes this assertion is compatible with.
+     * When omitted, the assertion is compatible with all modes.
+     * When specified, acts as a whitelist of mode IDs.
+     */
+    compatibleModes?: string[];
+    /** Assertion handler module path */
+    handlerModule: string;
+}
+/** A registered rubric template */
+export interface RubricTemplateRegistration {
+    /** Template ID (e.g., "api-accuracy") */
+    id: string;
+    /** Scoring dimension this template contributes to */
+    dimension: string;
+    /** Scale header text */
+    header: string;
+    /** Scale entries */
+    scale: string[];
+    /** Criteria label */
+    criteriaLabel?: string;
+}
+/** A registered fixture resolver */
+export interface FixtureResolverRegistration {
+    /** URI scheme this resolver handles (e.g., "graphql://") */
+    scheme: string;
+    /** Resolver module path */
+    handlerModule: string;
+}
+/** A registered report sink */
+export interface ReportSinkRegistration {
+    /** Sink identifier (e.g., "bigquery", "slack") */
+    id: string;
+    /** Sink module path */
+    handlerModule: string;
+}
+/**
+ * ModeBase — shared evaluation methodology for a mode.
+ *
+ * Defines HOW you evaluate (rubrics, scoring, prompts) independently of
+ * WHAT you're evaluating (sources, features, docs). Multiple domain presets
+ * can target the same mode base and inherit its defaults.
+ *
+ * Example: the "literacy" mode base defines rubric templates for
+ * task-completion, code-correctness, and doc-coverage. Both a Sanity docs
+ * preset and an external docs preset can target "literacy" and inherit
+ * these rubrics without redefining them.
+ */
+export interface ModeBase {
+    /** The mode registration (handler, provider patterns, rubric template IDs) */
+    mode: ModeRegistration;
+    /** Default rubric templates for this mode */
+    rubricTemplates?: RubricTemplateRegistration[];
+    /** Default scoring profiles for this mode (profile name → dimension weights) */
+    scoringProfiles?: Record<string, Record<string, number>>;
+    /** Default prompt templates for this mode (template name → template) */
+    promptTemplates?: Record<string, PromptTemplate>;
+    /** Mode-specific assertion types (beyond framework builtins) */
+    assertions?: AssertionRegistration[];
+}
+/** Plugin manifest describing a single plugin */
+export interface PluginManifest {
+    /** Plugin name (npm package style) */
+    name: string;
+    /** Semver version */
+    version: string;
+    /** Human-readable description */
+    description?: string;
+    /** Plugin API version this plugin targets */
+    pluginApiVersion: number;
+    /** Minimum AILF version required */
+    minAILFVersion?: string;
+    /** Dependencies on other plugins */
+    requires?: string[];
+}
+/**
+ * A domain preset targets a mode base and adds domain-specific configuration.
+ *
+ * The preset inherits evaluation methodology (rubrics, scoring, prompts) from
+ * its mode base. It can optionally override any inherited values.
+ */
+export interface PresetDefinition {
+    /** Preset name (unique identifier) */
+    name: string;
+    /** Plugin manifest */
+    manifest: PluginManifest;
+    /**
+     * Lifecycle status — mirrors task status semantics.
+     *   active:   registered and used in evaluations (default)
+     *   draft:    registered but skipped unless explicitly targeted
+     *   paused:   registered but skipped (can be resumed)
+     *   archived: not registered
+     */
+    status?: "active" | "archived" | "draft" | "paused";
+    /**
+     * Which mode this preset targets (by mode ID).
+     * Links to a registered ModeBase. The preset inherits rubrics,
+     * scoring profiles, and prompt templates from the base.
+     */
+    mode: string;
+    /** Fixture resolvers */
+    fixtureResolvers?: FixtureResolverRegistration[];
+    /** Report sinks */
+    reportSinks?: ReportSinkRegistration[];
+    /** Factory function that creates a DocFetcher instance */
+    docFetcher?: () => DocFetcher;
+    /** Documentation source definitions (production, branch, local, etc.) */
+    sourceDefs?: SourceEntry[];
+    /** Product feature registry for coverage tracking */
+    featureDefs?: FeatureRegistry;
+    /** Override rubric templates (merged by ID with mode base) */
+    rubricTemplates?: RubricTemplateRegistration[];
+    /** Override scoring profiles (merged by name with mode base) */
+    scoringProfiles?: Record<string, Record<string, number>>;
+    /** Override prompt templates (merged by name with mode base) */
+    promptTemplates?: Record<string, PromptTemplate>;
+    /** Additional mode-specific assertions */
+    assertions?: AssertionRegistration[];
+}
+/**
+ * PluginRegistry — central registry for all AILF extensions.
+ *
+ * Plugins register their capabilities here. The pipeline queries the
+ * registry to discover available modes, assertions, templates, etc.
+ */
+export interface PluginRegistry {
+    /** Register a complete preset (bundles multiple extensions) */
+    registerPreset(preset: PresetDefinition): void;
+    /** Register a single evaluation mode */
+    registerMode(mode: ModeRegistration): void;
+    /** Register a single assertion type */
+    registerAssertion(assertion: AssertionRegistration): void;
+    /** Register a rubric template */
+    registerRubricTemplate(template: RubricTemplateRegistration): void;
+    /** Register a fixture resolver */
+    registerFixtureResolver(resolver: FixtureResolverRegistration): void;
+    /** Register a report sink */
+    registerReportSink(sink: ReportSinkRegistration): void;
+    /** Get all registered modes */
+    getModes(): ModeRegistration[];
+    /** Get a mode by ID */
+    getMode(id: string): ModeRegistration | undefined;
+    /** Get all registered assertion types */
+    getAssertions(): AssertionRegistration[];
+    /** Get all registered rubric templates */
+    getRubricTemplates(): RubricTemplateRegistration[];
+    /** Get all registered fixture resolvers */
+    getFixtureResolvers(): FixtureResolverRegistration[];
+    /** Get all registered report sinks */
+    getReportSinks(): ReportSinkRegistration[];
+    /** Register prompt templates (merged with existing) */
+    registerPromptTemplates(templates: Record<string, PromptTemplate>): void;
+    /** Get all registered prompt templates */
+    getPromptTemplates(): Record<string, PromptTemplate>;
+    /** Register scoring profiles (merged with existing) */
+    registerScoringProfiles(profiles: Record<string, Record<string, number>>): void;
+    /** Get all registered scoring profiles */
+    getScoringProfiles(): Record<string, Record<string, number>>;
+    /** Register a doc fetcher factory (last-write-wins) */
+    registerDocFetcherFactory(factory: () => DocFetcher): void;
+    /** Get the registered doc fetcher factory, if any */
+    getDocFetcherFactory(): (() => DocFetcher) | undefined;
+    /** Register source definitions (concatenated with existing) */
+    registerSourceDefs(sources: SourceEntry[]): void;
+    /** Get all registered source definitions */
+    getSourceDefs(): SourceEntry[];
+    /** Register a feature registry (merged by feature ID with existing) */
+    registerFeatureDefs(features: FeatureRegistry): void;
+    /** Get the registered feature registry, if any */
+    getFeatureDefs(): FeatureRegistry | undefined;
+    /** Register a mode base (evaluation methodology) */
+    registerModeBase(base: ModeBase): void;
+    /** Get a mode base by mode ID */
+    getModeBase(modeId: string): ModeBase | undefined;
+    /** Get all registered mode bases */
+    getModeBases(): ModeBase[];
+    /** Get all registered presets */
+    getPresets(): PresetDefinition[];
+}
+/**
+ * In-memory plugin registry implementation.
+ */
+export declare class InMemoryPluginRegistry implements PluginRegistry {
+    private readonly modes;
+    private readonly assertions_;
+    private readonly rubricTemplates_;
+    private readonly fixtureResolvers_;
+    private readonly reportSinks_;
+    private readonly modeBases_;
+    private readonly presets_;
+    private promptTemplates_;
+    private scoringProfiles_;
+    private docFetcherFactory_;
+    private sourceDefs_;
+    private featureDefs_;
+    registerPreset(preset: PresetDefinition): void;
+    registerMode(mode: ModeRegistration): void;
+    registerAssertion(assertion: AssertionRegistration): void;
+    registerRubricTemplate(template: RubricTemplateRegistration): void;
+    registerFixtureResolver(resolver: FixtureResolverRegistration): void;
+    registerReportSink(sink: ReportSinkRegistration): void;
+    getModes(): ModeRegistration[];
+    getMode(id: string): ModeRegistration | undefined;
+    getAssertions(): AssertionRegistration[];
+    getRubricTemplates(): RubricTemplateRegistration[];
+    getFixtureResolvers(): FixtureResolverRegistration[];
+    getReportSinks(): ReportSinkRegistration[];
+    getPresets(): PresetDefinition[];
+    registerPromptTemplates(templates: Record<string, PromptTemplate>): void;
+    getPromptTemplates(): Record<string, PromptTemplate>;
+    registerScoringProfiles(profiles: Record<string, Record<string, number>>): void;
+    getScoringProfiles(): Record<string, Record<string, number>>;
+    registerDocFetcherFactory(factory: () => DocFetcher): void;
+    getDocFetcherFactory(): (() => DocFetcher) | undefined;
+    registerSourceDefs(sources: SourceEntry[]): void;
+    getSourceDefs(): SourceEntry[];
+    registerFeatureDefs(features: FeatureRegistry): void;
+    getFeatureDefs(): FeatureRegistry | undefined;
+    registerModeBase(base: ModeBase): void;
+    getModeBase(modeId: string): ModeBase | undefined;
+    getModeBases(): ModeBase[];
+}

package/dist/_vendor/ailf-core/types/plugin-registry.js

@@ -0,0 +1,185 @@
+/**
+ * Plugin registry — typed extension points for AILF evaluation capabilities.
+ *
+ * Three-tier architecture:
+ * - **Mode bases** define evaluation methodology (rubrics, scoring, prompts)
+ * - **Domain presets** target a mode base and add domain config (sources,
+ *   features, doc fetcher)
+ * - **Framework assertions** are generic evaluation primitives available to
+ *   all modes
+ *
+ * @see docs/design-docs/architecture-overhaul/extensibility-plugins.md
+ */
+/**
+ * In-memory plugin registry implementation.
+ */
+export class InMemoryPluginRegistry {
+    modes = new Map();
+    assertions_ = new Map();
+    rubricTemplates_ = new Map();
+    fixtureResolvers_ = new Map();
+    reportSinks_ = new Map();
+    modeBases_ = new Map();
+    presets_ = new Map();
+    promptTemplates_ = {};
+    scoringProfiles_ = {};
+    docFetcherFactory_;
+    sourceDefs_ = [];
+    featureDefs_;
+    registerPreset(preset) {
+        // Skip archived presets entirely
+        if (preset.status === "archived")
+            return;
+        // Store draft/paused presets in the map (for later activation via
+        // --preset flag) but skip all side-effect registrations. This prevents
+        // a draft preset from silently overwriting the doc fetcher, merging
+        // scoring profiles, etc.
+        this.presets_.set(preset.name, preset);
+        if (preset.status === "draft" || preset.status === "paused")
+            return;
+        // Resolve mode base defaults
+        const base = this.modeBases_.get(preset.mode);
+        if (!base) {
+            throw new Error(`Preset "${preset.name}" targets mode "${preset.mode}" ` +
+                `but no mode base is registered for it. ` +
+                `Available mode bases: ${[...this.modeBases_.keys()].join(", ") || "(none)"}`);
+        }
+        // Mode is already registered by registerModeBase() — no need to re-register.
+        // Merge rubric templates: base defaults + preset overrides (by ID)
+        const baseRubrics = new Map((base.rubricTemplates ?? []).map((r) => [r.id, r]));
+        for (const r of preset.rubricTemplates ?? []) {
+            baseRubrics.set(r.id, r);
+        }
+        for (const r of baseRubrics.values()) {
+            this.registerRubricTemplate(r);
+        }
+        // Merge scoring profiles: base defaults + preset overrides (by name)
+        const profiles = {
+            ...base.scoringProfiles,
+            ...preset.scoringProfiles,
+        };
+        if (Object.keys(profiles).length > 0) {
+            this.registerScoringProfiles(profiles);
+        }
+        // Merge prompt templates: base defaults + preset overrides (by name)
+        const prompts = {
+            ...base.promptTemplates,
+            ...preset.promptTemplates,
+        };
+        if (Object.keys(prompts).length > 0) {
+            this.registerPromptTemplates(prompts);
+        }
+        // Merge assertions: base + preset (preset overrides by type)
+        for (const a of base.assertions ?? [])
+            this.registerAssertion(a);
+        if (preset.assertions) {
+            for (const a of preset.assertions)
+                this.registerAssertion(a);
+        }
+        // Register domain-specific fields
+        if (preset.fixtureResolvers) {
+            for (const r of preset.fixtureResolvers)
+                this.registerFixtureResolver(r);
+        }
+        if (preset.reportSinks) {
+            for (const s of preset.reportSinks)
+                this.registerReportSink(s);
+        }
+        if (preset.docFetcher) {
+            this.registerDocFetcherFactory(preset.docFetcher);
+        }
+        if (preset.sourceDefs) {
+            this.registerSourceDefs(preset.sourceDefs);
+        }
+        if (preset.featureDefs) {
+            this.registerFeatureDefs(preset.featureDefs);
+        }
+    }
+    registerMode(mode) {
+        this.modes.set(mode.id, mode);
+    }
+    registerAssertion(assertion) {
+        this.assertions_.set(assertion.type, assertion);
+    }
+    registerRubricTemplate(template) {
+        this.rubricTemplates_.set(template.id, template);
+    }
+    registerFixtureResolver(resolver) {
+        this.fixtureResolvers_.set(resolver.scheme, resolver);
+    }
+    registerReportSink(sink) {
+        this.reportSinks_.set(sink.id, sink);
+    }
+    getModes() {
+        return [...this.modes.values()];
+    }
+    getMode(id) {
+        return this.modes.get(id);
+    }
+    getAssertions() {
+        return [...this.assertions_.values()];
+    }
+    getRubricTemplates() {
+        return [...this.rubricTemplates_.values()];
+    }
+    getFixtureResolvers() {
+        return [...this.fixtureResolvers_.values()];
+    }
+    getReportSinks() {
+        return [...this.reportSinks_.values()];
+    }
+    getPresets() {
+        return [...this.presets_.values()];
+    }
+    registerPromptTemplates(templates) {
+        Object.assign(this.promptTemplates_, templates);
+    }
+    getPromptTemplates() {
+        return this.promptTemplates_;
+    }
+    registerScoringProfiles(profiles) {
+        Object.assign(this.scoringProfiles_, profiles);
+    }
+    getScoringProfiles() {
+        return this.scoringProfiles_;
+    }
+    registerDocFetcherFactory(factory) {
+        this.docFetcherFactory_ = factory;
+    }
+    getDocFetcherFactory() {
+        return this.docFetcherFactory_;
+    }
+    registerSourceDefs(sources) {
+        this.sourceDefs_ = [...this.sourceDefs_, ...sources];
+    }
+    getSourceDefs() {
+        return this.sourceDefs_;
+    }
+    registerFeatureDefs(features) {
+        if (!this.featureDefs_) {
+            this.featureDefs_ = features;
+            return;
+        }
+        // Merge by feature ID: new features override existing on ID collision,
+        // existing features not in new set are preserved.
+        const merged = new Map(this.featureDefs_.features.map((f) => [f.id, f]));
+        for (const f of features.features) {
+            merged.set(f.id, f);
+        }
+        this.featureDefs_ = { features: [...merged.values()] };
+    }
+    getFeatureDefs() {
+        return this.featureDefs_;
+    }
+    registerModeBase(base) {
+        this.modeBases_.set(base.mode.id, base);
+        // Also register the mode itself so getMode() works
+        this.registerMode(base.mode);
+    }
+    getModeBase(modeId) {
+        return this.modeBases_.get(modeId);
+    }
+    getModeBases() {
+        return [...this.modeBases_.values()];
+    }
+}