@gleanwork/mcp-server-tester 1.1.0 → 2.0.0-beta.0

package/dist/index.d.cts

@@ -1,13 +1,13 @@
-import { L as LLMProvider, M as MCPFixtureApi, a as MCPHostConfig, b as MCPHostSimulationResult, O as OAuthSetupConfig, V as ValidationResult, S as SchemaValidatorOptions, T as TextValidatorOptions, P as PatternValidatorOptions, c as SizeValidatorOptions, d as SnapshotSanitizer, R as RubricSpec, J as JudgeMatcherOptions, e as ToolPredicate, f as ToolCallExpectation, g as ToolCallCountOptions, A as AuthType, h as JudgeConfig, i as Judge } from './index-BcUWzQCx.cjs';
-export { B as BUILT_IN_RUBRICS, j as BuiltInRubric, k as BuiltInSanitizer, C as CLIConfig, l as CLIOAuthClient, m as CLIOAuthClientConfig, n as CLIOAuthResult, o as CLIOutputFormat, p as CaseComparisonResult, q as ClientCredentialsConfig, r as CompareEvalRunsOptions, s as ComparisonOutcome, t as ContentBlock, u as CreateMCPClientOptions, v as CustomJudgeExecutor, w as CustomJudgeResult, D as DiscoveryError, E as ENV_VAR_NAMES, x as EvalCase, y as EvalCaseComparison, z as EvalCaseComparisonOutcome, F as EvalCaseRequest, G as EvalCaseResult, H as EvalCaseSchema, I as EvalContext, K as EvalDataset, N as EvalDatasetSchema, Q as EvalExpectBlock, U as EvalExpectationResult, W as EvalMode, X as EvalResultStore, Y as EvalResultStoreConfig, Z as EvalResultStoreLike, _ as EvalRunComparisonLabels, $ as EvalRunComparisonResult, a0 as EvalRunMetadata, a1 as EvalRunnerOptions, a2 as EvalRunnerResult, a3 as ExpectationBreakdown, a4 as ExpectationResultMap, a5 as ExpectationType, a6 as ExperimentMetric, a7 as FieldRemovalSanitizer, a8 as FileEvalResultStore, a9 as FileEvalResultStoreConfig, aa as GCSEvalResultStore, ab as GCSEvalResultStoreConfig, ac as HostType, ad as HttpMCPConfig, ae as IterationResult, af as JudgeExpectConfig, ag as JudgeResult, ah as JudgeValidatorConfig, ai as LLMToolCall, aj as ListStoredArtifactsOptions, ak as LoadDatasetOptions, al as MCPAuthConfig, am as MCPAuthFixtures, an as MCPClientCredentialsConfig, ao as MCPConfig, ap as MCPConfigSchema, aq as MCPConformanceCheck, ar as MCPConformanceOptions, as as MCPConformanceRaw, at as MCPConformanceResult, au as MCPConformanceResultData, av as MCPEvalData, aw as MCPEvalHistoricalSummary, ax as MCPEvalReporterConfig, ay as MCPEvalRunData, az as MCPFixtureOptions, aA as MCPHostCapabilities, aB as MCPHostSimulator, aC as MCPOAuthConfig, aD as MCPServerCapabilitiesData, aE as MCP_PROTOCOL_VERSION, aF as NormalizedToolResponse, aG as PlaywrightOAuthClientProvider, aH as PlaywrightOAuthClientProviderConfig, aI as PredicateResult, aJ as ProposeVariantsContext, aK as ProtectedResourceDiscoveryResult, aL as ProtectedResourceMetadata, aM as ProviderKind, aN as RegexSanitizer, aO as ResultSource, aP as SaveBaselineOptions, aQ as SaveEvalRunComparisonOptions, aR as SaveServerComparisonOptions, aS as SchemaRegistry, aT as SerializedEvalDataset, aU as ServerComparisonOptions, aV as ServerComparisonResult, aW as SnapshotSanitizers, aX as StdioMCPConfig, aY as StoredArtifactKind, aZ as StoredArtifactSummary, a_ as StoredClientInfo, a$ as StoredEvalArtifact, b0 as StoredEvalArtifactMetadata, b1 as StoredEvalResultLoadOptions, b2 as StoredEvalResultRef, b3 as StoredEvalResultSaveOptions, b4 as StoredEvalRunRef, b5 as StoredOAuthState, b6 as StoredServerMetadata, b7 as StoredTokens, b8 as TokenResult, b9 as ToolMetadataOverride, ba as ToolOverrideVariant, bb as UsageMetrics, bc as VariantCandidateResult, bd as VariantExperimentOptions, be as VariantExperimentReason, bf as VariantExperimentResult, bg as VariantExperimentRound, bh as VariantImprovementProposal, bi as VariantRecommendation, bj as clearJudgeRegistry, bk as closeMCPClient, bl as compareEvalRuns, bm as createDefaultArtifactId, bn as createEvalResultStore, bo as createMCPClientForConfig, bp as createMCPFixture, bq as createStoredEvalArtifact, br as defaultEnvironmentMetadata, bs as discoverAuthorizationServer, bt as discoverProtectedResource, bu as extractText, bv as getRegisteredJudge, bw as hasValidTokens, bx as injectTokens, by as isBuiltInRubric, bz as isEvalResultStore, bA as isHttpConfig, bB as isStdioConfig, bC as loadBaseline, bD as loadEvalDataset, bE as loadEvalDatasetFromObject, bF as loadStoredEvalRunnerResult, bG as loadTokens, bH as loadTokensFromEnv, bI as mcpAuthTest, bJ as normalizeToolResponse, bK as performClientCredentialsFlow, bL as refreshAccessToken, bM as registerJudge, bN as resolveEvalResultStore, bO as resolveRubric, bP as runConformanceChecks, bQ as runEvalCase, bR as runEvalDataset, bS as runServerComparison, bT as runVariantExperiment, bU as saveBaseline, bV as saveEvalRunComparison, bW as saveServerComparison, bX as validateEvalCase, bY as validateEvalDataset, bZ as validateJudge, b_ as validateMCPConfig, b$ as validateToolCallCount, c0 as validateToolCalls } from './index-BcUWzQCx.cjs';
+import { L as LLMProvider, M as MCPFixtureApi, a as MCPHostConfig, b as MCPHostSimulationResult, O as OAuthSetupConfig, V as ValidationResult, S as SchemaValidatorOptions, T as TextValidatorOptions, P as PatternValidatorOptions, c as SizeValidatorOptions, d as SnapshotSanitizer, R as RubricSpec, J as JudgeMatcherOptions, e as ToolPredicate, f as ToolCallExpectation, g as ToolCallCountOptions, A as AuthType, D as DatasetSource, H as HostDefinition, h as JudgeDefinition, i as MetricDefinition, j as ResultStoreDefinition, E as ExtensionConfig, k as EvalManifest, l as EvalCaseResult, m as MetricValue, n as ResolvedMetric, o as EvalDataset, p as MCPConfig, q as DatasetConfig, r as EvalRunnerResult, s as EvaluationSummary, t as EvaluationBatchItem, u as EvaluationBatchOptions, v as EvaluationBatchResult, w as ExternalHostConfig, x as ExternalHostRunResult, y as ExternalHostCapabilityImplementation, z as HostDriverId, B as HostDriverConfig, C as ExternalHostCapabilitiesConfig, F as JudgeConfig, G as Judge } from './index-aX_tCfTD.cjs';
+export { I as BUILT_IN_RUBRICS, K as BuiltInRubric, N as BuiltInSanitizer, Q as CLIConfig, U as CLIOAuthClient, W as CLIOAuthClientConfig, X as CLIOAuthResult, Y as CLIOutputFormat, Z as CaseComparisonResult, _ as ClientCredentialsConfig, $ as CompareEvalRunsOptions, a0 as ComparisonOutcome, a1 as ContentBlock, a2 as CreateMCPClientOptions, a3 as CustomJudgeExecutor, a4 as CustomJudgeResult, a5 as DatasetSourceContext, a6 as DiscoveryError, a7 as ENV_VAR_NAMES, a8 as EvalArm, a9 as EvalCase, aa as EvalCaseComparison, ab as EvalCaseComparisonOutcome, ac as EvalCaseRequest, ad as EvalCaseSchema, ae as EvalContext, af as EvalDatasetSchema, ag as EvalExpectBlock, ah as EvalExpectationResult, ai as EvalManifestInput, aj as EvalManifestSchema, ak as EvalMode, al as EvalResultStore, am as EvalResultStoreConfig, an as EvalResultStoreLike, ao as EvalRunComparisonLabels, ap as EvalRunComparisonResult, aq as EvalRunMetadata, ar as EvalRunnerOptions, as as EvalSummaryGenerator, at as EvaluationArmResult, au as EvaluationHostRunContext, av as EvaluationSuiteOptions, aw as EvaluationSuiteResult, ax as EvidenceSource, ay as ExpectationBreakdown, az as ExpectationResultMap, aA as ExpectationType, aB as ExperimentMetric, aC as ExternalHostCapabilityBinding, aD as ExternalHostCapabilityContext, aE as ExternalHostFailureKind, aF as ExternalHostMetadata, aG as ExternalHostSession, aH as ExternalHostSimulationResult, aI as ExternalHostType, aJ as FieldRemovalSanitizer, aK as FileEvalResultStore, aL as FileEvalResultStoreConfig, aM as GCSEvalResultStore, aN as GCSEvalResultStoreConfig, aO as HostArtifact, aP as HostCapability, aQ as HostConfig, aR as HostConfigPatch, aS as HostEvent, aT as HostEvidence, aU as HostRunContext, aV as HostRunInput, aW as HostRunOptions, aX as HostRunResult, aY as HostType, aZ as HttpMCPConfig, a_ as IterationResult, a$ as JudgeExpectConfig, b0 as JudgeResult, b1 as JudgeValidatorConfig, b2 as LLMToolCall, b3 as ListStoredArtifactsOptions, b4 as LoadDatasetOptions, b5 as MCPAuthConfig, b6 as MCPAuthFixtures, b7 as MCPClientCredentialsConfig, b8 as MCPConfigSchema, b9 as MCPConformanceCheck, ba as MCPConformanceOptions, bb as MCPConformanceRaw, bc as MCPConformanceResult, bd as MCPConformanceResultData, be as MCPEvalData, bf as MCPEvalHistoricalSummary, bg as MCPEvalReporterConfig, bh as MCPEvalRunData, bi as MCPFixtureOptions, bj as MCPHostCapabilities, bk as MCPHostSimulator, bl as MCPOAuthConfig, bm as MCPServerCapabilitiesData, bn as MCP_PROTOCOL_VERSION, bo as MetricKind, bp as NormalizedToolResponse, bq as ObservationConfidence, br as PlaywrightOAuthClientProvider, bs as PlaywrightOAuthClientProviderConfig, bt as PredicateResult, bu as ProposeVariantsContext, bv as ProtectedResourceDiscoveryResult, bw as ProtectedResourceMetadata, bx as ProviderKind, by as RegexSanitizer, bz as ResultSource, bA as RunSummary, bB as RunTelemetry, bC as SaveBaselineOptions, bD as SaveEvalRunComparisonOptions, bE as SaveServerComparisonOptions, bF as SchemaRegistry, bG as SerializedEvalDataset, bH as ServerComparisonOptions, bI as ServerComparisonResult, bJ as SnapshotSanitizers, bK as StdioMCPConfig, bL as StoredArtifactKind, bM as StoredArtifactSummary, bN as StoredClientInfo, bO as StoredEvalArtifact, bP as StoredEvalArtifactMetadata, bQ as StoredEvalResultLoadOptions, bR as StoredEvalResultRef, bS as StoredEvalResultSaveOptions, bT as StoredEvalRunRef, bU as StoredOAuthState, bV as StoredServerMetadata, bW as StoredTokens, bX as TaggedConfig, bY as TokenResult, bZ as ToolMetadataOverride, b_ as ToolOverrideVariant, b$ as TraceSource, c0 as UsageMetrics, c1 as VariantCandidateResult, c2 as VariantExperimentOptions, c3 as VariantExperimentReason, c4 as VariantExperimentResult, c5 as VariantExperimentRound, c6 as VariantImprovementProposal, c7 as VariantRecommendation, c8 as closeMCPClient, c9 as compareEvalRuns, ca as createDefaultArtifactId, cb as createEvalResultStore, cc as createMCPClientForConfig, cd as createMCPFixture, ce as createStoredEvalArtifact, cf as defaultEnvironmentMetadata, cg as discoverAuthorizationServer, ch as discoverProtectedResource, ci as extractText, cj as hasValidTokens, ck as injectTokens, cl as isBuiltInRubric, cm as isEvalResultStore, cn as isHttpConfig, co as isStdioConfig, cp as loadBaseline, cq as loadEvalDataset, cr as loadEvalDatasetFromObject, cs as loadEvalManifest, ct as loadEvalManifestFromObject, cu as loadStoredEvalRunnerResult, cv as loadTokens, cw as loadTokensFromEnv, cx as mcpAuthTest, cy as normalizeToolResponse, cz as performClientCredentialsFlow, cA as refreshAccessToken, cB as registerJudge, cC as resolveDatasetPaths, cD as resolveEvalResultStore, cE as resolveRubric, cF as runConformanceChecks, cG as runEvalCase, cH as runEvalDataset, cI as runServerComparison, cJ as runVariantExperiment, cK as saveBaseline, cL as saveEvalRunComparison, cM as saveServerComparison, cN as validateEvalCase, cO as validateEvalDataset, cP as validateJudge, cQ as validateMCPConfig, cR as validateToolCallCount, cS as validateToolCalls } from './index-aX_tCfTD.cjs';
 import { ZodType } from 'zod';
 import * as playwright_test from 'playwright/test';
 import { Client } from '@modelcontextprotocol/sdk/client/index.js';
 import '@playwright/test';
+import '@modelcontextprotocol/sdk/types.js';
 import '@modelcontextprotocol/sdk/client/auth.js';
 import '@modelcontextprotocol/sdk/shared/auth.js';
 import 'oauth4webapi';
-import '@modelcontextprotocol/sdk/types.js';
 
 /**
  * MCP Host Simulation - Main entry point
@@ -62,7 +62,7 @@ import '@modelcontextprotocol/sdk/types.js';
  * );
  * ```
  */
-declare function simulateMCPHost(mcp: MCPFixtureApi, scenario: string, config: MCPHostConfig): Promise<MCPHostSimulationResult>;
+declare function simulateMCPHost(mcp: MCPFixtureApi, scenario: string, config: MCPHostConfig, signal?: AbortSignal): Promise<MCPHostSimulationResult>;
 /**
  * Returns true if the given provider is supported.
  *
@@ -709,6 +709,176 @@ type MCPFixtures = {
  */
 declare const test: playwright_test.TestType<playwright_test.PlaywrightTestArgs & playwright_test.PlaywrightTestOptions & MCPFixtures, playwright_test.PlaywrightWorkerArgs & playwright_test.PlaywrightWorkerOptions>;
 
+declare const registerDatasetSource: (source: DatasetSource) => void;
+declare const getDatasetSource: (name: string) => DatasetSource;
+declare const listDatasetSources: () => DatasetSource[];
+declare const clearDatasetSources: () => void;
+declare const registerHost: (host: HostDefinition) => void;
+declare const getHost: (name: string) => HostDefinition;
+declare const listHosts: () => HostDefinition[];
+declare const clearHosts: () => void;
+declare const getJudge: (name: string) => JudgeDefinition;
+declare const listJudges: () => JudgeDefinition[];
+declare const clearJudges: () => void;
+declare const registerMetric: (metric: MetricDefinition) => void;
+declare const getMetric: (name: string) => MetricDefinition;
+declare const listMetrics: () => MetricDefinition[];
+declare const clearMetrics: () => void;
+declare const registerResultStore: (store: ResultStoreDefinition) => void;
+declare const getResultStore: (name: string) => ResultStoreDefinition;
+declare const listResultStores: () => ResultStoreDefinition[];
+declare const clearResultStores: () => void;
+/** Resolve by implementation type and retain parsed defaults/transforms for consumers. */
+declare function resolveResultStoreConfig(config: ExtensionConfig): {
+    definition: ResultStoreDefinition;
+    config: ExtensionConfig;
+};
+/**
+ * Validate registered schemas and return parsed options, including effective arm
+ * inheritance. Callers must use the returned manifest to retain defaults and
+ * transforms. The input is not mutated, and each effective config is parsed once.
+ */
+declare function validateManifestRegistrations(manifest: EvalManifest): EvalManifest;
+
+/**
+ * Tagged options are passed to compute without routing keys (type/metric/name).
+ * Legacy params are flattened for compatibility; keys may not also occur at the
+ * top level, since choosing either value would silently discard configuration.
+ */
+type MetricSpec = string | {
+    metric?: string;
+    type?: string;
+    name?: string;
+    params?: Record<string, unknown>;
+    [option: string]: unknown;
+};
+
+interface MetricResult {
+    perCase: Record<string, Record<string, MetricValue>>;
+    aggregated: Record<string, unknown>;
+}
+/** Built-in metrics, including the metrics used by Scio's evaluations. */
+declare const BUILT_IN_METRICS: Record<string, MetricDefinition>;
+/**
+ * Live record-compatible view of the process-wide framework registry. Both
+ * registration APIs, validation, and computation use the same backing map.
+ */
+declare const METRIC_REGISTRY: Record<string, MetricDefinition>;
+/** Resolve one config metric, including parameterized judge metrics. */
+declare function resolveMetric(spec: MetricSpec, registry?: Record<string, MetricDefinition>): ResolvedMetric;
+declare function computeMetrics(specs: MetricSpec[], cases: EvalCaseResult[], registry?: Record<string, MetricDefinition>): MetricResult;
+
+/**
+ * Validate a canonical EvalDataset and apply the manifest case limit.
+ * The host argument is retained for API compatibility; sources never infer a
+ * mode, attach a host, or manufacture assertions. Use an opt-in dataset source
+ * adapter to migrate other formats before canonical validation.
+ */
+declare function buildEvalDataset(raw: unknown, _hostConfig: MCPHostConfig | undefined, manifest: EvalManifest): EvalDataset;
+
+type HostEnvironment = Record<string, string | undefined>;
+
+interface BuiltinHostOptions {
+    env?: HostEnvironment;
+    temperature?: number;
+    maxTokens?: number;
+    apiKeyEnvVar?: string;
+    model?: string;
+    maxToolCalls?: number;
+    timeout?: number;
+    provider?: string;
+    server?: MCPConfig;
+    servers?: MCPConfig[];
+    apiToken?: string;
+    pluginDir?: string;
+    pluginMcpUrl?: string;
+    mcpServers?: Record<string, Record<string, unknown>>;
+}
+declare function getBuiltinHostConfig(name: string, options?: BuiltinHostOptions): MCPHostConfig;
+
+interface RunEvalSuiteOptions {
+    manifestPath: string;
+    rootDir?: string;
+    pluginPaths?: string[];
+    outputDir?: string;
+    secretsFile?: string;
+    mcpConfig?: MCPConfig;
+    dryRun?: boolean;
+    arm?: string;
+    redactStoredResponses?: boolean;
+}
+interface RunEvalSuiteResult {
+    manifest: EvalManifest;
+    outputDir: string;
+    datasets: Array<{
+        source: DatasetConfig;
+        dataset?: EvalDataset;
+        result?: EvalRunnerResult;
+    }>;
+    summary: EvaluationSummary;
+}
+declare function runEvalSuite(options: RunEvalSuiteOptions): Promise<RunEvalSuiteResult>;
+
+type RunEvalBatchOptions = EvaluationBatchOptions;
+type EvalBatchItem = EvaluationBatchItem;
+type RunEvalBatchResult = EvaluationBatchResult;
+/** Run multiple manifests with bounded process-level concurrency. */
+declare function runEvalBatch(options: RunEvalBatchOptions): Promise<RunEvalBatchResult>;
+
+interface EvalPluginModule {
+    register?: () => void | Promise<void>;
+    default?: () => void | Promise<void>;
+    [exportName: string]: unknown;
+}
+interface LoadPluginsOptions {
+    /** Additional register function names to invoke after import. */
+    registerExportNames?: string[];
+}
+/**
+ * Dynamically import a plugin module and invoke its registration hooks.
+ *
+ * Convention (first match wins):
+ * 1. Named export `register`, `registerPlugins`, or `registerGleanJudges`
+ * 2. Default export function
+ */
+declare function loadPluginModule(pluginPath: string, options?: LoadPluginsOptions): Promise<void>;
+/**
+ * Load one or more plugin paths in order.
+ */
+declare function loadPlugins(pluginPaths: string[], options?: LoadPluginsOptions): Promise<void>;
+
+declare function runExternalHostScenario(scenario: string, config: ExternalHostConfig, options?: {
+    caseId?: string;
+    runId?: string;
+}): Promise<ExternalHostRunResult>;
+
+declare function registerExternalHostCapability(implementation: ExternalHostCapabilityImplementation): void;
+
+declare function driverToSlug(driver: HostDriverId): string;
+declare function parseDriverSlug(slug: string): HostDriverId;
+declare function normalizeHostDriver(driver: HostDriverConfig): HostDriverId;
+
+interface ExternalHostDriverReference {
+    slug: string;
+    driver: HostDriverId;
+    name: string;
+    description?: string;
+    builtIn: true;
+    defaultConfig: ExternalHostConfig;
+    capabilities?: ExternalHostCapabilitiesConfig;
+    example: {
+        mode: 'external_host';
+        scenario: string;
+        externalHost: Pick<ExternalHostConfig, 'driver' | 'timeoutMs'>;
+        expect: {
+            containsText: string;
+        };
+    };
+}
+declare function listExternalHostDriverReferences(): ExternalHostDriverReference[];
+declare function getExternalHostConfigJsonSchema(): Record<string, unknown>;
+declare function getExternalHostReference(): Record<string, unknown>;
+
 /**
  * Creates an LLM judge for evaluating tool responses
  *
@@ -743,4 +913,4 @@ declare const test: playwright_test.TestType<playwright_test.PlaywrightTestArgs
  */
 declare function createJudge(config?: JudgeConfig): Judge;
 
-export { AuthType, Judge, JudgeConfig, JudgeMatcherOptions, LLMProvider, MCPFixtureApi, MCPHostConfig, MCPHostSimulationResult, OAuthSetupConfig, PatternValidatorOptions, RubricSpec, SchemaValidatorOptions, SizeValidatorOptions, SnapshotSanitizer, TextValidatorOptions, ToolCallCountOptions, ToolCallExpectation, ToolPredicate, ValidationResult, createJudge, createTokenAuthHeaders, expect, getMissingDependencyMessage, getResponseSizeBytes, isProviderAvailable, isTokenExpired, isTokenExpiringSoon, normalizeWhitespace, performOAuthSetup, performOAuthSetupIfNeeded, simulateMCPHost, test, validateAccessToken, validateError, validatePattern, validateResponse, validateSchema, validateSize, validateText };
+export { AuthType, BUILT_IN_METRICS, DatasetConfig, DatasetSource, type EvalBatchItem, EvalCaseResult, EvalDataset, EvalManifest, type EvalPluginModule, EvalRunnerResult, EvaluationBatchItem, EvaluationBatchOptions, EvaluationBatchResult, EvaluationSummary, ExtensionConfig, ExternalHostCapabilitiesConfig, ExternalHostCapabilityImplementation, ExternalHostConfig, type ExternalHostDriverReference, ExternalHostRunResult, HostDefinition, HostDriverConfig, HostDriverId, Judge, JudgeConfig, JudgeDefinition, JudgeMatcherOptions, LLMProvider, type LoadPluginsOptions, MCPConfig, MCPFixtureApi, MCPHostConfig, MCPHostSimulationResult, METRIC_REGISTRY, MetricDefinition, MetricValue, OAuthSetupConfig, PatternValidatorOptions, ResultStoreDefinition, RubricSpec, type RunEvalBatchOptions, type RunEvalBatchResult, type RunEvalSuiteOptions, type RunEvalSuiteResult, SchemaValidatorOptions, SizeValidatorOptions, SnapshotSanitizer, TextValidatorOptions, ToolCallCountOptions, ToolCallExpectation, ToolPredicate, ValidationResult, buildEvalDataset, clearDatasetSources, clearHosts, clearJudges, clearMetrics, clearResultStores, computeMetrics, createJudge, createTokenAuthHeaders, driverToSlug, expect, getBuiltinHostConfig, getDatasetSource, getExternalHostConfigJsonSchema, getExternalHostReference, getHost, getJudge, getMetric, getMissingDependencyMessage, getResponseSizeBytes, getResultStore, isProviderAvailable, isTokenExpired, isTokenExpiringSoon, listDatasetSources, listExternalHostDriverReferences, listHosts, listJudges, listMetrics, listResultStores, loadPluginModule, loadPlugins, normalizeHostDriver, normalizeWhitespace, parseDriverSlug, performOAuthSetup, performOAuthSetupIfNeeded, registerDatasetSource, registerExternalHostCapability, registerHost, registerMetric, registerResultStore, resolveMetric, resolveResultStoreConfig, runEvalBatch, runEvalSuite, runExternalHostScenario, simulateMCPHost, test, validateAccessToken, validateError, validateManifestRegistrations, validatePattern, validateResponse, validateSchema, validateSize, validateText };

package/dist/index.d.ts

@@ -1,13 +1,13 @@
-import { L as LLMProvider, M as MCPFixtureApi, a as MCPHostConfig, b as MCPHostSimulationResult, O as OAuthSetupConfig, V as ValidationResult, S as SchemaValidatorOptions, T as TextValidatorOptions, P as PatternValidatorOptions, c as SizeValidatorOptions, d as SnapshotSanitizer, R as RubricSpec, J as JudgeMatcherOptions, e as ToolPredicate, f as ToolCallExpectation, g as ToolCallCountOptions, A as AuthType, h as JudgeConfig, i as Judge } from './index-BcUWzQCx.js';
-export { B as BUILT_IN_RUBRICS, j as BuiltInRubric, k as BuiltInSanitizer, C as CLIConfig, l as CLIOAuthClient, m as CLIOAuthClientConfig, n as CLIOAuthResult, o as CLIOutputFormat, p as CaseComparisonResult, q as ClientCredentialsConfig, r as CompareEvalRunsOptions, s as ComparisonOutcome, t as ContentBlock, u as CreateMCPClientOptions, v as CustomJudgeExecutor, w as CustomJudgeResult, D as DiscoveryError, E as ENV_VAR_NAMES, x as EvalCase, y as EvalCaseComparison, z as EvalCaseComparisonOutcome, F as EvalCaseRequest, G as EvalCaseResult, H as EvalCaseSchema, I as EvalContext, K as EvalDataset, N as EvalDatasetSchema, Q as EvalExpectBlock, U as EvalExpectationResult, W as EvalMode, X as EvalResultStore, Y as EvalResultStoreConfig, Z as EvalResultStoreLike, _ as EvalRunComparisonLabels, $ as EvalRunComparisonResult, a0 as EvalRunMetadata, a1 as EvalRunnerOptions, a2 as EvalRunnerResult, a3 as ExpectationBreakdown, a4 as ExpectationResultMap, a5 as ExpectationType, a6 as ExperimentMetric, a7 as FieldRemovalSanitizer, a8 as FileEvalResultStore, a9 as FileEvalResultStoreConfig, aa as GCSEvalResultStore, ab as GCSEvalResultStoreConfig, ac as HostType, ad as HttpMCPConfig, ae as IterationResult, af as JudgeExpectConfig, ag as JudgeResult, ah as JudgeValidatorConfig, ai as LLMToolCall, aj as ListStoredArtifactsOptions, ak as LoadDatasetOptions, al as MCPAuthConfig, am as MCPAuthFixtures, an as MCPClientCredentialsConfig, ao as MCPConfig, ap as MCPConfigSchema, aq as MCPConformanceCheck, ar as MCPConformanceOptions, as as MCPConformanceRaw, at as MCPConformanceResult, au as MCPConformanceResultData, av as MCPEvalData, aw as MCPEvalHistoricalSummary, ax as MCPEvalReporterConfig, ay as MCPEvalRunData, az as MCPFixtureOptions, aA as MCPHostCapabilities, aB as MCPHostSimulator, aC as MCPOAuthConfig, aD as MCPServerCapabilitiesData, aE as MCP_PROTOCOL_VERSION, aF as NormalizedToolResponse, aG as PlaywrightOAuthClientProvider, aH as PlaywrightOAuthClientProviderConfig, aI as PredicateResult, aJ as ProposeVariantsContext, aK as ProtectedResourceDiscoveryResult, aL as ProtectedResourceMetadata, aM as ProviderKind, aN as RegexSanitizer, aO as ResultSource, aP as SaveBaselineOptions, aQ as SaveEvalRunComparisonOptions, aR as SaveServerComparisonOptions, aS as SchemaRegistry, aT as SerializedEvalDataset, aU as ServerComparisonOptions, aV as ServerComparisonResult, aW as SnapshotSanitizers, aX as StdioMCPConfig, aY as StoredArtifactKind, aZ as StoredArtifactSummary, a_ as StoredClientInfo, a$ as StoredEvalArtifact, b0 as StoredEvalArtifactMetadata, b1 as StoredEvalResultLoadOptions, b2 as StoredEvalResultRef, b3 as StoredEvalResultSaveOptions, b4 as StoredEvalRunRef, b5 as StoredOAuthState, b6 as StoredServerMetadata, b7 as StoredTokens, b8 as TokenResult, b9 as ToolMetadataOverride, ba as ToolOverrideVariant, bb as UsageMetrics, bc as VariantCandidateResult, bd as VariantExperimentOptions, be as VariantExperimentReason, bf as VariantExperimentResult, bg as VariantExperimentRound, bh as VariantImprovementProposal, bi as VariantRecommendation, bj as clearJudgeRegistry, bk as closeMCPClient, bl as compareEvalRuns, bm as createDefaultArtifactId, bn as createEvalResultStore, bo as createMCPClientForConfig, bp as createMCPFixture, bq as createStoredEvalArtifact, br as defaultEnvironmentMetadata, bs as discoverAuthorizationServer, bt as discoverProtectedResource, bu as extractText, bv as getRegisteredJudge, bw as hasValidTokens, bx as injectTokens, by as isBuiltInRubric, bz as isEvalResultStore, bA as isHttpConfig, bB as isStdioConfig, bC as loadBaseline, bD as loadEvalDataset, bE as loadEvalDatasetFromObject, bF as loadStoredEvalRunnerResult, bG as loadTokens, bH as loadTokensFromEnv, bI as mcpAuthTest, bJ as normalizeToolResponse, bK as performClientCredentialsFlow, bL as refreshAccessToken, bM as registerJudge, bN as resolveEvalResultStore, bO as resolveRubric, bP as runConformanceChecks, bQ as runEvalCase, bR as runEvalDataset, bS as runServerComparison, bT as runVariantExperiment, bU as saveBaseline, bV as saveEvalRunComparison, bW as saveServerComparison, bX as validateEvalCase, bY as validateEvalDataset, bZ as validateJudge, b_ as validateMCPConfig, b$ as validateToolCallCount, c0 as validateToolCalls } from './index-BcUWzQCx.js';
+import { L as LLMProvider, M as MCPFixtureApi, a as MCPHostConfig, b as MCPHostSimulationResult, O as OAuthSetupConfig, V as ValidationResult, S as SchemaValidatorOptions, T as TextValidatorOptions, P as PatternValidatorOptions, c as SizeValidatorOptions, d as SnapshotSanitizer, R as RubricSpec, J as JudgeMatcherOptions, e as ToolPredicate, f as ToolCallExpectation, g as ToolCallCountOptions, A as AuthType, D as DatasetSource, H as HostDefinition, h as JudgeDefinition, i as MetricDefinition, j as ResultStoreDefinition, E as ExtensionConfig, k as EvalManifest, l as EvalCaseResult, m as MetricValue, n as ResolvedMetric, o as EvalDataset, p as MCPConfig, q as DatasetConfig, r as EvalRunnerResult, s as EvaluationSummary, t as EvaluationBatchItem, u as EvaluationBatchOptions, v as EvaluationBatchResult, w as ExternalHostConfig, x as ExternalHostRunResult, y as ExternalHostCapabilityImplementation, z as HostDriverId, B as HostDriverConfig, C as ExternalHostCapabilitiesConfig, F as JudgeConfig, G as Judge } from './index-aX_tCfTD.js';
+export { I as BUILT_IN_RUBRICS, K as BuiltInRubric, N as BuiltInSanitizer, Q as CLIConfig, U as CLIOAuthClient, W as CLIOAuthClientConfig, X as CLIOAuthResult, Y as CLIOutputFormat, Z as CaseComparisonResult, _ as ClientCredentialsConfig, $ as CompareEvalRunsOptions, a0 as ComparisonOutcome, a1 as ContentBlock, a2 as CreateMCPClientOptions, a3 as CustomJudgeExecutor, a4 as CustomJudgeResult, a5 as DatasetSourceContext, a6 as DiscoveryError, a7 as ENV_VAR_NAMES, a8 as EvalArm, a9 as EvalCase, aa as EvalCaseComparison, ab as EvalCaseComparisonOutcome, ac as EvalCaseRequest, ad as EvalCaseSchema, ae as EvalContext, af as EvalDatasetSchema, ag as EvalExpectBlock, ah as EvalExpectationResult, ai as EvalManifestInput, aj as EvalManifestSchema, ak as EvalMode, al as EvalResultStore, am as EvalResultStoreConfig, an as EvalResultStoreLike, ao as EvalRunComparisonLabels, ap as EvalRunComparisonResult, aq as EvalRunMetadata, ar as EvalRunnerOptions, as as EvalSummaryGenerator, at as EvaluationArmResult, au as EvaluationHostRunContext, av as EvaluationSuiteOptions, aw as EvaluationSuiteResult, ax as EvidenceSource, ay as ExpectationBreakdown, az as ExpectationResultMap, aA as ExpectationType, aB as ExperimentMetric, aC as ExternalHostCapabilityBinding, aD as ExternalHostCapabilityContext, aE as ExternalHostFailureKind, aF as ExternalHostMetadata, aG as ExternalHostSession, aH as ExternalHostSimulationResult, aI as ExternalHostType, aJ as FieldRemovalSanitizer, aK as FileEvalResultStore, aL as FileEvalResultStoreConfig, aM as GCSEvalResultStore, aN as GCSEvalResultStoreConfig, aO as HostArtifact, aP as HostCapability, aQ as HostConfig, aR as HostConfigPatch, aS as HostEvent, aT as HostEvidence, aU as HostRunContext, aV as HostRunInput, aW as HostRunOptions, aX as HostRunResult, aY as HostType, aZ as HttpMCPConfig, a_ as IterationResult, a$ as JudgeExpectConfig, b0 as JudgeResult, b1 as JudgeValidatorConfig, b2 as LLMToolCall, b3 as ListStoredArtifactsOptions, b4 as LoadDatasetOptions, b5 as MCPAuthConfig, b6 as MCPAuthFixtures, b7 as MCPClientCredentialsConfig, b8 as MCPConfigSchema, b9 as MCPConformanceCheck, ba as MCPConformanceOptions, bb as MCPConformanceRaw, bc as MCPConformanceResult, bd as MCPConformanceResultData, be as MCPEvalData, bf as MCPEvalHistoricalSummary, bg as MCPEvalReporterConfig, bh as MCPEvalRunData, bi as MCPFixtureOptions, bj as MCPHostCapabilities, bk as MCPHostSimulator, bl as MCPOAuthConfig, bm as MCPServerCapabilitiesData, bn as MCP_PROTOCOL_VERSION, bo as MetricKind, bp as NormalizedToolResponse, bq as ObservationConfidence, br as PlaywrightOAuthClientProvider, bs as PlaywrightOAuthClientProviderConfig, bt as PredicateResult, bu as ProposeVariantsContext, bv as ProtectedResourceDiscoveryResult, bw as ProtectedResourceMetadata, bx as ProviderKind, by as RegexSanitizer, bz as ResultSource, bA as RunSummary, bB as RunTelemetry, bC as SaveBaselineOptions, bD as SaveEvalRunComparisonOptions, bE as SaveServerComparisonOptions, bF as SchemaRegistry, bG as SerializedEvalDataset, bH as ServerComparisonOptions, bI as ServerComparisonResult, bJ as SnapshotSanitizers, bK as StdioMCPConfig, bL as StoredArtifactKind, bM as StoredArtifactSummary, bN as StoredClientInfo, bO as StoredEvalArtifact, bP as StoredEvalArtifactMetadata, bQ as StoredEvalResultLoadOptions, bR as StoredEvalResultRef, bS as StoredEvalResultSaveOptions, bT as StoredEvalRunRef, bU as StoredOAuthState, bV as StoredServerMetadata, bW as StoredTokens, bX as TaggedConfig, bY as TokenResult, bZ as ToolMetadataOverride, b_ as ToolOverrideVariant, b$ as TraceSource, c0 as UsageMetrics, c1 as VariantCandidateResult, c2 as VariantExperimentOptions, c3 as VariantExperimentReason, c4 as VariantExperimentResult, c5 as VariantExperimentRound, c6 as VariantImprovementProposal, c7 as VariantRecommendation, c8 as closeMCPClient, c9 as compareEvalRuns, ca as createDefaultArtifactId, cb as createEvalResultStore, cc as createMCPClientForConfig, cd as createMCPFixture, ce as createStoredEvalArtifact, cf as defaultEnvironmentMetadata, cg as discoverAuthorizationServer, ch as discoverProtectedResource, ci as extractText, cj as hasValidTokens, ck as injectTokens, cl as isBuiltInRubric, cm as isEvalResultStore, cn as isHttpConfig, co as isStdioConfig, cp as loadBaseline, cq as loadEvalDataset, cr as loadEvalDatasetFromObject, cs as loadEvalManifest, ct as loadEvalManifestFromObject, cu as loadStoredEvalRunnerResult, cv as loadTokens, cw as loadTokensFromEnv, cx as mcpAuthTest, cy as normalizeToolResponse, cz as performClientCredentialsFlow, cA as refreshAccessToken, cB as registerJudge, cC as resolveDatasetPaths, cD as resolveEvalResultStore, cE as resolveRubric, cF as runConformanceChecks, cG as runEvalCase, cH as runEvalDataset, cI as runServerComparison, cJ as runVariantExperiment, cK as saveBaseline, cL as saveEvalRunComparison, cM as saveServerComparison, cN as validateEvalCase, cO as validateEvalDataset, cP as validateJudge, cQ as validateMCPConfig, cR as validateToolCallCount, cS as validateToolCalls } from './index-aX_tCfTD.js';
 import { ZodType } from 'zod';
 import * as playwright_test from 'playwright/test';
 import { Client } from '@modelcontextprotocol/sdk/client/index.js';
 import '@playwright/test';
+import '@modelcontextprotocol/sdk/types.js';
 import '@modelcontextprotocol/sdk/client/auth.js';
 import '@modelcontextprotocol/sdk/shared/auth.js';
 import 'oauth4webapi';
-import '@modelcontextprotocol/sdk/types.js';
 
 /**
  * MCP Host Simulation - Main entry point
@@ -62,7 +62,7 @@ import '@modelcontextprotocol/sdk/types.js';
  * );
  * ```
  */
-declare function simulateMCPHost(mcp: MCPFixtureApi, scenario: string, config: MCPHostConfig): Promise<MCPHostSimulationResult>;
+declare function simulateMCPHost(mcp: MCPFixtureApi, scenario: string, config: MCPHostConfig, signal?: AbortSignal): Promise<MCPHostSimulationResult>;
 /**
  * Returns true if the given provider is supported.
  *
@@ -709,6 +709,176 @@ type MCPFixtures = {
  */
 declare const test: playwright_test.TestType<playwright_test.PlaywrightTestArgs & playwright_test.PlaywrightTestOptions & MCPFixtures, playwright_test.PlaywrightWorkerArgs & playwright_test.PlaywrightWorkerOptions>;
 
+declare const registerDatasetSource: (source: DatasetSource) => void;
+declare const getDatasetSource: (name: string) => DatasetSource;
+declare const listDatasetSources: () => DatasetSource[];
+declare const clearDatasetSources: () => void;
+declare const registerHost: (host: HostDefinition) => void;
+declare const getHost: (name: string) => HostDefinition;
+declare const listHosts: () => HostDefinition[];
+declare const clearHosts: () => void;
+declare const getJudge: (name: string) => JudgeDefinition;
+declare const listJudges: () => JudgeDefinition[];
+declare const clearJudges: () => void;
+declare const registerMetric: (metric: MetricDefinition) => void;
+declare const getMetric: (name: string) => MetricDefinition;
+declare const listMetrics: () => MetricDefinition[];
+declare const clearMetrics: () => void;
+declare const registerResultStore: (store: ResultStoreDefinition) => void;
+declare const getResultStore: (name: string) => ResultStoreDefinition;
+declare const listResultStores: () => ResultStoreDefinition[];
+declare const clearResultStores: () => void;
+/** Resolve by implementation type and retain parsed defaults/transforms for consumers. */
+declare function resolveResultStoreConfig(config: ExtensionConfig): {
+    definition: ResultStoreDefinition;
+    config: ExtensionConfig;
+};
+/**
+ * Validate registered schemas and return parsed options, including effective arm
+ * inheritance. Callers must use the returned manifest to retain defaults and
+ * transforms. The input is not mutated, and each effective config is parsed once.
+ */
+declare function validateManifestRegistrations(manifest: EvalManifest): EvalManifest;
+
+/**
+ * Tagged options are passed to compute without routing keys (type/metric/name).
+ * Legacy params are flattened for compatibility; keys may not also occur at the
+ * top level, since choosing either value would silently discard configuration.
+ */
+type MetricSpec = string | {
+    metric?: string;
+    type?: string;
+    name?: string;
+    params?: Record<string, unknown>;
+    [option: string]: unknown;
+};
+
+interface MetricResult {
+    perCase: Record<string, Record<string, MetricValue>>;
+    aggregated: Record<string, unknown>;
+}
+/** Built-in metrics, including the metrics used by Scio's evaluations. */
+declare const BUILT_IN_METRICS: Record<string, MetricDefinition>;
+/**
+ * Live record-compatible view of the process-wide framework registry. Both
+ * registration APIs, validation, and computation use the same backing map.
+ */
+declare const METRIC_REGISTRY: Record<string, MetricDefinition>;
+/** Resolve one config metric, including parameterized judge metrics. */
+declare function resolveMetric(spec: MetricSpec, registry?: Record<string, MetricDefinition>): ResolvedMetric;
+declare function computeMetrics(specs: MetricSpec[], cases: EvalCaseResult[], registry?: Record<string, MetricDefinition>): MetricResult;
+
+/**
+ * Validate a canonical EvalDataset and apply the manifest case limit.
+ * The host argument is retained for API compatibility; sources never infer a
+ * mode, attach a host, or manufacture assertions. Use an opt-in dataset source
+ * adapter to migrate other formats before canonical validation.
+ */
+declare function buildEvalDataset(raw: unknown, _hostConfig: MCPHostConfig | undefined, manifest: EvalManifest): EvalDataset;
+
+type HostEnvironment = Record<string, string | undefined>;
+
+interface BuiltinHostOptions {
+    env?: HostEnvironment;
+    temperature?: number;
+    maxTokens?: number;
+    apiKeyEnvVar?: string;
+    model?: string;
+    maxToolCalls?: number;
+    timeout?: number;
+    provider?: string;
+    server?: MCPConfig;
+    servers?: MCPConfig[];
+    apiToken?: string;
+    pluginDir?: string;
+    pluginMcpUrl?: string;
+    mcpServers?: Record<string, Record<string, unknown>>;
+}
+declare function getBuiltinHostConfig(name: string, options?: BuiltinHostOptions): MCPHostConfig;
+
+interface RunEvalSuiteOptions {
+    manifestPath: string;
+    rootDir?: string;
+    pluginPaths?: string[];
+    outputDir?: string;
+    secretsFile?: string;
+    mcpConfig?: MCPConfig;
+    dryRun?: boolean;
+    arm?: string;
+    redactStoredResponses?: boolean;
+}
+interface RunEvalSuiteResult {
+    manifest: EvalManifest;
+    outputDir: string;
+    datasets: Array<{
+        source: DatasetConfig;
+        dataset?: EvalDataset;
+        result?: EvalRunnerResult;
+    }>;
+    summary: EvaluationSummary;
+}
+declare function runEvalSuite(options: RunEvalSuiteOptions): Promise<RunEvalSuiteResult>;
+
+type RunEvalBatchOptions = EvaluationBatchOptions;
+type EvalBatchItem = EvaluationBatchItem;
+type RunEvalBatchResult = EvaluationBatchResult;
+/** Run multiple manifests with bounded process-level concurrency. */
+declare function runEvalBatch(options: RunEvalBatchOptions): Promise<RunEvalBatchResult>;
+
+interface EvalPluginModule {
+    register?: () => void | Promise<void>;
+    default?: () => void | Promise<void>;
+    [exportName: string]: unknown;
+}
+interface LoadPluginsOptions {
+    /** Additional register function names to invoke after import. */
+    registerExportNames?: string[];
+}
+/**
+ * Dynamically import a plugin module and invoke its registration hooks.
+ *
+ * Convention (first match wins):
+ * 1. Named export `register`, `registerPlugins`, or `registerGleanJudges`
+ * 2. Default export function
+ */
+declare function loadPluginModule(pluginPath: string, options?: LoadPluginsOptions): Promise<void>;
+/**
+ * Load one or more plugin paths in order.
+ */
+declare function loadPlugins(pluginPaths: string[], options?: LoadPluginsOptions): Promise<void>;
+
+declare function runExternalHostScenario(scenario: string, config: ExternalHostConfig, options?: {
+    caseId?: string;
+    runId?: string;
+}): Promise<ExternalHostRunResult>;
+
+declare function registerExternalHostCapability(implementation: ExternalHostCapabilityImplementation): void;
+
+declare function driverToSlug(driver: HostDriverId): string;
+declare function parseDriverSlug(slug: string): HostDriverId;
+declare function normalizeHostDriver(driver: HostDriverConfig): HostDriverId;
+
+interface ExternalHostDriverReference {
+    slug: string;
+    driver: HostDriverId;
+    name: string;
+    description?: string;
+    builtIn: true;
+    defaultConfig: ExternalHostConfig;
+    capabilities?: ExternalHostCapabilitiesConfig;
+    example: {
+        mode: 'external_host';
+        scenario: string;
+        externalHost: Pick<ExternalHostConfig, 'driver' | 'timeoutMs'>;
+        expect: {
+            containsText: string;
+        };
+    };
+}
+declare function listExternalHostDriverReferences(): ExternalHostDriverReference[];
+declare function getExternalHostConfigJsonSchema(): Record<string, unknown>;
+declare function getExternalHostReference(): Record<string, unknown>;
+
 /**
  * Creates an LLM judge for evaluating tool responses
  *
@@ -743,4 +913,4 @@ declare const test: playwright_test.TestType<playwright_test.PlaywrightTestArgs
  */
 declare function createJudge(config?: JudgeConfig): Judge;
 
-export { AuthType, Judge, JudgeConfig, JudgeMatcherOptions, LLMProvider, MCPFixtureApi, MCPHostConfig, MCPHostSimulationResult, OAuthSetupConfig, PatternValidatorOptions, RubricSpec, SchemaValidatorOptions, SizeValidatorOptions, SnapshotSanitizer, TextValidatorOptions, ToolCallCountOptions, ToolCallExpectation, ToolPredicate, ValidationResult, createJudge, createTokenAuthHeaders, expect, getMissingDependencyMessage, getResponseSizeBytes, isProviderAvailable, isTokenExpired, isTokenExpiringSoon, normalizeWhitespace, performOAuthSetup, performOAuthSetupIfNeeded, simulateMCPHost, test, validateAccessToken, validateError, validatePattern, validateResponse, validateSchema, validateSize, validateText };
+export { AuthType, BUILT_IN_METRICS, DatasetConfig, DatasetSource, type EvalBatchItem, EvalCaseResult, EvalDataset, EvalManifest, type EvalPluginModule, EvalRunnerResult, EvaluationBatchItem, EvaluationBatchOptions, EvaluationBatchResult, EvaluationSummary, ExtensionConfig, ExternalHostCapabilitiesConfig, ExternalHostCapabilityImplementation, ExternalHostConfig, type ExternalHostDriverReference, ExternalHostRunResult, HostDefinition, HostDriverConfig, HostDriverId, Judge, JudgeConfig, JudgeDefinition, JudgeMatcherOptions, LLMProvider, type LoadPluginsOptions, MCPConfig, MCPFixtureApi, MCPHostConfig, MCPHostSimulationResult, METRIC_REGISTRY, MetricDefinition, MetricValue, OAuthSetupConfig, PatternValidatorOptions, ResultStoreDefinition, RubricSpec, type RunEvalBatchOptions, type RunEvalBatchResult, type RunEvalSuiteOptions, type RunEvalSuiteResult, SchemaValidatorOptions, SizeValidatorOptions, SnapshotSanitizer, TextValidatorOptions, ToolCallCountOptions, ToolCallExpectation, ToolPredicate, ValidationResult, buildEvalDataset, clearDatasetSources, clearHosts, clearJudges, clearMetrics, clearResultStores, computeMetrics, createJudge, createTokenAuthHeaders, driverToSlug, expect, getBuiltinHostConfig, getDatasetSource, getExternalHostConfigJsonSchema, getExternalHostReference, getHost, getJudge, getMetric, getMissingDependencyMessage, getResponseSizeBytes, getResultStore, isProviderAvailable, isTokenExpired, isTokenExpiringSoon, listDatasetSources, listExternalHostDriverReferences, listHosts, listJudges, listMetrics, listResultStores, loadPluginModule, loadPlugins, normalizeHostDriver, normalizeWhitespace, parseDriverSlug, performOAuthSetup, performOAuthSetupIfNeeded, registerDatasetSource, registerExternalHostCapability, registerHost, registerMetric, registerResultStore, resolveMetric, resolveResultStoreConfig, runEvalBatch, runEvalSuite, runExternalHostScenario, simulateMCPHost, test, validateAccessToken, validateError, validateManifestRegistrations, validatePattern, validateResponse, validateSchema, validateSize, validateText };