@tangle-network/agent-eval 0.1.0 → 0.2.0

package/dist/index.d.ts

@@ -470,6 +470,31 @@ declare function mannWhitneyU(a: number[], b: number[]): {
 };
 /** Partial credit: returns 0-1 ratio of current toward target */
 declare function partialCredit(current: number, target: number): number;
+/**
+ * Paired t-test — before/after measurements on the SAME items.
+ * Pairing removes inter-item variance, giving tighter significance than
+ * an unpaired test when comparing prompt v1 vs prompt v2 on identical
+ * scenarios.
+ */
+declare function pairedTTest(before: number[], after: number[]): {
+    t: number;
+    df: number;
+    p: number;
+};
+/**
+ * Wilcoxon signed-rank test — paired non-parametric alternative.
+ * Use when the differences aren't normally distributed.
+ */
+declare function wilcoxonSignedRank(before: number[], after: number[]): {
+    w: number;
+    p: number;
+};
+/**
+ * Cohen's d — standardized effect size for two independent groups.
+ * Positive d means group b has higher mean than group a.
+ * Rule of thumb: |d| < 0.2 negligible, 0.2–0.5 small, 0.5–0.8 medium, > 0.8 large.
+ */
+declare function cohensD(a: number[], b: number[]): number;
 
 /**
  * ConvergenceTracker — tracks completion percentage over turns.
@@ -576,4 +601,587 @@ declare function formatDriverReport(results: DriverResult[]): string;
 /** Print a compact summary to console */
 declare function printDriverSummary(results: DriverResult[]): void;
 
-export { AgentDriver, type AgentDriverConfig, type ArtifactCheck, type ArtifactResult, type BenchmarkReport, BenchmarkRunner, type BenchmarkRunnerConfig, type CheckResult, type CollectedArtifacts, type CompletionCriterion, ConvergenceTracker, type DriverResult, type DriverState, type EvalResult, type ExecutorConfig, type FeedbackPattern, type JudgeConfig, type JudgeFn, type JudgeInput, type JudgeRubric, type JudgeScore, MODEL_PRICING, MetricsCollector, type PersonaConfig, ProductClient, type ProductClientConfig, type RouteMap, type RubricDimension, type Scenario, type ScenarioFile, ScenarioRegistry, type ScenarioResult, type TestResult, TokenCounter, type Turn, type TurnMetrics, type TurnResult, adversarialJudge, codeExecutionJudge, coherenceJudge, confidenceInterval, createCustomJudge, createDomainExpertJudge, defaultJudges, estimateCost, estimateTokens, executeScenario, formatBenchmarkReport, formatDriverReport, interRaterReliability, mannWhitneyU, normalizeScores, partialCredit, printDriverSummary, runE2EWorkflow, weightedMean };
+/**
+ * Versioned prompt registry.
+ *
+ * Every prompt used in an eval run is registered with an explicit version.
+ * Reports include the content hash so A/B compares are rigorous: if the
+ * hash changes between two reports, the prompt actually changed; if it
+ * matches, the variance is elsewhere.
+ *
+ * Hash is SHA-256(content), truncated to 12 hex chars for readability.
+ * Uses the Web Crypto API (works in Workers, Node 22+, browsers).
+ */
+interface PromptHandle {
+    /** Stable human-readable id, e.g. 'legal.system' */
+    id: string;
+    /** Caller-chosen version string, e.g. 'v3' or '2026-04-20' */
+    version: string;
+    /** SHA-256 of content, 12-hex-char prefix */
+    hash: string;
+    /** Full prompt body */
+    content: string;
+}
+declare class PromptRegistry {
+    private readonly entries;
+    /**
+     * Register a prompt. Re-registering the same id+version with DIFFERENT
+     * content throws — versions are immutable. Re-registering with the SAME
+     * content is a no-op (idempotent).
+     */
+    register(id: string, version: string, content: string): Promise<PromptHandle>;
+    /** Look up a registered prompt. Throws if unknown — no implicit defaults. */
+    get(id: string, version: string): PromptHandle;
+    /** Return all versions of an id, newest-first (lex-descending on version). */
+    listVersions(id: string): PromptHandle[];
+    /** Snapshot the whole registry — useful for including in reports. */
+    list(): PromptHandle[];
+    /** Verify a hash against registered content. Returns null if not found. */
+    verifyHash(id: string, version: string, expectedHash: string): boolean | null;
+}
+/** SHA-256(content) → first 12 hex chars. Stable across runtimes. */
+declare function hashContent(content: string): Promise<string>;
+
+/**
+ * LLM trace store — one record per model call.
+ *
+ * Sink for the full eval data-plane: what got sent, what came back, what it
+ * cost, how long it took. Replayable, queryable, diff-able.
+ *
+ * Two built-in stores:
+ *   - `MemoryTraceStore` — fast, ephemeral, useful in tests and short runs
+ *   - `FileSystemTraceStore` — NDJSON files per-run, grepable, committable
+ *
+ * Consumers plug in custom stores for Langfuse / OTEL / D1 / Postgres.
+ */
+interface LlmTrace {
+    id: string;
+    runId: string;
+    scenarioId?: string;
+    turnIndex?: number;
+    role: 'driver' | 'judge' | 'product' | 'optimizer' | string;
+    model: string;
+    prompt: string;
+    output: string;
+    inputTokens?: number;
+    outputTokens?: number;
+    costUsd?: number;
+    durationMs?: number;
+    timestamp: string;
+    metadata?: Record<string, unknown>;
+}
+interface TraceQuery {
+    runId?: string;
+    scenarioId?: string;
+    role?: string;
+    model?: string;
+    sinceMs?: number;
+    limit?: number;
+}
+interface TraceStore {
+    record(trace: LlmTrace): Promise<void>;
+    query(query: TraceQuery): Promise<LlmTrace[]>;
+    count(query?: TraceQuery): Promise<number>;
+}
+declare class MemoryTraceStore implements TraceStore {
+    private traces;
+    record(trace: LlmTrace): Promise<void>;
+    query(query: TraceQuery): Promise<LlmTrace[]>;
+    count(query?: TraceQuery): Promise<number>;
+    /** Clear the store — test helper. */
+    reset(): void;
+    private filter;
+}
+interface FileSystemTraceStoreOptions {
+    dir: string;
+    /** Max file size before rolling to a new segment (default 32 MB). */
+    rolloverBytes?: number;
+    /** Function to write the file — defaults to node:fs/promises.appendFile */
+    append?: (path: string, data: string) => Promise<void>;
+    read?: (path: string) => Promise<string>;
+    list?: (dir: string) => Promise<string[]>;
+    stat?: (path: string) => Promise<{
+        size: number;
+    }>;
+    mkdir?: (dir: string) => Promise<void>;
+}
+declare class FileSystemTraceStore implements TraceStore {
+    private readonly opts;
+    constructor(opts: FileSystemTraceStoreOptions);
+    record(trace: LlmTrace): Promise<void>;
+    query(query: TraceQuery): Promise<LlmTrace[]>;
+    count(query?: TraceQuery): Promise<number>;
+    private segments;
+    private currentSegment;
+}
+
+/**
+ * Anti-slop quality judge.
+ *
+ * Deterministic pattern-based quality check — no LLM call. Catches the
+ * 80% of AI slop that every production agent leaks:
+ *   - Banned phrases (voice-specific: "delve", "it's worth noting", etc.)
+ *   - N-gram repetition (same phrase over and over)
+ *   - Hedging overuse ("I could be wrong, but...")
+ *   - Apology padding ("I'm so sorry for the confusion...")
+ *   - Unused opening formulas ("Great question!")
+ *   - Length bounds (too short to be useful, too long to be read)
+ *
+ * Produces a JudgeScore in the same shape as LLM judges so it composes into
+ * `BenchmarkRunner`'s judge array transparently.
+ */
+
+interface AntiSlopConfig {
+    /** Domain label — appears in the JudgeScore output */
+    domain?: string;
+    /** Case-insensitive substrings that must not appear. Each occurrence = penalty. */
+    bannedPhrases?: string[];
+    /** Regexes matching opening formulas to penalize (e.g. /^great question/i). */
+    bannedOpenings?: RegExp[];
+    /** Regexes matching hedges (e.g. /i could be wrong/i). Ratio of hedged sentences drives score. */
+    hedgingPatterns?: RegExp[];
+    /** Regexes matching apology padding. */
+    apologyPatterns?: RegExp[];
+    /** Fraction of sentences that can be duplicates before penalty (default 0.15 = 15%). */
+    repetitionThreshold?: number;
+    /** Min output length in chars; below this the turn is deemed too terse. */
+    minLength?: number;
+    /** Max output length in chars; above this the turn is deemed too verbose. */
+    maxLength?: number;
+    /** How heavily each violation class reduces the score (default 1). */
+    penaltyWeights?: Partial<Record<SlopCategory, number>>;
+}
+type SlopCategory = 'banned_phrase' | 'banned_opening' | 'hedging' | 'apology' | 'repetition' | 'length';
+/** Create a reusable Judge function from an anti-slop config. */
+declare function createAntiSlopJudge(config?: AntiSlopConfig): JudgeFn;
+interface AntiSlopIssue {
+    category: SlopCategory;
+    detail: string;
+    example?: string;
+}
+interface AntiSlopReport {
+    /** 0–10 score; 10 is clean, lower values mean more slop. */
+    score: number;
+    issues: AntiSlopIssue[];
+    /** Count of each category for programmatic aggregation. */
+    counts: Record<SlopCategory, number>;
+}
+/**
+ * Pure function — analyze one or more outputs against the config. Exposed
+ * separately so consumers can build their own reporters on top.
+ */
+declare function analyzeAntiSlop(outputs: string[], config: Omit<Required<AntiSlopConfig>, 'domain'> & {
+    penaltyWeights: Record<SlopCategory, number>;
+}): AntiSlopReport;
+
+/**
+ * Artifact validators.
+ *
+ * Generic "score a produced artifact" primitive. Tax uses it for PDF form
+ * correctness, legal for contract clauses, film for script breakdowns, GTM
+ * for social posts. One interface, many validators; all plug into
+ * `BenchmarkRunner` the same way.
+ *
+ * A validator receives an `Artifact` (file on disk, JSON blob, text, binary)
+ * plus a `ValidationContext` (scenario id, the turns that produced it) and
+ * returns a `ValidationResult` with pass/fail + 0..1 score + structured
+ * issues.
+ */
+interface Artifact {
+    /** Logical kind — validators type-guard on this */
+    kind: 'file' | 'json' | 'text' | 'binary' | string;
+    /** Filesystem-style path, optional */
+    path?: string;
+    /** String content for text/json/file kinds */
+    content?: string;
+    /** Binary content (if kind === 'binary') */
+    bytes?: Uint8Array;
+    /** Caller-supplied metadata (mimeType, sha256, size, etc.) */
+    metadata?: Record<string, unknown>;
+}
+interface ValidationContext {
+    scenarioId: string;
+    turnIndex?: number;
+    /** Prior artifacts for multi-artifact scenarios */
+    priorArtifacts?: Artifact[];
+    /** Free-form hints the validator uses for domain-specific checks */
+    hints?: Record<string, unknown>;
+}
+interface ValidationIssue {
+    severity: 'error' | 'warning' | 'info';
+    message: string;
+    /** Optional path into the artifact (e.g. JSON path or byte offset) */
+    locus?: string;
+}
+interface ValidationResult {
+    pass: boolean;
+    /** 0–1 normalized score. Validators should be monotonic in pass-ness. */
+    score: number;
+    issues: ValidationIssue[];
+    /** Diagnostic payload for reporters */
+    evidence?: Record<string, unknown>;
+}
+interface ArtifactValidator {
+    /** Stable identifier for the validator; appears in reports. */
+    name: string;
+    /** Optional description for human-facing reports. */
+    description?: string;
+    /** Called once per artifact; validators are expected to be pure + idempotent. */
+    validate(artifact: Artifact, context: ValidationContext): Promise<ValidationResult>;
+}
+/**
+ * Run every validator on the same artifact; aggregate pass as AND, score as
+ * (weighted) mean, issues concatenated. Weights default to 1 each.
+ */
+declare function composeValidators(validators: ArtifactValidator[], options?: {
+    name?: string;
+    weights?: number[];
+}): ArtifactValidator;
+/** Pass if the artifact body matches a provided regex. */
+declare function regexMatch(name: string, pattern: RegExp): ArtifactValidator;
+/** Pass if JSON parses and every required key is present. */
+declare function jsonHasKeys(name: string, requiredPaths: string[]): ArtifactValidator;
+/** Pass if min ≤ byte length ≤ max. */
+declare function byteLengthRange(name: string, min: number, max: number): ArtifactValidator;
+/** Pass if the artifact contains every required substring (case-insensitive by default). */
+declare function containsAll(name: string, required: string[], options?: {
+    caseSensitive?: boolean;
+}): ArtifactValidator;
+
+/**
+ * Workspace inspector — score the persisted state of an agent after a run.
+ *
+ * Many evals don't ask "did the response say the right thing" but "did the
+ * agent put the right rows in the DB / files in the vault / entities on the
+ * canvas". This is the primitive for that.
+ *
+ * Implementations read from D1, KV, filesystem, or any store — the interface
+ * is deliberately small so consumers plug in their own backends.
+ */
+interface WorkspaceSnapshot {
+    /** Vault files: logical path → content */
+    files: Record<string, string>;
+    /** DB rows: table name → array of rows (post-validation) */
+    rows: Record<string, Array<Record<string, unknown>>>;
+    /** KV entries: key → value (scoped to whatever prefix the inspector chose) */
+    kv: Record<string, string>;
+    /** Free-form blob metadata: for large binaries the inspector stores summary, not bytes */
+    blobs?: Record<string, {
+        size: number;
+        hash?: string;
+        mimeType?: string;
+    }>;
+}
+interface InspectorContext {
+    /** Workspace / agent / thread id — whatever the backend uses to scope the snapshot */
+    scopeId: string;
+    /** Optional scenario id — allows scenario-specific snapshot shaping */
+    scenarioId?: string;
+}
+interface WorkspaceInspector {
+    name: string;
+    snapshot(context: InspectorContext): Promise<WorkspaceSnapshot>;
+}
+declare class InMemoryWorkspaceInspector implements WorkspaceInspector {
+    readonly name = "in-memory";
+    private readonly snapshots;
+    set(scopeId: string, snapshot: WorkspaceSnapshot): void;
+    snapshot(context: InspectorContext): Promise<WorkspaceSnapshot>;
+}
+interface WorkspaceAssertion {
+    name: string;
+    description?: string;
+    check(snapshot: WorkspaceSnapshot): WorkspaceAssertionResult;
+}
+interface WorkspaceAssertionResult {
+    pass: boolean;
+    /** 0..1 — partial credit for assertions that admit it */
+    score: number;
+    detail?: string;
+}
+declare function fileExists(path: string): WorkspaceAssertion;
+declare function fileContains(path: string, needle: string): WorkspaceAssertion;
+declare function rowCount(table: string, min: number, max?: number): WorkspaceAssertion;
+declare function rowWhere<T extends Record<string, unknown>>(table: string, predicate: (row: T) => boolean, options?: {
+    min?: number;
+}): WorkspaceAssertion;
+/** Run many assertions; return aggregate pass + mean score + per-assertion details. */
+declare function runAssertions(snapshot: WorkspaceSnapshot, assertions: WorkspaceAssertion[]): {
+    pass: boolean;
+    score: number;
+    results: Array<{
+        assertion: string;
+        result: WorkspaceAssertionResult;
+    }>;
+};
+
+/**
+ * Experiment tracker — group runs, diff them, watch scores move over time.
+ *
+ * Not MLflow. Not Weights & Biases. Just the 20% that actually ships:
+ *   - A run has a config (prompt hash, model, scenario ids, seed)
+ *   - Runs belong to experiments (named groups)
+ *   - The store is pluggable (in-memory for tests, filesystem for local,
+ *     custom for Langfuse/D1)
+ *   - Diffs show score deltas, new/dropped scenarios, and config changes
+ *
+ * The output plugs directly into `BenchmarkReport` — runs archive the full
+ * report, diff operates on the summary.
+ */
+
+interface RunConfig {
+    experimentId: string;
+    name?: string;
+    model?: string;
+    promptHash?: string;
+    promptVersion?: string;
+    seed?: number;
+    metadata?: Record<string, unknown>;
+}
+interface Run {
+    id: string;
+    experimentId: string;
+    name?: string;
+    config: RunConfig;
+    startedAt: string;
+    completedAt?: string;
+    status: 'running' | 'completed' | 'failed';
+    report?: BenchmarkReport;
+    error?: string;
+}
+interface Experiment {
+    id: string;
+    name: string;
+    createdAt: string;
+    metadata?: Record<string, unknown>;
+}
+interface ExperimentStore {
+    saveExperiment(exp: Experiment): Promise<void>;
+    getExperiment(id: string): Promise<Experiment | null>;
+    listExperiments(): Promise<Experiment[]>;
+    saveRun(run: Run): Promise<void>;
+    getRun(id: string): Promise<Run | null>;
+    listRuns(experimentId: string): Promise<Run[]>;
+}
+declare class InMemoryExperimentStore implements ExperimentStore {
+    private readonly experiments;
+    private readonly runs;
+    saveExperiment(exp: Experiment): Promise<void>;
+    getExperiment(id: string): Promise<Experiment | null>;
+    listExperiments(): Promise<Experiment[]>;
+    saveRun(run: Run): Promise<void>;
+    getRun(id: string): Promise<Run | null>;
+    listRuns(experimentId: string): Promise<Run[]>;
+}
+declare class ExperimentTracker {
+    private readonly store;
+    constructor(store: ExperimentStore);
+    startExperiment(name: string, metadata?: Record<string, unknown>): Promise<Experiment>;
+    startRun(config: RunConfig): Promise<Run>;
+    completeRun(runId: string, report: BenchmarkReport): Promise<void>;
+    failRun(runId: string, error: string): Promise<void>;
+    /**
+     * Diff two completed runs. Returns per-scenario deltas, aggregate delta,
+     * and config changes that may explain the movement.
+     */
+    diff(runIdA: string, runIdB: string): Promise<RunDiff>;
+    /** Timeline of aggregate scores for an experiment. */
+    timeline(experimentId: string): Promise<Array<{
+        runId: string;
+        startedAt: string;
+        overall: number | null;
+    }>>;
+}
+interface RunDiff {
+    before: {
+        runId: string;
+        name?: string;
+        startedAt: string;
+    };
+    after: {
+        runId: string;
+        name?: string;
+        startedAt: string;
+    };
+    aggregateDelta: number;
+    scenarios: Array<{
+        scenarioId: string;
+        before: number | null;
+        after: number | null;
+        delta: number | null;
+        status: 'improved' | 'regressed' | 'unchanged' | 'added' | 'removed';
+    }>;
+    configChanges: Record<string, {
+        before: unknown;
+        after: unknown;
+    }>;
+}
+
+/**
+ * Prompt optimizer — A/B test prompt variants with statistical rigor.
+ *
+ * Runs N prompt variants against a fixed scenario set, collects per-scenario
+ * scores via the user-provided `scoreVariant` callback, and returns:
+ *   - per-variant mean + bootstrap CI
+ *   - pairwise significance (Mann-Whitney, non-parametric — works on any
+ *     score distribution, not just normal)
+ *   - a winner (highest mean, flagged if the lead is not significant)
+ *
+ * Deliberately generic — the `scoreVariant` callback does whatever domain
+ * work the consumer needs (invoke the agent, judge the output, whatever),
+ * and returns a number per scenario. This lets the optimizer stay small +
+ * testable.
+ */
+interface PromptVariant {
+    id: string;
+    prompt: string;
+    metadata?: Record<string, unknown>;
+}
+interface OptimizationConfig {
+    variants: PromptVariant[];
+    /** How many trials per (variant, scenario) — controls CI tightness. Default 3. */
+    trialsPerScenario?: number;
+    /** Significance threshold for pairwise comparison (default 0.05). */
+    significanceLevel?: number;
+    /**
+     * The scoring callback. For each (variant, scenarioId, trialIndex), produce
+     * a score in 0..1 (or any numeric range — the optimizer only cares about
+     * monotonicity).
+     */
+    scoreVariant: (args: {
+        variant: PromptVariant;
+        scenarioId: string;
+        trialIndex: number;
+    }) => Promise<number>;
+    /** Scenario ids to run against. */
+    scenarioIds: string[];
+    /** Optional hook — fires after each (variant, scenario) fully scored. */
+    onScenarioComplete?: (info: {
+        variantId: string;
+        scenarioId: string;
+        scores: number[];
+    }) => void;
+}
+interface VariantScore {
+    variantId: string;
+    mean: number;
+    ci95: {
+        lower: number;
+        upper: number;
+    };
+    n: number;
+    perScenario: Record<string, {
+        mean: number;
+        n: number;
+        samples: number[];
+    }>;
+}
+interface PairwiseComparison {
+    variantA: string;
+    variantB: string;
+    pValue: number;
+    significant: boolean;
+    meanDelta: number;
+}
+interface OptimizationResult {
+    winner: {
+        variantId: string;
+        /** True when the winner's lead vs every other variant is statistically significant. */
+        significant: boolean;
+        ciLowerBoundExceedsSecondMean: boolean;
+    };
+    scores: VariantScore[];
+    pairwise: PairwiseComparison[];
+    config: {
+        trialsPerScenario: number;
+        significanceLevel: number;
+        variants: string[];
+        scenarios: string[];
+    };
+}
+declare class PromptOptimizer {
+    run(config: OptimizationConfig): Promise<OptimizationResult>;
+}
+
+/**
+ * Dual-agent convergence bench.
+ *
+ * Pattern lifted from tax-agent + legal-agent: two agents take turns until
+ * they converge on a consensus artifact. One proposes, the other critiques;
+ * the proposer revises; repeat until a score threshold is hit or max rounds.
+ *
+ * Generalized so any two "agents" (gateways, local functions, anything with
+ * `propose` + `critique`) compose in. Returns convergence rounds per
+ * scenario + whether convergence happened.
+ */
+interface DualAgentScenario {
+    id: string;
+    initialPrompt: string;
+    /** Optional context the agents can read (e.g. source documents). */
+    context?: Record<string, unknown>;
+}
+interface DualAgentRound {
+    roundIndex: number;
+    proposal: string;
+    critique: string;
+    convergenceScore: number;
+}
+interface DualAgentScenarioResult {
+    scenarioId: string;
+    converged: boolean;
+    roundsToConverge: number | null;
+    finalProposal: string;
+    history: DualAgentRound[];
+    finalScore: number;
+}
+interface DualAgentBenchConfig {
+    scenarios: DualAgentScenario[];
+    maxRounds?: number;
+    /** Convergence threshold in 0..1 (default 0.85). */
+    convergenceThreshold?: number;
+    /**
+     * Propose an answer given the scenario + the critic's prior critique (if any).
+     * Returns the proposal string.
+     */
+    propose: (args: {
+        scenario: DualAgentScenario;
+        roundIndex: number;
+        priorProposal?: string;
+        priorCritique?: string;
+    }) => Promise<string>;
+    /**
+     * Critique the proposer's current output. Returns a structured critique
+     * (free text) plus a convergence score: how close the proposal is to
+     * acceptable. 1.0 = accept, 0.0 = totally off.
+     */
+    critique: (args: {
+        scenario: DualAgentScenario;
+        roundIndex: number;
+        proposal: string;
+    }) => Promise<{
+        critique: string;
+        convergenceScore: number;
+    }>;
+    /** Optional per-round hook for progress + tracing. */
+    onRoundComplete?: (info: {
+        scenarioId: string;
+        round: DualAgentRound;
+    }) => void;
+}
+interface DualAgentReport {
+    scenarios: DualAgentScenarioResult[];
+    aggregate: {
+        convergenceRate: number;
+        avgRoundsToConverge: number | null;
+        avgFinalScore: number;
+    };
+    config: {
+        maxRounds: number;
+        convergenceThreshold: number;
+    };
+}
+declare class DualAgentBench {
+    run(config: DualAgentBenchConfig): Promise<DualAgentReport>;
+}
+
+export { AgentDriver, type AgentDriverConfig, type AntiSlopConfig, type AntiSlopIssue, type AntiSlopReport, type Artifact, type ArtifactCheck, type ArtifactResult, type ArtifactValidator, type BenchmarkReport, BenchmarkRunner, type BenchmarkRunnerConfig, type CheckResult, type CollectedArtifacts, type CompletionCriterion, ConvergenceTracker, type DriverResult, type DriverState, DualAgentBench, type DualAgentBenchConfig, type DualAgentReport, type DualAgentRound, type DualAgentScenario, type DualAgentScenarioResult, type EvalResult, type ExecutorConfig, type Experiment, type ExperimentStore, ExperimentTracker, type FeedbackPattern, FileSystemTraceStore, type FileSystemTraceStoreOptions, InMemoryExperimentStore, InMemoryWorkspaceInspector, type InspectorContext, type JudgeConfig, type JudgeFn, type JudgeInput, type JudgeRubric, type JudgeScore, type LlmTrace, MODEL_PRICING, MemoryTraceStore, MetricsCollector, type OptimizationConfig, type OptimizationResult, type PairwiseComparison, type PersonaConfig, ProductClient, type ProductClientConfig, type PromptHandle, PromptOptimizer, PromptRegistry, type PromptVariant, type RouteMap, type RubricDimension, type Run, type RunConfig, type RunDiff, type Scenario, type ScenarioFile, ScenarioRegistry, type ScenarioResult, type SlopCategory, type TestResult, TokenCounter, type TraceQuery, type TraceStore, type Turn, type TurnMetrics, type TurnResult, type ValidationContext, type ValidationIssue, type ValidationResult, type VariantScore, type WorkspaceAssertion, type WorkspaceAssertionResult, type WorkspaceInspector, type WorkspaceSnapshot, adversarialJudge, analyzeAntiSlop, byteLengthRange, codeExecutionJudge, cohensD, coherenceJudge, composeValidators, confidenceInterval, containsAll, createAntiSlopJudge, createCustomJudge, createDomainExpertJudge, defaultJudges, estimateCost, estimateTokens, executeScenario, fileContains, fileExists, formatBenchmarkReport, formatDriverReport, hashContent, interRaterReliability, jsonHasKeys, mannWhitneyU, normalizeScores, pairedTTest, partialCredit, printDriverSummary, regexMatch, rowCount, rowWhere, runAssertions, runE2EWorkflow, weightedMean, wilcoxonSignedRank };