@tangle-network/agent-eval 0.107.0 → 0.108.0

package/dist/index-pPtfoIJO.d.ts

@@ -0,0 +1,423 @@
+import { S as Scenario, b as DispatchContext, C as CampaignResult } from './types-BdIv5dvA.js';
+import { b as RunSplitTag } from './run-record-I-Z3JNvO.js';
+import { C as CampaignStorage } from './storage-Dw_f7WMt.js';
+
+/**
+ * Shared types for the reference benchmark wrappers under
+ * `src/benchmarks/`. Each wrapper exports the three functions in
+ * `BenchmarkAdapter` plus its own typed `DatasetItem` shape.
+ */
+
+type BenchmarkTaskKind = 'retrieval' | 'rag-answer' | 'hallucination' | 'kb-improvement' | 'routing' | 'custom';
+type BenchmarkFamily = 'beir' | 'mteb-retrieval' | 'msmarco' | 'trec-dl' | 'miracl' | 'lotte' | 'bright' | 'crag' | 'hotpotqa' | 'kilt' | 'ragtruth' | 'faithbench' | 'first-party' | 'custom';
+interface BenchmarkDatasetItem<TPayload = unknown> {
+    /** Stable dataset-local item id (used for split assignment + paper
+     *  references). Unique within a benchmark. */
+    id: string;
+    /** Free-form payload. Each benchmark defines its own shape. */
+    payload: TPayload;
+    /** Optional precomputed split. When absent, adapters use assignSplit(id). */
+    split?: RunSplitTag;
+    /** Benchmark family this row came from, e.g. `beir` or `crag`. */
+    family?: BenchmarkFamily | string;
+    /** Benchmark-local task kind, e.g. retrieval vs answer quality. */
+    taskKind?: BenchmarkTaskKind | string;
+    /** Slice labels such as language, domain, freshness, multihop, long-tail. */
+    tags?: string[];
+    /** Dataset provenance, version, URL, or license notes. */
+    source?: BenchmarkSource;
+    metadata?: Record<string, unknown>;
+}
+interface BenchmarkEvaluation {
+    /** [0, 1] score for the response on this item. Exact-match
+     *  benchmarks use 0/1; partial-credit benchmarks may return
+     *  fractional values. */
+    score: number;
+    /** Optional pass/fail projection. Defaults to `score > 0` when absent. */
+    passed?: boolean;
+    /** Numeric sub-metrics. These become report dimensions. */
+    dimensions?: Record<string, number>;
+    /** Optional bag of raw scoring signals — e.g. parsed numeric
+     *  answer, regex match, judge sub-scores. */
+    raw: Record<string, unknown>;
+    notes?: string;
+}
+interface BenchmarkSource {
+    name?: string;
+    url?: string;
+    version?: string;
+    license?: string;
+    citation?: string;
+}
+/** Common signature implemented by every adapter under `src/benchmarks/*`. */
+interface BenchmarkAdapter<_TItem = unknown, TPayload = unknown, TArtifact = string> {
+    /** Stable benchmark id such as `beir/nfcorpus` or `crag/smoke`. */
+    id?: string;
+    family?: BenchmarkFamily | string;
+    taskKind?: BenchmarkTaskKind | string;
+    description?: string;
+    source?: BenchmarkSource;
+    defaultMetric?: string;
+    /** Load the dataset for the given split. May hit the network on
+     *  first call but should be cache-friendly. Adapters that don't
+     *  ship the dataset itself MUST throw a clearly-marked error
+     *  pointing the caller at the loader script. */
+    loadDataset(split: RunSplitTag): Promise<BenchmarkDatasetItem<TPayload>[]>;
+    /** Score a single response. Pure with respect to the inputs. */
+    evaluate(item: BenchmarkDatasetItem<TPayload>, artifact: TArtifact): Promise<BenchmarkEvaluation>;
+    /** Deterministic split assignment via item id hashing. The
+     *  fraction of items in each split is implementation-defined but
+     *  MUST be stable across processes and platforms. */
+    assignSplit(itemId: string): RunSplitTag;
+}
+interface BenchmarkScenario<TPayload = unknown> extends Scenario {
+    kind: 'benchmark';
+    benchmarkId: string;
+    family: BenchmarkFamily | string;
+    taskKind: BenchmarkTaskKind | string;
+    splitTag: RunSplitTag;
+    item: BenchmarkDatasetItem<TPayload>;
+}
+type BenchmarkResponder<TPayload = unknown, TArtifact = string> = (input: {
+    scenario: BenchmarkScenario<TPayload>;
+    item: BenchmarkDatasetItem<TPayload>;
+    context: DispatchContext;
+}) => Promise<TArtifact> | TArtifact;
+/** Split-assignment seed shared across all benchmarks. Bumping this
+ *  value reshuffles every split — do NOT do that lightly. */
+declare const BENCHMARK_SPLIT_SEED = "agent-eval-v1";
+/**
+ * Assign an item id to one of `'search' | 'dev' | 'holdout'` using a
+ * stable 32-bit hash of `${seed}::${id}`. Default proportions:
+ *
+ *   search: 60%   (optimization-readable)
+ *   dev:    20%   (held-out for tuning, leak-on-purpose during dev)
+ *   holdout:20%   (paper-grade held-out, gated reads)
+ */
+declare function deterministicSplit(itemId: string, seed?: string): RunSplitTag;
+
+interface BenchmarkMetricCalibrationOptions<TPayload = unknown, TArtifact = string> {
+    adapter: BenchmarkAdapter<BenchmarkDatasetItem<TPayload>, TPayload, TArtifact>;
+    item: BenchmarkDatasetItem<TPayload>;
+    weakArtifact: TArtifact;
+    strongArtifact: TArtifact;
+    maxWeakScore?: number;
+    minStrongScore?: number;
+    minGap?: number;
+}
+interface BenchmarkMetricCalibrationResult {
+    passed: boolean;
+    weak: BenchmarkEvaluation;
+    strong: BenchmarkEvaluation;
+    weakScore: number;
+    strongScore: number;
+    gap: number;
+    reasons: string[];
+}
+declare function calibrateBenchmarkMetric<TPayload = unknown, TArtifact = string>(options: BenchmarkMetricCalibrationOptions<TPayload, TArtifact>): Promise<BenchmarkMetricCalibrationResult>;
+
+/**
+ * Synthetic routing dataset. 16 tasks across 4 categories. Used as a
+ * deterministic, dependency-free benchmark for any router that maps a
+ * natural-language request to one of a fixed set of route labels.
+ *
+ * Format (see `routing/README.md` for prose):
+ *
+ *   {
+ *     id:           stable per-task ID (matches across processes).
+ *     category:     one of the four route labels.
+ *     prompt:       the user-facing request the router must classify.
+ *     route:        the ground-truth route the router should pick.
+ *     synonyms:     other strings that count as a correct answer.
+ *     hardNegatives:close-but-wrong route labels — used to detect the
+ *                   "always picks the popular route" failure mode.
+ *   }
+ *
+ * The four categories are intentionally cross-domain (file ops,
+ * math, search, conversation) so a router that collapses to one
+ * category is easy to spot.
+ */
+interface RoutingItem {
+    id: string;
+    category: 'file' | 'math' | 'search' | 'chat';
+    prompt: string;
+    /** Canonical correct route label. */
+    route: string;
+    /** Alternate route labels that also count as correct. */
+    synonyms: string[];
+    /** Wrong-but-tempting route labels (for analysis, not grading). */
+    hardNegatives: string[];
+}
+declare const ROUTING_DATASET: RoutingItem[];
+
+/**
+ * Routing benchmark — synthetic, dependency-free, ships in the
+ * package. 16 cross-category items in `dataset.ts`. See
+ * `routing/README.md` for the format.
+ *
+ * `evaluate` does case-insensitive exact match against the canonical
+ * route plus declared synonyms. The first valid route token in the
+ * response wins; everything else is ignored. Wrong answers also
+ * report whether they hit a hard negative — useful when triaging
+ * "always picks the popular route" failure modes.
+ */
+
+type RoutingPayload = RoutingItem;
+type RoutingDatasetItem = BenchmarkDatasetItem<RoutingPayload>;
+declare class RoutingAdapter implements BenchmarkAdapter<RoutingDatasetItem, RoutingPayload> {
+    readonly id = "first-party/routing";
+    readonly family = "first-party";
+    readonly taskKind = "routing";
+    readonly description = "Synthetic fixed-route classification smoke benchmark";
+    readonly defaultMetric = "route_exact_match";
+    loadDataset(split: RunSplitTag): Promise<RoutingDatasetItem[]>;
+    evaluate(item: RoutingDatasetItem, response: string): Promise<BenchmarkEvaluation>;
+    assignSplit(itemId: string): RunSplitTag;
+}
+/**
+ * Pull route-shaped tokens out of a model response. Routes look like
+ * `category.action` (`fs.write`, `chat.reply`). Bare alphanumerics
+ * are not routes, but `category.action` patterns are robust to most
+ * model wrappers (JSON output, prose explanations, code fences).
+ */
+declare function extractRouteTokens(response: string): string[];
+declare const loadDataset: (split: RunSplitTag) => Promise<RoutingDatasetItem[]>;
+declare const evaluate: (item: RoutingDatasetItem, response: string) => Promise<BenchmarkEvaluation>;
+declare const assignSplit: (itemId: string) => RunSplitTag;
+
+declare const index$1_ROUTING_DATASET: typeof ROUTING_DATASET;
+type index$1_RoutingAdapter = RoutingAdapter;
+declare const index$1_RoutingAdapter: typeof RoutingAdapter;
+type index$1_RoutingDatasetItem = RoutingDatasetItem;
+type index$1_RoutingItem = RoutingItem;
+type index$1_RoutingPayload = RoutingPayload;
+declare const index$1_assignSplit: typeof assignSplit;
+declare const index$1_evaluate: typeof evaluate;
+declare const index$1_extractRouteTokens: typeof extractRouteTokens;
+declare const index$1_loadDataset: typeof loadDataset;
+declare namespace index$1 {
+  export { index$1_ROUTING_DATASET as ROUTING_DATASET, index$1_RoutingAdapter as RoutingAdapter, type index$1_RoutingDatasetItem as RoutingDatasetItem, type index$1_RoutingItem as RoutingItem, type index$1_RoutingPayload as RoutingPayload, index$1_assignSplit as assignSplit, index$1_evaluate as evaluate, index$1_extractRouteTokens as extractRouteTokens, index$1_loadDataset as loadDataset };
+}
+
+interface BenchmarkRunOptions<TPayload = unknown, TArtifact = string> {
+    adapter: BenchmarkAdapter<BenchmarkDatasetItem<TPayload>, TPayload, TArtifact>;
+    respond: BenchmarkResponder<TPayload, TArtifact>;
+    splits?: readonly RunSplitTag[];
+    runDir: string;
+    repo?: string;
+    seed?: number;
+    reps?: number;
+    resumable?: boolean;
+    costCeiling?: number;
+    maxConcurrency?: number;
+    dispatchTimeoutMs?: number;
+    expectUsage?: 'assert' | 'warn' | 'off';
+    storage?: CampaignStorage;
+    now?: () => Date;
+}
+interface BenchmarkReport {
+    benchmarkId: string;
+    family: BenchmarkFamily | string;
+    taskKind: BenchmarkTaskKind | string;
+    source?: BenchmarkAdapter['source'];
+    runDir: string;
+    manifestHash: string;
+    seed: number;
+    startedAt: string;
+    endedAt: string;
+    durationMs: number;
+    totalItems: number;
+    totalCells: number;
+    cellsFailed: number;
+    cellsCached: number;
+    totalCostUsd: number;
+    splits: Record<string, BenchmarkSliceSummary>;
+    tags: Record<string, BenchmarkSliceSummary>;
+    dimensions: Record<string, BenchmarkDistribution>;
+    score: BenchmarkDistribution;
+    costUsd: BenchmarkDistribution;
+    latencyMs: BenchmarkDistribution;
+}
+interface BenchmarkSliceSummary {
+    n: number;
+    meanScore: number;
+    passRate: number;
+    score: BenchmarkDistribution;
+    costUsd: BenchmarkDistribution;
+    latencyMs: BenchmarkDistribution;
+}
+interface BenchmarkDistribution {
+    n: number;
+    min: number;
+    mean: number;
+    median: number;
+    p90: number;
+    max: number;
+}
+interface BenchmarkRunResult<TPayload = unknown, TArtifact = string> {
+    scenarios: Array<BenchmarkScenario<TPayload>>;
+    campaign: CampaignResult<TArtifact, BenchmarkScenario<TPayload>>;
+    report: BenchmarkReport;
+    reportJsonPath: string;
+    reportMarkdownPath: string;
+}
+declare function runBenchmarkAdapter<TPayload = unknown, TArtifact = string>(options: BenchmarkRunOptions<TPayload, TArtifact>): Promise<BenchmarkRunResult<TPayload, TArtifact>>;
+declare function summarizeBenchmarkCampaign<TPayload, TArtifact>(input: {
+    adapter: BenchmarkAdapter<BenchmarkDatasetItem<TPayload>, TPayload, TArtifact>;
+    scenarios: Array<BenchmarkScenario<TPayload>>;
+    campaign: CampaignResult<TArtifact, BenchmarkScenario<TPayload>>;
+}): BenchmarkReport;
+declare function renderBenchmarkReportMarkdown(report: BenchmarkReport): string;
+
+interface StandardRetrievalDocument {
+    id: string;
+    title?: string;
+    text: string;
+    metadata?: Record<string, unknown>;
+}
+interface StandardRetrievalQuery {
+    id: string;
+    text: string;
+    metadata?: Record<string, unknown>;
+}
+interface StandardRetrievalQrel {
+    queryId: string;
+    documentId: string;
+    score: number;
+}
+interface StandardRetrievalPayload {
+    queryId: string;
+    query: string;
+    expectedDocumentIds: string[];
+    expectedScores: Record<string, number>;
+    corpus?: Record<string, StandardRetrievalDocument>;
+    metadata?: Record<string, unknown>;
+}
+interface BuildStandardRetrievalItemsOptions {
+    benchmarkId: string;
+    family: BenchmarkFamily | string;
+    queries: readonly StandardRetrievalQuery[];
+    qrels: readonly StandardRetrievalQrel[];
+    corpus?: readonly StandardRetrievalDocument[];
+    includeCorpusInPayload?: boolean;
+    source?: BenchmarkSource;
+    tags?: readonly string[];
+    splitOf?: (queryId: string) => RunSplitTag;
+}
+interface RetrievalIdAdapterOptions extends BuildStandardRetrievalItemsOptions {
+    responseIdPattern?: RegExp;
+    cutoffs?: readonly number[];
+    primaryMetric?: string;
+    passMetric?: string;
+    passThreshold?: number;
+}
+interface StandardRetrievalResult {
+    id?: string;
+    documentId?: string;
+    docId?: string;
+    score?: number;
+}
+type StandardRetrievalArtifact = string | readonly string[] | readonly StandardRetrievalResult[] | {
+    ids?: readonly string[];
+    documentIds?: readonly string[];
+    results?: readonly StandardRetrievalResult[];
+};
+interface StandardRetrievalEvaluationOptions {
+    responseIdPattern?: RegExp;
+    cutoffs?: readonly number[];
+    primaryMetric?: string;
+    passMetric?: string;
+    passThreshold?: number;
+}
+declare function parseJsonlRows<T = unknown>(text: string): T[];
+declare function parseTsvRows(text: string): string[][];
+declare function parseQrels(text: string): StandardRetrievalQrel[];
+declare function parseBeirCorpusJsonl(text: string): StandardRetrievalDocument[];
+declare function parseBeirQueriesJsonl(text: string): StandardRetrievalQuery[];
+declare function buildStandardRetrievalItems(options: BuildStandardRetrievalItemsOptions): Array<BenchmarkDatasetItem<StandardRetrievalPayload>>;
+declare function createRetrievalIdBenchmarkAdapter(options: RetrievalIdAdapterOptions): BenchmarkAdapter<BenchmarkDatasetItem<StandardRetrievalPayload>, StandardRetrievalPayload, StandardRetrievalArtifact>;
+declare function evaluateStandardRetrieval(payload: StandardRetrievalPayload, artifact: StandardRetrievalArtifact, options?: StandardRetrievalEvaluationOptions): {
+    score: number;
+    passed: boolean;
+    dimensions: Record<string, number>;
+    raw: {
+        rankedDocumentIds: string[];
+        expectedDocumentIds: string[];
+        expectedScores: Record<string, number>;
+    };
+};
+declare function normalizeRetrievedDocumentIds(artifact: StandardRetrievalArtifact, responseIdPattern?: RegExp): string[];
+declare function retrievalMetricsAtCutoff(input: {
+    rankedDocumentIds: readonly string[];
+    expectedScores: Record<string, number>;
+    cutoff: number;
+}): Record<string, number>;
+
+/**
+ * Reference benchmark wrappers — entry point.
+ *
+ * Core surface (exported here):
+ *   - The `BenchmarkAdapter` contract.
+ *   - `runBenchmarkAdapter` for campaign-backed benchmark execution.
+ *   - `calibrateBenchmarkMetric` for weak/strong metric checks.
+ *   - Standard retrieval parsers for BEIR/MTEB/MS MARCO/TREC/MIRACL-style files.
+ *   - `deterministicSplit` + `BENCHMARK_SPLIT_SEED` for split assignment.
+ *   - `routing` — synthetic 16-task router benchmark. The only novel
+ *     benchmark we built; ships in the package.
+ *
+ * Example wrappers (under `examples/benchmarks/`, NOT in the bundle):
+ *   - `gsm8k`         — exact-match math reasoning (HF mirror, dataset
+ *                       not bundled).
+ *   - `swebench-lite` — 30-instance SWE-Bench subset via an external
+ *                       grader command.
+ *
+ * The example wrappers are reference implementations of `BenchmarkAdapter`.
+ * Read them, copy them, adapt them. They're intentionally not in the main
+ * entry — every team will configure them differently.
+ */
+
+declare const index_BENCHMARK_SPLIT_SEED: typeof BENCHMARK_SPLIT_SEED;
+type index_BenchmarkAdapter<_TItem = unknown, TPayload = unknown, TArtifact = string> = BenchmarkAdapter<_TItem, TPayload, TArtifact>;
+type index_BenchmarkDatasetItem<TPayload = unknown> = BenchmarkDatasetItem<TPayload>;
+type index_BenchmarkDistribution = BenchmarkDistribution;
+type index_BenchmarkEvaluation = BenchmarkEvaluation;
+type index_BenchmarkFamily = BenchmarkFamily;
+type index_BenchmarkMetricCalibrationOptions<TPayload = unknown, TArtifact = string> = BenchmarkMetricCalibrationOptions<TPayload, TArtifact>;
+type index_BenchmarkMetricCalibrationResult = BenchmarkMetricCalibrationResult;
+type index_BenchmarkReport = BenchmarkReport;
+type index_BenchmarkResponder<TPayload = unknown, TArtifact = string> = BenchmarkResponder<TPayload, TArtifact>;
+type index_BenchmarkRunOptions<TPayload = unknown, TArtifact = string> = BenchmarkRunOptions<TPayload, TArtifact>;
+type index_BenchmarkRunResult<TPayload = unknown, TArtifact = string> = BenchmarkRunResult<TPayload, TArtifact>;
+type index_BenchmarkScenario<TPayload = unknown> = BenchmarkScenario<TPayload>;
+type index_BenchmarkSliceSummary = BenchmarkSliceSummary;
+type index_BenchmarkSource = BenchmarkSource;
+type index_BenchmarkTaskKind = BenchmarkTaskKind;
+type index_BuildStandardRetrievalItemsOptions = BuildStandardRetrievalItemsOptions;
+type index_RetrievalIdAdapterOptions = RetrievalIdAdapterOptions;
+type index_StandardRetrievalArtifact = StandardRetrievalArtifact;
+type index_StandardRetrievalDocument = StandardRetrievalDocument;
+type index_StandardRetrievalEvaluationOptions = StandardRetrievalEvaluationOptions;
+type index_StandardRetrievalPayload = StandardRetrievalPayload;
+type index_StandardRetrievalQrel = StandardRetrievalQrel;
+type index_StandardRetrievalQuery = StandardRetrievalQuery;
+type index_StandardRetrievalResult = StandardRetrievalResult;
+declare const index_buildStandardRetrievalItems: typeof buildStandardRetrievalItems;
+declare const index_calibrateBenchmarkMetric: typeof calibrateBenchmarkMetric;
+declare const index_createRetrievalIdBenchmarkAdapter: typeof createRetrievalIdBenchmarkAdapter;
+declare const index_deterministicSplit: typeof deterministicSplit;
+declare const index_evaluateStandardRetrieval: typeof evaluateStandardRetrieval;
+declare const index_normalizeRetrievedDocumentIds: typeof normalizeRetrievedDocumentIds;
+declare const index_parseBeirCorpusJsonl: typeof parseBeirCorpusJsonl;
+declare const index_parseBeirQueriesJsonl: typeof parseBeirQueriesJsonl;
+declare const index_parseJsonlRows: typeof parseJsonlRows;
+declare const index_parseQrels: typeof parseQrels;
+declare const index_parseTsvRows: typeof parseTsvRows;
+declare const index_renderBenchmarkReportMarkdown: typeof renderBenchmarkReportMarkdown;
+declare const index_retrievalMetricsAtCutoff: typeof retrievalMetricsAtCutoff;
+declare const index_runBenchmarkAdapter: typeof runBenchmarkAdapter;
+declare const index_summarizeBenchmarkCampaign: typeof summarizeBenchmarkCampaign;
+declare namespace index {
+  export { index_BENCHMARK_SPLIT_SEED as BENCHMARK_SPLIT_SEED, type index_BenchmarkAdapter as BenchmarkAdapter, type index_BenchmarkDatasetItem as BenchmarkDatasetItem, type index_BenchmarkDistribution as BenchmarkDistribution, type index_BenchmarkEvaluation as BenchmarkEvaluation, type index_BenchmarkFamily as BenchmarkFamily, type index_BenchmarkMetricCalibrationOptions as BenchmarkMetricCalibrationOptions, type index_BenchmarkMetricCalibrationResult as BenchmarkMetricCalibrationResult, type index_BenchmarkReport as BenchmarkReport, type index_BenchmarkResponder as BenchmarkResponder, type index_BenchmarkRunOptions as BenchmarkRunOptions, type index_BenchmarkRunResult as BenchmarkRunResult, type index_BenchmarkScenario as BenchmarkScenario, type index_BenchmarkSliceSummary as BenchmarkSliceSummary, type index_BenchmarkSource as BenchmarkSource, type index_BenchmarkTaskKind as BenchmarkTaskKind, type index_BuildStandardRetrievalItemsOptions as BuildStandardRetrievalItemsOptions, type index_RetrievalIdAdapterOptions as RetrievalIdAdapterOptions, type index_StandardRetrievalArtifact as StandardRetrievalArtifact, type index_StandardRetrievalDocument as StandardRetrievalDocument, type index_StandardRetrievalEvaluationOptions as StandardRetrievalEvaluationOptions, type index_StandardRetrievalPayload as StandardRetrievalPayload, type index_StandardRetrievalQrel as StandardRetrievalQrel, type index_StandardRetrievalQuery as StandardRetrievalQuery, type index_StandardRetrievalResult as StandardRetrievalResult, index_buildStandardRetrievalItems as buildStandardRetrievalItems, index_calibrateBenchmarkMetric as calibrateBenchmarkMetric, index_createRetrievalIdBenchmarkAdapter as createRetrievalIdBenchmarkAdapter, index_deterministicSplit as deterministicSplit, index_evaluateStandardRetrieval as evaluateStandardRetrieval, index_normalizeRetrievedDocumentIds as normalizeRetrievedDocumentIds, index_parseBeirCorpusJsonl as parseBeirCorpusJsonl, index_parseBeirQueriesJsonl as parseBeirQueriesJsonl, index_parseJsonlRows as parseJsonlRows, index_parseQrels as parseQrels, index_parseTsvRows as parseTsvRows, index_renderBenchmarkReportMarkdown as renderBenchmarkReportMarkdown, index_retrievalMetricsAtCutoff as retrievalMetricsAtCutoff, index$1 as routing, index_runBenchmarkAdapter as runBenchmarkAdapter, index_summarizeBenchmarkCampaign as summarizeBenchmarkCampaign };
+}
+
+export { createRetrievalIdBenchmarkAdapter as A, BENCHMARK_SPLIT_SEED as B, evaluateStandardRetrieval as C, normalizeRetrievedDocumentIds as D, parseBeirCorpusJsonl as E, parseBeirQueriesJsonl as F, parseJsonlRows as G, parseQrels as H, parseTsvRows as I, renderBenchmarkReportMarkdown as J, retrievalMetricsAtCutoff as K, index$1 as L, runBenchmarkAdapter as M, summarizeBenchmarkCampaign as N, type RetrievalIdAdapterOptions as R, type StandardRetrievalArtifact as S, type BenchmarkAdapter as a, type BenchmarkDatasetItem as b, type BenchmarkEvaluation as c, type BenchmarkFamily as d, type BenchmarkResponder as e, type BenchmarkScenario as f, type BenchmarkSource as g, type BenchmarkTaskKind as h, deterministicSplit as i, index as j, type BenchmarkDistribution as k, type BenchmarkMetricCalibrationOptions as l, type BenchmarkMetricCalibrationResult as m, type BenchmarkReport as n, type BenchmarkRunOptions as o, type BenchmarkRunResult as p, type BenchmarkSliceSummary as q, type BuildStandardRetrievalItemsOptions as r, type StandardRetrievalDocument as s, type StandardRetrievalEvaluationOptions as t, type StandardRetrievalPayload as u, type StandardRetrievalQrel as v, type StandardRetrievalQuery as w, type StandardRetrievalResult as x, buildStandardRetrievalItems as y, calibrateBenchmarkMetric as z };

package/dist/index.d.ts

@@ -18,8 +18,8 @@ import { A as AgentEvalError, J as JudgeError, a as ConfigError } from './errors
 export { b as AgentEvalErrorCode, C as CaptureIntegrityError, N as NotFoundError, R as ReplayError, V as ValidationError, c as VerificationError } from './errors-oeQrLqXC.js';
 import { b as FeedbackLabel, F as FeedbackTrajectoryStore, a as FeedbackTrajectory } from './feedback-trajectory-C9KCo8ag.js';
 export { c as FeedbackArtifactType, d as FeedbackAttempt, e as FeedbackLabelKind, f as FeedbackLabelSource, g as FeedbackOptimizerRow, h as FeedbackOutcome, i as FeedbackReplayAdapter, j as FeedbackReplayResult, k as FeedbackSeverity, l as FeedbackSplitPolicy, m as FeedbackTask, n as FeedbackTrajectoryFilter, o as FileSystemFeedbackTrajectoryStore, I as InMemoryFeedbackTrajectoryStore, P as PreferenceMemoryEntry, p as ProposedSideEffect, q as assignFeedbackSplit, r as controlRunToFeedbackTrajectory, s as createFeedbackTrajectory, t as feedbackTrajectoriesToDatasetScenarios, u as feedbackTrajectoriesToOptimizerRows, v as feedbackTrajectoryToDatasetScenario, w as feedbackTrajectoryToOptimizerRow, x as parseFeedbackTrajectoriesJsonl, y as renderPreferenceMemoryMarkdown, z as replayFeedbackTrajectories, A as replayFeedbackTrajectory, B as serializeFeedbackTrajectoriesJsonl, C as summarizePreferenceMemory, D as withAssignedFeedbackSplit } from './feedback-trajectory-C9KCo8ag.js';
-import { b as CorrectnessChecker } from './pre-registration-x3f0VpxT.js';
-export { A as ArtifactCheckArtifact, d as ArtifactEventLike, e as ArtifactValidator, f as BackendIntegrityError, B as BackendIntegrityReport, C as CompletionRequirement, a as CompletionVerdict, H as HypothesisManifest, g as HypothesisResult, h as LlmCorrectnessCheckerOpts, L as LlmJudgeDimension, c as LlmJudgeOptions, i as ProducedProposal, P as ProducedState, j as ProposalEventLike, k as RequirementCheck, R as RuntimeEventLike, m as SatisfiedBy, S as SignedManifest, n as SignedManifestAlgo, T as TaskGold, o as ToolCallEventLike, V as ValidationContext, p as ValidationIssue, q as ValidationResult, r as assertRealBackend, s as byteLengthRange, t as canonicalize, u as completionVerdict, v as composeValidators, w as containsAll, x as createLlmCorrectnessChecker, y as createTokenRecallChecker, z as evaluateHypothesis, D as extractProducedState, E as hashJson, F as jsonHasKeys, l as llmJudge, G as parseCorrectnessResponse, I as regexMatch, J as signManifest, K as summarizeBackendIntegrity, M as verifyCompletion, N as verifyManifest } from './pre-registration-x3f0VpxT.js';
+import { b as CorrectnessChecker } from './pre-registration-BUhVPzE7.js';
+export { A as ArtifactCheckArtifact, d as ArtifactEventLike, e as ArtifactValidator, f as BackendIntegrityError, B as BackendIntegrityReport, C as CompletionRequirement, a as CompletionVerdict, H as HypothesisManifest, g as HypothesisResult, h as LlmCorrectnessCheckerOpts, L as LlmJudgeDimension, c as LlmJudgeOptions, i as ProducedProposal, P as ProducedState, j as ProposalEventLike, k as RequirementCheck, R as RuntimeEventLike, m as SatisfiedBy, S as SignedManifest, n as SignedManifestAlgo, T as TaskGold, o as ToolCallEventLike, V as ValidationContext, p as ValidationIssue, q as ValidationResult, r as assertRealBackend, s as byteLengthRange, t as canonicalize, u as completionVerdict, v as composeValidators, w as containsAll, x as createLlmCorrectnessChecker, y as createTokenRecallChecker, z as evaluateHypothesis, D as extractProducedState, E as hashJson, F as jsonHasKeys, l as llmJudge, G as parseCorrectnessResponse, I as regexMatch, J as signManifest, K as summarizeBackendIntegrity, M as verifyCompletion, N as verifyManifest } from './pre-registration-BUhVPzE7.js';
 export { DataAcquisitionPlan, KnowledgeAcquisitionMode, KnowledgeBundle, KnowledgeFallbackPolicy, KnowledgeFreshness, KnowledgeImportance, KnowledgeReadinessReport, KnowledgeRecommendedAction, KnowledgeRequirement, KnowledgeRequirementCategory, KnowledgeResponsibleSurface, KnowledgeSensitivity, ScoreKnowledgeReadinessOptions, UserQuestion, acquisitionPlansForKnowledgeGaps, blockingKnowledgeEval, knowledgeReadinessTracePayload, scoreKnowledgeReadiness, userQuestionsForKnowledgeGaps } from './knowledge/index.js';
 import { h as ReleaseConfidenceThresholds, f as ReleaseConfidenceScorecard } from './release-report-DeJpsBiA.js';
 export { A as ActionableSideInfo, o as AsiSeverity, B as BootstrapOptions, a as BootstrapResult, J as JudgeReplayGateArgs, R as ReleaseConfidenceAxis, b as ReleaseConfidenceAxisName, c as ReleaseConfidenceInput, d as ReleaseConfidenceIssue, e as ReleaseConfidenceMetrics, g as ReleaseConfidenceStatus, i as ReleaseTraceEvidence, j as RenderReleaseReportOptions, V as Verdict, k as assertReleaseConfidence, l as bootstrapCi, m as evaluateReleaseConfidence, n as judgeReplayGate, r as renderReleaseReport } from './release-report-DeJpsBiA.js';
@@ -31,7 +31,7 @@ import { a as AnalyzeTracesInput, A as AnalyzeTracesOptions, b as AnalyzeTracesR
 export { c as AnalyzeTracesTurnSnapshot, d as analyzeTraces } from './analyst-C8HHvfJp.js';
 import { a as TraceAnalystSpan } from './store-C1YxJDEK.js';
 export { D as DEFAULT_TRACE_ANALYST_BUDGETS, b as DatasetOverview, E as ErrorCluster, Q as QueryTracesPage, S as SearchSpanResult, c as SearchTraceResult, d as SpanMatchRecord, e as TRACE_ANALYST_TRUNCATION_MARKER_PREFIX, T as TraceAnalysisStore, f as TraceAnalystByteBudgets, g as TraceAnalystFilters, h as TraceAnalystSpanKind, i as TraceAnalystSpanStatus, j as TraceAnalystTraceSummary, V as ViewSpansResult, k as ViewTraceOversized, l as ViewTraceResult } from './store-C1YxJDEK.js';
-import { a as JudgeConfig, J as JudgeScore, S as Scenario$1, g as Gate } from './types-CNZ0tHET.js';
+import { a as JudgeConfig, J as JudgeScore, S as Scenario$1, f as Gate } from './types-BdIv5dvA.js';
 import { A as AnalyzeRunsOptions } from './analyze-runs-DJYpep3L.js';
 import { S as SteeringBundle } from './harness-optimizer-mOl9XX_O.js';
 export { D as DEFAULT_HARNESS_OBJECTIVES, H as HarnessAdapter, a as HarnessExperimentConfig, b as HarnessExperimentResult, c as HarnessIntervention, d as HarnessRunRequest, e as HarnessRunResult, f as HarnessScenario, g as HarnessSelection, h as HarnessVariant, i as HarnessVariantReport, M as MeasurementPolicy, j as SteeringDelta, k as SteeringRolePrompt, W as WorkflowTopology, m as mergeSteeringBundle, r as renderSteeringText, l as runHarnessExperiment, s as selectHarnessVariant, n as summarizeHarnessResults } from './harness-optimizer-mOl9XX_O.js';
@@ -71,18 +71,19 @@ import { b as Layer, S as Severity, L as LayerResult, c as VerifyContext } from
 export { F as Finding, d as LayerStatus, M as MultiLayerVerifier, a as VerificationReport, V as VerifyOptions, g as gradeSemanticStatus } from './multi-layer-verifier-CI4jdX-q.js';
 import { L as LlmClientOptions } from './llm-client-DyqEH4jH.js';
 export { d as LlmCallError, b as LlmCallRequest, c as LlmCallResult, e as LlmClient, f as LlmMessage, g as LlmRouteAssertionError, a as LlmRouteRequirements, h as LlmUsage, i as assertLlmRoute, j as backoffMs, k as callLlm, l as callLlmJson, m as isTransientLlmError, p as probeLlm, s as stripFencedJson } from './llm-client-DyqEH4jH.js';
-export { B as BENCHMARK_SPLIT_SEED, a as BenchmarkAdapter, b as BenchmarkDatasetItem, c as BenchmarkEvaluation, d as benchmarkDeterministicSplit, i as benchmarks } from './index-C2CC_dry.js';
+export { B as BENCHMARK_SPLIT_SEED, a as BenchmarkAdapter, b as BenchmarkDatasetItem, c as BenchmarkEvaluation, d as BenchmarkFamily, e as BenchmarkResponder, f as BenchmarkScenario, g as BenchmarkSource, h as BenchmarkTaskKind, i as benchmarkDeterministicSplit, j as benchmarks } from './index-pPtfoIJO.js';
 export { C as CallbackResearcher, d as CallbackResearcherOptions, e as CampaignFactoryParams, f as CampaignIntegrityPolicy, g as CampaignRunContext, h as CampaignRunOutcome, i as CampaignRunner, j as CampaignScenario, k as CampaignVariant, c as EvalCampaignOptions, b as EvalCampaignResult, E as ExperimentPlan, a as ExperimentResult, l as FailedRun, F as FailureMode, N as NoopResearcher, R as Researcher, S as SteeringChange, r as runEvalCampaign } from './researcher-Wc7dx6GM.js';
 export { G as GainDistributionBin, a as GainDistributionFigureSpec, b as GainDistributionOptions, m as GateDecision, n as GateEvidence, H as HeldOutGate, o as HeldOutGateConfig, q as HeldOutGateRejectionCode, P as ParetoFigureSpec, c as ParetoPoint, R as RESEARCH_REPORT_HARD_PAIR_FLOOR, d as ResearchReport, e as ResearchReportCandidate, f as ResearchReportDecision, g as ResearchReportMethodology, h as ResearchReportOptions, i as ResearchReportRecommendation, S as SummaryTable, j as SummaryTableOptions, k as SummaryTableRow, l as gainHistogram, p as paretoChart, r as researchReport, s as summaryTable } from './summary-report-QMZVe3P-.js';
 export { L as LockedJsonlAppender } from './testing-C21CHsq2.js';
 export { I as InterimReleaseConfidence, a as InterimReleaseConfidenceInput, P as PairedEvalueOptions, b as PairedEvalueSequence, c as PairedEvalueStep, S as SequentialDecision, e as evaluateInterimReleaseConfidence, p as pairedEvalueSequence } from './sequential-5iSVfzl2.js';
-import { j as GepaProposerConstraints, a as RunImprovementLoopResult } from './gepa-BmqTrdtg.js';
+import { f as GepaProposerConstraints, a as RunImprovementLoopResult } from './gepa-B3x5Ulcv.js';
 export { IntegrityResult, IntegrityViolation, JourneySpec, PerfBaseline, PerfGateResult, PerfRegression, PerfScenario, PerfStat, ScenarioAxes, assertRecordIntegrity, checkRecordIntegrity, expandMatrix, gatePerf, scenarioKey, summarizeRecords } from './perf/index.js';
 export { AgentProfileRuntimeReceipt, ProductBenchmarkArm, ProductBenchmarkArtifactPaths, ProductBenchmarkBudgets, ProductBenchmarkExportOptions, ProductBenchmarkExportResult, ProductBenchmarkManifest, ProductBenchmarkProfileRef, ProductBenchmarkRecord, ProductBenchmarkRepoRef, ProductBenchmarkRunInput, ProductBenchmarkScenario, ProductBenchmarkSingleRunExportOptions, ProductBenchmarkSplit, ProductBenchmarkSubstrateVersions, ProductBenchmarkValidationReport, RuntimeResolution, assertProductBenchmarkRun, buildProductBenchmarkManifest, exportProductBenchmark, exportProductBenchmarkRuns, findProductBenchmarkArtifacts, productBenchmarkIntegrityFailures, productBenchmarkMutableSurfaces, productBenchmarkRepoIdentity, productBenchmarkSplits, readProductBenchmarkManifest, readProductBenchmarkRecords, runRecordToProductBenchmarkRecord, validateProductBenchmarkManifest, validateProductBenchmarkRecord, validateProductBenchmarkRun } from './product-benchmark/index.js';
 import '@ax-llm/ax';
 import 'zod';
 import './insight-report-B4xrdwEK.js';
 import './outcome-store-rnXLEqSn.js';
+import './storage-Dw_f7WMt.js';
 
 /**
  * Automated pull-request transports for the production loop.

package/dist/index.js

@@ -50,6 +50,11 @@ import {
   validateProductBenchmarkRecord,
   validateProductBenchmarkRun
 } from "./chunk-FOUG2VVS.js";
+import {
+  BENCHMARK_SPLIT_SEED,
+  benchmarks_exports,
+  deterministicSplit
+} from "./chunk-T6W5ADLG.js";
 import {
   CODING_HARNESSES,
   HARNESS_NATIVE_MODEL,
@@ -72,7 +77,7 @@ import {
   llmJudge,
   parseCorrectnessResponse,
   verifyCompletion
-} from "./chunk-6PF6LXRY.js";
+} from "./chunk-4VLZEPJ3.js";
 import {
   DEFAULT_MUTATION_PRIMITIVES,
   DEFAULT_RED_TEAM_CORPUS,
@@ -96,7 +101,7 @@ import {
   scoreRedTeamOutput,
   surfaceContentHash,
   toolNamesForRun
-} from "./chunk-G4DLZAV5.js";
+} from "./chunk-OVPVM4JC.js";
 import {
   BackendIntegrityError,
   assertRealBackend,
@@ -116,11 +121,6 @@ import {
   isModelPriced,
   resolveModelPricing
 } from "./chunk-VI2UW6B6.js";
-import {
-  BENCHMARK_SPLIT_SEED,
-  benchmarks_exports,
-  deterministicSplit
-} from "./chunk-6QDKWHLS.js";
 import {
   DEFAULT_RULES,
   classifyFailure,
@@ -228,6 +228,7 @@ import {
 import {
   runEvalCampaign
 } from "./chunk-LIEJUH2I.js";
+import "./chunk-N22ZO7FV.js";
 import {
   LlmCallError,
   LlmClient,