@tangle-network/agent-eval 0.128.2 → 0.130.0

package/dist/index-CAPUUKaM.d.ts

@@ -0,0 +1,335 @@
+import { s as RunSplitTag } from "./run-record-CnZu_gjl.js";
+import { R as Scenario, a as CampaignResult, d as DispatchContext } from "./types-k9tZGKUg.js";
+import { sn as CampaignStorage } from "./skillopt-optimization-method-B7wX7XkF.js";
+import "./index-DE5fb3EC.js";
+//#region src/benchmarks/types.d.ts
+type BenchmarkTaskKind = 'retrieval' | 'rag-answer' | 'hallucination' | 'kb-improvement' | 'routing' | 'custom';
+type BenchmarkFamily = 'beir' | 'mteb-retrieval' | 'msmarco' | 'trec-dl' | 'miracl' | 'lotte' | 'bright' | 'crag' | 'hotpotqa' | 'kilt' | 'ragtruth' | 'faithbench' | 'first-party' | 'custom';
+interface BenchmarkDatasetItem<TPayload = unknown> {
+  /** Stable dataset-local item id (used for split assignment + paper
+   *  references). Unique within a benchmark. */
+  id: string;
+  /** Free-form payload. Each benchmark defines its own shape. */
+  payload: TPayload;
+  /** Optional precomputed split. When absent, adapters use assignSplit(id). */
+  split?: RunSplitTag;
+  /** Benchmark family this row came from, e.g. `beir` or `crag`. */
+  family?: BenchmarkFamily | string;
+  /** Benchmark-local task kind, e.g. retrieval vs answer quality. */
+  taskKind?: BenchmarkTaskKind | string;
+  /** Slice labels such as language, domain, freshness, multihop, long-tail. */
+  tags?: string[];
+  /** Dataset provenance, version, URL, or license notes. */
+  source?: BenchmarkSource;
+  metadata?: Record<string, unknown>;
+}
+interface BenchmarkEvaluation {
+  /** [0, 1] score for the response on this item. Exact-match
+   *  benchmarks use 0/1; partial-credit benchmarks may return
+   *  fractional values. */
+  score: number;
+  /** Optional pass/fail projection. Defaults to `score > 0` when absent. */
+  passed?: boolean;
+  /** Numeric sub-metrics. These become report dimensions. */
+  dimensions?: Record<string, number>;
+  /** Optional bag of raw scoring signals — e.g. parsed numeric
+   *  answer, regex match, judge sub-scores. */
+  raw: Record<string, unknown>;
+  notes?: string;
+}
+interface BenchmarkSource {
+  name?: string;
+  url?: string;
+  version?: string;
+  license?: string;
+  citation?: string;
+}
+/** Common signature implemented by every adapter under `src/benchmarks/*`. */
+interface BenchmarkAdapter<_TItem = unknown, TPayload = unknown, TArtifact = string> {
+  /** Stable benchmark id such as `beir/nfcorpus` or `crag/smoke`. */
+  id?: string;
+  family?: BenchmarkFamily | string;
+  taskKind?: BenchmarkTaskKind | string;
+  description?: string;
+  source?: BenchmarkSource;
+  defaultMetric?: string;
+  /** Load the dataset for the given split. May hit the network on
+   *  first call but should be cache-friendly. Adapters that don't
+   *  ship the dataset itself MUST throw a clearly-marked error
+   *  pointing the caller at the loader script. */
+  loadDataset(split: RunSplitTag): Promise<BenchmarkDatasetItem<TPayload>[]>;
+  /** Score a single response. Pure with respect to the inputs. */
+  evaluate(item: BenchmarkDatasetItem<TPayload>, artifact: TArtifact): Promise<BenchmarkEvaluation>;
+  /** Deterministic split assignment via item id hashing. The
+   *  fraction of items in each split is implementation-defined but
+   *  MUST be stable across processes and platforms. */
+  assignSplit(itemId: string): RunSplitTag;
+}
+interface BenchmarkScenario<TPayload = unknown> extends Scenario {
+  kind: 'benchmark';
+  benchmarkId: string;
+  family: BenchmarkFamily | string;
+  taskKind: BenchmarkTaskKind | string;
+  splitTag: RunSplitTag;
+  item: BenchmarkDatasetItem<TPayload>;
+}
+type BenchmarkResponder<TPayload = unknown, TArtifact = string> = (input: {
+  scenario: BenchmarkScenario<TPayload>;
+  item: BenchmarkDatasetItem<TPayload>;
+  context: DispatchContext;
+}) => Promise<TArtifact> | TArtifact;
+/** Split-assignment seed shared across all benchmarks. Bumping this
+ *  value reshuffles every split — do NOT do that lightly. */
+declare const BENCHMARK_SPLIT_SEED = "agent-eval-v1";
+/**
+ * Assign an item id to one of `'search' | 'dev' | 'holdout'` using a
+ * stable 32-bit hash of `${seed}::${id}`. Default proportions:
+ *
+ *   search: 60%   (optimization-readable)
+ *   dev:    20%   (held-out for tuning, leak-on-purpose during dev)
+ *   holdout:20%   (paper-grade held-out, gated reads)
+ */
+declare function deterministicSplit(itemId: string, seed?: string): RunSplitTag;
+//#endregion
+//#region src/benchmarks/calibration.d.ts
+interface BenchmarkMetricCalibrationOptions<TPayload = unknown, TArtifact = string> {
+  adapter: BenchmarkAdapter<BenchmarkDatasetItem<TPayload>, TPayload, TArtifact>;
+  item: BenchmarkDatasetItem<TPayload>;
+  weakArtifact: TArtifact;
+  strongArtifact: TArtifact;
+  maxWeakScore?: number;
+  minStrongScore?: number;
+  minGap?: number;
+}
+interface BenchmarkMetricCalibrationResult {
+  passed: boolean;
+  weak: BenchmarkEvaluation;
+  strong: BenchmarkEvaluation;
+  weakScore: number;
+  strongScore: number;
+  gap: number;
+  reasons: string[];
+}
+declare function calibrateBenchmarkMetric<TPayload = unknown, TArtifact = string>(options: BenchmarkMetricCalibrationOptions<TPayload, TArtifact>): Promise<BenchmarkMetricCalibrationResult>;
+//#endregion
+//#region src/benchmarks/routing/dataset.d.ts
+/**
+ * Synthetic routing dataset. 16 tasks across 4 categories. Used as a
+ * deterministic, dependency-free benchmark for any router that maps a
+ * natural-language request to one of a fixed set of route labels.
+ *
+ * Format (see `routing/README.md` for prose):
+ *
+ *   {
+ *     id:           stable per-task ID (matches across processes).
+ *     category:     one of the four route labels.
+ *     prompt:       the user-facing request the router must classify.
+ *     route:        the ground-truth route the router should pick.
+ *     synonyms:     other strings that count as a correct answer.
+ *     hardNegatives:close-but-wrong route labels — used to detect the
+ *                   "always picks the popular route" failure mode.
+ *   }
+ *
+ * The four categories are intentionally cross-domain (file ops,
+ * math, search, conversation) so a router that collapses to one
+ * category is easy to spot.
+ */
+interface RoutingItem {
+  id: string;
+  category: 'file' | 'math' | 'search' | 'chat';
+  prompt: string;
+  /** Canonical correct route label. */
+  route: string;
+  /** Alternate route labels that also count as correct. */
+  synonyms: string[];
+  /** Wrong-but-tempting route labels (for analysis, not grading). */
+  hardNegatives: string[];
+}
+declare const ROUTING_DATASET: RoutingItem[];
+declare namespace index_d_exports$1 {
+  export { ROUTING_DATASET, RoutingAdapter, RoutingDatasetItem, RoutingItem, RoutingPayload, assignSplit, evaluate, extractRouteTokens, loadDataset };
+}
+type RoutingPayload = RoutingItem;
+type RoutingDatasetItem = BenchmarkDatasetItem<RoutingPayload>;
+declare class RoutingAdapter implements BenchmarkAdapter<RoutingDatasetItem, RoutingPayload> {
+  readonly id = "first-party/routing";
+  readonly family = "first-party";
+  readonly taskKind = "routing";
+  readonly description = "Synthetic fixed-route classification smoke benchmark";
+  readonly defaultMetric = "route_exact_match";
+  loadDataset(split: RunSplitTag): Promise<RoutingDatasetItem[]>;
+  evaluate(item: RoutingDatasetItem, response: string): Promise<BenchmarkEvaluation>;
+  assignSplit(itemId: string): RunSplitTag;
+}
+/**
+ * Pull route-shaped tokens out of a model response. Routes look like
+ * `category.action` (`fs.write`, `chat.reply`). Bare alphanumerics
+ * are not routes, but `category.action` patterns are robust to most
+ * model wrappers (JSON output, prose explanations, code fences).
+ */
+declare function extractRouteTokens(response: string): string[];
+declare const loadDataset: (split: RunSplitTag) => Promise<RoutingDatasetItem[]>;
+declare const evaluate: (item: RoutingDatasetItem, response: string) => Promise<BenchmarkEvaluation>;
+declare const assignSplit: (itemId: string) => RunSplitTag;
+//#endregion
+//#region src/benchmarks/runner.d.ts
+interface BenchmarkRunOptions<TPayload = unknown, TArtifact = string> {
+  adapter: BenchmarkAdapter<BenchmarkDatasetItem<TPayload>, TPayload, TArtifact>;
+  respond: BenchmarkResponder<TPayload, TArtifact>;
+  splits?: readonly RunSplitTag[];
+  runDir: string;
+  repo?: string;
+  seed?: number;
+  reps?: number;
+  resumable?: boolean;
+  costCeiling?: number;
+  maxConcurrency?: number;
+  dispatchTimeoutMs?: number;
+  expectUsage?: 'assert' | 'warn' | 'off';
+  storage?: CampaignStorage;
+  now?: () => Date;
+}
+interface BenchmarkReport {
+  benchmarkId: string;
+  family: BenchmarkFamily | string;
+  taskKind: BenchmarkTaskKind | string;
+  source?: BenchmarkAdapter['source'];
+  runDir: string;
+  manifestHash: string;
+  seed: number;
+  startedAt: string;
+  endedAt: string;
+  durationMs: number;
+  totalItems: number;
+  totalCells: number;
+  cellsFailed: number;
+  cellsCached: number;
+  totalCostUsd: number;
+  splits: Record<string, BenchmarkSliceSummary>;
+  tags: Record<string, BenchmarkSliceSummary>;
+  dimensions: Record<string, BenchmarkDistribution>;
+  score: BenchmarkDistribution;
+  costUsd: BenchmarkDistribution;
+  latencyMs: BenchmarkDistribution;
+}
+interface BenchmarkSliceSummary {
+  n: number;
+  meanScore: number;
+  passRate: number;
+  score: BenchmarkDistribution;
+  costUsd: BenchmarkDistribution;
+  latencyMs: BenchmarkDistribution;
+}
+interface BenchmarkDistribution {
+  n: number;
+  min: number;
+  mean: number;
+  median: number;
+  p90: number;
+  max: number;
+}
+interface BenchmarkRunResult<TPayload = unknown, TArtifact = string> {
+  scenarios: Array<BenchmarkScenario<TPayload>>;
+  campaign: CampaignResult<TArtifact, BenchmarkScenario<TPayload>>;
+  report: BenchmarkReport;
+  reportJsonPath: string;
+  reportMarkdownPath: string;
+}
+declare function runBenchmarkAdapter<TPayload = unknown, TArtifact = string>(options: BenchmarkRunOptions<TPayload, TArtifact>): Promise<BenchmarkRunResult<TPayload, TArtifact>>;
+declare function summarizeBenchmarkCampaign<TPayload, TArtifact>(input: {
+  adapter: BenchmarkAdapter<BenchmarkDatasetItem<TPayload>, TPayload, TArtifact>;
+  scenarios: Array<BenchmarkScenario<TPayload>>;
+  campaign: CampaignResult<TArtifact, BenchmarkScenario<TPayload>>;
+}): BenchmarkReport;
+declare function renderBenchmarkReportMarkdown(report: BenchmarkReport): string;
+//#endregion
+//#region src/benchmarks/standard-formats.d.ts
+interface StandardRetrievalDocument {
+  id: string;
+  title?: string;
+  text: string;
+  metadata?: Record<string, unknown>;
+}
+interface StandardRetrievalQuery {
+  id: string;
+  text: string;
+  metadata?: Record<string, unknown>;
+}
+interface StandardRetrievalQrel {
+  queryId: string;
+  documentId: string;
+  score: number;
+}
+interface StandardRetrievalPayload {
+  queryId: string;
+  query: string;
+  expectedDocumentIds: string[];
+  expectedScores: Record<string, number>;
+  corpus?: Record<string, StandardRetrievalDocument>;
+  metadata?: Record<string, unknown>;
+}
+interface BuildStandardRetrievalItemsOptions {
+  benchmarkId: string;
+  family: BenchmarkFamily | string;
+  queries: readonly StandardRetrievalQuery[];
+  qrels: readonly StandardRetrievalQrel[];
+  corpus?: readonly StandardRetrievalDocument[];
+  includeCorpusInPayload?: boolean;
+  source?: BenchmarkSource;
+  tags?: readonly string[];
+  splitOf?: (queryId: string) => RunSplitTag;
+}
+interface RetrievalIdAdapterOptions extends BuildStandardRetrievalItemsOptions {
+  responseIdPattern?: RegExp;
+  cutoffs?: readonly number[];
+  primaryMetric?: string;
+  passMetric?: string;
+  passThreshold?: number;
+}
+interface StandardRetrievalResult {
+  id?: string;
+  documentId?: string;
+  docId?: string;
+  score?: number;
+}
+type StandardRetrievalArtifact = string | readonly string[] | readonly StandardRetrievalResult[] | {
+  ids?: readonly string[];
+  documentIds?: readonly string[];
+  results?: readonly StandardRetrievalResult[];
+};
+interface StandardRetrievalEvaluationOptions {
+  responseIdPattern?: RegExp;
+  cutoffs?: readonly number[];
+  primaryMetric?: string;
+  passMetric?: string;
+  passThreshold?: number;
+}
+declare function parseJsonlRows<T = unknown>(text: string): T[];
+declare function parseTsvRows(text: string): string[][];
+declare function parseQrels(text: string): StandardRetrievalQrel[];
+declare function parseBeirCorpusJsonl(text: string): StandardRetrievalDocument[];
+declare function parseBeirQueriesJsonl(text: string): StandardRetrievalQuery[];
+declare function buildStandardRetrievalItems(options: BuildStandardRetrievalItemsOptions): Array<BenchmarkDatasetItem<StandardRetrievalPayload>>;
+declare function createRetrievalIdBenchmarkAdapter(options: RetrievalIdAdapterOptions): BenchmarkAdapter<BenchmarkDatasetItem<StandardRetrievalPayload>, StandardRetrievalPayload, StandardRetrievalArtifact>;
+declare function evaluateStandardRetrieval(payload: StandardRetrievalPayload, artifact: StandardRetrievalArtifact, options?: StandardRetrievalEvaluationOptions): {
+  score: number;
+  passed: boolean;
+  dimensions: Record<string, number>;
+  raw: {
+    rankedDocumentIds: string[];
+    expectedDocumentIds: string[];
+    expectedScores: Record<string, number>;
+  };
+};
+declare function normalizeRetrievedDocumentIds(artifact: StandardRetrievalArtifact, responseIdPattern?: RegExp): string[];
+declare function retrievalMetricsAtCutoff(input: {
+  rankedDocumentIds: readonly string[];
+  expectedScores: Record<string, number>;
+  cutoff: number;
+}): Record<string, number>;
+declare namespace index_d_exports {
+  export { BENCHMARK_SPLIT_SEED, BenchmarkAdapter, BenchmarkDatasetItem, BenchmarkDistribution, BenchmarkEvaluation, BenchmarkFamily, BenchmarkMetricCalibrationOptions, BenchmarkMetricCalibrationResult, BenchmarkReport, BenchmarkResponder, BenchmarkRunOptions, BenchmarkRunResult, BenchmarkScenario, BenchmarkSliceSummary, BenchmarkSource, BenchmarkTaskKind, BuildStandardRetrievalItemsOptions, RetrievalIdAdapterOptions, StandardRetrievalArtifact, StandardRetrievalDocument, StandardRetrievalEvaluationOptions, StandardRetrievalPayload, StandardRetrievalQrel, StandardRetrievalQuery, StandardRetrievalResult, buildStandardRetrievalItems, calibrateBenchmarkMetric, createRetrievalIdBenchmarkAdapter, deterministicSplit, evaluateStandardRetrieval, normalizeRetrievedDocumentIds, parseBeirCorpusJsonl, parseBeirQueriesJsonl, parseJsonlRows, parseQrels, parseTsvRows, renderBenchmarkReportMarkdown, retrievalMetricsAtCutoff, index_d_exports$1 as routing, runBenchmarkAdapter, summarizeBenchmarkCampaign };
+}
+//#endregion
+export { BenchmarkMetricCalibrationOptions as A, BenchmarkSource as B, BenchmarkRunOptions as C, runBenchmarkAdapter as D, renderBenchmarkReportMarkdown as E, BenchmarkDatasetItem as F, deterministicSplit as H, BenchmarkEvaluation as I, BenchmarkFamily as L, calibrateBenchmarkMetric as M, BENCHMARK_SPLIT_SEED as N, summarizeBenchmarkCampaign as O, BenchmarkAdapter as P, BenchmarkResponder as R, BenchmarkReport as S, BenchmarkSliceSummary as T, BenchmarkTaskKind as V, parseJsonlRows as _, StandardRetrievalDocument as a, retrievalMetricsAtCutoff as b, StandardRetrievalQrel as c, buildStandardRetrievalItems as d, createRetrievalIdBenchmarkAdapter as f, parseBeirQueriesJsonl as g, parseBeirCorpusJsonl as h, StandardRetrievalArtifact as i, BenchmarkMetricCalibrationResult as j, index_d_exports$1 as k, StandardRetrievalQuery as l, normalizeRetrievedDocumentIds as m, BuildStandardRetrievalItemsOptions as n, StandardRetrievalEvaluationOptions as o, evaluateStandardRetrieval as p, RetrievalIdAdapterOptions as r, StandardRetrievalPayload as s, index_d_exports as t, StandardRetrievalResult as u, parseQrels as v, BenchmarkRunResult as w, BenchmarkDistribution as x, parseTsvRows as y, BenchmarkScenario as z };
+//# sourceMappingURL=index-CAPUUKaM.d.ts.map

package/dist/index-CAPUUKaM.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index-CAPUUKaM.d.ts","names":[],"sources":["../src/benchmarks/types.ts","../src/benchmarks/calibration.ts","../src/benchmarks/routing/dataset.ts","../src/benchmarks/routing/index.ts","../src/benchmarks/runner.ts","../src/benchmarks/standard-formats.ts","../src/benchmarks/index.ts"],"mappings":";;;;;KASY;KAQA;UAgBK,qBAAqB;;;EAGpC;;EAEA,SAAS;;EAET,QAAQ;;EAER,SAAS;;EAET,WAAW;;EAEX;;EAEA,SAAS;EACT,WAAW;;UAGI;;;;EAIf;;EAEA;;EAEA,aAAa;;;EAGb,KAAK;EACL;;UAGe;EACf;EACA;EACA;EACA;EACA;;;UAQe,iBAAiB,kBAAkB,oBAAoB;;EAEtE;EACA,SAAS;EACT,WAAW;EACX;EACA,SAAS;EACT;;;;;EAKA,YAAY,OAAO,cAAc,QAAQ,qBAAqB;;EAE9D,SAAS,MAAM,qBAAqB,WAAW,UAAU,YAAY,QAAQ;;;;EAI7E,YAAY,iBAAiB;;UAGd,kBAAkB,4BAA4B;EAC7D;EACA;EACA,QAAQ;EACR,UAAU;EACV,UAAU;EACV,MAAM,qBAAqB;;KAGjB,mBAAmB,oBAAoB,uBAAuB;EACxE,UAAU,kBAAkB;EAC5B,MAAM,qBAAqB;EAC3B,SAAS;MACL,QAAQ,aAAa;;;cAoBd;;;;;;;;;iBAUG,mBACd,gBACA,gBACC;;;UCjJc,kCAAkC,oBAAoB;EACrE,SAAS,iBAAiB,qBAAqB,WAAW,UAAU;EACpE,MAAM,qBAAqB;EAC3B,cAAc;EACd,gBAAgB;EAChB;EACA;EACA;;UAGe;EACf;EACA,MAAM;EACN,QAAQ;EACR;EACA;EACA;EACA;;iBAGoB,yBAAyB,oBAAoB,oBACjE,SAAS,kCAAkC,UAAU,aACpD,QAAQ;;;;;;;;;;;;;;;;;;;;;;;;UCFM;EACf;EACA;EACA;;EAEA;;EAEA;;EAEA;;cAGW,iBAAiB;;;;KChBlB,iBAAiB;KACjB,qBAAqB,qBAAqB;cAEhD,0BAA0B,iBAAiB,oBAAoB;WAC1D;WACA;WACA;WACA;WACA;EAEH,YAAY,OAAO,cAAc,QAAQ;EAMzC,SAAS,MAAM,oBAAoB,mBAAmB,QAAQ;EAqBpE,YAAY,iBAAiB;;;;;;;;iBAef,mBAAmB;cAOtB,cAAW,OAjDG,gBAAc,QAAQ;cAkDpC,WAAQ,MA5CE,oBAAkB,qBAAqB,QAAQ;cA6CzD,cAAW,mBAxBO;;;UClCd,oBAAoB,oBAAoB;EACvD,SAAS,iBAAiB,qBAAqB,WAAW,UAAU;EACpE,SAAS,mBAAmB,UAAU;EACtC,kBAAkB;EAClB;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA,UAAU;EACV,YAAY;;UAGG;EACf;EACA,QAAQ;EACR,UAAU;EACV,SAAS;EACT;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA,QAAQ,eAAe;EACvB,MAAM,eAAe;EACrB,YAAY,eAAe;EAC3B,OAAO;EACP,SAAS;EACT,WAAW;;UAGI;EACf;EACA;EACA;EACA,OAAO;EACP,SAAS;EACT,WAAW;;UAGI;EACf;EACA;EACA;EACA;EACA;EACA;;UAGe,mBAAmB,oBAAoB;EACtD,WAAW,MAAM,kBAAkB;EACnC,UAAU,eAAe,WAAW,kBAAkB;EACtD,QAAQ;EACR;EACA;;iBAGoB,oBAAoB,oBAAoB,oBAC5D,SAAS,oBAAoB,UAAU,aACtC,QAAQ,mBAAmB,UAAU;iBAmGxB,2BAA2B,UAAU,WAAW;EAC9D,SAAS,iBAAiB,qBAAqB,WAAW,UAAU;EACpE,WAAW,MAAM,kBAAkB;EACnC,UAAU,eAAe,WAAW,kBAAkB;IACpD;iBA6CY,8BAA8B,QAAQ;;;UCpOrC;EACf;EACA;EACA;EACA,WAAW;;UAGI;EACf;EACA;EACA,WAAW;;UAGI;EACf;EACA;EACA;;UAGe;EACf;EACA;EACA;EACA,gBAAgB;EAChB,SAAS,eAAe;EACxB,WAAW;;UAGI;EACf;EACA,QAAQ;EACR,kBAAkB;EAClB,gBAAgB;EAChB,kBAAkB;EAClB;EACA,SAAS;EACT;EACA,WAAW,oBAAoB;;UAGhB,kCAAkC;EACjD,oBAAoB;EACpB;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA;EACA;;KAGU,kEAGC;EAEP;EACA;EACA,mBAAmB;;UAGR;EACf,oBAAoB;EACpB;EACA;EACA;EACA;;iBAGc,eAAe,aAAa,eAAe;iBAc3C,aAAa;iBASb,WAAW,eAAe;iBAgB1B,qBAAqB,eAAe;iBAgBpC,sBAAsB,eAAe;iBAerC,4BACd,SAAS,qCACR,MAAM,qBAAqB;iBAuCd,kCACd,SAAS,4BACR,iBACD,qBAAqB,2BACrB,0BACA;iBAqBc,0BACd,SAAS,0BACT,UAAU,2BACV,UAAS;;;;;;;;;;iBAsCK,8BACd,UAAU,2BACV,oBAAmB;iBA0BL,yBAAyB;EACvC;EACA,gBAAgB;EAChB;IACE"}