@tangle-network/agent-eval 0.144.11 → 0.144.13

package/dist/benchmarks/index.d.ts

@@ -1,2 +1,244 @@
-import { A as BenchmarkMetricCalibrationOptions, B as BenchmarkSource, C as BenchmarkRunOptions, D as runBenchmarkAdapter, E as renderBenchmarkReportMarkdown, F as BenchmarkDatasetItem, H as deterministicSplit, I as BenchmarkEvaluation, L as BenchmarkFamily, M as calibrateBenchmarkMetric, N as BENCHMARK_SPLIT_SEED, O as summarizeBenchmarkCampaign, P as BenchmarkAdapter, R as BenchmarkResponder, S as BenchmarkReport, T as BenchmarkSliceSummary, V as BenchmarkTaskKind, _ as parseJsonlRows, a as StandardRetrievalDocument, b as retrievalMetricsAtCutoff, c as StandardRetrievalQrel, d as buildStandardRetrievalItems, f as createRetrievalIdBenchmarkAdapter, g as parseBeirQueriesJsonl, h as parseBeirCorpusJsonl, i as StandardRetrievalArtifact, j as BenchmarkMetricCalibrationResult, k as index_d_exports, l as StandardRetrievalQuery, m as normalizeRetrievedDocumentIds, n as BuildStandardRetrievalItemsOptions, o as StandardRetrievalEvaluationOptions, p as evaluateStandardRetrieval, r as RetrievalIdAdapterOptions, s as StandardRetrievalPayload, u as StandardRetrievalResult, v as parseQrels, w as BenchmarkRunResult, x as BenchmarkDistribution, y as parseTsvRows, z as BenchmarkScenario } from "../index-YE4KdKbO2.js";
-export { BENCHMARK_SPLIT_SEED, type BenchmarkAdapter, type BenchmarkDatasetItem, type BenchmarkDistribution, type BenchmarkEvaluation, type BenchmarkFamily, type BenchmarkMetricCalibrationOptions, type BenchmarkMetricCalibrationResult, type BenchmarkReport, type BenchmarkResponder, type BenchmarkRunOptions, type BenchmarkRunResult, type BenchmarkScenario, type BenchmarkSliceSummary, type BenchmarkSource, type BenchmarkTaskKind, type BuildStandardRetrievalItemsOptions, type RetrievalIdAdapterOptions, type StandardRetrievalArtifact, type StandardRetrievalDocument, type StandardRetrievalEvaluationOptions, type StandardRetrievalPayload, type StandardRetrievalQrel, type StandardRetrievalQuery, type StandardRetrievalResult, buildStandardRetrievalItems, calibrateBenchmarkMetric, createRetrievalIdBenchmarkAdapter, deterministicSplit, evaluateStandardRetrieval, normalizeRetrievedDocumentIds, parseBeirCorpusJsonl, parseBeirQueriesJsonl, parseJsonlRows, parseQrels, parseTsvRows, renderBenchmarkReportMarkdown, retrievalMetricsAtCutoff, index_d_exports as routing, runBenchmarkAdapter, summarizeBenchmarkCampaign };
+import { s as RunSplitTag } from "../run-record-CKiihE6f.js";
+import { a as CampaignResult } from "../types-BZ59Ahr8.js";
+import { a as BenchmarkFamily, c as BenchmarkSource, i as BenchmarkEvaluation, l as BenchmarkTaskKind, n as BenchmarkAdapter, o as BenchmarkResponder, r as BenchmarkDatasetItem, s as BenchmarkScenario, t as BENCHMARK_SPLIT_SEED, u as deterministicSplit } from "../types-BEPZc6eo.js";
+import { xt as CampaignStorage } from "../provenance-CCdxgLDT.js";
+import "../index-DSmEylT9.js";
+//#region src/benchmarks/calibration.d.ts
+interface BenchmarkMetricCalibrationOptions<TPayload = unknown, TArtifact = string> {
+  adapter: BenchmarkAdapter<BenchmarkDatasetItem<TPayload>, TPayload, TArtifact>;
+  item: BenchmarkDatasetItem<TPayload>;
+  weakArtifact: TArtifact;
+  strongArtifact: TArtifact;
+  maxWeakScore?: number;
+  minStrongScore?: number;
+  minGap?: number;
+}
+interface BenchmarkMetricCalibrationResult {
+  passed: boolean;
+  weak: BenchmarkEvaluation;
+  strong: BenchmarkEvaluation;
+  weakScore: number;
+  strongScore: number;
+  gap: number;
+  reasons: string[];
+}
+declare function calibrateBenchmarkMetric<TPayload = unknown, TArtifact = string>(options: BenchmarkMetricCalibrationOptions<TPayload, TArtifact>): Promise<BenchmarkMetricCalibrationResult>;
+//#endregion
+//#region src/benchmarks/routing/dataset.d.ts
+/**
+ * Synthetic routing dataset. 16 tasks across 4 categories. Used as a
+ * deterministic, dependency-free benchmark for any router that maps a
+ * natural-language request to one of a fixed set of route labels.
+ *
+ * Format (see `routing/README.md` for prose):
+ *
+ *   {
+ *     id:           stable per-task ID (matches across processes).
+ *     category:     one of the four route labels.
+ *     prompt:       the user-facing request the router must classify.
+ *     route:        the ground-truth route the router should pick.
+ *     synonyms:     other strings that count as a correct answer.
+ *     hardNegatives:close-but-wrong route labels — used to detect the
+ *                   "always picks the popular route" failure mode.
+ *   }
+ *
+ * The four categories are intentionally cross-domain (file ops,
+ * math, search, conversation) so a router that collapses to one
+ * category is easy to spot.
+ */
+interface RoutingItem {
+  id: string;
+  category: 'file' | 'math' | 'search' | 'chat';
+  prompt: string;
+  /** Canonical correct route label. */
+  route: string;
+  /** Alternate route labels that also count as correct. */
+  synonyms: string[];
+  /** Wrong-but-tempting route labels (for analysis, not grading). */
+  hardNegatives: string[];
+}
+declare const ROUTING_DATASET: RoutingItem[];
+declare namespace index_d_exports {
+  export { ROUTING_DATASET, RoutingAdapter, RoutingDatasetItem, RoutingItem, RoutingPayload, assignSplit, evaluate, extractRouteTokens, loadDataset };
+}
+type RoutingPayload = RoutingItem;
+type RoutingDatasetItem = BenchmarkDatasetItem<RoutingPayload>;
+declare class RoutingAdapter implements BenchmarkAdapter<RoutingDatasetItem, RoutingPayload> {
+  readonly id = "first-party/routing";
+  readonly family = "first-party";
+  readonly taskKind = "routing";
+  readonly description = "Synthetic fixed-route classification smoke benchmark";
+  readonly defaultMetric = "route_exact_match";
+  loadDataset(split: RunSplitTag): Promise<RoutingDatasetItem[]>;
+  evaluate(item: RoutingDatasetItem, response: string): Promise<BenchmarkEvaluation>;
+  assignSplit(itemId: string): RunSplitTag;
+}
+/**
+ * Pull route-shaped tokens out of a model response. Routes look like
+ * `category.action` (`fs.write`, `chat.reply`). Bare alphanumerics
+ * are not routes, but `category.action` patterns are robust to most
+ * model wrappers (JSON output, prose explanations, code fences).
+ */
+declare function extractRouteTokens(response: string): string[];
+declare const loadDataset: (split: RunSplitTag) => Promise<RoutingDatasetItem[]>;
+declare const evaluate: (item: RoutingDatasetItem, response: string) => Promise<BenchmarkEvaluation>;
+declare const assignSplit: (itemId: string) => RunSplitTag;
+//#endregion
+//#region src/benchmarks/runner.d.ts
+interface BenchmarkRunOptions<TPayload = unknown, TArtifact = string> {
+  adapter: BenchmarkAdapter<BenchmarkDatasetItem<TPayload>, TPayload, TArtifact>;
+  respond: BenchmarkResponder<TPayload, TArtifact>;
+  splits?: readonly RunSplitTag[];
+  runDir: string;
+  repo?: string;
+  seed?: number;
+  reps?: number;
+  resumable?: boolean;
+  costCeiling?: number;
+  maxConcurrency?: number;
+  dispatchTimeoutMs?: number;
+  expectUsage?: 'assert' | 'warn' | 'off';
+  storage?: CampaignStorage;
+  now?: () => Date;
+}
+interface BenchmarkReport {
+  benchmarkId: string;
+  family: BenchmarkFamily | string;
+  taskKind: BenchmarkTaskKind | string;
+  source?: BenchmarkAdapter['source'];
+  runDir: string;
+  manifestHash: string;
+  seed: number;
+  startedAt: string;
+  endedAt: string;
+  durationMs: number;
+  totalItems: number;
+  totalCells: number;
+  cellsFailed: number;
+  cellsCached: number;
+  totalCostUsd: number;
+  splits: Record<string, BenchmarkSliceSummary>;
+  tags: Record<string, BenchmarkSliceSummary>;
+  dimensions: Record<string, BenchmarkDistribution>;
+  score: BenchmarkDistribution;
+  costUsd: BenchmarkDistribution;
+  latencyMs: BenchmarkDistribution;
+}
+interface BenchmarkSliceSummary {
+  n: number;
+  meanScore: number;
+  passRate: number;
+  score: BenchmarkDistribution;
+  costUsd: BenchmarkDistribution;
+  latencyMs: BenchmarkDistribution;
+}
+interface BenchmarkDistribution {
+  n: number;
+  min: number;
+  mean: number;
+  median: number;
+  p90: number;
+  max: number;
+}
+interface BenchmarkRunResult<TPayload = unknown, TArtifact = string> {
+  scenarios: Array<BenchmarkScenario<TPayload>>;
+  campaign: CampaignResult<TArtifact, BenchmarkScenario<TPayload>>;
+  report: BenchmarkReport;
+  reportJsonPath: string;
+  reportMarkdownPath: string;
+}
+declare function runBenchmarkAdapter<TPayload = unknown, TArtifact = string>(options: BenchmarkRunOptions<TPayload, TArtifact>): Promise<BenchmarkRunResult<TPayload, TArtifact>>;
+declare function summarizeBenchmarkCampaign<TPayload, TArtifact>(input: {
+  adapter: BenchmarkAdapter<BenchmarkDatasetItem<TPayload>, TPayload, TArtifact>;
+  scenarios: Array<BenchmarkScenario<TPayload>>;
+  campaign: CampaignResult<TArtifact, BenchmarkScenario<TPayload>>;
+}): BenchmarkReport;
+declare function renderBenchmarkReportMarkdown(report: BenchmarkReport): string;
+//#endregion
+//#region src/benchmarks/standard-formats.d.ts
+interface StandardRetrievalDocument {
+  id: string;
+  title?: string;
+  text: string;
+  metadata?: Record<string, unknown>;
+}
+interface StandardRetrievalQuery {
+  id: string;
+  text: string;
+  metadata?: Record<string, unknown>;
+}
+interface StandardRetrievalQrel {
+  queryId: string;
+  documentId: string;
+  score: number;
+}
+interface StandardRetrievalPayload {
+  queryId: string;
+  query: string;
+  expectedDocumentIds: string[];
+  expectedScores: Record<string, number>;
+  corpus?: Record<string, StandardRetrievalDocument>;
+  metadata?: Record<string, unknown>;
+}
+interface BuildStandardRetrievalItemsOptions {
+  benchmarkId: string;
+  family: BenchmarkFamily | string;
+  queries: readonly StandardRetrievalQuery[];
+  qrels: readonly StandardRetrievalQrel[];
+  corpus?: readonly StandardRetrievalDocument[];
+  includeCorpusInPayload?: boolean;
+  source?: BenchmarkSource;
+  tags?: readonly string[];
+  splitOf?: (queryId: string) => RunSplitTag;
+}
+interface RetrievalIdAdapterOptions extends BuildStandardRetrievalItemsOptions {
+  responseIdPattern?: RegExp;
+  cutoffs?: readonly number[];
+  primaryMetric?: string;
+  passMetric?: string;
+  passThreshold?: number;
+}
+interface StandardRetrievalResult {
+  id?: string;
+  documentId?: string;
+  docId?: string;
+  score?: number;
+}
+type StandardRetrievalArtifact = string | readonly string[] | readonly StandardRetrievalResult[] | {
+  ids?: readonly string[];
+  documentIds?: readonly string[];
+  results?: readonly StandardRetrievalResult[];
+};
+interface StandardRetrievalEvaluationOptions {
+  responseIdPattern?: RegExp;
+  cutoffs?: readonly number[];
+  primaryMetric?: string;
+  passMetric?: string;
+  passThreshold?: number;
+}
+declare function parseJsonlRows<T = unknown>(text: string): T[];
+declare function parseTsvRows(text: string): string[][];
+declare function parseQrels(text: string): StandardRetrievalQrel[];
+declare function parseBeirCorpusJsonl(text: string): StandardRetrievalDocument[];
+declare function parseBeirQueriesJsonl(text: string): StandardRetrievalQuery[];
+declare function buildStandardRetrievalItems(options: BuildStandardRetrievalItemsOptions): Array<BenchmarkDatasetItem<StandardRetrievalPayload>>;
+declare function createRetrievalIdBenchmarkAdapter(options: RetrievalIdAdapterOptions): BenchmarkAdapter<BenchmarkDatasetItem<StandardRetrievalPayload>, StandardRetrievalPayload, StandardRetrievalArtifact>;
+declare function evaluateStandardRetrieval(payload: StandardRetrievalPayload, artifact: StandardRetrievalArtifact, options?: StandardRetrievalEvaluationOptions): {
+  score: number;
+  passed: boolean;
+  dimensions: Record<string, number>;
+  raw: {
+    rankedDocumentIds: string[];
+    expectedDocumentIds: string[];
+    expectedScores: Record<string, number>;
+  };
+};
+declare function normalizeRetrievedDocumentIds(artifact: StandardRetrievalArtifact, responseIdPattern?: RegExp): string[];
+declare function retrievalMetricsAtCutoff(input: {
+  rankedDocumentIds: readonly string[];
+  expectedScores: Record<string, number>;
+  cutoff: number;
+}): Record<string, number>;
+//#endregion
+export { BENCHMARK_SPLIT_SEED, type BenchmarkAdapter, type BenchmarkDatasetItem, type BenchmarkDistribution, type BenchmarkEvaluation, type BenchmarkFamily, type BenchmarkMetricCalibrationOptions, type BenchmarkMetricCalibrationResult, type BenchmarkReport, type BenchmarkResponder, type BenchmarkRunOptions, type BenchmarkRunResult, type BenchmarkScenario, type BenchmarkSliceSummary, type BenchmarkSource, type BenchmarkTaskKind, type BuildStandardRetrievalItemsOptions, type RetrievalIdAdapterOptions, type StandardRetrievalArtifact, type StandardRetrievalDocument, type StandardRetrievalEvaluationOptions, type StandardRetrievalPayload, type StandardRetrievalQrel, type StandardRetrievalQuery, type StandardRetrievalResult, buildStandardRetrievalItems, calibrateBenchmarkMetric, createRetrievalIdBenchmarkAdapter, deterministicSplit, evaluateStandardRetrieval, normalizeRetrievedDocumentIds, parseBeirCorpusJsonl, parseBeirQueriesJsonl, parseJsonlRows, parseQrels, parseTsvRows, renderBenchmarkReportMarkdown, retrievalMetricsAtCutoff, index_d_exports as routing, runBenchmarkAdapter, summarizeBenchmarkCampaign };
+//# sourceMappingURL=index.d.ts.map

package/dist/benchmarks/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","names":[],"sources":["../../src/benchmarks/calibration.ts","../../src/benchmarks/routing/dataset.ts","../../src/benchmarks/routing/index.ts","../../src/benchmarks/runner.ts","../../src/benchmarks/standard-formats.ts"],"mappings":";;;;;;UAEiB,kCAAkC,oBAAoB;EACrE,SAAS,iBAAiB,qBAAqB,WAAW,UAAU;EACpE,MAAM,qBAAqB;EAC3B,cAAc;EACd,gBAAgB;EAChB;EACA;EACA;;UAGe;EACf;EACA,MAAM;EACN,QAAQ;EACR;EACA;EACA;EACA;;iBAGoB,yBAAyB,oBAAoB,oBACjE,SAAS,kCAAkC,UAAU,aACpD,QAAQ;;;;;;;;;;;;;;;;;;;;;;;;UCFM;EACf;EACA;EACA;;EAEA;;EAEA;;EAEA;;cAGW,iBAAiB;;;;KChBlB,iBAAiB;KACjB,qBAAqB,qBAAqB;cAEhD,0BAA0B,iBAAiB,oBAAoB;WAC1D;WACA;WACA;WACA;WACA;EAEH,YAAY,OAAO,cAAc,QAAQ;EAMzC,SAAS,MAAM,oBAAoB,mBAAmB,QAAQ;EAqBpE,YAAY,iBAAiB;;;;;;;;iBAef,mBAAmB;cAOtB,cAAW,OAjDG,gBAAc,QAAQ;cAkDpC,WAAQ,MA5CE,oBAAkB,qBAAqB,QAAQ;cA6CzD,cAAW,mBAxBO;;;UClCd,oBAAoB,oBAAoB;EACvD,SAAS,iBAAiB,qBAAqB,WAAW,UAAU;EACpE,SAAS,mBAAmB,UAAU;EACtC,kBAAkB;EAClB;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA,UAAU;EACV,YAAY;;UAGG;EACf;EACA,QAAQ;EACR,UAAU;EACV,SAAS;EACT;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA,QAAQ,eAAe;EACvB,MAAM,eAAe;EACrB,YAAY,eAAe;EAC3B,OAAO;EACP,SAAS;EACT,WAAW;;UAGI;EACf;EACA;EACA;EACA,OAAO;EACP,SAAS;EACT,WAAW;;UAGI;EACf;EACA;EACA;EACA;EACA;EACA;;UAGe,mBAAmB,oBAAoB;EACtD,WAAW,MAAM,kBAAkB;EACnC,UAAU,eAAe,WAAW,kBAAkB;EACtD,QAAQ;EACR;EACA;;iBAGoB,oBAAoB,oBAAoB,oBAC5D,SAAS,oBAAoB,UAAU,aACtC,QAAQ,mBAAmB,UAAU;iBAmGxB,2BAA2B,UAAU,WAAW;EAC9D,SAAS,iBAAiB,qBAAqB,WAAW,UAAU;EACpE,WAAW,MAAM,kBAAkB;EACnC,UAAU,eAAe,WAAW,kBAAkB;IACpD;iBA6CY,8BAA8B,QAAQ;;;UCpOrC;EACf;EACA;EACA;EACA,WAAW;;UAGI;EACf;EACA;EACA,WAAW;;UAGI;EACf;EACA;EACA;;UAGe;EACf;EACA;EACA;EACA,gBAAgB;EAChB,SAAS,eAAe;EACxB,WAAW;;UAGI;EACf;EACA,QAAQ;EACR,kBAAkB;EAClB,gBAAgB;EAChB,kBAAkB;EAClB;EACA,SAAS;EACT;EACA,WAAW,oBAAoB;;UAGhB,kCAAkC;EACjD,oBAAoB;EACpB;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA;EACA;;KAGU,kEAGC;EAEP;EACA;EACA,mBAAmB;;UAGR;EACf,oBAAoB;EACpB;EACA;EACA;EACA;;iBAGc,eAAe,aAAa,eAAe;iBAc3C,aAAa;iBASb,WAAW,eAAe;iBAgB1B,qBAAqB,eAAe;iBAgBpC,sBAAsB,eAAe;iBAerC,4BACd,SAAS,qCACR,MAAM,qBAAqB;iBAuCd,kCACd,SAAS,4BACR,iBACD,qBAAqB,2BACrB,0BACA;iBAqBc,0BACd,SAAS,0BACT,UAAU,2BACV,UAAS;;;;;;;;;;iBAsCK,8BACd,UAAU,2BACV,oBAAmB;iBA0BL,yBAAyB;EACvC;EACA,gBAAgB;EAChB;IACE"}