@tangle-network/agent-eval 0.107.0 → 0.108.1

package/dist/index-C2CC_dry.d.ts

@@ -1,159 +0,0 @@
-import { b as RunSplitTag } from './run-record-I-Z3JNvO.js';
-
-/**
- * Shared types for the reference benchmark wrappers under
- * `src/benchmarks/`. Each wrapper exports the three functions in
- * `BenchmarkAdapter` plus its own typed `DatasetItem` shape.
- */
-
-interface BenchmarkDatasetItem<TPayload = unknown> {
-    /** Stable dataset-local item id (used for split assignment + paper
-     *  references). Unique within a benchmark. */
-    id: string;
-    /** Free-form payload. Each benchmark defines its own shape. */
-    payload: TPayload;
-}
-interface BenchmarkEvaluation {
-    /** [0, 1] score for the response on this item. Exact-match
-     *  benchmarks use 0/1; partial-credit benchmarks may return
-     *  fractional values. */
-    score: number;
-    /** Optional bag of raw scoring signals — e.g. parsed numeric
-     *  answer, regex match, judge sub-scores. */
-    raw: Record<string, unknown>;
-}
-/** Common signature implemented by every adapter under `src/benchmarks/*`. */
-interface BenchmarkAdapter<_TItem = unknown, TPayload = unknown> {
-    /** Load the dataset for the given split. May hit the network on
-     *  first call but should be cache-friendly. Adapters that don't
-     *  ship the dataset itself MUST throw a clearly-marked error
-     *  pointing the caller at the loader script. */
-    loadDataset(split: RunSplitTag): Promise<BenchmarkDatasetItem<TPayload>[]>;
-    /** Score a single response. Pure with respect to the inputs. */
-    evaluate(item: BenchmarkDatasetItem<TPayload>, response: string): Promise<BenchmarkEvaluation>;
-    /** Deterministic split assignment via item id hashing. The
-     *  fraction of items in each split is implementation-defined but
-     *  MUST be stable across processes and platforms. */
-    assignSplit(itemId: string): RunSplitTag;
-}
-/** Split-assignment seed shared across all benchmarks. Bumping this
- *  value reshuffles every split — do NOT do that lightly. */
-declare const BENCHMARK_SPLIT_SEED = "agent-eval-v1";
-/**
- * Assign an item id to one of `'search' | 'dev' | 'holdout'` using a
- * stable 32-bit hash of `${seed}::${id}`. Default proportions:
- *
- *   search: 60%   (optimization-readable)
- *   dev:    20%   (held-out for tuning, leak-on-purpose during dev)
- *   holdout:20%   (paper-grade held-out, gated reads)
- */
-declare function deterministicSplit(itemId: string, seed?: string): RunSplitTag;
-
-/**
- * Synthetic routing dataset. 16 tasks across 4 categories. Used as a
- * deterministic, dependency-free benchmark for any router that maps a
- * natural-language request to one of a fixed set of route labels.
- *
- * Format (see `routing/README.md` for prose):
- *
- *   {
- *     id:           stable per-task ID (matches across processes).
- *     category:     one of the four route labels.
- *     prompt:       the user-facing request the router must classify.
- *     route:        the ground-truth route the router should pick.
- *     synonyms:     other strings that count as a correct answer.
- *     hardNegatives:close-but-wrong route labels — used to detect the
- *                   "always picks the popular route" failure mode.
- *   }
- *
- * The four categories are intentionally cross-domain (file ops,
- * math, search, conversation) so a router that collapses to one
- * category is easy to spot.
- */
-interface RoutingItem {
-    id: string;
-    category: 'file' | 'math' | 'search' | 'chat';
-    prompt: string;
-    /** Canonical correct route label. */
-    route: string;
-    /** Alternate route labels that also count as correct. */
-    synonyms: string[];
-    /** Wrong-but-tempting route labels (for analysis, not grading). */
-    hardNegatives: string[];
-}
-declare const ROUTING_DATASET: RoutingItem[];
-
-/**
- * Routing benchmark — synthetic, dependency-free, ships in the
- * package. 16 cross-category items in `dataset.ts`. See
- * `routing/README.md` for the format.
- *
- * `evaluate` does case-insensitive exact match against the canonical
- * route plus declared synonyms. The first valid route token in the
- * response wins; everything else is ignored. Wrong answers also
- * report whether they hit a hard negative — useful when triaging
- * "always picks the popular route" failure modes.
- */
-
-type RoutingPayload = RoutingItem;
-type RoutingDatasetItem = BenchmarkDatasetItem<RoutingPayload>;
-declare class RoutingAdapter implements BenchmarkAdapter<RoutingDatasetItem, RoutingPayload> {
-    loadDataset(split: RunSplitTag): Promise<RoutingDatasetItem[]>;
-    evaluate(item: RoutingDatasetItem, response: string): Promise<BenchmarkEvaluation>;
-    assignSplit(itemId: string): RunSplitTag;
-}
-/**
- * Pull route-shaped tokens out of a model response. Routes look like
- * `category.action` (`fs.write`, `chat.reply`). Bare alphanumerics
- * are not routes, but `category.action` patterns are robust to most
- * model wrappers (JSON output, prose explanations, code fences).
- */
-declare function extractRouteTokens(response: string): string[];
-declare const loadDataset: (split: RunSplitTag) => Promise<RoutingDatasetItem[]>;
-declare const evaluate: (item: RoutingDatasetItem, response: string) => Promise<BenchmarkEvaluation>;
-declare const assignSplit: (itemId: string) => RunSplitTag;
-
-declare const index$1_ROUTING_DATASET: typeof ROUTING_DATASET;
-type index$1_RoutingAdapter = RoutingAdapter;
-declare const index$1_RoutingAdapter: typeof RoutingAdapter;
-type index$1_RoutingDatasetItem = RoutingDatasetItem;
-type index$1_RoutingItem = RoutingItem;
-type index$1_RoutingPayload = RoutingPayload;
-declare const index$1_assignSplit: typeof assignSplit;
-declare const index$1_evaluate: typeof evaluate;
-declare const index$1_extractRouteTokens: typeof extractRouteTokens;
-declare const index$1_loadDataset: typeof loadDataset;
-declare namespace index$1 {
-  export { index$1_ROUTING_DATASET as ROUTING_DATASET, index$1_RoutingAdapter as RoutingAdapter, type index$1_RoutingDatasetItem as RoutingDatasetItem, type index$1_RoutingItem as RoutingItem, type index$1_RoutingPayload as RoutingPayload, index$1_assignSplit as assignSplit, index$1_evaluate as evaluate, index$1_extractRouteTokens as extractRouteTokens, index$1_loadDataset as loadDataset };
-}
-
-/**
- * Reference benchmark wrappers — entry point.
- *
- * Core surface (exported here):
- *   - The `BenchmarkAdapter` contract.
- *   - `deterministicSplit` + `BENCHMARK_SPLIT_SEED` for split assignment.
- *   - `routing` — synthetic 16-task router benchmark. The only novel
- *     benchmark we built; ships in the package.
- *
- * Example wrappers (under `examples/benchmarks/`, NOT in the bundle):
- *   - `gsm8k`         — exact-match math reasoning (HF mirror, dataset
- *                       not bundled).
- *   - `swebench-lite` — 30-instance SWE-Bench subset via an external
- *                       grader command.
- *
- * The example wrappers are reference implementations of `BenchmarkAdapter`.
- * Read them, copy them, adapt them. They're intentionally not in the main
- * entry — every team will configure them differently.
- */
-
-declare const index_BENCHMARK_SPLIT_SEED: typeof BENCHMARK_SPLIT_SEED;
-type index_BenchmarkAdapter<_TItem = unknown, TPayload = unknown> = BenchmarkAdapter<_TItem, TPayload>;
-type index_BenchmarkDatasetItem<TPayload = unknown> = BenchmarkDatasetItem<TPayload>;
-type index_BenchmarkEvaluation = BenchmarkEvaluation;
-declare const index_deterministicSplit: typeof deterministicSplit;
-declare namespace index {
-  export { index_BENCHMARK_SPLIT_SEED as BENCHMARK_SPLIT_SEED, type index_BenchmarkAdapter as BenchmarkAdapter, type index_BenchmarkDatasetItem as BenchmarkDatasetItem, type index_BenchmarkEvaluation as BenchmarkEvaluation, index_deterministicSplit as deterministicSplit, index$1 as routing };
-}
-
-export { BENCHMARK_SPLIT_SEED as B, type BenchmarkAdapter as a, type BenchmarkDatasetItem as b, type BenchmarkEvaluation as c, deterministicSplit as d, index$1 as e, index as i };