@tangle-network/agent-eval 0.94.0 → 0.95.0

package/dist/harness-optimizer-mOl9XX_O.d.ts

@@ -1,106 +0,0 @@
-import { O as Objective, P as ParetoResult } from './pareto-E-pembql.js';
-import { R as RunScore, a as RunTrace, b as RunScoreWeights } from './run-critic-CmMf05uV.js';
-
-interface SteeringRolePrompt {
-    system?: string;
-    append?: string;
-}
-interface SteeringBundle {
-    id: string;
-    coderPrompt?: string;
-    continuePrompt?: string;
-    reviewerPrompts?: Record<string, string>;
-    skills?: string[];
-    rolePrompts?: Record<string, SteeringRolePrompt>;
-    metadata?: Record<string, unknown>;
-}
-interface SteeringDelta {
-    coderPrompt?: string;
-    continuePrompt?: string;
-    reviewerPrompts?: Record<string, string>;
-    skills?: string[];
-    rolePrompts?: Record<string, SteeringRolePrompt>;
-    metadata?: Record<string, unknown>;
-}
-declare function mergeSteeringBundle(base: SteeringBundle, delta: SteeringDelta): SteeringBundle;
-declare function renderSteeringText(bundle: SteeringBundle): string;
-
-type HarnessIntervention = 'continue' | 'plan' | 'audit' | 'recover' | 'repair' | 'verify' | 'final_gate' | 'wait_for_measurement' | 'abort';
-interface WorkflowTopology {
-    id: string;
-    interventions: HarnessIntervention[];
-    maxParallelBranches?: number;
-    metadata?: Record<string, unknown>;
-}
-interface MeasurementPolicy {
-    required: string[];
-    optional?: string[];
-    promoteOn?: Array<keyof RunScore | 'aggregate'>;
-}
-interface HarnessVariant {
-    id: string;
-    steering?: SteeringBundle;
-    topology?: WorkflowTopology;
-    measurement?: MeasurementPolicy;
-    budgets?: Record<string, number>;
-    models?: Record<string, string>;
-    reviewers?: Record<string, string>;
-    metadata?: Record<string, unknown>;
-}
-interface HarnessScenario {
-    id: string;
-    task: string;
-    split?: 'train' | 'validation' | 'test' | string;
-    metadata?: Record<string, unknown>;
-}
-interface HarnessRunRequest {
-    variant: HarnessVariant;
-    scenario: HarnessScenario;
-    trialIndex: number;
-}
-interface HarnessAdapter {
-    run(request: HarnessRunRequest): Promise<RunTrace>;
-}
-interface HarnessRunResult {
-    variant: HarnessVariant;
-    scenario: HarnessScenario;
-    trialIndex: number;
-    trace: RunTrace;
-    score: RunScore;
-    aggregate: number;
-}
-interface HarnessVariantReport {
-    variant: HarnessVariant;
-    runs: HarnessRunResult[];
-    aggregateMean: number;
-    passRate: number;
-    costUsdMean: number;
-    wallSecondsMean: number;
-    scoreMean: RunScore;
-}
-interface HarnessSelection {
-    winner: HarnessVariantReport;
-    frontier: ParetoResult<HarnessVariantReport>;
-    reports: HarnessVariantReport[];
-}
-interface HarnessExperimentResult {
-    results: HarnessRunResult[];
-    selection: HarnessSelection;
-}
-interface HarnessExperimentConfig {
-    adapter: HarnessAdapter;
-    variants: HarnessVariant[];
-    scenarios: HarnessScenario[];
-    trialsPerScenario?: number;
-    parallelism?: number;
-    weights?: Partial<RunScoreWeights>;
-    objectives?: Array<Objective<HarnessVariantReport>>;
-    score?: (trace: RunTrace, request: HarnessRunRequest) => RunScore | Promise<RunScore>;
-    onResult?: (result: HarnessRunResult) => void | Promise<void>;
-}
-declare const DEFAULT_HARNESS_OBJECTIVES: Array<Objective<HarnessVariantReport>>;
-declare function runHarnessExperiment(config: HarnessExperimentConfig): Promise<HarnessExperimentResult>;
-declare function selectHarnessVariant(results: HarnessRunResult[], objectives?: Array<Objective<HarnessVariantReport>>): HarnessSelection;
-declare function summarizeHarnessResults(results: HarnessRunResult[]): HarnessVariantReport[];
-
-export { DEFAULT_HARNESS_OBJECTIVES as D, type HarnessAdapter as H, type MeasurementPolicy as M, type SteeringBundle as S, type WorkflowTopology as W, type HarnessExperimentConfig as a, type HarnessExperimentResult as b, type HarnessIntervention as c, type HarnessRunRequest as d, type HarnessRunResult as e, type HarnessScenario as f, type HarnessSelection as g, type HarnessVariant as h, type HarnessVariantReport as i, type SteeringDelta as j, type SteeringRolePrompt as k, runHarnessExperiment as l, mergeSteeringBundle as m, summarizeHarnessResults as n, renderSteeringText as r, selectHarnessVariant as s };

package/dist/index-Bx3gZ8xl.d.ts

@@ -1,159 +0,0 @@
-import { a as RunSplitTag } from './run-record-e7vj1uZQ.js';
-
-/**
- * Shared types for the reference benchmark wrappers under
- * `src/benchmarks/`. Each wrapper exports the three functions in
- * `BenchmarkAdapter` plus its own typed `DatasetItem` shape.
- */
-
-interface BenchmarkDatasetItem<TPayload = unknown> {
-    /** Stable dataset-local item id (used for split assignment + paper
-     *  references). Unique within a benchmark. */
-    id: string;
-    /** Free-form payload. Each benchmark defines its own shape. */
-    payload: TPayload;
-}
-interface BenchmarkEvaluation {
-    /** [0, 1] score for the response on this item. Exact-match
-     *  benchmarks use 0/1; partial-credit benchmarks may return
-     *  fractional values. */
-    score: number;
-    /** Optional bag of raw scoring signals — e.g. parsed numeric
-     *  answer, regex match, judge sub-scores. */
-    raw: Record<string, unknown>;
-}
-/** Common signature implemented by every adapter under `src/benchmarks/*`. */
-interface BenchmarkAdapter<_TItem = unknown, TPayload = unknown> {
-    /** Load the dataset for the given split. May hit the network on
-     *  first call but should be cache-friendly. Adapters that don't
-     *  ship the dataset itself MUST throw a clearly-marked error
-     *  pointing the caller at the loader script. */
-    loadDataset(split: RunSplitTag): Promise<BenchmarkDatasetItem<TPayload>[]>;
-    /** Score a single response. Pure with respect to the inputs. */
-    evaluate(item: BenchmarkDatasetItem<TPayload>, response: string): Promise<BenchmarkEvaluation>;
-    /** Deterministic split assignment via item id hashing. The
-     *  fraction of items in each split is implementation-defined but
-     *  MUST be stable across processes and platforms. */
-    assignSplit(itemId: string): RunSplitTag;
-}
-/** Split-assignment seed shared across all benchmarks. Bumping this
- *  value reshuffles every split — do NOT do that lightly. */
-declare const BENCHMARK_SPLIT_SEED = "agent-eval-v1";
-/**
- * Assign an item id to one of `'search' | 'dev' | 'holdout'` using a
- * stable 32-bit hash of `${seed}::${id}`. Default proportions:
- *
- *   search: 60%   (optimization-readable)
- *   dev:    20%   (held-out for tuning, leak-on-purpose during dev)
- *   holdout:20%   (paper-grade held-out, gated reads)
- */
-declare function deterministicSplit(itemId: string, seed?: string): RunSplitTag;
-
-/**
- * Synthetic routing dataset. 16 tasks across 4 categories. Used as a
- * deterministic, dependency-free benchmark for any router that maps a
- * natural-language request to one of a fixed set of route labels.
- *
- * Format (see `routing/README.md` for prose):
- *
- *   {
- *     id:           stable per-task ID (matches across processes).
- *     category:     one of the four route labels.
- *     prompt:       the user-facing request the router must classify.
- *     route:        the ground-truth route the router should pick.
- *     synonyms:     other strings that count as a correct answer.
- *     hardNegatives:close-but-wrong route labels — used to detect the
- *                   "always picks the popular route" failure mode.
- *   }
- *
- * The four categories are intentionally cross-domain (file ops,
- * math, search, conversation) so a router that collapses to one
- * category is easy to spot.
- */
-interface RoutingItem {
-    id: string;
-    category: 'file' | 'math' | 'search' | 'chat';
-    prompt: string;
-    /** Canonical correct route label. */
-    route: string;
-    /** Alternate route labels that also count as correct. */
-    synonyms: string[];
-    /** Wrong-but-tempting route labels (for analysis, not grading). */
-    hardNegatives: string[];
-}
-declare const ROUTING_DATASET: RoutingItem[];
-
-/**
- * Routing benchmark — synthetic, dependency-free, ships in the
- * package. 16 cross-category items in `dataset.ts`. See
- * `routing/README.md` for the format.
- *
- * `evaluate` does case-insensitive exact match against the canonical
- * route plus declared synonyms. The first valid route token in the
- * response wins; everything else is ignored. Wrong answers also
- * report whether they hit a hard negative — useful when triaging
- * "always picks the popular route" failure modes.
- */
-
-type RoutingPayload = RoutingItem;
-type RoutingDatasetItem = BenchmarkDatasetItem<RoutingPayload>;
-declare class RoutingAdapter implements BenchmarkAdapter<RoutingDatasetItem, RoutingPayload> {
-    loadDataset(split: RunSplitTag): Promise<RoutingDatasetItem[]>;
-    evaluate(item: RoutingDatasetItem, response: string): Promise<BenchmarkEvaluation>;
-    assignSplit(itemId: string): RunSplitTag;
-}
-/**
- * Pull route-shaped tokens out of a model response. Routes look like
- * `category.action` (`fs.write`, `chat.reply`). Bare alphanumerics
- * are not routes, but `category.action` patterns are robust to most
- * model wrappers (JSON output, prose explanations, code fences).
- */
-declare function extractRouteTokens(response: string): string[];
-declare const loadDataset: (split: RunSplitTag) => Promise<RoutingDatasetItem[]>;
-declare const evaluate: (item: RoutingDatasetItem, response: string) => Promise<BenchmarkEvaluation>;
-declare const assignSplit: (itemId: string) => RunSplitTag;
-
-declare const index$1_ROUTING_DATASET: typeof ROUTING_DATASET;
-type index$1_RoutingAdapter = RoutingAdapter;
-declare const index$1_RoutingAdapter: typeof RoutingAdapter;
-type index$1_RoutingDatasetItem = RoutingDatasetItem;
-type index$1_RoutingItem = RoutingItem;
-type index$1_RoutingPayload = RoutingPayload;
-declare const index$1_assignSplit: typeof assignSplit;
-declare const index$1_evaluate: typeof evaluate;
-declare const index$1_extractRouteTokens: typeof extractRouteTokens;
-declare const index$1_loadDataset: typeof loadDataset;
-declare namespace index$1 {
-  export { index$1_ROUTING_DATASET as ROUTING_DATASET, index$1_RoutingAdapter as RoutingAdapter, type index$1_RoutingDatasetItem as RoutingDatasetItem, type index$1_RoutingItem as RoutingItem, type index$1_RoutingPayload as RoutingPayload, index$1_assignSplit as assignSplit, index$1_evaluate as evaluate, index$1_extractRouteTokens as extractRouteTokens, index$1_loadDataset as loadDataset };
-}
-
-/**
- * Reference benchmark wrappers — entry point.
- *
- * Core surface (exported here):
- *   - The `BenchmarkAdapter` contract.
- *   - `deterministicSplit` + `BENCHMARK_SPLIT_SEED` for split assignment.
- *   - `routing` — synthetic 16-task router benchmark. The only novel
- *     benchmark we built; ships in the package.
- *
- * Example wrappers (under `examples/benchmarks/`, NOT in the bundle):
- *   - `gsm8k`         — exact-match math reasoning (HF mirror, dataset
- *                       not bundled).
- *   - `swebench-lite` — 30-instance SWE-Bench subset via an external
- *                       grader command.
- *
- * The example wrappers are reference implementations of `BenchmarkAdapter`.
- * Read them, copy them, adapt them. They're intentionally not in the main
- * entry — every team will configure them differently.
- */
-
-declare const index_BENCHMARK_SPLIT_SEED: typeof BENCHMARK_SPLIT_SEED;
-type index_BenchmarkAdapter<_TItem = unknown, TPayload = unknown> = BenchmarkAdapter<_TItem, TPayload>;
-type index_BenchmarkDatasetItem<TPayload = unknown> = BenchmarkDatasetItem<TPayload>;
-type index_BenchmarkEvaluation = BenchmarkEvaluation;
-declare const index_deterministicSplit: typeof deterministicSplit;
-declare namespace index {
-  export { index_BENCHMARK_SPLIT_SEED as BENCHMARK_SPLIT_SEED, type index_BenchmarkAdapter as BenchmarkAdapter, type index_BenchmarkDatasetItem as BenchmarkDatasetItem, type index_BenchmarkEvaluation as BenchmarkEvaluation, index_deterministicSplit as deterministicSplit, index$1 as routing };
-}
-
-export { BENCHMARK_SPLIT_SEED as B, type BenchmarkAdapter as a, type BenchmarkDatasetItem as b, type BenchmarkEvaluation as c, deterministicSplit as d, index$1 as e, index as i };