@tangle-network/agent-eval 0.93.0 → 0.95.0

package/dist/oracle.d.ts

@@ -0,0 +1,55 @@
+/**
+ * Declarative oracles — ground-truth assertions without an LLM.
+ *
+ * Lifted from browser-agent-driver's _oracle.mjs. When you know the
+ * expected outcome exactly (a URL, a text fragment, a JSON shape), you
+ * don't need an LLM judge — you need a regex. These oracles are
+ * composable pass/fail checks over an observation bundle.
+ *
+ * Each oracle returns { pass, detail, evidence? } and has a short
+ * `id` for reporting. `evaluateOracles` runs a batch and aggregates.
+ */
+export interface OracleObservation {
+    /** Final observable text output from the agent (response, page snapshot, stdout). */
+    text?: string;
+    /** Final URL — for browser-style scenarios. */
+    url?: string;
+    /** Any structured JSON the agent produced. */
+    json?: unknown;
+    /** Free-form context used by custom oracles. */
+    context?: Record<string, unknown>;
+}
+export interface OracleResult {
+    id: string;
+    pass: boolean;
+    detail: string;
+    evidence?: string;
+}
+export interface Oracle {
+    id: string;
+    check(obs: OracleObservation): OracleResult;
+}
+export declare function textInSnapshot(needle: string, opts?: {
+    caseSensitive?: boolean;
+}): Oracle;
+export declare function urlContains(fragment: string): Oracle;
+export declare function jsonShape(expected: Record<string, unknown>): Oracle;
+export declare function regexMatches(pattern: RegExp): Oracle;
+/**
+ * Anti-bot detector — distinguishes genuine failures from blocked navigation
+ * (cloudflare, recaptcha, etc). Returns an Oracle that PASSES when no block
+ * marker is present; on block, detail names the blocker so runners can tag
+ * results as "blocked" rather than "failed". Lifted from browser-agent-driver.
+ */
+export declare function notBlocked(): Oracle;
+export interface OracleReport {
+    results: OracleResult[];
+    pass: boolean;
+    passCount: number;
+    failCount: number;
+    /** 0-1 ratio of oracles passed. */
+    score: number;
+}
+/** Run all oracles against one observation and aggregate. */
+export declare function evaluateOracles(obs: OracleObservation, oracles: Oracle[]): OracleReport;
+//# sourceMappingURL=oracle.d.ts.map

package/dist/oracle.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"oracle.d.ts","sourceRoot":"","sources":["../src/oracle.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,MAAM,WAAW,iBAAiB;IAChC,qFAAqF;IACrF,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,+CAA+C;IAC/C,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,8CAA8C;IAC9C,IAAI,CAAC,EAAE,OAAO,CAAA;IACd,gDAAgD;IAChD,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CAClC;AAED,MAAM,WAAW,YAAY;IAC3B,EAAE,EAAE,MAAM,CAAA;IACV,IAAI,EAAE,OAAO,CAAA;IACb,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,CAAC,EAAE,MAAM,CAAA;CAClB;AAED,MAAM,WAAW,MAAM;IACrB,EAAE,EAAE,MAAM,CAAA;IACV,KAAK,CAAC,GAAG,EAAE,iBAAiB,GAAG,YAAY,CAAA;CAC5C;AAED,wBAAgB,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,GAAE;IAAE,aAAa,CAAC,EAAE,OAAO,CAAA;CAAO,GAAG,MAAM,CAiB7F;AAED,wBAAgB,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAepD;AAED,wBAAgB,SAAS,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,CA4BnE;AAED,wBAAgB,YAAY,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAepD;AAED;;;;;GAKG;AACH,wBAAgB,UAAU,IAAI,MAAM,CA4BnC;AAED,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE,YAAY,EAAE,CAAA;IACvB,IAAI,EAAE,OAAO,CAAA;IACb,SAAS,EAAE,MAAM,CAAA;IACjB,SAAS,EAAE,MAAM,CAAA;IACjB,mCAAmC;IACnC,KAAK,EAAE,MAAM,CAAA;CACd;AAED,6DAA6D;AAC7D,wBAAgB,eAAe,CAAC,GAAG,EAAE,iBAAiB,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,YAAY,CAWvF"}

package/dist/orthogonality.d.ts

@@ -0,0 +1,35 @@
+/**
+ * Inter-critic / inter-pass orthogonality.
+ *
+ * Detects redundant ensembles. When you run N critics (or N audit passes,
+ * or N specialized agents) on the same input, you want them to disagree —
+ * each contributing distinct signal. If they all converge on the same set
+ * of findings, you're paying N× cost for ~1× signal.
+ *
+ * The metric is `1 − mean pairwise cosine similarity` over bags of words
+ * extracted from each pass's outputs. 1.0 = fully orthogonal,
+ * 0.0 = fully redundant.
+ *
+ * Universal primitive: pass anything that produces text (findings, tool
+ * calls rendered as JSON, verdict strings) and the matcher derives its own
+ * vocabulary.
+ */
+export interface OrthogonalityInput<T> {
+    passes: Array<{
+        findings: T[];
+    }>;
+    /** Render one element to text. Default: defaultRender (concatenates string fields). */
+    text?: (item: T) => string;
+    /** Minimum token length kept in the bag. Default 4 (drops short fillers). */
+    minTokenLength?: number;
+}
+export interface OrthogonalityResult {
+    /** 1 − mean pairwise cosine similarity across passes. 1=fully orthogonal, 0=fully redundant. */
+    orthogonality: number;
+    /** Number of passes considered. */
+    passCount: number;
+    /** Pairwise cosine similarities, in upper-triangular order (for debugging). */
+    similarities: number[];
+}
+export declare function passOrthogonality<T>(input: OrthogonalityInput<T>): OrthogonalityResult;
+//# sourceMappingURL=orthogonality.d.ts.map

package/dist/orthogonality.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"orthogonality.d.ts","sourceRoot":"","sources":["../src/orthogonality.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,MAAM,WAAW,kBAAkB,CAAC,CAAC;IACnC,MAAM,EAAE,KAAK,CAAC;QAAE,QAAQ,EAAE,CAAC,EAAE,CAAA;KAAE,CAAC,CAAA;IAChC,uFAAuF;IACvF,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,MAAM,CAAA;IAC1B,6EAA6E;IAC7E,cAAc,CAAC,EAAE,MAAM,CAAA;CACxB;AAED,MAAM,WAAW,mBAAmB;IAClC,gGAAgG;IAChG,aAAa,EAAE,MAAM,CAAA;IACrB,mCAAmC;IACnC,SAAS,EAAE,MAAM,CAAA;IACjB,+EAA+E;IAC/E,YAAY,EAAE,MAAM,EAAE,CAAA;CACvB;AAED,wBAAgB,iBAAiB,CAAC,CAAC,EAAE,KAAK,EAAE,kBAAkB,CAAC,CAAC,CAAC,GAAG,mBAAmB,CAoBtF"}

package/dist/otel-pipeline.d.ts

@@ -0,0 +1,31 @@
+/**
+ * Pipeline-level OTEL integration — auto-attaches an OTEL exporter when
+ * OTEL_EXPORTER_OTLP_ENDPOINT is set. Pipelines call `withOtelPipeline()`
+ * to get a configured exporter + shutdown handle without manual wiring.
+ */
+import { type OtelExportConfig, type OtelExporter } from './trace/otel-export';
+export interface OtelPipelineHandle {
+    /** The active exporter, or undefined if no endpoint is configured. */
+    exporter: OtelExporter | undefined;
+    /** Call at pipeline end to flush + shutdown. Safe to call even if exporter is undefined. */
+    shutdown(): Promise<void>;
+}
+export interface OtelPipelineOptions {
+    /** Override OTEL config. */
+    otelConfig?: OtelExportConfig;
+    /** Pipeline-specific resource attributes. */
+    pipelineKind?: string;
+    pipelineId?: string;
+}
+/**
+ * Create an OTEL exporter scoped to a pipeline run. Auto-reads
+ * OTEL_EXPORTER_OTLP_ENDPOINT from env when no explicit config is passed.
+ *
+ * Returns a handle with `exporter` (possibly undefined) and `shutdown()`.
+ */
+export declare function withOtelPipeline(opts?: OtelPipelineOptions): OtelPipelineHandle;
+/**
+ * Check if OTEL export is configured (endpoint is set).
+ */
+export declare function isOtelConfigured(): boolean;
+//# sourceMappingURL=otel-pipeline.d.ts.map

package/dist/otel-pipeline.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"otel-pipeline.d.ts","sourceRoot":"","sources":["../src/otel-pipeline.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAsB,KAAK,gBAAgB,EAAE,KAAK,YAAY,EAAE,MAAM,qBAAqB,CAAA;AAElG,MAAM,WAAW,kBAAkB;IACjC,sEAAsE;IACtE,QAAQ,EAAE,YAAY,GAAG,SAAS,CAAA;IAClC,4FAA4F;IAC5F,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC,CAAA;CAC1B;AAED,MAAM,WAAW,mBAAmB;IAClC,4BAA4B;IAC5B,UAAU,CAAC,EAAE,gBAAgB,CAAA;IAC7B,6CAA6C;IAC7C,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,UAAU,CAAC,EAAE,MAAM,CAAA;CACpB;AAED;;;;;GAKG;AACH,wBAAgB,gBAAgB,CAAC,IAAI,CAAC,EAAE,mBAAmB,GAAG,kBAAkB,CAkB/E;AAED;;GAEG;AACH,wBAAgB,gBAAgB,IAAI,OAAO,CAE1C"}

package/dist/paraphrase.d.ts

@@ -0,0 +1,107 @@
+/**
+ * Paraphrase robustness — mutates a scenario prompt in structure-
+ * preserving ways, re-scores, and reports score variance.
+ *
+ * Mutators are pure functions `(prompt: string) => string`. Ship a
+ * default set; consumers add domain-specific ones.
+ *
+ * Robustness score: 1 - stdDev(scores) / (mean if positive else 1).
+ * A perfect agent returns the same answer regardless of typo / case /
+ * reordering — any variance signals a brittle prompt.
+ */
+export type Mutator = (prompt: string, seed: number) => string;
+export interface RobustnessResult {
+    originalScore: number;
+    variantScores: Array<{
+        mutator: string;
+        score: number;
+        mutated: string;
+    }>;
+    meanScore: number;
+    stdDev: number;
+    robustness: number;
+}
+export declare function paraphraseRobustness(prompt: string, mutators: Array<{
+    id: string;
+    fn: Mutator;
+}>, scoreFn: (prompt: string) => Promise<number>, options?: {
+    seed?: number;
+}): Promise<RobustnessResult>;
+/** Lowercase the whole prompt. Robust models ignore case. */
+export declare const lowercaseMutator: Mutator;
+/** Reorder sentences. Robust models don't depend on sentence order. */
+export declare const sentenceReorderMutator: Mutator;
+/** Swap adjacent letter pairs (1 per 40 chars, min 1). Robust models tolerate typos. */
+export declare const typoMutator: Mutator;
+/** Add a benign politeness prefix. Robust models ignore flattery. */
+export declare const politenessPrefixMutator: Mutator;
+/** Compact whitespace, strip newlines. Robust models don't depend on formatting. */
+export declare const whitespaceCollapseMutator: Mutator;
+export declare const DEFAULT_MUTATORS: Array<{
+    id: string;
+    fn: Mutator;
+}>;
+export interface ParaphraseRobustnessScenarioInput {
+    scenarios: Array<{
+        id: string;
+        userTurns: string[];
+    }>;
+    /**
+     * Mutators applied to every user turn in every scenario. Each
+     * scenario is paraphrased once per mutator (so `reps` × `scenarios`
+     * × `mutators` total paraphrased runs).
+     */
+    mutators: Array<{
+        name: string;
+        mutator: (text: string) => string;
+    }>;
+    /**
+     * Run a (possibly mutated) scenario and return its score in [0,1].
+     * Called once for the original turns of each scenario, and once per
+     * (scenario × mutator × rep) for the paraphrased variants.
+     */
+    runScenario: (args: {
+        id: string;
+        userTurns: string[];
+    }) => Promise<{
+        score: number;
+    }>;
+    /** Times to repeat each (scenario × mutator) pair. Default 1. */
+    reps?: number;
+}
+export interface ParaphraseRobustnessScenarioResult {
+    /**
+     * Aggregate robustness: `mean(paraphrased) / mean(original)`,
+     * clipped to `[0, 1]`. `1` = paraphrasing didn't degrade the agent;
+     * `0` = paraphrasing destroyed it (or original was 0).
+     */
+    score: number;
+    perScenario: Array<{
+        id: string;
+        originalScore: number;
+        paraphrasedMean: number;
+        /** Per-mutator delta (paraphrased − original); negative = mutator hurt. */
+        deltas: Record<string, number>;
+    }>;
+    mutators: string[];
+}
+/**
+ * Multi-turn convenience wrapper around {@link paraphraseRobustness}.
+ *
+ * Consumers with a list of multi-turn scenarios were hand-wrapping the
+ * single-prompt runner per scenario; this iterates for them. Mutators
+ * are applied to every user turn (mutator runs once per turn with a
+ * stable seed derived from the rep index).
+ *
+ * Contract:
+ *   - Calls `runScenario` once with the original `userTurns` to
+ *     establish the baseline `originalScore`.
+ *   - For each `(scenario, mutator, rep)` combination, builds a
+ *     mutated copy of `userTurns` (every turn passed through
+ *     `mutator.mutator`) and calls `runScenario` again.
+ *   - Aggregates per-scenario means, then computes the overall
+ *     `mean(paraphrasedMean) / mean(originalScore)`, clipped to
+ *     `[0, 1]`. If every original score is 0 the aggregate is 0.
+ */
+export declare function paraphraseRobustnessScenarios(args: ParaphraseRobustnessScenarioInput): Promise<ParaphraseRobustnessScenarioResult>;
+//# sourceMappingURL=paraphrase.d.ts.map

package/dist/paraphrase.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"paraphrase.d.ts","sourceRoot":"","sources":["../src/paraphrase.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,MAAM,MAAM,OAAO,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,KAAK,MAAM,CAAA;AAE9D,MAAM,WAAW,gBAAgB;IAC/B,aAAa,EAAE,MAAM,CAAA;IACrB,aAAa,EAAE,KAAK,CAAC;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC,CAAA;IACzE,SAAS,EAAE,MAAM,CAAA;IACjB,MAAM,EAAE,MAAM,CAAA;IACd,UAAU,EAAE,MAAM,CAAA;CACnB;AAED,wBAAsB,oBAAoB,CACxC,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,KAAK,CAAC;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,EAAE,EAAE,OAAO,CAAA;CAAE,CAAC,EAC5C,OAAO,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,EAC5C,OAAO,GAAE;IAAE,IAAI,CAAC,EAAE,MAAM,CAAA;CAAO,GAC9B,OAAO,CAAC,gBAAgB,CAAC,CAiB3B;AAID,6DAA6D;AAC7D,eAAO,MAAM,gBAAgB,EAAE,OAAgC,CAAA;AAE/D,uEAAuE;AACvE,eAAO,MAAM,sBAAsB,EAAE,OAWpC,CAAA;AAED,wFAAwF;AACxF,eAAO,MAAM,WAAW,EAAE,OAmBzB,CAAA;AAED,qEAAqE;AACrE,eAAO,MAAM,uBAAuB,EAAE,OAAwD,CAAA;AAE9F,oFAAoF;AACpF,eAAO,MAAM,yBAAyB,EAAE,OAA8C,CAAA;AAEtF,eAAO,MAAM,gBAAgB,EAAE,KAAK,CAAC;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,EAAE,EAAE,OAAO,CAAA;CAAE,CAM/D,CAAA;AAID,MAAM,WAAW,iCAAiC;IAChD,SAAS,EAAE,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC,CAAA;IACrD;;;;OAIG;IACH,QAAQ,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,CAAA;KAAE,CAAC,CAAA;IACpE;;;;OAIG;IACH,WAAW,EAAE,CAAC,IAAI,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,EAAE,CAAA;KAAE,KAAK,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAA;IACtF,iEAAiE;IACjE,IAAI,CAAC,EAAE,MAAM,CAAA;CACd;AAED,MAAM,WAAW,kCAAkC;IACjD;;;;OAIG;IACH,KAAK,EAAE,MAAM,CAAA;IACb,WAAW,EAAE,KAAK,CAAC;QACjB,EAAE,EAAE,MAAM,CAAA;QACV,aAAa,EAAE,MAAM,CAAA;QACrB,eAAe,EAAE,MAAM,CAAA;QACvB,2EAA2E;QAC3E,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;KAC/B,CAAC,CAAA;IACF,QAAQ,EAAE,MAAM,EAAE,CAAA;CACnB;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAsB,6BAA6B,CACjD,IAAI,EAAE,iCAAiC,GACtC,OAAO,CAAC,kCAAkC,CAAC,CAoD7C"}

package/dist/{pareto-E-pembql.d.ts → pareto.d.ts}

@@ -13,14 +13,14 @@
  * or 'minimize' (lower=better). Candidates are any object; pass an
  * `objective(candidate)` accessor.
  */
-type Direction = 'maximize' | 'minimize';
-interface Objective<T> {
+export type Direction = 'maximize' | 'minimize';
+export interface Objective<T> {
     /** Stable label used in reports. */
     name: string;
     direction: Direction;
     value: (candidate: T) => number;
 }
-interface ParetoResult<T> {
+export interface ParetoResult<T> {
     frontier: T[];
     dominated: T[];
     /** Index map: frontier[i] dominates each of dominatedBy[i]. */
@@ -30,13 +30,13 @@ interface ParetoResult<T> {
     }>;
 }
 /** Does candidate A weakly dominate B — strictly better on at least one objective and no worse on any? */
-declare function dominates<T>(a: T, b: T, objectives: Objective<T>[]): boolean;
+export declare function dominates<T>(a: T, b: T, objectives: Objective<T>[]): boolean;
 /**
  * Compute the non-dominated frontier. Candidates with NaN/Infinity on any
  * objective are excluded (can't rank them). A candidate enters the frontier
  * iff no other candidate dominates it.
  */
-declare function paretoFrontier<T>(candidates: T[], objectives: Objective<T>[]): ParetoResult<T>;
+export declare function paretoFrontier<T>(candidates: T[], objectives: Objective<T>[]): ParetoResult<T>;
 /**
  * Weighted-sum scalarisation. Use as a tie-break / single-winner selector
  * when callers don't want to consume a frontier. Each objective contributes
@@ -46,7 +46,7 @@ declare function paretoFrontier<T>(candidates: T[], objectives: Objective<T>[]):
  * Direction is honoured automatically — `minimize` axes have their values
  * inverted before scaling so "higher scalar = better" always holds.
  */
-declare function scalarScore<T>(candidates: T[], objectives: Objective<T>[], options?: {
+export declare function scalarScore<T>(candidates: T[], objectives: Objective<T>[], options?: {
     weights?: Partial<Record<string, number>>;
 }): Array<{
     candidate: T;
@@ -64,7 +64,7 @@ declare function scalarScore<T>(candidates: T[], objectives: Objective<T>[], opt
  * input. Higher distance = more isolated = should be preferred when
  * preserving diversity.
  */
-declare function crowdingDistance<T>(candidates: T[], objectives: Objective<T>[]): Array<{
+export declare function crowdingDistance<T>(candidates: T[], objectives: Objective<T>[]): Array<{
     candidate: T;
     distance: number;
 }>;
@@ -73,9 +73,8 @@ declare function crowdingDistance<T>(candidates: T[], objectives: Objective<T>[]
  * NSGA-II selection step. Returns the frontier sorted by descending crowding
  * distance so callers can `.slice(0, k)` to pick K diverse winners.
  */
-declare function paretoFrontierWithCrowding<T>(candidates: T[], objectives: Objective<T>[]): Array<{
+export declare function paretoFrontierWithCrowding<T>(candidates: T[], objectives: Objective<T>[]): Array<{
     candidate: T;
     distance: number;
 }>;
-
-export { type Direction as D, type Objective as O, type ParetoResult as P, paretoFrontierWithCrowding as a, crowdingDistance as c, dominates as d, paretoFrontier as p, scalarScore as s };
+//# sourceMappingURL=pareto.d.ts.map

package/dist/pareto.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pareto.d.ts","sourceRoot":"","sources":["../src/pareto.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,MAAM,MAAM,SAAS,GAAG,UAAU,GAAG,UAAU,CAAA;AAE/C,MAAM,WAAW,SAAS,CAAC,CAAC;IAC1B,oCAAoC;IACpC,IAAI,EAAE,MAAM,CAAA;IACZ,SAAS,EAAE,SAAS,CAAA;IACpB,KAAK,EAAE,CAAC,SAAS,EAAE,CAAC,KAAK,MAAM,CAAA;CAChC;AAED,MAAM,WAAW,YAAY,CAAC,CAAC;IAC7B,QAAQ,EAAE,CAAC,EAAE,CAAA;IACb,SAAS,EAAE,CAAC,EAAE,CAAA;IACd,+DAA+D;IAC/D,YAAY,EAAE,KAAK,CAAC;QAAE,SAAS,EAAE,CAAC,CAAC;QAAC,SAAS,EAAE,CAAC,EAAE,CAAA;KAAE,CAAC,CAAA;CACtD;AAED,0GAA0G;AAC1G,wBAAgB,SAAS,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,UAAU,EAAE,SAAS,CAAC,CAAC,CAAC,EAAE,GAAG,OAAO,CAY5E;AAED;;;;GAIG;AACH,wBAAgB,cAAc,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,EAAE,UAAU,EAAE,SAAS,CAAC,CAAC,CAAC,EAAE,GAAG,YAAY,CAAC,CAAC,CAAC,CAiB9F;AAED;;;;;;;;GAQG;AACH,wBAAgB,WAAW,CAAC,CAAC,EAC3B,UAAU,EAAE,CAAC,EAAE,EACf,UAAU,EAAE,SAAS,CAAC,CAAC,CAAC,EAAE,EAC1B,OAAO,GAAE;IAAE,OAAO,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAA;CAAO,GAC1D,KAAK,CAAC;IAAE,SAAS,EAAE,CAAC,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CA2BxC;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,gBAAgB,CAAC,CAAC,EAChC,UAAU,EAAE,CAAC,EAAE,EACf,UAAU,EAAE,SAAS,CAAC,CAAC,CAAC,EAAE,GACzB,KAAK,CAAC;IAAE,SAAS,EAAE,CAAC,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,CAAC,CAsB3C;AAED;;;;GAIG;AACH,wBAAgB,0BAA0B,CAAC,CAAC,EAC1C,UAAU,EAAE,CAAC,EAAE,EACf,UAAU,EAAE,SAAS,CAAC,CAAC,CAAC,EAAE,GACzB,KAAK,CAAC;IAAE,SAAS,EAAE,CAAC,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,CAAC,CAK3C"}

package/dist/partition-held-out.d.ts

@@ -0,0 +1,70 @@
+/**
+ * Deterministic held-out partitioning over stable ids.
+ *
+ * A promotion gate's paired-delta math needs a fixed partition: the same
+ * scenario id lands in the same bucket forever, regardless of when it ran or
+ * which loop emitted the record — otherwise the gate has nothing paired to
+ * compare against. Picking one global split per run is wrong: every run then
+ * stamps either ALL search or ALL holdout, and no id is ever observed on both
+ * sides. Per-id deterministic hashing fixes that.
+ *
+ * Two consumers hand-roll this (agent-builder's `deterministicSplit` and the
+ * frontier persona-splitter). The substrate already has a 3-way benchmark
+ * `deterministicSplit` in `./benchmarks`; this is the generic, validated batch
+ * partitioner: it takes a list of ids, splits them by a stable FNV-1a hash, and
+ * fails loud when the inputs can't support a trustworthy held-out comparison
+ * (duplicate ids, or a holdout set below the significance floor).
+ */
+/** 32-bit FNV-1a hash. Stable, allocation-free, deterministic across runtimes —
+ *  the same id+seed maps to the same bucket on every machine and process. */
+export declare function fnv1a32(input: string): number;
+/** Map an id+seed to the half-open unit interval [0, 1) deterministically. */
+export declare function hashToUnit(id: string, seed: string): number;
+export interface PartitionHeldOutOptions {
+    /** Partition seed. Bumping it reshuffles every assignment — do that only when
+     *  the corpus/policy changes meaningfully so old and new records can't pair.
+     *  Default 'held-out-v1'. */
+    seed?: string;
+    /** Fraction routed to the held-out bucket, in (0, 1). Default 0.5. */
+    holdoutFraction?: number;
+    /**
+     * Minimum held-out ids required for the split to be admissible. Below this,
+     * a paired comparison has too few observations to be significant, so the
+     * partition throws rather than hand back an underpowered holdout. Default 1
+     * (only the empty-holdout degenerate case is rejected). Raise it to enforce
+     * real statistical power. */
+    minHoldout?: number;
+    /** Same floor for the search side. Default 1. */
+    minSearch?: number;
+}
+export interface HeldOutPartition {
+    /** Ids the optimizer is allowed to read/optimize against. */
+    search: string[];
+    /** Ids reserved for gated, held-out evaluation. */
+    holdout: string[];
+    seed: string;
+    holdoutFraction: number;
+}
+/**
+ * Assign a single id to `'search'` or `'holdout'` deterministically. Same
+ * `(id, seed)` always returns the same tag — the invariant the gate relies on.
+ * Use this in a `splitTag:` field literal when stamping records one at a time;
+ * use `partitionHeldOut` when you have the whole id list up front and want the
+ * validated disjoint split.
+ */
+export declare function assignHeldOutTag(id: string, options?: {
+    seed?: string;
+    holdoutFraction?: number;
+}): 'search' | 'holdout';
+/**
+ * Partition a list of stable ids into disjoint `search` / `holdout` sets by a
+ * deterministic hash. Fail-loud:
+ *   - empty input throws,
+ *   - duplicate ids throw (a dup would be observed on both sides, double-counting),
+ *   - a holdout (or search) set below its floor throws (underpowered comparison).
+ *
+ * Order within each bucket follows the input order, so the result is stable and
+ * reproducible.
+ */
+export declare function partitionHeldOut(ids: string[], options?: PartitionHeldOutOptions): HeldOutPartition;
+//# sourceMappingURL=partition-held-out.d.ts.map

package/dist/partition-held-out.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"partition-held-out.d.ts","sourceRoot":"","sources":["../src/partition-held-out.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAIH;6EAC6E;AAC7E,wBAAgB,OAAO,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAO7C;AAED,8EAA8E;AAC9E,wBAAgB,UAAU,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CAG3D;AAED,MAAM,WAAW,uBAAuB;IACtC;;iCAE6B;IAC7B,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,sEAAsE;IACtE,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB;;;;;iCAK6B;IAC7B,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,iDAAiD;IACjD,SAAS,CAAC,EAAE,MAAM,CAAA;CACnB;AAED,MAAM,WAAW,gBAAgB;IAC/B,6DAA6D;IAC7D,MAAM,EAAE,MAAM,EAAE,CAAA;IAChB,mDAAmD;IACnD,OAAO,EAAE,MAAM,EAAE,CAAA;IACjB,IAAI,EAAE,MAAM,CAAA;IACZ,eAAe,EAAE,MAAM,CAAA;CACxB;AAED;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAC9B,EAAE,EAAE,MAAM,EACV,OAAO,GAAE;IAAE,IAAI,CAAC,EAAE,MAAM,CAAC;IAAC,eAAe,CAAC,EAAE,MAAM,CAAA;CAAO,GACxD,QAAQ,GAAG,SAAS,CAKtB;AAED;;;;;;;;;GASG;AACH,wBAAgB,gBAAgB,CAC9B,GAAG,EAAE,MAAM,EAAE,EACb,OAAO,GAAE,uBAA4B,GACpC,gBAAgB,CA2ClB"}

package/dist/partition-held-out.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=partition-held-out.test.d.ts.map

package/dist/partition-held-out.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"partition-held-out.test.d.ts","sourceRoot":"","sources":["../src/partition-held-out.test.ts"],"names":[],"mappings":""}

package/dist/perf/index.d.ts

@@ -1,123 +1,17 @@
 /**
- * Journey × axes matrix for infra performance benchmarks.
+ * @tangle-network/agent-eval/perf
  *
- * A journey is one measurable user path ("provision.cold", "chat.ttft");
- * axes are free-form scenario dimensions (driver, region, image…). The
- * matrix expansion is pure bookkeeping — running the scenarios and
- * recording metrics is the caller's job. This module complements the
- * judge-panel `BenchmarkRunner` (src/benchmark.ts): that one scores
- * QUALITY via judges, this one structures LATENCY / RELIABILITY runs
- * over flat metric records.
- */
-/** One measurable user journey (e.g. "provision.cold", "chat.ttft"). */
-interface JourneySpec {
-    id: string;
-    description: string;
-    /** Needs a real LLM call — schedule nightly, not per-PR. */
-    requiresLLM: boolean;
-    /**
-     * Fields that MUST be non-null on a passing record of this journey.
-     * A "passing" record missing one is an integrity violation, not a pass.
-     */
-    requiredFields: ReadonlyArray<string>;
-    /** Numeric floors, e.g. {field: 'event_count', min: 1} for streaming. */
-    minimums?: ReadonlyArray<{
-        field: string;
-        min: number;
-    }>;
-    /** Per-phase breakdown fields expected non-null (subset of requiredFields semantics, reported separately). */
-    phaseFields?: ReadonlyArray<string>;
-}
-interface ScenarioAxes {
-    /** e.g. driver: ['docker','firecracker'] — every key is a free-form dimension. */
-    [dimension: string]: ReadonlyArray<string>;
-}
-interface PerfScenario {
-    /** `${journeyId}|${dim1}=${v1}|${dim2}=${v2}` (dims sorted). */
-    key: string;
-    journey: JourneySpec;
-    axes: Record<string, string>;
-}
-/** Stable scenario key: journey id then `dim=value` pairs in sorted-dim order. */
-declare function scenarioKey(journeyId: string, axes: Record<string, string>): string;
-/** Cartesian expansion; `filter` lets callers drop invalid combos (e.g. firecracker×resume). */
-declare function expandMatrix(journeys: ReadonlyArray<JourneySpec>, axes: ScenarioAxes, filter?: (journeyId: string, combo: Record<string, string>) => boolean): PerfScenario[];
-
-/**
- * Record-integrity contracts for perf metric records.
- *
- * A record that claims `pass === true` must actually carry the journey's
- * required measurements — a "passing" provision run with a null
- * `total_ms` is a lying record, not a pass. Failed records are exempt:
- * a run that errored mid-flight legitimately has nulls.
- */
-
-interface IntegrityViolation {
-    recordIndex: number;
-    journeyId: string;
-    field: string;
-    reason: 'null-required-field' | 'below-minimum';
-    detail: string;
-}
-interface IntegrityResult {
-    succeeded: boolean;
-    violations: IntegrityViolation[];
-}
-/**
- * Validates flat metric records (Record<string, unknown> with a boolean
- * `pass` field) against their journey contract. Only records with
- * pass === true are checked — a failed record may legitimately have nulls.
- * resolveJourney maps a record to its JourneySpec (or null to skip).
- */
-declare function checkRecordIntegrity(records: ReadonlyArray<Record<string, unknown>>, resolveJourney: (record: Record<string, unknown>) => JourneySpec | null): IntegrityResult;
-/** Throws an Error listing every violation when the result fails. */
-declare function assertRecordIntegrity(records: ReadonlyArray<Record<string, unknown>>, resolveJourney: (record: Record<string, unknown>) => JourneySpec | null): void;
-
-/**
- * Percentile ratchet over perf metric records.
+ * Domain-agnostic infra-performance benchmarking substrate: a journeys ×
+ * axes scenario matrix, record-integrity contracts over flat metric
+ * records, and a percentile ratchet (summarize → baseline → gate).
  *
- * `summarizeRecords` folds flat records into per-scenario p50/p90 stats;
- * `gatePerf` compares a current summary against a committed baseline and
- * trips on regressions beyond tolerance. Percentiles use nearest-rank on
- * sorted values. Null / non-numeric metric values are excluded from the
- * stat (n reflects only real samples); a field with zero samples is
- * omitted entirely — no fake zeros.
+ * Complements the judge-panel `BenchmarkRunner` (root): that one scores
+ * QUALITY; this one scores LATENCY / RELIABILITY over flat metric records.
  */
-interface PerfStat {
-    p50: number;
-    p90: number;
-    n: number;
-}
-interface PerfBaseline {
-    version: 1;
-    /** key → metric field → stat. */
-    scenarios: Record<string, Record<string, PerfStat>>;
-}
-interface PerfRegression {
-    scenarioKey: string;
-    field: string;
-    baseline: PerfStat;
-    current: PerfStat;
-    /** percent over baseline p50 / p90, whichever tripped. */
-    overBy: {
-        p50Pct: number;
-        p90Pct: number;
-    };
-}
-interface PerfGateResult {
-    succeeded: boolean;
-    regressions: PerfRegression[];
-    /** Negative overBy: strictly better than baseline on both percentiles. */
-    improvements: PerfRegression[];
-    /** In baseline but absent (or under-sampled, n < minSamples) in current. */
-    missingScenarios: string[];
-    /** In records but absent from baseline. */
-    newScenarios: string[];
-}
-declare function summarizeRecords(records: ReadonlyArray<Record<string, unknown>>, keyOf: (record: Record<string, unknown>) => string | null, metricFields: ReadonlyArray<string>): PerfBaseline;
-declare function gatePerf(current: PerfBaseline, baseline: PerfBaseline, options?: {
-    tolerancePct?: number;
-    minSamples?: number;
-}): PerfGateResult;
-
-export { type IntegrityResult, type IntegrityViolation, type JourneySpec, type PerfBaseline, type PerfGateResult, type PerfRegression, type PerfScenario, type PerfStat, type ScenarioAxes, assertRecordIntegrity, checkRecordIntegrity, expandMatrix, gatePerf, scenarioKey, summarizeRecords };
+export type { IntegrityResult, IntegrityViolation } from './integrity';
+export { assertRecordIntegrity, checkRecordIntegrity } from './integrity';
+export type { JourneySpec, PerfScenario, ScenarioAxes } from './journey';
+export { expandMatrix, scenarioKey } from './journey';
+export type { PerfBaseline, PerfGateResult, PerfRegression, PerfStat } from './ratchet';
+export { gatePerf, summarizeRecords } from './ratchet';
+//# sourceMappingURL=index.d.ts.map

package/dist/perf/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/perf/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,YAAY,EAAE,eAAe,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAA;AACtE,OAAO,EAAE,qBAAqB,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAA;AACzE,YAAY,EAAE,WAAW,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,WAAW,CAAA;AACxE,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,WAAW,CAAA;AACrD,YAAY,EAAE,YAAY,EAAE,cAAc,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAA;AACvF,OAAO,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAA"}

package/dist/perf/integrity.d.ts

@@ -0,0 +1,30 @@
+/**
+ * Record-integrity contracts for perf metric records.
+ *
+ * A record that claims `pass === true` must actually carry the journey's
+ * required measurements — a "passing" provision run with a null
+ * `total_ms` is a lying record, not a pass. Failed records are exempt:
+ * a run that errored mid-flight legitimately has nulls.
+ */
+import type { JourneySpec } from './journey';
+export interface IntegrityViolation {
+    recordIndex: number;
+    journeyId: string;
+    field: string;
+    reason: 'null-required-field' | 'below-minimum';
+    detail: string;
+}
+export interface IntegrityResult {
+    succeeded: boolean;
+    violations: IntegrityViolation[];
+}
+/**
+ * Validates flat metric records (Record<string, unknown> with a boolean
+ * `pass` field) against their journey contract. Only records with
+ * pass === true are checked — a failed record may legitimately have nulls.
+ * resolveJourney maps a record to its JourneySpec (or null to skip).
+ */
+export declare function checkRecordIntegrity(records: ReadonlyArray<Record<string, unknown>>, resolveJourney: (record: Record<string, unknown>) => JourneySpec | null): IntegrityResult;
+/** Throws an Error listing every violation when the result fails. */
+export declare function assertRecordIntegrity(records: ReadonlyArray<Record<string, unknown>>, resolveJourney: (record: Record<string, unknown>) => JourneySpec | null): void;
+//# sourceMappingURL=integrity.d.ts.map

package/dist/perf/integrity.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"integrity.d.ts","sourceRoot":"","sources":["../../src/perf/integrity.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,WAAW,CAAA;AAE5C,MAAM,WAAW,kBAAkB;IACjC,WAAW,EAAE,MAAM,CAAA;IACnB,SAAS,EAAE,MAAM,CAAA;IACjB,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,qBAAqB,GAAG,eAAe,CAAA;IAC/C,MAAM,EAAE,MAAM,CAAA;CACf;AAED,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,OAAO,CAAA;IAClB,UAAU,EAAE,kBAAkB,EAAE,CAAA;CACjC;AAMD;;;;;GAKG;AACH,wBAAgB,oBAAoB,CAClC,OAAO,EAAE,aAAa,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,EAC/C,cAAc,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,WAAW,GAAG,IAAI,GACtE,eAAe,CAqDjB;AAED,qEAAqE;AACrE,wBAAgB,qBAAqB,CACnC,OAAO,EAAE,aAAa,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,EAC/C,cAAc,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,WAAW,GAAG,IAAI,GACtE,IAAI,CASN"}

package/dist/perf/journey.d.ts

@@ -0,0 +1,45 @@
+/**
+ * Journey × axes matrix for infra performance benchmarks.
+ *
+ * A journey is one measurable user path ("provision.cold", "chat.ttft");
+ * axes are free-form scenario dimensions (driver, region, image…). The
+ * matrix expansion is pure bookkeeping — running the scenarios and
+ * recording metrics is the caller's job. This module complements the
+ * judge-panel `BenchmarkRunner` (src/benchmark.ts): that one scores
+ * QUALITY via judges, this one structures LATENCY / RELIABILITY runs
+ * over flat metric records.
+ */
+/** One measurable user journey (e.g. "provision.cold", "chat.ttft"). */
+export interface JourneySpec {
+    id: string;
+    description: string;
+    /** Needs a real LLM call — schedule nightly, not per-PR. */
+    requiresLLM: boolean;
+    /**
+     * Fields that MUST be non-null on a passing record of this journey.
+     * A "passing" record missing one is an integrity violation, not a pass.
+     */
+    requiredFields: ReadonlyArray<string>;
+    /** Numeric floors, e.g. {field: 'event_count', min: 1} for streaming. */
+    minimums?: ReadonlyArray<{
+        field: string;
+        min: number;
+    }>;
+    /** Per-phase breakdown fields expected non-null (subset of requiredFields semantics, reported separately). */
+    phaseFields?: ReadonlyArray<string>;
+}
+export interface ScenarioAxes {
+    /** e.g. driver: ['docker','firecracker'] — every key is a free-form dimension. */
+    [dimension: string]: ReadonlyArray<string>;
+}
+export interface PerfScenario {
+    /** `${journeyId}|${dim1}=${v1}|${dim2}=${v2}` (dims sorted). */
+    key: string;
+    journey: JourneySpec;
+    axes: Record<string, string>;
+}
+/** Stable scenario key: journey id then `dim=value` pairs in sorted-dim order. */
+export declare function scenarioKey(journeyId: string, axes: Record<string, string>): string;
+/** Cartesian expansion; `filter` lets callers drop invalid combos (e.g. firecracker×resume). */
+export declare function expandMatrix(journeys: ReadonlyArray<JourneySpec>, axes: ScenarioAxes, filter?: (journeyId: string, combo: Record<string, string>) => boolean): PerfScenario[];
+//# sourceMappingURL=journey.d.ts.map

package/dist/perf/journey.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"journey.d.ts","sourceRoot":"","sources":["../../src/perf/journey.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,wEAAwE;AACxE,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,MAAM,CAAA;IACV,WAAW,EAAE,MAAM,CAAA;IACnB,4DAA4D;IAC5D,WAAW,EAAE,OAAO,CAAA;IACpB;;;OAGG;IACH,cAAc,EAAE,aAAa,CAAC,MAAM,CAAC,CAAA;IACrC,yEAAyE;IACzE,QAAQ,CAAC,EAAE,aAAa,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,CAAC,CAAA;IACxD,8GAA8G;IAC9G,WAAW,CAAC,EAAE,aAAa,CAAC,MAAM,CAAC,CAAA;CACpC;AAED,MAAM,WAAW,YAAY;IAC3B,kFAAkF;IAClF,CAAC,SAAS,EAAE,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC,CAAA;CAC3C;AAED,MAAM,WAAW,YAAY;IAC3B,gEAAgE;IAChE,GAAG,EAAE,MAAM,CAAA;IACX,OAAO,EAAE,WAAW,CAAA;IACpB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CAC7B;AAED,kFAAkF;AAClF,wBAAgB,WAAW,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,MAAM,CAKnF;AAED,gGAAgG;AAChG,wBAAgB,YAAY,CAC1B,QAAQ,EAAE,aAAa,CAAC,WAAW,CAAC,EACpC,IAAI,EAAE,YAAY,EAClB,MAAM,CAAC,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,KAAK,OAAO,GACrE,YAAY,EAAE,CAqBhB"}