@tangle-network/agent-eval 0.98.0 → 0.100.0

package/dist/matrix/index.d.ts

@@ -1,21 +1,7 @@
-import { M as MatrixResult, a as MatrixAxis, A as AxisSummary, b as MatrixCell, C as CellResult, R as RunAgentMatrixOptions } from '../types-BUxNaJ8c.js';
+import { R as RunAgentMatrixOptions, M as MatrixResult } from '../types-BUxNaJ8c.js';
+export { A as AxisSummary, C as CellResult, a as MatrixAxis, b as MatrixCell } from '../types-BUxNaJ8c.js';
 export { D as DefaultVerdict } from '../verdict-C9MlYujm.js';
 
-/**
- * Per-axis aggregation of cell runs into `AxisSummary` rows.
- *
- * Pure: consumes the final `cells: [{cell, runs}]` array and returns the
- * `byAxis` table. Error runs contribute 0 to passRate and meanScore. Cost
- * and duration always count — the budget was spent regardless.
- */
-
-interface Row<Output> {
-    cell: MatrixCell;
-    result: CellResult<Output>;
-}
-declare function summariseRows<Output>(rows: Row<Output>[], axisName: string, axisValue: string): AxisSummary;
-declare function buildByAxis<Output>(cells: MatrixResult<Output>['cells'], axes: MatrixAxis<unknown>[], aggregateBy: string[]): Record<string, Record<string, AxisSummary>>;
-
 /**
  * N-axis cartesian runner.
  *
@@ -31,4 +17,4 @@ declare function buildByAxis<Output>(cells: MatrixResult<Output>['cells'], axes:
 
 declare function runAgentMatrix<Output>(opts: RunAgentMatrixOptions<Output>): Promise<MatrixResult<Output>>;
 
-export { AxisSummary, CellResult, MatrixAxis, MatrixCell, MatrixResult, RunAgentMatrixOptions, buildByAxis, runAgentMatrix, summariseRows };
+export { MatrixResult, RunAgentMatrixOptions, runAgentMatrix };

package/dist/matrix/index.js

@@ -1,12 +1,8 @@
 import {
-  buildByAxis,
-  runAgentMatrix,
-  summariseRows
-} from "../chunk-QWV226SL.js";
+  runAgentMatrix
+} from "../chunk-JHCHEVET.js";
 import "../chunk-PZ5AY32C.js";
 export {
-  buildByAxis,
-  runAgentMatrix,
-  summariseRows
+  runAgentMatrix
 };
 //# sourceMappingURL=index.js.map

package/dist/multishot/index.d.ts

@@ -1,4 +1,4 @@
-import { J as JudgeScore } from '../types-D9tuReDv.js';
+import { J as JudgeScore } from '../types-BTI16iFl.js';
 import { AgentProfile } from '@tangle-network/agent-interface';
 import { M as MatrixResult } from '../types-BUxNaJ8c.js';
 import '../run-record-CP2ObebC.js';

package/dist/multishot/index.js

@@ -1,6 +1,6 @@
 import {
   runAgentMatrix
-} from "../chunk-QWV226SL.js";
+} from "../chunk-JHCHEVET.js";
 import "../chunk-PZ5AY32C.js";
 
 // src/multishot/router.ts

package/dist/openapi.json

@@ -2,7 +2,7 @@
   "openapi": "3.1.0",
   "info": {
     "title": "@tangle-network/agent-eval — wire protocol",
-    "version": "0.98.0",
+    "version": "0.100.0",
     "description": "HTTP and stdio RPC interface to agent-eval. The TypeScript runtime is the source of truth; this spec is the contract that cross-language clients (Python, Rust, Go) generate from.\n\nWire-protocol version: 1.0.0. Bumps on breaking changes to request/response schemas.",
     "contact": {
       "name": "Tangle Network",

package/dist/{pre-registration-nfUdc9EQ.d.ts → pre-registration-CMm8cvrh.d.ts}

@@ -1,5 +1,7 @@
 import { A as AgentEvalError } from './errors-CzMUYo7b.js';
 import { R as RunRecord } from './run-record-CP2ObebC.js';
+import { C as ChatClient } from './types-B5x54y6n.js';
+import { c as JudgeDimension, S as Scenario, a as JudgeConfig } from './types-BTI16iFl.js';
 import { TCloud } from '@tangle-network/tcloud';
 import { D as DefaultVerdict } from './verdict-C9MlYujm.js';
 
@@ -292,6 +294,76 @@ declare function createTokenRecallChecker(opts?: {
     minContentLength?: number;
 }): CorrectnessChecker;
 
+/**
+ * `llmJudge` — the single-LLM-call bridge that turns a rubric prompt into a
+ * canonical campaign `JudgeConfig`.
+ *
+ * The `JudgeConfig` contract (src/campaign/types.ts) is deliberately a
+ * function, not a fixed LLM-prompt shape: real consumers judge with
+ * ensembles, deterministic checks, or one LLM call. `ensembleJudge`
+ * (src/judge-panel.ts) covers the multi-model case; `buildAgreementJudge`
+ * (src/campaign/distillation) covers the pure-comparator case. `llmJudge`
+ * covers the common single-call case the `JudgeConfig` doc-comment names:
+ * one model call against `prompt`, parsed into the canonical `JudgeScore`
+ * (`{ dimensions, composite, notes }`) on the campaign [0,1] scale.
+ *
+ * Transport is injected as a `ChatClient` (src/analyst/chat-client.ts) — the
+ * substrate's transport-agnostic LLM seam — so the judge stays decoupled from
+ * router-vs-sandbox-vs-cli-bridge and is unit-testable with the `mock`
+ * transport. The composite is computed by `weightedComposite` (the same
+ * sum-normalized weighting `ensembleJudge` uses), so a lift is attributable to
+ * the dimension scores, not to a bespoke reducer.
+ *
+ * Fail-loud throughout: an unparseable model response throws `JudgeParseError`;
+ * a response missing a declared dimension throws; an out-of-range score throws.
+ * A thrown judge is recorded by the campaign engine as a failed cell, never
+ * folded into a silent zero.
+ */
+
+/** A rubric dimension as a bare key or the full `{ key, description }` shape. A
+ *  bare string uses the key as its own description. */
+type LlmJudgeDimension = string | JudgeDimension;
+interface LlmJudgeOptions<TArtifact, TScenario extends Scenario = Scenario> {
+    /** The injected LLM transport. One `chat()` call per `score()`. Required —
+     *  there is no default route, so a misconfigured judge fails at construction,
+     *  never silently against the free-tier router. */
+    chat: ChatClient;
+    /** Rubric dimensions the model scores. Each becomes a `[0,1]` field of the
+     *  returned `JudgeScore.dimensions`. Defaults to a single `quality` dimension. */
+    dimensions?: LlmJudgeDimension[];
+    /** Model id. Falls back to `chat.defaultModel`; one of the two MUST resolve. */
+    model?: string;
+    temperature?: number;
+    maxTokens?: number;
+    /** Composite weights forwarded to `weightedComposite`: a partial map selects
+     *  AND weights exactly the named dimensions. Omit for a uniform mean. */
+    weights?: Record<string, number>;
+    /** Scale the model is prompted to score on, normalized into `[0,1]`:
+     *   - `'unit'`  (default): the model returns `[0,1]` directly.
+     *   - `'ten'`:  the model returns `[0,10]`; divided by 10 here.
+     *  The prompt is annotated with the expected range either way. */
+    scale?: 'unit' | 'ten';
+    /** Run this judge only on matching scenarios (mirrors `JudgeConfig.appliesTo`). */
+    appliesTo?: (scenario: TScenario) => boolean;
+    /** Render the artifact + scenario into the user message. Default:
+     *  pretty-printed JSON of `{ scenario, artifact }`. */
+    renderUser?: (input: {
+        artifact: TArtifact;
+        scenario: TScenario;
+    }) => string;
+}
+/**
+ * Build a campaign-shaped `JudgeConfig` whose `score()` makes ONE LLM call
+ * against `prompt` and reduces the model's per-dimension scores to a canonical
+ * `JudgeScore` in `[0,1]`.
+ *
+ * The model is instructed to return JSON `{ "dimensions": { <key>: <number>, … },
+ * "notes": "…" }`; the helper strips fenced JSON, validates every declared
+ * dimension is present and in range, normalizes by `scale`, and composites via
+ * `weightedComposite`.
+ */
+declare function llmJudge<TArtifact = unknown, TScenario extends Scenario = Scenario>(name: string, prompt: string, opts: LlmJudgeOptions<TArtifact, TScenario>): JudgeConfig<TArtifact, TScenario>;
+
 /**
  * Produced-state extraction — normalize a run's runtime event stream into the
  * typed `ProducedState` the completion oracle consumes.
@@ -480,4 +552,4 @@ declare function evaluateHypothesis(manifest: SignedManifest, observed: {
     pValue: number;
 }): Promise<HypothesisResult>;
 
-export { type Artifact as A, type BackendIntegrityReport as B, type CompletionRequirement as C, parseCorrectnessResponse as D, regexMatch as E, signManifest as F, summarizeBackendIntegrity as G, type HypothesisManifest as H, verifyCompletion as I, verifyManifest as J, type LlmCorrectnessCheckerOpts as L, type ProducedState as P, type RuntimeEventLike as R, type SignedManifest as S, type TaskGold as T, type ValidationContext as V, type CompletionVerdict as a, type CorrectnessChecker as b, type ArtifactEventLike as c, type ArtifactValidator as d, BackendIntegrityError as e, type HypothesisResult as f, type ProducedProposal as g, type ProposalEventLike as h, type RequirementCheck as i, type SatisfiedBy as j, type SignedManifestAlgo as k, type ToolCallEventLike as l, type ValidationIssue as m, type ValidationResult as n, assertRealBackend as o, byteLengthRange as p, canonicalize as q, completionVerdict as r, composeValidators as s, containsAll as t, createLlmCorrectnessChecker as u, createTokenRecallChecker as v, evaluateHypothesis as w, extractProducedState as x, hashJson as y, jsonHasKeys as z };
+export { type Artifact as A, type BackendIntegrityReport as B, type CompletionRequirement as C, extractProducedState as D, hashJson as E, jsonHasKeys as F, parseCorrectnessResponse as G, type HypothesisManifest as H, regexMatch as I, signManifest as J, summarizeBackendIntegrity as K, type LlmJudgeDimension as L, verifyCompletion as M, verifyManifest as N, type ProducedState as P, type RuntimeEventLike as R, type SignedManifest as S, type TaskGold as T, type ValidationContext as V, type CompletionVerdict as a, type CorrectnessChecker as b, type LlmJudgeOptions as c, type ArtifactEventLike as d, type ArtifactValidator as e, BackendIntegrityError as f, type HypothesisResult as g, type LlmCorrectnessCheckerOpts as h, type ProducedProposal as i, type ProposalEventLike as j, type RequirementCheck as k, llmJudge as l, type SatisfiedBy as m, type SignedManifestAlgo as n, type ToolCallEventLike as o, type ValidationIssue as p, type ValidationResult as q, assertRealBackend as r, byteLengthRange as s, canonicalize as t, completionVerdict as u, composeValidators as v, containsAll as w, createLlmCorrectnessChecker as x, createTokenRecallChecker as y, evaluateHypothesis as z };

package/dist/{provenance-C5KAhiom.d.ts → provenance-Bg_RttR8.d.ts}

@@ -1,9 +1,9 @@
-import { S as Scenario, f as Gate, G as GateResult, g as GateContext, C as CampaignResult, h as Mutator, e as SurfaceProposer, M as MutableSurface, i as GateDecision } from './types-D9tuReDv.js';
+import { S as Scenario, g as Gate, G as GateResult, h as GateContext, C as CampaignResult, i as Mutator, f as SurfaceProposer, M as MutableSurface, j as GateDecision } from './types-BTI16iFl.js';
 import { R as RedTeamCase } from './red-team-BWdoyleI.js';
 import { R as RunRecord } from './run-record-CP2ObebC.js';
 import { D as Direction } from './pareto-E-pembql.js';
 import { a as PairedBootstrapResult } from './statistics-xP-cWc5k.js';
-import { R as RunCampaignOptions, C as CampaignStorage } from './gepa-3TG16SYf.js';
+import { R as RunCampaignOptions, C as CampaignStorage } from './gepa-H6mlM0KN.js';
 import { HostedClient, TraceSpanEvent } from './hosted/index.js';
 
 /**

package/dist/rl.d.ts

@@ -1,5 +1,5 @@
 import { R as RunRecord, a as RunSplitTag } from './run-record-CP2ObebC.js';
-export { A as AdversarialMutation, a as AdversarialScenario, b as AdversarialSearchOptions, c as AdversarialSearchReport, d as adversarialScenarioSearch } from './adversarial-DIVcDoI_.js';
+export { A as AdversarialMutation } from './adversarial-B7loGVVX.js';
 import { P as PreferenceExtractionReport, D as DpoExportRow, G as GrpoExportRow, S as SftExportRow, E as ExtractPreferencesOptions, a as DpoLookups, b as GrpoLookups, c as SftLookups } from './corpus-D4YW9UoJ.js';
 export { d as CorpusAppendResult, C as CorpusRecord, e as DatasetFormat, f as ExtractStepRewardsOptions, H as HarvestOptions, g as PreferenceStrategy, h as PreferenceTriple, i as PrmExportRow, j as PrmLookups, k as PrmTrainingTriple, R as RewardKind, l as RewardStats, m as RlDatasetBundle, n as RlDatasetConfig, o as RlDatasetManifest, p as RlDatasetStats, q as RunwiseStepSummary, r as StepReward, s as StepRewardJsonlRow, t as StepScorer, u as appendToCorpus, v as buildDatasetFromCorpus, w as buildRlDataset, x as datasheetToMarkdown, y as extractPreferences, z as extractStepRewards, A as prmTrainingPairs, B as readCorpus, F as runwiseStepRewardSummary, I as stepRewardsToJsonl, J as toAnthropicFormat, K as toDpoJsonl, L as toDpoRows, M as toGrpoJsonl, N as toGrpoRows, O as toPrmJsonl, Q as toPrmRows, T as toSftJsonl, U as toSftRows, V as toTRLFormat } from './corpus-D4YW9UoJ.js';
 export { O as OffPolicyEstimate, a as OffPolicyOptions, b as OffPolicyTrajectory, d as doublyRobust, i as inverseProbabilityWeighting, o as offPolicyEstimateAll, s as selfNormalizedImportanceWeighting } from './off-policy-DiwuKKg7.js';
@@ -10,7 +10,7 @@ import { R as Researcher, F as FailureMode, S as SteeringChange, E as Experiment
 export { r as runEvalCampaign } from './researcher-Jr8ME1dZ.js';
 import { a as VerificationReport } from './multi-layer-verifier-DUZXrPDA.js';
 import { I as InterimReleaseConfidence } from './sequential-5iSVfzl2.js';
-import { C as CampaignResult } from './types-D9tuReDv.js';
+import { C as CampaignResult } from './types-BTI16iFl.js';
 import '@tangle-network/agent-interface';
 import './errors-CzMUYo7b.js';
 import './schema-m0gsnbt3.js';

package/dist/rl.js

@@ -157,84 +157,6 @@ function bootstrapMeanCi(xs, resamples, confidence, rng) {
   };
 }
 
-// src/rl/adversarial.ts
-async function adversarialScenarioSearch(opts) {
-  const failureThreshold = opts.failureThreshold ?? 0.5;
-  const rounds = opts.rounds ?? 3;
-  const children = opts.childrenPerParent ?? 4;
-  const budget = opts.budget ?? Number.POSITIVE_INFINITY;
-  const seed = opts.seed ?? 1;
-  const rng = mulberry32(seed);
-  const scenarios = [];
-  const seen = /* @__PURE__ */ new Set();
-  let scoreCalls = 0;
-  for (const s of opts.seeds) {
-    const id = opts.mutateScenarioId(s);
-    if (seen.has(id)) continue;
-    seen.add(id);
-    if (scoreCalls >= budget) break;
-    const score = await opts.scoreFn(s);
-    scoreCalls++;
-    scenarios.push({
-      id,
-      generation: 0,
-      parentId: null,
-      scenario: s,
-      score,
-      mutationStrategy: null
-    });
-  }
-  for (let g = 1; g <= rounds; g++) {
-    if (scoreCalls >= budget) break;
-    const parents = scenarios.filter((s) => s.generation === g - 1);
-    for (const parent of parents) {
-      for (const mutation of opts.mutations) {
-        if (scoreCalls >= budget) break;
-        const produced = await mutation.mutate(parent.scenario, rng);
-        const childArr = Array.isArray(produced) ? produced : [produced];
-        for (let k = 0; k < Math.min(children, childArr.length); k++) {
-          if (scoreCalls >= budget) break;
-          const child = childArr[k];
-          const cid = opts.mutateScenarioId(child);
-          if (seen.has(cid)) continue;
-          seen.add(cid);
-          const cscore = await opts.scoreFn(child);
-          scoreCalls++;
-          scenarios.push({
-            id: cid,
-            generation: g,
-            parentId: parent.id,
-            scenario: child,
-            score: cscore,
-            mutationStrategy: mutation.id
-          });
-        }
-      }
-    }
-  }
-  const failures = scenarios.filter((s) => s.score !== null && s.score < failureThreshold).sort((a, b) => (a.score ?? 0) - (b.score ?? 0));
-  const byGeneration = [];
-  const maxGen = scenarios.reduce((m, s) => Math.max(m, s.generation), 0);
-  for (let g = 0; g <= maxGen; g++) {
-    const gens = scenarios.filter((s) => s.generation === g);
-    if (gens.length === 0) continue;
-    const fails = gens.filter((s) => s.score !== null && s.score < failureThreshold).length;
-    const meanScore = gens.reduce((sum, s) => sum + (s.score ?? 0), 0) / gens.length;
-    byGeneration.push({ generation: g, total: gens.length, failures: fails, meanScore });
-  }
-  return { scenarios, failures, byGeneration, scoreCalls };
-}
-function mulberry32(seed) {
-  let s = seed >>> 0;
-  return () => {
-    s = s + 1831565813 >>> 0;
-    let t = s;
-    t = Math.imul(t ^ t >>> 15, t | 1);
-    t ^= t + Math.imul(t ^ t >>> 7, t | 61);
-    return ((t ^ t >>> 14) >>> 0) / 4294967296;
-  };
-}
-
 // src/rl/compute-curves.ts
 async function runComputeCurve(opts) {
   const points = [];
@@ -1741,7 +1663,6 @@ export {
   InMemoryOutcomeStore,
   PredictiveValidityResearcher,
   REPRESENTATIVE_MIN_FIDELITY,
-  adversarialScenarioSearch,
   appendToCorpus,
   applyEloUpdate,
   bestOfN,