@tangle-network/agent-eval 0.106.0 → 0.106.1

package/dist/campaign/index.d.ts

@@ -5,7 +5,7 @@ import { S as Scenario, M as MutableSurface, b as DispatchContext, a as JudgeCon
 export { i as CampaignAggregates, j as CampaignArtifactWriter, k as CampaignCellResult, l as CampaignCostMeter, z as CampaignTokenUsage, d as CampaignTraceWriter, D as DispatchFn, n as GateContext, h as GateDecision, G as GateResult, o as GenerationCandidate, A as JudgeAggregate, c as JudgeDimension, p as Mutator, O as OptimizationProposer, q as OptimizerConfig, P as ParetoParent, R as RedactionStatus, B as ScenarioAggregate, r as SessionScript, T as TraceSpan, E as isProposedCandidate, F as labelTrustRank } from '../types-Ctf7XIAL.js';
 import { e as CampaignRunPlan, P as PlanCampaignRunOptions, C as CampaignStorage, b as RunCampaignOptions, c as RunImprovementLoopOptions } from '../gepa-DQ3ruj18.js';
 export { h as CampaignRunPlanCell, j as GepaProposerConstraints, G as GepaProposerOptions, O as OpenAutoPrOptions, k as OpenAutoPrResult, a as RunImprovementLoopResult, R as RunOptimizationOptions, l as RunOptimizationResult, m as countSentenceEdits, n as defaultRenderDiff, o as extractH2Sections, f as fsCampaignStorage, g as gepaProposer, i as inMemoryCampaignStorage, p as openAutoPr, q as planCampaignRun, r as runCampaign, d as runImprovementLoop, s as runOptimization, t as surfaceHash } from '../gepa-DQ3ruj18.js';
-export { A as AxisEvidence, a as AxisVerdict, B as BuildEvidenceVectorOptions, l as BuildLoopProvenanceArgs, D as DefaultProductionGateOptions, m as EmitLoopProvenanceArgs, n as EmitLoopProvenanceResult, E as EvidenceVector, b as EvolutionaryProposerOptions, H as HeldOutGateOptions, o as LoopProvenanceBackend, q as LoopProvenanceCandidate, L as LoopProvenanceRecord, O as ObjectiveSource, c as ParetoSignificanceGateOptions, P as PowerPreflight, s as PowerPreflightOptions, d as PromotionObjective, e as PromotionPolicy, R as RunEvalOptions, f as buildEvidenceVector, t as buildLoopProvenanceRecord, g as composeGate, h as defaultProductionGate, u as emitLoopProvenance, i as evolutionaryProposer, j as heldOutGate, v as loopProvenanceSpans, p as paretoPolicy, k as paretoSignificanceGate, w as powerPreflight, x as provenanceRecordPath, y as provenanceSpansPath, r as runEval, z as surfaceContentHash } from '../provenance-HDy5TtOY.js';
+export { A as AxisEvidence, a as AxisVerdict, B as BuildEvidenceVectorOptions, l as BuildLoopProvenanceArgs, D as DefaultProductionGateOptions, m as EmitLoopProvenanceArgs, n as EmitLoopProvenanceResult, E as EvidenceVector, b as EvolutionaryProposerOptions, H as HeldOutGateOptions, o as LoopProvenanceBackend, q as LoopProvenanceCandidate, L as LoopProvenanceRecord, O as ObjectiveSource, c as ParetoSignificanceGateOptions, P as PowerPreflight, s as PowerPreflightOptions, d as PromotionObjective, e as PromotionPolicy, R as RunEvalOptions, f as buildEvidenceVector, t as buildLoopProvenanceRecord, g as composeGate, h as defaultProductionGate, u as emitLoopProvenance, i as evolutionaryProposer, j as heldOutGate, v as loopProvenanceSpans, p as paretoPolicy, k as paretoSignificanceGate, w as powerPreflight, x as provenanceRecordPath, y as provenanceSpansPath, r as runEval, z as surfaceContentHash } from '../provenance-BGPBEFoV.js';
 import { E as EProcessState, a as PairedBootstrapResult } from '../statistics-D88peojY.js';
 import { L as LlmClientOptions } from '../llm-client-DyqEH4jH.js';
 import { AgentProfile } from '@tangle-network/agent-interface';
@@ -391,7 +391,17 @@ interface PairedHoldout {
 declare function pairHoldout(candidate: Map<string, Record<string, JudgeScore>>, baseline: Map<string, Record<string, JudgeScore>>, scenarioIds: Set<string>, select: (s: JudgeScore) => number | undefined): PairedHoldout;
 interface HeldoutSignificance {
     paired: PairedHoldout;
+    /** The bootstrap the ship decision keys on — of the MEAN paired delta by
+     *  default (see the tie note on `heldoutSignificance`). */
     bootstrap: PairedBootstrapResult;
+    /** The MEDIAN paired-delta bootstrap, reported as a diagnostic. When many
+     *  scenarios are tied (both sides solve them), the median is pinned near 0
+     *  regardless of the mean lift — comparing the two exposes tie-domination. */
+    medianBootstrap: PairedBootstrapResult;
+    /** Fraction of paired observations that are exact ties (|delta| < 1e-9). A
+     *  high tie fraction is WHY a median-based gate would have missed a real lift;
+     *  it is the observability the tie fix adds. */
+    tieFraction: number;
     /** n paired observations. */
     n: number;
     /** True iff n >= minProductiveRuns AND the CI lower bound clears the threshold. */

package/dist/campaign/index.js

@@ -7,7 +7,7 @@ import {
   paretoSignificanceGate,
   powerPreflight,
   runEval
-} from "../chunk-JTSA52OJ.js";
+} from "../chunk-IA3X6CSF.js";
 import {
   HARNESS_NATIVE_MODEL,
   agentProfileHash,
@@ -17,7 +17,7 @@ import {
   harnessAxisOf,
   llmJudge,
   verifyCompletion
-} from "../chunk-W7AJBSNO.js";
+} from "../chunk-2HLM4SJJ.js";
 import {
   buildLoopProvenanceRecord,
   campaignBreakdown,
@@ -43,7 +43,7 @@ import {
   runOptimization,
   surfaceContentHash,
   surfaceHash
-} from "../chunk-Y7RY553D.js";
+} from "../chunk-3E5KUXYZ.js";
 import {
   assertRealBackend,
   contentHash,

package/dist/{chunk-W7AJBSNO.js → chunk-2HLM4SJJ.js}

@@ -1,6 +1,6 @@
 import {
   recoverTruncatedJson
-} from "./chunk-Y7RY553D.js";
+} from "./chunk-3E5KUXYZ.js";
 import {
   clamp01
 } from "./chunk-LOJ2QVCE.js";
@@ -858,4 +858,4 @@ export {
   agentProfileModelId,
   agentProfileHash
 };
-//# sourceMappingURL=chunk-W7AJBSNO.js.map
+//# sourceMappingURL=chunk-2HLM4SJJ.js.map

package/dist/{chunk-Y7RY553D.js → chunk-3E5KUXYZ.js}

@@ -1055,6 +1055,7 @@ function parseReflectionResponse(raw, maxProposals) {
 }
 
 // src/campaign/gates/statistical-heldout.ts
+var TIE_WARN_FRACTION = 0.4;
 function pairHoldout(candidate, baseline, scenarioIds, select) {
   const cellValue = (byCell, cellId) => {
     const scores = byCell.get(cellId);
@@ -1091,16 +1092,33 @@ function pairHoldout(candidate, baseline, scenarioIds, select) {
 function heldoutSignificance(paired, opts = {}) {
   const deltaThreshold = opts.deltaThreshold ?? 0;
   const minProductiveRuns = opts.minProductiveRuns ?? 3;
+  const confidence = opts.confidence ?? 0.95;
+  const resamples = opts.resamples ?? 2e3;
+  const seed = opts.seed ?? 1337;
+  const statistic = opts.statistic ?? "mean";
   const bootstrap = pairedBootstrap(paired.before, paired.after, {
-    confidence: opts.confidence ?? 0.95,
-    resamples: opts.resamples ?? 2e3,
-    statistic: opts.statistic ?? "median",
-    seed: opts.seed ?? 1337
+    confidence,
+    resamples,
+    statistic,
+    seed
+  });
+  const medianBootstrap = statistic === "median" ? bootstrap : pairedBootstrap(paired.before, paired.after, {
+    confidence,
+    resamples,
+    statistic: "median",
+    seed
   });
   const n = paired.before.length;
+  let ties = 0;
+  for (let i = 0; i < n; i += 1) {
+    const after = paired.after[i] ?? 0;
+    const before = paired.before[i] ?? 0;
+    if (Math.abs(after - before) < 1e-9) ties += 1;
+  }
+  const tieFraction = n === 0 ? 0 : ties / n;
   const fewRuns = n < minProductiveRuns;
   const significant = !fewRuns && bootstrap.low > deltaThreshold;
-  return { paired, bootstrap, n, significant, fewRuns };
+  return { paired, bootstrap, medianBootstrap, tieFraction, n, significant, fewRuns };
 }
 function detectScale(values) {
   return values.some((v) => Math.abs(v) > 1.5) ? 100 : 1;
@@ -1154,22 +1172,25 @@ function heldOutGate(options) {
           seed
         }
       );
-      const delta = sig.bootstrap.median;
+      const delta = sig.bootstrap.mean;
       const passed = sig.significant;
+      const tieNote = sig.tieFraction >= TIE_WARN_FRACTION ? `, ${(sig.tieFraction * 100).toFixed(0)}% tied` : "";
       const ci = `${(sig.bootstrap.confidence * 100).toFixed(0)}% CI [${sig.bootstrap.low.toFixed(3)}, ${sig.bootstrap.high.toFixed(3)}]`;
       return {
         decision: passed ? "ship" : "hold",
         reasons: passed ? [
-          `held-out paired \u0394 median ${delta.toFixed(3)}, CI.low ${sig.bootstrap.low.toFixed(3)} > ${deltaThreshold} (${ci}, n=${sig.n})`
+          `held-out mean \u0394 ${delta.toFixed(3)}, CI.low ${sig.bootstrap.low.toFixed(3)} > ${deltaThreshold} (${ci}, n=${sig.n}${tieNote})`
         ] : [
-          sig.fewRuns ? `held-out: only ${sig.n} paired runs \u2014 too few to claim significance` : `held-out paired \u0394 median ${delta.toFixed(3)}, CI.low ${sig.bootstrap.low.toFixed(3)} \u2264 ${deltaThreshold} (${ci}, n=${sig.n})`
+          sig.fewRuns ? `held-out: only ${sig.n} paired runs \u2014 too few to claim significance` : `held-out mean \u0394 ${delta.toFixed(3)}, CI.low ${sig.bootstrap.low.toFixed(3)} \u2264 ${deltaThreshold} (${ci}, n=${sig.n}${tieNote})`
         ],
         contributingGates: [
           {
             name: "heldOutGate",
             passed,
             detail: {
-              deltaMedian: delta,
+              deltaMean: delta,
+              deltaMedianDiagnostic: sig.medianBootstrap.median,
+              tieFraction: sig.tieFraction,
               ciLow: sig.bootstrap.low,
               ciHigh: sig.bootstrap.high,
               confidence: sig.bootstrap.confidence,
@@ -2111,6 +2132,7 @@ export {
   buildReflectionPrompt,
   renderAnalystEvidence,
   parseReflectionResponse,
+  TIE_WARN_FRACTION,
   pairHoldout,
   heldoutSignificance,
   detectScale,
@@ -2135,4 +2157,4 @@ export {
   provenanceSpansPath,
   emitLoopProvenance
 };
-//# sourceMappingURL=chunk-Y7RY553D.js.map
+//# sourceMappingURL=chunk-3E5KUXYZ.js.map