@tangle-network/agent-eval 0.104.0 → 0.106.0

package/dist/contract/index.d.ts

@@ -1,9 +1,9 @@
-import { S as Scenario, M as MutableSurface, b as DispatchContext, a as JudgeConfig, f as SurfaceProposer, g as Gate, L as LabeledScenarioStore, C as CampaignResult, j as GateDecision } from '../types-Mh6I44ub.js';
-export { k as CampaignAggregates, l as CampaignArtifactWriter, m as CampaignCellResult, n as CampaignCostMeter, d as CampaignTraceWriter, o as CodeSurface, D as Dispatch, h as GateContext, G as GateResult, p as GenerationCandidate, e as GenerationRecord, c as JudgeDimension, J as JudgeScore, i as Mutator, O as OptimizationProposer, q as OptimizerConfig, r as SessionScript } from '../types-Mh6I44ub.js';
-import { L as LoopProvenanceRecord, R as RunEvalOptions } from '../provenance-phkRBl4h.js';
-export { A as AxisEvidence, a as AxisVerdict, B as BuildEvidenceVectorOptions, D as DefaultProductionGateOptions, E as EvidenceVector, b as EvolutionaryProposerOptions, H as HeldOutGateOptions, O as ObjectiveSource, P as ParetoSignificanceGateOptions, c as PromotionObjective, d as PromotionPolicy, e as buildEvidenceVector, f as composeGate, g as defaultProductionGate, h as evolutionaryProposer, i as heldOutGate, p as paretoPolicy, j as paretoSignificanceGate, r as runEval } from '../provenance-phkRBl4h.js';
-import { C as CampaignStorage, a as RunOptimizationOptions, b as RunImprovementLoopResult } from '../gepa-Dn0gtj9n.js';
-export { G as GepaProposerOptions, R as RunCampaignOptions, c as RunImprovementLoopOptions, f as fsCampaignStorage, g as gepaProposer, i as inMemoryCampaignStorage, r as runCampaign, d as runImprovementLoop } from '../gepa-Dn0gtj9n.js';
+import { S as Scenario, M as MutableSurface, b as DispatchContext, a as JudgeConfig, f as SurfaceProposer, g as Gate, L as LabeledScenarioStore, C as CampaignResult, h as GateDecision } from '../types-Ctf7XIAL.js';
+export { i as CampaignAggregates, j as CampaignArtifactWriter, k as CampaignCellResult, l as CampaignCostMeter, d as CampaignTraceWriter, m as CodeSurface, D as Dispatch, n as GateContext, G as GateResult, o as GenerationCandidate, e as GenerationRecord, c as JudgeDimension, J as JudgeScore, p as Mutator, O as OptimizationProposer, q as OptimizerConfig, r as SessionScript } from '../types-Ctf7XIAL.js';
+import { L as LoopProvenanceRecord, P as PowerPreflight, R as RunEvalOptions } from '../provenance-HDy5TtOY.js';
+export { A as AxisEvidence, a as AxisVerdict, B as BuildEvidenceVectorOptions, D as DefaultProductionGateOptions, E as EvidenceVector, b as EvolutionaryProposerOptions, H as HeldOutGateOptions, O as ObjectiveSource, c as ParetoSignificanceGateOptions, d as PromotionObjective, e as PromotionPolicy, f as buildEvidenceVector, g as composeGate, h as defaultProductionGate, i as evolutionaryProposer, j as heldOutGate, p as paretoPolicy, k as paretoSignificanceGate, r as runEval } from '../provenance-HDy5TtOY.js';
+import { C as CampaignStorage, R as RunOptimizationOptions, a as RunImprovementLoopResult } from '../gepa-DQ3ruj18.js';
+export { G as GepaProposerOptions, b as RunCampaignOptions, c as RunImprovementLoopOptions, f as fsCampaignStorage, g as gepaProposer, i as inMemoryCampaignStorage, r as runCampaign, d as runImprovementLoop } from '../gepa-DQ3ruj18.js';
 export { D as DeploymentOutcome, F as FileSystemOutcomeStore, a as FileSystemOutcomeStoreOptions, I as InMemoryOutcomeStore, b as OutcomeStore } from '../outcome-store-rnXLEqSn.js';
 import { HostedTenant, EvalRunCellScore, EvalRunGenerationSnapshot, EvalRunEvent, TraceSpanEvent } from '../hosted/index.js';
 import { R as RunRecord, b as RunSplitTag } from '../run-record-I-Z3JNvO.js';
@@ -45,7 +45,7 @@ import 'zod';
  *
  * Defaults:
  *   - In-memory storage (no filesystem touch).
- *   - `gepaProposer` reflective mutation with copywriting-flavored primitives
+ *   - `gepaProposer` reflective mutation with domain-neutral engineering primitives
  *     (override `proposer` or `mutationPrimitives` for any domain).
  *   - `defaultProductionGate` with `deltaThreshold: 0.05`.
  *   - Held-out split = 25% of scenarios, deterministic by id hash.
@@ -108,6 +108,12 @@ type SelfImproveProgressEvent = {
     kind: 'gate.decided';
     decision: string;
     lift: number;
+} | {
+    kind: 'power.estimated';
+    n: number;
+    sd: number;
+    mde: number;
+    underpowered: boolean;
 };
 interface SelfImproveOptions<TScenario extends Scenario, TArtifact> {
     /**
@@ -277,6 +283,11 @@ interface SelfImproveResult<TScenario extends Scenario, TArtifact> {
      * Hosted-tier dashboards render this as the v3-vs-v4 decision view.
      */
     insight: InsightReport;
+    /** Minimum-detectable-lift analysis from the baseline holdout cells: could this
+     *  budget have shipped ANY plausible effect? Absent when the baseline produced
+     *  fewer than 3 scored holdout cells. See `powerPreflight` for the standalone
+     *  pre-run version (run `gate: 'none'` first, budget the real search after). */
+    power?: PowerPreflight;
     /**
      * Raw substrate result for advanced inspection — full per-generation
      * candidates, full campaign artifacts, all judge scores. Useful for

package/dist/contract/index.js

@@ -17,8 +17,9 @@ import {
   evolutionaryProposer,
   paretoPolicy,
   paretoSignificanceGate,
+  powerPreflight,
   runEval
-} from "../chunk-Z7FK73R5.js";
+} from "../chunk-JTSA52OJ.js";
 import {
   analyzeRuns
 } from "../chunk-GDZAWO2I.js";
@@ -28,7 +29,7 @@ import {
   heldOutGate,
   runImprovementLoop,
   surfaceContentHash
-} from "../chunk-ZKIL3VOD.js";
+} from "../chunk-Y7RY553D.js";
 import {
   fsCampaignStorage,
   inMemoryCampaignStorage,
@@ -95,12 +96,6 @@ function meanComposite(byScenario) {
     perScenario
   };
 }
-var DEFAULT_MUTATION_PRIMITIVES = [
-  "Tighten the hook: lead with the specific user outcome.",
-  "Replace generic adjectives with specific verbs or proof numbers.",
-  "Anchor every claim in something the scenario's brief literally supports.",
-  "Honor the surface-shape constraint (length, register, audience vocabulary)."
-];
 async function selfImprove(opts) {
   const startedAt = Date.now();
   const budget = opts.budget ?? {};
@@ -130,7 +125,9 @@ async function selfImprove(opts) {
     },
     model: opts.llm?.model ?? "anthropic/claude-sonnet-4.6",
     target: opts.proposerTarget ?? "agent surface (system prompt or config) being optimized by selfImprove",
-    mutationPrimitives: opts.mutationPrimitives ?? DEFAULT_MUTATION_PRIMITIVES
+    // Pass-through: when unset, gepaProposer falls back to its own
+    // domain-neutral engineering primitives.
+    mutationPrimitives: opts.mutationPrimitives
   });
   const gate = opts.gate ?? defaultProductionGate({
     holdoutScenarios: holdout,
@@ -170,6 +167,29 @@ async function selfImprove(opts) {
   });
   const baseline = meanComposite(result.baselineOnHoldout.aggregates.byScenario);
   const winnerStats = meanComposite(result.winnerOnHoldout.aggregates.byScenario);
+  let power;
+  const baselineHoldoutComposites = result.baselineOnHoldout.cells.filter((cell) => !cell.error).map((cell) => {
+    const scores = Object.values(cell.judgeScores);
+    return scores.length === 0 ? Number.NaN : scores.reduce((sum, s) => sum + s.composite, 0) / scores.length;
+  }).filter((v) => Number.isFinite(v));
+  if (baselineHoldoutComposites.length >= 3) {
+    power = powerPreflight({
+      baselineComposites: baselineHoldoutComposites,
+      sharedScorerChannel: true
+    });
+    if (opts.onProgress) {
+      opts.onProgress({
+        kind: "power.estimated",
+        n: power.n,
+        sd: power.sd,
+        mde: power.mde,
+        underpowered: power.underpowered
+      });
+    }
+    if (power.underpowered && generations > 0) {
+      console.warn(`[selfImprove] ${power.recommendation}`);
+    }
+  }
   if (opts.onProgress) {
     opts.onProgress({
       kind: "baseline.completed",
@@ -237,6 +257,7 @@ async function selfImprove(opts) {
     durationMs,
     totalCostUsd: totalCost,
     insight,
+    ...power ? { power } : {},
     raw: result
   };
   if (opts.hostedTenant) {