@tangle-network/agent-eval 0.79.0 → 0.81.0

package/dist/campaign/index.d.ts

@@ -1,26 +1,26 @@
-import { A as AnalyzeTracesOptions, a as AnalyzeTracesInput, b as AnalyzeTracesResult } from '../analyst-t7zZS3TV.js';
-import { S as Scenario, M as MutableSurface, b as DispatchContext, a as JudgeConfig, I as ImprovementDriver, q as ProposeContext, J as JudgeScore, L as LabeledScenarioStore, r as LabeledScenarioWrite, s as LabeledScenarioSampleArgs, t as LabeledScenarioRecord, u as LabelTrust, v as LabeledScenarioSource, f as CampaignResult, h as CodeSurface } from '../types-QHG0KnkF.js';
-export { C as CampaignAggregates, c as CampaignArtifactWriter, d as CampaignCellResult, e as CampaignCostMeter, w as CampaignTokenUsage, g as CampaignTraceWriter, D as DispatchFn, G as Gate, i as GateContext, j as GateDecision, k as GateResult, l as GenerationCandidate, m as GenerationRecord, x as JudgeAggregate, n as JudgeDimension, o as Mutator, O as OptimizerConfig, P as ParetoParent, y as ProposedCandidate, R as RedactionStatus, z as ScenarioAggregate, p as SessionScript, T as TraceSpan, A as isProposedCandidate, B as labelTrustRank } from '../types-QHG0KnkF.js';
-import { a as RunCampaignOptions, b as RunImprovementLoopOptions, C as CampaignStorage } from '../run-improvement-loop-Bgu4C59E.js';
-export { d as GepaDriverConstraints, G as GepaDriverOptions, O as OpenAutoPrOptions, e as OpenAutoPrResult, R as RunImprovementLoopResult, h as RunOptimizationOptions, j as RunOptimizationResult, k as countSentenceEdits, l as defaultRenderDiff, m as extractH2Sections, f as fsCampaignStorage, g as gepaDriver, i as inMemoryCampaignStorage, o as openAutoPr, r as runCampaign, c as runImprovementLoop, n as runOptimization, s as surfaceHash } from '../run-improvement-loop-Bgu4C59E.js';
-export { A as AxisEvidence, a as AxisVerdict, B as BuildEvidenceVectorOptions, k as BuildLoopProvenanceArgs, D as DefaultProductionGateOptions, l as EmitLoopProvenanceArgs, m as EmitLoopProvenanceResult, E as EvidenceVector, b as EvolutionaryDriverOptions, H as HeldOutGateOptions, n as LoopProvenanceBackend, o as LoopProvenanceCandidate, L as LoopProvenanceRecord, O as ObjectiveSource, P as ParetoSignificanceGateOptions, c as PromotionObjective, d as PromotionPolicy, R as RunEvalOptions, e as buildEvidenceVector, q as buildLoopProvenanceRecord, f as composeGate, g as defaultProductionGate, s as emitLoopProvenance, h as evolutionaryDriver, i as heldOutGate, t as loopProvenanceSpans, p as paretoPolicy, j as paretoSignificanceGate, u as provenanceRecordPath, v as provenanceSpansPath, r as runEval, w as surfaceContentHash } from '../provenance-CEAJI9rm.js';
-import { L as LlmClientOptions } from '../llm-client-DbjLfz-K.js';
-import { c as TraceAnalystKindSpec } from '../kind-factory-DqV2t1Xk.js';
-import { c as AnalystFinding } from '../types-DRvV0zRo.js';
-import { a as PairedBootstrapResult } from '../statistics-B7yCbi9i.js';
-import { A as AgentProfile, B as BackendIntegrityReport, C as CompletionRequirement, R as RuntimeEventLike, a as CompletionVerdict, P as ProducedState, b as CorrectnessChecker } from '../agent-profile-aSEaJ9Pl.js';
+import { A as AnalyzeTracesOptions, a as AnalyzeTracesInput, b as AnalyzeTracesResult } from '../analyst-C8HHvfJp.js';
+import { S as Scenario, M as MutableSurface, b as DispatchContext, a as JudgeConfig, I as ImprovementDriver, q as ProposeContext, J as JudgeScore, L as LabeledScenarioStore, r as LabeledScenarioWrite, s as LabeledScenarioSampleArgs, t as LabeledScenarioRecord, u as LabelTrust, v as LabeledScenarioSource, g as CampaignResult, i as CodeSurface } from '../types-D7lLRYe9.js';
+export { C as CampaignAggregates, d as CampaignArtifactWriter, e as CampaignCellResult, f as CampaignCostMeter, w as CampaignTokenUsage, h as CampaignTraceWriter, D as DispatchFn, G as Gate, j as GateContext, c as GateDecision, k as GateResult, l as GenerationCandidate, m as GenerationRecord, x as JudgeAggregate, n as JudgeDimension, o as Mutator, O as OptimizerConfig, P as ParetoParent, y as ProposedCandidate, R as RedactionStatus, z as ScenarioAggregate, p as SessionScript, T as TraceSpan, A as isProposedCandidate, B as labelTrustRank } from '../types-D7lLRYe9.js';
+import { a as RunCampaignOptions, b as RunImprovementLoopOptions, C as CampaignStorage } from '../run-improvement-loop-D6PZOoQL.js';
+export { d as GepaDriverConstraints, G as GepaDriverOptions, O as OpenAutoPrOptions, e as OpenAutoPrResult, R as RunImprovementLoopResult, h as RunOptimizationOptions, j as RunOptimizationResult, k as countSentenceEdits, l as defaultRenderDiff, m as extractH2Sections, f as fsCampaignStorage, g as gepaDriver, i as inMemoryCampaignStorage, o as openAutoPr, r as runCampaign, c as runImprovementLoop, n as runOptimization, s as surfaceHash } from '../run-improvement-loop-D6PZOoQL.js';
+export { A as AxisEvidence, a as AxisVerdict, B as BuildEvidenceVectorOptions, k as BuildLoopProvenanceArgs, D as DefaultProductionGateOptions, l as EmitLoopProvenanceArgs, m as EmitLoopProvenanceResult, E as EvidenceVector, b as EvolutionaryDriverOptions, H as HeldOutGateOptions, n as LoopProvenanceBackend, o as LoopProvenanceCandidate, L as LoopProvenanceRecord, O as ObjectiveSource, P as ParetoSignificanceGateOptions, c as PromotionObjective, d as PromotionPolicy, R as RunEvalOptions, e as buildEvidenceVector, q as buildLoopProvenanceRecord, f as composeGate, g as defaultProductionGate, s as emitLoopProvenance, h as evolutionaryDriver, i as heldOutGate, t as loopProvenanceSpans, p as paretoPolicy, j as paretoSignificanceGate, u as provenanceRecordPath, v as provenanceSpansPath, r as runEval, w as surfaceContentHash } from '../provenance-B9Q4886D.js';
+import { L as LlmClientOptions } from '../llm-client-CuUg2Mn3.js';
+import { c as TraceAnalystKindSpec } from '../kind-factory-CVecZZG_.js';
+import { c as AnalystFinding } from '../types-Cu3u_x59.js';
+import { a as PairedBootstrapResult } from '../statistics-CnC1FMbx.js';
+import { A as AgentProfile, B as BackendIntegrityReport, C as CompletionRequirement, R as RuntimeEventLike, a as CompletionVerdict, P as ProducedState, b as CorrectnessChecker } from '../agent-profile-D0PBIWlV.js';
 import { A as AgentEvalError } from '../errors-Dwqw-T_m.js';
-import { b as RunSplitTag, R as RunRecord } from '../run-record-sItO5ftF.js';
+import { a as RunSplitTag, R as RunRecord } from '../run-record-De9VarXR.js';
 import '@ax-llm/ax';
-import '../store-GmBE2pZZ.js';
+import '../store-C1YxJDEK.js';
 import '../red-team-DW9Ca_tj.js';
 import '../dataset-B2kL-fSM.js';
 import '../store-CKUAgsJz.js';
 import '../schema-m0gsnbt3.js';
 import '../pareto-E-pembql.js';
 import '../hosted/index.js';
-import '../insight-report-dlpEzQDi.js';
-import '../summary-report-BTaXq1TS.js';
+import '../insight-report-3ADTfClO.js';
+import '../summary-report-Db0dDSWP.js';
 import '../failure-cluster-CL7IVgkJ.js';
 import '../judge-calibration-DilmB3Ml.js';
 import '../raw-provider-sink-C46HDghv.js';
@@ -172,79 +172,23 @@ interface AceDriverOptions {
 }
 declare function aceDriver(opts?: AceDriverOptions): ImprovementDriver;
 
-/**
- * Driver selection guide — "which `ImprovementDriver` do I pick, and why?"
- *
- * The substrate ships seven drivers with overlapping shapes. This is the
- * decision table (data, not behavior): each entry says what a driver mutates,
- * how it proposes changes, when to reach for it, and its relative cost.
- * `selectDriver()` turns a goal + surface into a ranked recommendation.
- *
- * Import the actual driver functions from `@tangle-network/agent-eval/campaign`
- * (gepaDriver, skillOptDriver, aceDriver, memoryCurationDriver, haloDriver,
- * traceAnalystDriver, evolutionaryDriver); this module only helps you choose.
- */
-type DriverName = 'gepa' | 'skillOpt' | 'ace' | 'memoryCuration' | 'halo' | 'traceAnalyst' | 'evolutionary';
-/** The mutable surface a driver targets. */
-type DriverSurface = 'prompt' | 'skill-doc' | 'playbook' | 'memory' | 'any';
-/** How a driver turns evidence into the next candidate. */
-type DriverStrategy = 'reflective-rewrite' | 'anchored-patch' | 'append-only' | 'dedup-curate' | 'analysis-edit' | 'population-mutate';
-/** What a caller is trying to do this run. */
-type DriverGoal = 'explore' | 'refine' | 'accumulate' | 'benchmark';
-interface DriverGuideEntry {
-    /** One-line description of the mechanism. */
-    summary: string;
-    /** The surface the driver edits. */
-    surface: DriverSurface;
-    /** How it proposes the next candidate. */
-    strategy: DriverStrategy;
-    /** When to reach for this driver. */
-    whenUse: string;
-    /** Relative LLM cost per generation. */
-    cost: 'low' | 'medium' | 'high';
-    /** True when the driver shells out to an external engine (extra setup). */
-    external?: boolean;
-}
-declare const DRIVER_GUIDE: Record<DriverName, DriverGuideEntry>;
-interface SelectDriverCriteria {
-    /** What you're trying to do this run. */
-    goal: DriverGoal;
-    /** Restrict to drivers that edit this surface (optional). */
-    surface?: DriverSurface;
-}
-interface DriverRecommendation {
-    name: DriverName;
-    entry: DriverGuideEntry;
-    reason: string;
-}
-/**
- * Rank the drivers for a goal (and optional surface filter), best first.
- * Returns the recommendation list, not instances — import the chosen driver
- * function yourself. Always returns at least the goal's primary driver.
- */
-declare function selectDriver(criteria: SelectDriverCriteria): DriverRecommendation[];
-
 /**
  * @experimental
  *
  * `haloDriver` — wraps the REAL halo-engine (Inference.net's hierarchical
  * agentic trace analyzer, `pip install halo-engine`, repo context-labs/halo)
  * as an agent-eval `ImprovementDriver`, so HALO competes head-to-head with
- * `gepaDriver` — and with our own `traceAnalystDriver` — inside
- * `compareDrivers` on identical traces / scenarios / held-out scoring.
+ * `gepaDriver` — and with our own `traceAnalystDriver` — inside `compareDrivers`
+ * on identical traces / scenarios / held-out scoring.
  *
- * It PRESERVES halo's actual working usage — `propose()` shells out to the
- * published CLI (`halo <traces.jsonl> -p <prompt> -m <model> --base-url
- * --api-key`) and uses its real RLM findings verbatim. We do NOT reimplement
- * its analysis; that would make the benchmark meaningless. The only adaptation
- * is applying HALO's findings to the current prompt surface via one LLM edit —
- * exactly what makes the comparison prompt-tier apples-to-apples with
- * `gepaDriver` (which also mutates the prompt). The analysis is HALO's; only
- * the surface-application is ours, and it is identical in spirit to how HALO's
- * own loop feeds findings to a coding agent.
+ * It PRESERVES halo's actual working usage — `analyze` shells out to the
+ * published CLI (`halo <traces.jsonl> -p <prompt> -m <model>`) and uses its real
+ * RLM findings verbatim. We do NOT reimplement its analysis; that would make the
+ * benchmark meaningless. The materialize/apply pipeline is the shared
+ * `analysisEditDriver` — identical to `traceAnalystDriver`, which is what makes
+ * the comparison apples-to-apples.
  *
  * Fail-loud: no traces → throw; halo errors → throw; empty findings → throw.
- * Never fabricate a candidate (that would silently flatter or penalize HALO).
  */
 
 interface HaloDriverOptions {
@@ -259,11 +203,8 @@ interface HaloDriverOptions {
     applyModel?: string;
     /** The real halo binary. Default 'halo' (from `pip install halo-engine`). */
     haloBin?: string;
-    /**
-     * Resolve the OTLP traces (JSONL string) halo should analyze for THIS
-     * generation — wired by the bench to the captured AppWorld OTLP for the
-     * current surface. Returning empty throws (halo has nothing to analyze).
-     */
+    /** Resolve the OTLP traces (JSONL string) halo should analyze for THIS
+     *  generation. Returning empty throws (halo has nothing to analyze). */
     resolveTraces: (ctx: ProposeContext) => string | Promise<string>;
     /** halo's analysis prompt (`-p`). Default targets the failure taxonomy. */
     analysisPrompt?: string;
@@ -481,28 +422,18 @@ declare function parseSkillPatchResponse(raw: string, maxPatches: number, editBu
  *
  * `traceAnalystDriver` — wraps agent-eval's OWN trace-analyst engine
  * (`AnalystRegistry` over the agentic OTLP reader) as an `ImprovementDriver`.
- * It is the symmetric opponent to `haloDriver`: both consume the SAME OTLP
- * corpus and apply their findings to the prompt surface via one IDENTICAL
- * LLM edit, so a `compareDrivers` lift delta isolates a single variable —
- * ANALYSIS QUALITY. The benchmark answers "is our HALO clone as good as the
- * real HALO?" as a held-out lift CI, not a vibe.
- *
- * The fairness contract (the only thing that makes the head-to-head honest):
- *   - SAME input: both engines read the identical `traces.jsonl` (haloDriver
- *     hands it to the halo CLI; this driver wraps it in an `OtlpFileTraceStore`).
- *   - SAME application: the apply-step here is byte-for-byte the apply-step in
- *     `haloDriver` (same `APPLY_SYSTEM`, same one-shot `callLlm` prompt edit).
- *   - ONLY difference: who produced the findings — the real halo-engine vs our
- *     `AnalystRegistry` (whose actor prompt is a near-verbatim port of HALO's).
+ * It is the symmetric opponent to `haloDriver`: both run the SAME shared
+ * `analysisEditDriver` pipeline (materialize identical traces → apply via one
+ * identical LLM edit), so a `compareDrivers` lift delta isolates a single
+ * variable — ANALYSIS QUALITY. The benchmark answers "is our HALO clone as good
+ * as the real HALO?" as a held-out lift CI, not a vibe.
  *
  * Findings come from the REGISTRY (structured `AnalystFinding[]` carrying
- * area / severity / recommended_action), NOT bare `analyzeTraces` (which emits
- * `string[]`). The registry is the productized engine; raw `analyzeTraces` is
- * the unstructured escape hatch.
+ * area / severity / recommended_action), rendered into the report the shared
+ * apply step consumes.
  *
  * Fail-loud: no traces → throw; analyst run errors → throw; zero findings →
- * throw. Never fabricate a candidate (that would silently flatter or penalize
- * our engine relative to HALO).
+ * throw. Never fabricate a candidate.
  */
 
 interface TraceAnalystDriverOptions {
@@ -519,26 +450,15 @@ interface TraceAnalystDriverOptions {
     /** Ax provider name. Default 'openai' — works for any OpenAI-compatible base
      *  via `apiURL`. Use 'deepseek' to hit DeepSeek's native provider. */
     provider?: string;
-    /** Which analyst kinds to run. Default = the full shipped suite
-     *  (`DEFAULT_TRACE_ANALYST_KINDS`: failure-mode, knowledge-gap,
-     *  knowledge-poisoning, improvement). Narrow it for cost-parity runs. */
+    /** Which analyst kinds to run. Default = the full shipped suite. */
     kinds?: readonly TraceAnalystKindSpec[];
-    /**
-     * Resolve the OTLP traces (JSONL string) the analyst should read for THIS
-     * generation — identical contract to `haloDriver.resolveTraces`, wired by
-     * the bench to the captured AppWorld OTLP for the current surface. Returning
-     * empty throws (the analyst has nothing to read).
-     */
+    /** Resolve the OTLP traces (JSONL string) the analyst should read for THIS
+     *  generation — identical contract to `haloDriver.resolveTraces`. */
     resolveTraces: (ctx: ProposeContext) => string | Promise<string>;
-    /**
-     * Override the findings producer. Default: the shipped `AnalystRegistry`
-     * over `kinds`, reading the resolved traces as an `OtlpFileTraceStore`. A
-     * consumer may inject a pre-built registry / alternate engine here; the
-     * unit suite injects canned findings to exercise the apply path without
-     * driving the agentic loop.
-     */
+    /** Override the findings producer. Default: the shipped `AnalystRegistry`
+     *  over `kinds`. The unit suite injects canned findings here. */
     analyze?: (tracePath: string, ctx: ProposeContext) => Promise<ReadonlyArray<AnalystFinding>>;
-    /** Test seam: inject a fetch for the apply-step `callLlm` (no network in unit tests). */
+    /** Test seam: inject a fetch for the apply-step `callLlm`. */
     fetchImpl?: LlmClientOptions['fetch'];
 }
 /** Wrap agent-eval's trace-analyst registry as an ImprovementDriver (prompt-tier). */
@@ -1272,4 +1192,4 @@ declare function gitWorktreeAdapter(opts: GitWorktreeAdapterOptions): WorktreeAd
  *  as a ref under the adapter's worktree dir. */
 declare function resolveWorktreePath(surface: CodeSurface, worktreeDir?: string): string;
 
-export { type AcceptedEdit, type AceDriverOptions, type AnalystArtifact, type AnalystScenario, type ApplySkillPatchResult, type BuildAnalystSurfaceDispatchOptions, type CampaignBreakdown, CampaignResult, CampaignStorage, CodeSurface, type CompareDriversOptions, DRIVER_GUIDE, type DimensionRegression, DispatchContext, type DriverComparison, type DriverEntry, type DriverGoal, type DriverGuideEntry, type DriverName, type DriverPairwise, type DriverRecommendation, type DriverScore, type DriverStrategy, type DriverSurface, type FailureModeRecallJudgeOptions, FsLabeledScenarioStore, type FsLabeledScenarioStoreOptions, type GitWorktreeAdapterOptions, type HaloDriverOptions, type HeldoutSignificance, type HeldoutSignificanceOptions, ImprovementDriver, JudgeConfig, JudgeScore, LabelTrust, LabeledScenarioRecord, LabeledScenarioSampleArgs, LabeledScenarioSource, LabeledScenarioStore, LabeledScenarioStoreError, LabeledScenarioWrite, type MemoryCurationDriverOptions, MutableSurface, type OptimizerEntryConfig, type PairedHoldout, type PlaybackContext, type PlaybackDriver, type PlaybackStep, type ProfileDispatchFn, ProfileMatrixError, type ProfileSummary, ProposeContext, type ProposePatchesArgs, type RejectedEdit, RunCampaignOptions, RunImprovementLoopOptions, type RunProfileMatrixOptions, type RunProfileMatrixResult, type RunSkillOptOptions, type RunSkillOptResult, Scenario, type ScenarioRollup, type ScoreboardRenderOptions, type ScoreboardRow, type ScoreboardSummary, type SelectDriverCriteria, type SkillOptDriver, type SkillOptDriverOptions, type SkillOptEpochRecord, type SkillOptEvidence, type SkillPatch, type SkillPatchOp, SkillPatchParseError, type SkillPatchRejection, type TraceAnalystDriverOptions, type UserStory, type UserStoryVerdict, type Worktree, type WorktreeAdapter, WorktreeAdapterError, aceDriver, applySkillPatch, buildAnalystSurfaceDispatch, campaignBreakdown, campaignMeanComposite, compareDrivers, detectScale, dimensionRegressions, failureModeRecallJudge, gepaParetoEntry, gepaReflectionEntry, gitWorktreeAdapter, haloDriver, heldoutSignificance, makePlaybackDispatch, memoryCurationDriver, pairHoldout, parseSkillPatchResponse, patchEditCount, renderScoreboardMarkdown, resolveWorktreePath, runProfileMatrix, runSkillOpt, scoreUserStory, scoreboardSummary, selectDriver, skillOptDriver, skillOptEntry, traceAnalystDriver, userStoryScoreboard };
+export { type AcceptedEdit, type AceDriverOptions, type AnalystArtifact, type AnalystScenario, type ApplySkillPatchResult, type BuildAnalystSurfaceDispatchOptions, type CampaignBreakdown, CampaignResult, CampaignStorage, CodeSurface, type CompareDriversOptions, type DimensionRegression, DispatchContext, type DriverComparison, type DriverEntry, type DriverPairwise, type DriverScore, type FailureModeRecallJudgeOptions, FsLabeledScenarioStore, type FsLabeledScenarioStoreOptions, type GitWorktreeAdapterOptions, type HaloDriverOptions, type HeldoutSignificance, type HeldoutSignificanceOptions, ImprovementDriver, JudgeConfig, JudgeScore, LabelTrust, LabeledScenarioRecord, LabeledScenarioSampleArgs, LabeledScenarioSource, LabeledScenarioStore, LabeledScenarioStoreError, LabeledScenarioWrite, type MemoryCurationDriverOptions, MutableSurface, type OptimizerEntryConfig, type PairedHoldout, type PlaybackContext, type PlaybackDriver, type PlaybackStep, type ProfileDispatchFn, ProfileMatrixError, type ProfileSummary, ProposeContext, type ProposePatchesArgs, type RejectedEdit, RunCampaignOptions, RunImprovementLoopOptions, type RunProfileMatrixOptions, type RunProfileMatrixResult, type RunSkillOptOptions, type RunSkillOptResult, Scenario, type ScenarioRollup, type ScoreboardRenderOptions, type ScoreboardRow, type ScoreboardSummary, type SkillOptDriver, type SkillOptDriverOptions, type SkillOptEpochRecord, type SkillOptEvidence, type SkillPatch, type SkillPatchOp, SkillPatchParseError, type SkillPatchRejection, type TraceAnalystDriverOptions, type UserStory, type UserStoryVerdict, type Worktree, type WorktreeAdapter, WorktreeAdapterError, aceDriver, applySkillPatch, buildAnalystSurfaceDispatch, campaignBreakdown, campaignMeanComposite, compareDrivers, detectScale, dimensionRegressions, failureModeRecallJudge, gepaParetoEntry, gepaReflectionEntry, gitWorktreeAdapter, haloDriver, heldoutSignificance, makePlaybackDispatch, memoryCurationDriver, pairHoldout, parseSkillPatchResponse, patchEditCount, renderScoreboardMarkdown, resolveWorktreePath, runProfileMatrix, runSkillOpt, scoreUserStory, scoreboardSummary, skillOptDriver, skillOptEntry, traceAnalystDriver, userStoryScoreboard };