@tangle-network/agent-eval 0.20.12 → 0.22.0

package/dist/optimization.d.ts

@@ -1,145 +1,8 @@
-import { G as GateDecision } from './multi-shot-optimization-Bvtz294B.js';
-export { A as ActionableSideInfo, b as AsiSeverity, D as DEFAULT_MUTATION_PRIMITIVES, E as EvolvableVariant, e as GenerationReport, I as InMemoryTrialCache, h as MultiShotGateConfig, i as MultiShotGateResult, j as MultiShotMutateAdapter, k as MultiShotOptimizationConfig, l as MultiShotOptimizationResult, m as MultiShotRun, n as MultiShotRunInput, o as MultiShotRunner, p as MultiShotScore, q as MultiShotScorer, r as MultiShotSplit, s as MultiShotTrace, t as MultiShotTrialResult, u as MultiShotVariant, M as MutateAdapter, v as PromptEvolutionConfig, w as PromptEvolutionEvent, x as PromptEvolutionResult, R as ReflectionContext, y as ReflectionProposal, S as ScenarioAggregate, z as ScoreAdapter, T as TrialCache, a as TrialResult, B as TrialTrace, V as VariantAggregate, C as buildReflectionPrompt, J as defaultMultiShotObjectives, Q as parseReflectionResponse, U as runMultiShotOptimization, W as runPromptEvolution, Y as trialTraceFromMultiShotTrial } from './multi-shot-optimization-Bvtz294B.js';
-import { a as RunRecord } from './run-record-CX_jcAyr.js';
-export { n as FeedbackArtifactType, o as FeedbackAttempt, F as FeedbackLabel, p as FeedbackLabelKind, q as FeedbackLabelSource, r as FeedbackOptimizerRow, s as FeedbackOutcome, t as FeedbackReplayAdapter, u as FeedbackReplayResult, v as FeedbackSeverity, w as FeedbackSplitPolicy, x as FeedbackTask, b as FeedbackTrajectory, y as FeedbackTrajectoryFilter, a as FeedbackTrajectoryStore, z as FileSystemFeedbackTrajectoryStore, I as InMemoryFeedbackTrajectoryStore, P as PreferenceMemoryEntry, A as ProposedSideEffect, D as assignFeedbackSplit, E as controlRunToFeedbackTrajectory, G as createFeedbackTrajectory, H as feedbackTrajectoriesToDatasetScenarios, J as feedbackTrajectoriesToOptimizerRows, K as feedbackTrajectoryToDatasetScenario, L as feedbackTrajectoryToOptimizerRow, N as parseFeedbackTrajectoriesJsonl, O as renderPreferenceMemoryMarkdown, Q as replayFeedbackTrajectories, R as replayFeedbackTrajectory, U as serializeFeedbackTrajectoriesJsonl, Y as summarizePreferenceMemory, Z as withAssignedFeedbackSplit } from './feedback-trajectory-BGQ_ANCN.js';
+export { C as CallbackResearcher, a as CallbackResearcherOptions, b as CampaignFactoryParams, c as CampaignIntegrityPolicy, d as CampaignRunContext, e as CampaignRunOutcome, f as CampaignRunner, g as CampaignScenario, h as CampaignVariant, E as EvalCampaignOptions, i as EvalCampaignResult, j as ExperimentPlan, k as ExperimentResult, F as FailedRun, l as FailureMode, N as NoopResearcher, R as Researcher, S as SteeringChange, y as runEvalCampaign } from './optimization-UVDNKaO6.js';
+export { A as ActionableSideInfo, b as AsiSeverity, c as DEFAULT_MUTATION_PRIMITIVES, E as EvolvableVariant, m as GenerationReport, I as InMemoryTrialCache, p as MultiShotGateConfig, q as MultiShotGateResult, r as MultiShotMutateAdapter, s as MultiShotOptimizationConfig, t as MultiShotOptimizationResult, u as MultiShotRun, v as MultiShotRunInput, w as MultiShotRunner, x as MultiShotScore, y as MultiShotScorer, z as MultiShotSplit, B as MultiShotTrace, C as MultiShotTrialResult, J as MultiShotVariant, M as MutateAdapter, N as PromptEvolutionConfig, Q as PromptEvolutionEvent, R as PromptEvolutionResult, U as ReflectionContext, W as ReflectionProposal, a1 as ScenarioAggregate, a2 as ScoreAdapter, T as TrialCache, a as TrialResult, a6 as TrialTrace, V as VariantAggregate, a7 as buildReflectionPrompt, aa as defaultMultiShotObjectives, ah as parseReflectionResponse, aj as runMultiShotOptimization, ak as runPromptEvolution, an as trialTraceFromMultiShotTrial } from './summary-report-D4p7RlDu.js';
+export { n as FeedbackArtifactType, o as FeedbackAttempt, F as FeedbackLabel, p as FeedbackLabelKind, q as FeedbackLabelSource, r as FeedbackOptimizerRow, s as FeedbackOutcome, t as FeedbackReplayAdapter, u as FeedbackReplayResult, v as FeedbackSeverity, w as FeedbackSplitPolicy, x as FeedbackTask, b as FeedbackTrajectory, y as FeedbackTrajectoryFilter, a as FeedbackTrajectoryStore, z as FileSystemFeedbackTrajectoryStore, I as InMemoryFeedbackTrajectoryStore, P as PreferenceMemoryEntry, A as ProposedSideEffect, D as assignFeedbackSplit, E as controlRunToFeedbackTrajectory, G as createFeedbackTrajectory, H as feedbackTrajectoriesToDatasetScenarios, J as feedbackTrajectoriesToOptimizerRows, K as feedbackTrajectoryToDatasetScenario, L as feedbackTrajectoryToOptimizerRow, N as parseFeedbackTrajectoriesJsonl, O as renderPreferenceMemoryMarkdown, Q as replayFeedbackTrajectories, R as replayFeedbackTrajectory, U as serializeFeedbackTrajectoriesJsonl, Y as summarizePreferenceMemory, Z as withAssignedFeedbackSplit } from './feedback-trajectory-CB0A32o3.js';
+import './integrity-K2oVlF57.js';
+import './store-u47QaJ9G.js';
+import './emitter-B2XqDKFU.js';
+import './run-record-CX_jcAyr.js';
 import './dataset-B9qvlm_o.js';
-import './emitter-BYO2nSDA.js';
-
-/**
- * Researcher interface — stable hook for an external autonomous-research
- * agent to drive the meta-loop.
- *
- * Implementations live downstream (typically in a private repo that
- * runs the actual LLM). This package ships only the contract + a
- * `NoopResearcher` so consumers can wire the surface without being
- * forced to implement every method up front.
- *
- * The four methods mirror the four stages of the paper "Two Loops,
- * Three Roles":
- *
- *   inspectFailures   — given the observed runs, what failure modes
- *                       are present? (data → diagnosis)
- *   proposeChange     — given diagnosed failure modes, what
- *                       structural changes should we try?
- *                       (diagnosis → plan delta)
- *   applyChange       — fold the proposed deltas into a concrete
- *                       experiment plan against an existing baseline.
- *                       (plan delta → executable plan)
- *   evaluateChange    — run the plan, return runs + the gate verdict.
- *                       (executable plan → verdict)
- *
- * Composition is the discipline: a Researcher implementation MUST
- * keep these four steps separate and inspectable. Conflating
- * "diagnose + propose + run" into a single LLM call defeats the
- * point of the framework — you can't audit which step lied.
- *
- * THIS INTERFACE IS STABLE. Breaking changes require a new module
- * (e.g. `Researcher2`) so existing implementations keep working.
- */
-
-/** A diagnosed failure mode with the run-IDs that exhibit it. */
-interface FailureMode {
-    /** Short machine-readable code. Must be stable across runs of the
-     *  same researcher to enable longitudinal tracking. */
-    code: string;
-    /** Human-readable description for the paper / dashboard. */
-    description: string;
-    evidence: {
-        /** Run IDs (from `RunRecord.runId`) where this failure mode was
-         *  observed. */
-        runIds: string[];
-        /** Number of run samples that informed the diagnosis. */
-        samples: number;
-    };
-}
-/** A single steering change the researcher wants to try. */
-interface SteeringChange {
-    kind: 'reviewer_prompt' | 'skill_add' | 'skill_remove' | 'threshold' | 'budget';
-    /** Implementation-specific payload. Researcher implementations
-     *  define the schema — keep this `unknown` here to avoid coupling
-     *  the public interface to any one researcher's internal model. */
-    payload: unknown;
-    /** Why the researcher proposed this change. Goes into the audit
-     *  trail next to the failure-mode evidence. */
-    rationale: string;
-    /** Optional self-reported expected delta on the headline metric. */
-    expectedDelta?: number;
-}
-/** A single experiment plan, mapped onto the search/holdout splits. */
-interface ExperimentPlan {
-    baselineCandidateId: string;
-    proposedCandidateId: string;
-    changes: SteeringChange[];
-    /** USD ceiling for the entire experiment. The runner must stop
-     *  before exceeding this and report a partial result. */
-    evaluationBudgetUsd: number;
-    /** Item IDs (your dataset keys) for the search vs holdout splits. */
-    splits: {
-        search: string[];
-        holdout: string[];
-    };
-}
-/** Result of running a plan: every run, plus the gate verdict. */
-interface ExperimentResult {
-    plan: ExperimentPlan;
-    runs: RunRecord[];
-    gateDecision: GateDecision;
-}
-/**
- * The researcher loop. Stable, four-step, inspectable.
- *
- *   ┌──────────┐  inspectFailures  ┌──────────┐  proposeChange ┌──────────┐
- *   │   runs   │ ─────────────────▶│ failures │ ──────────────▶│ changes  │
- *   └──────────┘                   └──────────┘                └────┬─────┘
- *                                                                   │
- *                                                                   ▼
- *                              ┌────────────────┐  applyChange ┌────────┐
- *                              │ ExperimentPlan │ ◀────────────│  base  │
- *                              └────────┬───────┘              └────────┘
- *                                       │
- *                       evaluateChange  ▼
- *                              ┌────────────────┐
- *                              │ ExperimentResult│
- *                              └────────────────┘
- */
-interface Researcher {
-    inspectFailures(runs: RunRecord[]): Promise<FailureMode[]>;
-    proposeChange(failures: FailureMode[]): Promise<SteeringChange[]>;
-    applyChange(changes: SteeringChange[], baseline: ExperimentPlan): Promise<ExperimentPlan>;
-    evaluateChange(plan: ExperimentPlan): Promise<ExperimentResult>;
-}
-interface CallbackResearcherOptions {
-    inspectFailures: Researcher['inspectFailures'];
-    proposeChange: Researcher['proposeChange'];
-    applyChange: Researcher['applyChange'];
-    evaluateChange: Researcher['evaluateChange'];
-}
-/**
- * Minimal concrete researcher for tests, scripts, and small integrations.
- * Larger autonomous researchers can still implement `Researcher` directly.
- */
-declare class CallbackResearcher implements Researcher {
-    private readonly callbacks;
-    constructor(callbacks: CallbackResearcherOptions);
-    inspectFailures(runs: RunRecord[]): Promise<FailureMode[]>;
-    proposeChange(failures: FailureMode[]): Promise<SteeringChange[]>;
-    applyChange(changes: SteeringChange[], baseline: ExperimentPlan): Promise<ExperimentPlan>;
-    evaluateChange(plan: ExperimentPlan): Promise<ExperimentResult>;
-}
-/**
- * No-op researcher — fails loud on every method. Use as a placeholder
- * in code paths that wire the interface but don't have an implementation
- * yet. Importantly, this does NOT silently succeed: a no-op researcher
- * that returned empty arrays would muffle the loop's signal that
- * nobody implemented the brain.
- */
-declare class NoopResearcher implements Researcher {
-    private readonly hint;
-    constructor(hint?: string);
-    inspectFailures(_runs: RunRecord[]): Promise<FailureMode[]>;
-    proposeChange(_failures: FailureMode[]): Promise<SteeringChange[]>;
-    applyChange(_changes: SteeringChange[], _baseline: ExperimentPlan): Promise<ExperimentPlan>;
-    evaluateChange(_plan: ExperimentPlan): Promise<ExperimentResult>;
-}
-
-export { CallbackResearcher, type CallbackResearcherOptions, type ExperimentPlan, type ExperimentResult, type FailureMode, NoopResearcher, type Researcher, type SteeringChange };

package/dist/optimization.js

@@ -19,15 +19,21 @@ import {
   renderPreferenceMemoryMarkdown,
   replayFeedbackTrajectories,
   replayFeedbackTrajectory,
+  runEvalCampaign,
   runMultiShotOptimization,
   runPromptEvolution,
   serializeFeedbackTrajectoriesJsonl,
   summarizePreferenceMemory,
   trialTraceFromMultiShotTrial,
   withAssignedFeedbackSplit
-} from "./chunk-HKYRWNHV.js";
+} from "./chunk-USHQBPMH.js";
 import "./chunk-YUFXO3TU.js";
-import "./chunk-ODFINDLQ.js";
+import "./chunk-IOXMGMHQ.js";
+import "./chunk-QUKKGHTZ.js";
+import "./chunk-5IIQKMD5.js";
+import "./chunk-6M774GY6.js";
+import "./chunk-KAO3Q65R.js";
+import "./chunk-SQQLHODJ.js";
 import "./chunk-PZ5AY32C.js";
 export {
   CallbackResearcher,
@@ -50,6 +56,7 @@ export {
   renderPreferenceMemoryMarkdown,
   replayFeedbackTrajectories,
   replayFeedbackTrajectory,
+  runEvalCampaign,
   runMultiShotOptimization,
   runPromptEvolution,
   serializeFeedbackTrajectoriesJsonl,