@tangle-network/agent-runtime 0.36.0 → 0.38.0

package/dist/optimize-prompt-cmH9wZdH.d.ts

@@ -0,0 +1,129 @@
+import { LlmClientOptions } from '@tangle-network/agent-eval';
+import { Scenario, DispatchContext, JudgeConfig, ImprovementDriver, Gate, CampaignStorage, GateResult, RunImprovementLoopResult } from '@tangle-network/agent-eval/campaign';
+
+/**
+ * @experimental
+ *
+ * `optimizePrompt` — identity-gated optimization for any TEXT prompt surface
+ * (system prompt, planner prompt, judge rubric, skill doc).
+ *
+ * The text-surface sibling to this module's `improvementDriver` (the
+ * CODE-surface / worktree path). Both feed agent-eval's `runImprovementLoop`;
+ * this one defaults the driver to agent-eval's `gepaDriver` (reflective text
+ * mutator) and the gate to `heldOutGate`.
+ *
+ * IDENTITY-GATED BY CONSTRUCTION — the whole point. The loop runs evals,
+ * collects per-scenario signal, proposes candidates, and the gate compares
+ * candidate-vs-baseline ON THE HELDOUT. `result.prompt` is the baseline
+ * (identity) UNLESS the gate decided `'ship'`. So wiring a surface up is safe:
+ * a surface with no beneficial mutation simply keeps its baseline. You never
+ * regress by registering a prompt — you only ever improve when the held-out
+ * data earns it.
+ *
+ * Generic over the runtime: `runWithPrompt` is the only domain seam — given a
+ * candidate prompt + scenario, run it however the surface runs (sandbox
+ * `streamPrompt`, a `runLoop`, a direct model call) and return the artifact the
+ * judges score. The optimizer never assumes how a prompt is executed.
+ */
+
+/** Reflection config for the default `gepaDriver`. Omit when passing a custom
+ *  `driver`. */
+interface OptimizePromptReflection {
+    /** Router transport for the reflection model. */
+    llm: LlmClientOptions;
+    /** Model that performs the reflective rewrite. */
+    model: string;
+    /** What is being optimized — orients the reflection prompt. Default
+     *  `'system prompt'`. */
+    target?: string;
+    /** Surface-specific mutation levers offered to the reflector. */
+    mutationPrimitives?: string[];
+    /** H2 (`## Foo`) headings that MUST survive every candidate. gepaDriver's
+     *  only structural guard — load-bearing sections of the prompt should be
+     *  `##` headings so a rewrite cannot drop them. */
+    preserveSections?: string[];
+    /** Max sentence-level edits per candidate vs the parent (a textual learning
+     *  rate). Caps a rewrite from wiping prior rules in one generation. */
+    maxSentenceEdits?: number;
+}
+/** @experimental */
+interface OptimizePromptOptions<TScenario extends Scenario, TArtifact> {
+    /** The prompt being optimized — the identity baseline the gate protects. */
+    baselinePrompt: string;
+    /** Domain seam: run a candidate prompt against a scenario → artifact the
+     *  judges score. The optimizer is agnostic to HOW the prompt runs. */
+    runWithPrompt: (prompt: string, scenario: TScenario, ctx: DispatchContext) => Promise<TArtifact>;
+    /** Training pool — scored each generation to rank candidates. */
+    scenarios: TScenario[];
+    /** Held out of training — scored ONLY for the gate's baseline-vs-winner
+     *  delta. Disjoint from `scenarios`; this is what makes promotion measure
+     *  generalization, not memorization. */
+    holdoutScenarios: TScenario[];
+    /** Scorers — deterministic checks or LLM judges. */
+    judges: JudgeConfig<TArtifact, TScenario>[];
+    /** Where artifacts + traces land (opaque key under in-memory storage). */
+    runDir: string;
+    /** Default driver = `gepaDriver` built from this. Required UNLESS `driver`
+     *  is supplied. */
+    reflection?: OptimizePromptReflection;
+    /** Override the improvement strategy (custom driver / deterministic tests). */
+    driver?: ImprovementDriver;
+    /** Override the promotion gate. Default `heldOutGate` over `holdoutScenarios`
+     *  — zero extra LLM. Wrap `defaultProductionGate` for red-team/reward-hacking
+     *  hardening on production wiring. */
+    gate?: Gate<TArtifact, TScenario>;
+    /** Minimum held-out composite lift to ship, forwarded to the default
+     *  `heldOutGate`. When omitted the gate uses its own default. */
+    deltaThreshold?: number;
+    /** Candidates proposed per generation. Default 4. */
+    populationSize?: number;
+    /** Generations to run. Default 3. */
+    maxGenerations?: number;
+    /** Candidates carried to the next generation. Default 2. */
+    promoteTopK?: number;
+    /** Storage backend. Pass `inMemoryCampaignStorage()` for filesystem-less /
+     *  test runs. Default: Node filesystem. */
+    storage?: CampaignStorage;
+    /** Reproducibility seed. Default 42. */
+    seed?: number;
+    /** Per-scenario replicates for CI bands. Default 1. */
+    reps?: number;
+    /** Max concurrent cells. Default 2. */
+    maxConcurrency?: number;
+    /** Test seam — override the wall clock. */
+    now?: () => Date;
+    /** On a shipped gate: `'pr'` opens a PR, `'none'` just reports. Default
+     *  `'none'`. */
+    autoOnPromote?: 'pr' | 'none';
+    ghOwner?: string;
+    ghRepo?: string;
+}
+/** @experimental */
+interface OptimizePromptResult<TArtifact, TScenario extends Scenario> {
+    /** The prompt to USE. Identity (the baseline) unless the gate shipped a
+     *  winner — so a caller can always assign `result.prompt` unconditionally. */
+    prompt: string;
+    /** True only when the gate promoted a candidate over baseline on holdout. */
+    improved: boolean;
+    /** The gate's verdict (`'ship' | 'hold' | 'need_more_work' | ...`). */
+    decision: GateResult['decision'];
+    /** Human-readable reasons the gate gave. */
+    reasons: string[];
+    /** Mean held-out composite of the baseline. */
+    baselineComposite: number;
+    /** Mean held-out composite of the winner candidate. */
+    winnerComposite: number;
+    /** Held-out lift (winner − baseline); the gate's `delta` when it reported one. */
+    delta: number;
+    /** Why the winner was proposed — present when a shipped winner carried a
+     *  driver rationale. */
+    rationale?: string;
+    /** Unified baseline→winner diff (empty when the winner is the baseline). */
+    diff: string;
+    /** The full loop result for callers that need generations / campaigns. */
+    raw: RunImprovementLoopResult<TArtifact, TScenario>;
+}
+/** @experimental */
+declare function optimizePrompt<TScenario extends Scenario, TArtifact>(opts: OptimizePromptOptions<TScenario, TArtifact>): Promise<OptimizePromptResult<TArtifact, TScenario>>;
+
+export { type OptimizePromptOptions as O, type OptimizePromptReflection as a, type OptimizePromptResult as b, optimizePrompt as o };