@crewhaus/prompt-optimizer-claude 0.1.4 → 0.1.5

package/dist/index.d.ts

@@ -0,0 +1,132 @@
+/**
+ * Pillar 2 — `prompt-optimizer-claude`. The model-driven MutationProvider
+ * that closes the gap flagged at
+ * [packages/prompt-optimizer/src/index.ts:91](../../prompt-optimizer/src/index.ts):
+ * "Real-world prompt tuners (DSPy, OPRO) use model-driven rewriting; v0
+ * of this module ships rule-based mutations." This package IS that
+ * model-driven rewriter.
+ *
+ * Behaviour:
+ *   1. Receives the current best Candidate via the MutationProvider
+ *      seam (state.best.prompt + state.trajectory).
+ *   2. Selects a window of failing dev samples (the ones whose recent
+ *      grades suggest the prompt is the blocker — heuristic: the
+ *      lowest-scoring fitness deltas from the most recent iteration).
+ *   3. Calls Claude with a meta-prompt asking for a structured
+ *      `{ rewrite: string, rationale: string }` rewrite. The model
+ *      sees: the current prompt, a sample of failures, and an
+ *      explicit instruction to produce a better prompt without
+ *      leaking dev-set answers.
+ *   4. Validates the response shape with Zod. Falls back to the
+ *      current best on any error (so a model outage doesn't abort
+ *      the search — see the rule-based provider as the deterministic
+ *      fallback path).
+ *
+ * Cost: each call is one Claude request (~1-3K tokens). Each `next()`
+ * now surfaces the call's actual token `usage` on the returned
+ * `ProviderMutation`, and the provider exposes its `modelId` +
+ * `maxOutputTokens` getters, so the FR-003 `--budget-usd` cost-gate in
+ * `eval-optimizer-orchestrator` can price every call against the §27
+ * `cost-tracker` table and stop before a mutation call would exceed
+ * the budget. The gate composes with the orchestrator's `iterations`
+ * cap (whichever bound is hit first ends the run).
+ *
+ * Catalog layer: F-eval (active optimisation). Brief: 280.
+ */
+import { type ProviderAdapter } from "@crewhaus/adapter-anthropic";
+import { CrewhausError } from "@crewhaus/errors";
+import type { MutationProvider, OptimizerState, ProviderMutation } from "@crewhaus/prompt-optimizer";
+export declare class ClaudeMutationProviderError extends CrewhausError {
+    readonly name = "ClaudeMutationProviderError";
+    constructor(message: string, cause?: unknown);
+}
+export type ClaudeMutationProviderOptions = {
+    /** Provider adapter (typically the Anthropic adapter). */
+    readonly adapter: ProviderAdapter;
+    /** Model id, e.g. "claude-sonnet-4-5". */
+    readonly model: string;
+    /** Maximum failures to include in the meta-prompt (default 5). */
+    readonly maxFailuresInPrompt?: number;
+    /** Lowest score threshold below which a sample counts as a "failure" (default 0.5). */
+    readonly failureThreshold?: number;
+    /** Maximum tokens for the rewrite response (default 2048). */
+    readonly maxTokens?: number;
+    /**
+     * Override the meta-prompt's system block. Useful for evals that
+     * have domain-specific rewrite constraints. Defaults to the
+     * production prompt above.
+     */
+    readonly systemOverride?: string;
+};
+/**
+ * Build a `MutationProvider` that delegates each candidate generation
+ * to a Claude (or any `ProviderAdapter`-compatible) model call.
+ */
+export declare class ClaudeMutationProvider implements MutationProvider {
+    readonly name = "claude";
+    private readonly adapter;
+    private readonly model;
+    private readonly maxFailuresInPrompt;
+    private readonly failureThreshold;
+    private readonly maxTokens;
+    private readonly systemBlock;
+    constructor(opts: ClaudeMutationProviderOptions);
+    /**
+     * The model id this provider calls, exposed read-only so the FR-003
+     * cost-gate can price each call via `resolvePricing(DEFAULT_PRICING,
+     * "anthropic", modelId)`. The `MutationProvider` interface only
+     * guarantees `name` + `next()`; the orchestrator feature-detects this
+     * getter and falls back to a zero-cost meter for providers that don't
+     * expose it.
+     */
+    get modelId(): string;
+    /**
+     * The provider id of the injected adapter, exposed read-only so the
+     * FR-003 cost-gate prices calls against the REAL provider's pricing
+     * table instead of assuming "anthropic". Feature-detected by the
+     * orchestrator like `modelId` — providers without the getter price as
+     * Anthropic (the historical behaviour).
+     */
+    get providerId(): string;
+    /**
+     * The output-token ceiling for each call, exposed read-only so the
+     * cost-gate can compute a worst-case pre-call estimate (the gate must
+     * decide BEFORE a call whether it would exceed the budget).
+     */
+    get maxOutputTokens(): number;
+    /**
+     * FR-003 — exact serialized INPUT character count this provider would
+     * transmit for `state`, so the cost-gate prices the *real* meta-prompt
+     * rather than just `best.prompt.length + a fixed overhead`. The naive
+     * estimate (prompt length only) under-counts the system block AND the
+     * rendered dev-set failure block (each failure's input + expected_output)
+     * that `next()` actually sends; with a large dev window + small maxTokens
+     * that deficit could let a gate-passing call exceed the budget after the
+     * fact. Returning the full system+user char count here makes the
+     * `chars/4` token estimate cover everything the model is billed for, so
+     * the orchestrator's estimate-before guarantee holds unconditionally
+     * (input is now bounded from above, output is already the ceiling).
+     *
+     * The orchestrator feature-detects this method (it is not part of the
+     * `MutationProvider` interface); providers that omit it fall back to the
+     * `best.prompt.length + metaOverheadChars` heuristic.
+     */
+    estimateInputChars(state: OptimizerState): number;
+    next(state: OptimizerState): Promise<ProviderMutation>;
+    /** Build the user message for the meta-prompt. */
+    private buildUserMessage;
+    /** Identify failure samples from the trajectory. */
+    private selectFailures;
+    /**
+     * Fallback when the model can't produce a usable rewrite. Returns
+     * the current best verbatim so the search loop records a no-op
+     * iteration. The orchestrator logs the fallback reason. When the
+     * model DID round-trip (returned text/JSON that turned out unusable),
+     * the `usage` actually consumed is forwarded so the cost-gate still
+     * accounts for the spend; a stream error (no completed call) passes
+     * no usage and is therefore charged zero.
+     */
+    private fallback;
+}
+/** Convenience factory mirroring `createAnthropicAdapter` ergonomics. */
+export declare function createClaudeMutationProvider(opts: ClaudeMutationProviderOptions): ClaudeMutationProvider;

package/dist/index.js

@@ -0,0 +1,233 @@
+/**
+ * Pillar 2 — `prompt-optimizer-claude`. The model-driven MutationProvider
+ * that closes the gap flagged at
+ * [packages/prompt-optimizer/src/index.ts:91](../../prompt-optimizer/src/index.ts):
+ * "Real-world prompt tuners (DSPy, OPRO) use model-driven rewriting; v0
+ * of this module ships rule-based mutations." This package IS that
+ * model-driven rewriter.
+ *
+ * Behaviour:
+ *   1. Receives the current best Candidate via the MutationProvider
+ *      seam (state.best.prompt + state.trajectory).
+ *   2. Selects a window of failing dev samples (the ones whose recent
+ *      grades suggest the prompt is the blocker — heuristic: the
+ *      lowest-scoring fitness deltas from the most recent iteration).
+ *   3. Calls Claude with a meta-prompt asking for a structured
+ *      `{ rewrite: string, rationale: string }` rewrite. The model
+ *      sees: the current prompt, a sample of failures, and an
+ *      explicit instruction to produce a better prompt without
+ *      leaking dev-set answers.
+ *   4. Validates the response shape with Zod. Falls back to the
+ *      current best on any error (so a model outage doesn't abort
+ *      the search — see the rule-based provider as the deterministic
+ *      fallback path).
+ *
+ * Cost: each call is one Claude request (~1-3K tokens). Each `next()`
+ * now surfaces the call's actual token `usage` on the returned
+ * `ProviderMutation`, and the provider exposes its `modelId` +
+ * `maxOutputTokens` getters, so the FR-003 `--budget-usd` cost-gate in
+ * `eval-optimizer-orchestrator` can price every call against the §27
+ * `cost-tracker` table and stop before a mutation call would exceed
+ * the budget. The gate composes with the orchestrator's `iterations`
+ * cap (whichever bound is hit first ends the run).
+ *
+ * Catalog layer: F-eval (active optimisation). Brief: 280.
+ */
+import { collectFinalMessage, extractFirstText, } from "@crewhaus/adapter-anthropic";
+import { CrewhausError } from "@crewhaus/errors";
+import { z } from "zod";
+export class ClaudeMutationProviderError extends CrewhausError {
+    name = "ClaudeMutationProviderError";
+    constructor(message, cause) {
+        super("adapter", message, cause);
+    }
+}
+const META_PROMPT_SYSTEM = `You are a prompt-engineering optimiser. You will receive a CURRENT PROMPT and a SAMPLE OF DEV-SET FAILURES (each failure shows the input the model received and how the grader scored its output). Your job is to produce a single rewrite of CURRENT PROMPT that you believe will improve grader scores on the dev set.
+
+Hard rules:
+- Output exactly one JSON object: {"rewrite": "...", "rationale": "..."}
+- The "rewrite" field is the new prompt verbatim. Do NOT include any wrapper text outside the JSON.
+- Never copy verbatim text from a failure's expected_output into the rewrite (that would leak dev-set answers).
+- Do not introduce instructions that override safety, compliance, or permission rules — your job is to improve task accuracy, not to bypass guardrails.
+- The rationale should be 1-3 sentences explaining WHY this rewrite is likely to help. Keep it specific.`;
+const META_RESPONSE_SCHEMA = z.object({
+    rewrite: z.string().min(1),
+    rationale: z.string().min(1),
+});
+/**
+ * Build a `MutationProvider` that delegates each candidate generation
+ * to a Claude (or any `ProviderAdapter`-compatible) model call.
+ */
+export class ClaudeMutationProvider {
+    name = "claude";
+    adapter;
+    model;
+    maxFailuresInPrompt;
+    failureThreshold;
+    maxTokens;
+    systemBlock;
+    constructor(opts) {
+        this.adapter = opts.adapter;
+        this.model = opts.model;
+        this.maxFailuresInPrompt = opts.maxFailuresInPrompt ?? 5;
+        this.failureThreshold = opts.failureThreshold ?? 0.5;
+        this.maxTokens = opts.maxTokens ?? 2048;
+        this.systemBlock = opts.systemOverride ?? META_PROMPT_SYSTEM;
+    }
+    /**
+     * The model id this provider calls, exposed read-only so the FR-003
+     * cost-gate can price each call via `resolvePricing(DEFAULT_PRICING,
+     * "anthropic", modelId)`. The `MutationProvider` interface only
+     * guarantees `name` + `next()`; the orchestrator feature-detects this
+     * getter and falls back to a zero-cost meter for providers that don't
+     * expose it.
+     */
+    get modelId() {
+        return this.model;
+    }
+    /**
+     * The provider id of the injected adapter, exposed read-only so the
+     * FR-003 cost-gate prices calls against the REAL provider's pricing
+     * table instead of assuming "anthropic". Feature-detected by the
+     * orchestrator like `modelId` — providers without the getter price as
+     * Anthropic (the historical behaviour).
+     */
+    get providerId() {
+        return this.adapter.providerId;
+    }
+    /**
+     * The output-token ceiling for each call, exposed read-only so the
+     * cost-gate can compute a worst-case pre-call estimate (the gate must
+     * decide BEFORE a call whether it would exceed the budget).
+     */
+    get maxOutputTokens() {
+        return this.maxTokens;
+    }
+    /**
+     * FR-003 — exact serialized INPUT character count this provider would
+     * transmit for `state`, so the cost-gate prices the *real* meta-prompt
+     * rather than just `best.prompt.length + a fixed overhead`. The naive
+     * estimate (prompt length only) under-counts the system block AND the
+     * rendered dev-set failure block (each failure's input + expected_output)
+     * that `next()` actually sends; with a large dev window + small maxTokens
+     * that deficit could let a gate-passing call exceed the budget after the
+     * fact. Returning the full system+user char count here makes the
+     * `chars/4` token estimate cover everything the model is billed for, so
+     * the orchestrator's estimate-before guarantee holds unconditionally
+     * (input is now bounded from above, output is already the ceiling).
+     *
+     * The orchestrator feature-detects this method (it is not part of the
+     * `MutationProvider` interface); providers that omit it fall back to the
+     * `best.prompt.length + metaOverheadChars` heuristic.
+     */
+    estimateInputChars(state) {
+        const failures = this.selectFailures(state);
+        const userMessage = this.buildUserMessage(state.best.prompt, failures);
+        return this.systemBlock.length + userMessage.length;
+    }
+    async next(state) {
+        const failures = this.selectFailures(state);
+        const userMessage = this.buildUserMessage(state.best.prompt, failures);
+        // The call's actual token usage, captured so the FR-003 cost-gate
+        // can fold it into a running spend total. Stays undefined until the
+        // model call round-trips; a failed/unusable response still records
+        // whatever was consumed (the model DID run) so spend is accounted.
+        let callUsage;
+        let rawText;
+        try {
+            const final = await collectFinalMessage(this.adapter.stream({
+                model: this.model,
+                system: [{ type: "text", text: this.systemBlock }],
+                messages: [{ role: "user", content: userMessage }],
+                maxTokens: this.maxTokens,
+            }));
+            callUsage = {
+                input: final.usage.input,
+                output: final.usage.output,
+                ...(final.usage.cacheRead !== undefined ? { cacheRead: final.usage.cacheRead } : {}),
+            };
+            rawText = extractFirstText(final);
+        }
+        catch (err) {
+            // Mutator unavailability is not fatal — fall back to current best.
+            // The orchestrator's outer loop will record a degenerate iteration
+            // (score unchanged) and the search continues. No usage is available
+            // on a stream error (the call did not complete), so spend is zero.
+            return this.fallback(state, `model error: ${err.message}`);
+        }
+        if (rawText === undefined) {
+            return this.fallback(state, "model returned no text block", callUsage);
+        }
+        // Extract JSON: tolerate ```json fences and leading prose. We
+        // search for the first balanced `{...}` substring.
+        const jsonMatch = rawText.match(/\{[\s\S]*\}/);
+        if (jsonMatch === null) {
+            return this.fallback(state, "model response did not contain a JSON object", callUsage);
+        }
+        let parsed;
+        try {
+            const raw = JSON.parse(jsonMatch[0]);
+            parsed = META_RESPONSE_SCHEMA.parse(raw);
+        }
+        catch (err) {
+            return this.fallback(state, `model response failed schema validation: ${err.message}`, callUsage);
+        }
+        const mutation = { kind: "rephrase-instruction" };
+        // We use the rephrase-instruction kind to record the mutation in
+        // the trajectory; the actual rewrite is full-replacement, not the
+        // rule-based provider's "append a sentence" behaviour. Future work:
+        // add a distinct `{ kind: "model-rewrite"; rationale: string }`
+        // variant to the Mutation union for better trajectory metadata.
+        return {
+            prompt: parsed.rewrite,
+            mutations: [mutation],
+            rationale: parsed.rationale,
+            ...(callUsage !== undefined ? { usage: callUsage } : {}),
+        };
+    }
+    /** Build the user message for the meta-prompt. */
+    buildUserMessage(currentPrompt, failures) {
+        const failureBlock = failures.length === 0
+            ? "(No dev-set failures available yet — propose a refinement that improves clarity, specificity, or instruction-following.)"
+            : failures
+                .map((f, i) => `--- Failure ${i + 1} (observed score ${f.observedScore.toFixed(2)}) ---\nInput: ${f.input}\n${f.expected !== undefined ? `Expected output (do NOT copy this verbatim into the rewrite): ${f.expected}\n` : ""}`)
+                .join("\n");
+        return `CURRENT PROMPT:\n${currentPrompt}\n\nSAMPLE OF DEV-SET FAILURES:\n${failureBlock}\n\nReturn one JSON object: {"rewrite": "...", "rationale": "..."}`;
+    }
+    /** Identify failure samples from the trajectory. */
+    selectFailures(state) {
+        // The trajectory records aggregate scores per candidate, not per
+        // sample. For v0 we surface the dev set as raw inputs and let
+        // the model reason about them generically; future iterations can
+        // wire a per-sample grade map through the OptimizerState. The
+        // result is still useful because the model sees the dev set
+        // distribution even without per-sample grades.
+        return state.devSet.slice(0, this.maxFailuresInPrompt).map((s) => ({
+            input: s.input,
+            // Use the trajectory's most recent score as a coarse signal.
+            observedScore: state.best.score,
+            ...(s.expected_output !== undefined ? { expected: s.expected_output } : {}),
+        }));
+    }
+    /**
+     * Fallback when the model can't produce a usable rewrite. Returns
+     * the current best verbatim so the search loop records a no-op
+     * iteration. The orchestrator logs the fallback reason. When the
+     * model DID round-trip (returned text/JSON that turned out unusable),
+     * the `usage` actually consumed is forwarded so the cost-gate still
+     * accounts for the spend; a stream error (no completed call) passes
+     * no usage and is therefore charged zero.
+     */
+    fallback(state, reason, usage) {
+        return {
+            prompt: state.best.prompt,
+            mutations: [],
+            rationale: `claude-fallback: ${reason}`,
+            ...(usage !== undefined ? { usage } : {}),
+        };
+    }
+}
+/** Convenience factory mirroring `createAnthropicAdapter` ergonomics. */
+export function createClaudeMutationProvider(opts) {
+    return new ClaudeMutationProvider(opts);
+}

package/package.json

@@ -1,25 +1,28 @@
 {
   "name": "@crewhaus/prompt-optimizer-claude",
-  "version": "0.1.4",
+  "version": "0.1.5",
   "type": "module",
   "description": "Pillar-2 model-driven MutationProvider — asks Claude to rewrite the prompt given dev-set failures. Closes the v0 prompt-optimizer's L91 'rule-based-only' gap.",
-  "main": "src/index.ts",
-  "types": "src/index.ts",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
   "exports": {
-    ".": "./src/index.ts"
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
   },
   "scripts": {
     "test": "bun test src"
   },
   "dependencies": {
     "@anthropic-ai/sdk": "^0.96.0",
-    "@crewhaus/adapter-anthropic": "0.1.4",
-    "@crewhaus/errors": "0.1.4",
-    "@crewhaus/prompt-optimizer": "0.1.4",
+    "@crewhaus/adapter-anthropic": "0.1.5",
+    "@crewhaus/errors": "0.1.5",
+    "@crewhaus/prompt-optimizer": "0.1.5",
     "zod": "^3.23.8"
   },
   "devDependencies": {
-    "@crewhaus/eval-dataset": "0.1.4"
+    "@crewhaus/eval-dataset": "0.1.5"
   },
   "license": "Apache-2.0",
   "author": {
@@ -39,5 +42,5 @@
   "publishConfig": {
     "access": "public"
   },
-  "files": ["src", "README.md", "LICENSE", "NOTICE"]
+  "files": ["dist", "README.md", "LICENSE", "NOTICE"]
 }

package/src/index.test.ts

@@ -1,324 +0,0 @@
-import { describe, expect, test } from "bun:test";
-import type { ProviderAdapter } from "@crewhaus/adapter-anthropic";
-import type { Sample } from "@crewhaus/eval-dataset";
-import type { OptimizerState } from "@crewhaus/prompt-optimizer";
-import {
-  ClaudeMutationProvider,
-  ClaudeMutationProviderError,
-  createClaudeMutationProvider,
-} from "./index";
-
-const SAMPLE_TRAIN: ReadonlyArray<Sample> = [
-  { id: "t1", input: "What is 2+2?", expected_output: "4" },
-  { id: "t2", input: "Capital of France?", expected_output: "Paris" },
-];
-
-const SAMPLE_DEV: ReadonlyArray<Sample> = [
-  { id: "d1", input: "What is 3+3?", expected_output: "6" },
-  { id: "d2", input: "Capital of Germany?", expected_output: "Berlin" },
-];
-
-/**
- * Build a mock provider adapter whose `.stream()` yields the StreamEvent
- * sequence that produces a single text message with the given content.
- * Matches the canonical event protocol consumed by
- * `consumeStream`/`collectFinalMessage`.
- */
-function mockAdapter(
-  content: string,
-  usage: { input: number; output: number; cacheRead?: number } = { input: 0, output: 0 },
-): ProviderAdapter {
-  return {
-    id: "mock",
-    features: {
-      caching: "none",
-      thinking: false,
-      multimodal: { input: false, output: false },
-    },
-    // biome-ignore lint/suspicious/noExplicitAny: minimal mock
-    stream(_params: any): AsyncIterable<any> {
-      return (async function* () {
-        // message_start carries input (+ cache) counts; message_delta
-        // carries the running output count — matches the real protocol.
-        yield {
-          kind: "message_start",
-          usage: {
-            input: usage.input,
-            output: 0,
-            ...(usage.cacheRead !== undefined ? { cacheRead: usage.cacheRead } : {}),
-          },
-        };
-        yield {
-          kind: "content_block_start",
-          index: 0,
-          block: { type: "text", text: "" },
-        };
-        yield {
-          kind: "content_block_delta",
-          index: 0,
-          delta: { type: "text_delta", text: content },
-        };
-        yield { kind: "content_block_stop", index: 0 };
-        yield {
-          kind: "message_delta",
-          stopReason: "end_turn",
-          usage: { input: usage.input, output: usage.output },
-        };
-        yield { kind: "message_stop" };
-      })();
-    },
-  } as unknown as ProviderAdapter;
-}
-
-const baseState: OptimizerState = {
-  iteration: 1,
-  best: {
-    id: "candidate-0",
-    prompt: "You are a helpful assistant.",
-    mutations: [],
-    score: 0.4,
-  },
-  trajectory: [],
-  trainSet: SAMPLE_TRAIN,
-  devSet: SAMPLE_DEV,
-};
-
-describe("ClaudeMutationProvider", () => {
-  test("exposes the adapter's providerId for the FR-003 cost-gate (provider-agnostic pricing)", () => {
-    const adapter = {
-      ...mockAdapter("{}"),
-      providerId: "openai",
-    } as unknown as ProviderAdapter;
-    const provider = new ClaudeMutationProvider({ adapter, model: "gpt-4o-mini" });
-    expect(provider.providerId).toBe("openai");
-    expect(provider.modelId).toBe("gpt-4o-mini");
-  });
-
-  test("parses a clean JSON response and returns it as the rewrite", async () => {
-    const adapter = mockAdapter(
-      `{"rewrite": "Think step by step before answering. Be concise.", "rationale": "Adds explicit chain-of-thought scaffolding which helps on multi-step problems."}`,
-    );
-    const provider = new ClaudeMutationProvider({
-      adapter,
-      model: "claude-sonnet-4-5",
-    });
-    const result = await provider.next(baseState);
-    expect(result.prompt).toBe("Think step by step before answering. Be concise.");
-    expect(result.rationale).toContain("chain-of-thought");
-    expect(result.mutations).toHaveLength(1);
-  });
-
-  test("tolerates code-fence wrapping around the JSON", async () => {
-    const adapter = mockAdapter(
-      "```json\n" +
-        `{"rewrite": "Be precise and direct.", "rationale": "Removes ambiguity."}` +
-        "\n```",
-    );
-    const provider = new ClaudeMutationProvider({
-      adapter,
-      model: "claude-sonnet-4-5",
-    });
-    const result = await provider.next(baseState);
-    expect(result.prompt).toBe("Be precise and direct.");
-  });
-
-  test("falls back to current best on malformed JSON (no abort)", async () => {
-    const adapter = mockAdapter("not actually json");
-    const provider = new ClaudeMutationProvider({
-      adapter,
-      model: "claude-sonnet-4-5",
-    });
-    const result = await provider.next(baseState);
-    expect(result.prompt).toBe(baseState.best.prompt);
-    expect(result.rationale).toContain("claude-fallback");
-  });
-
-  test("falls back when the model errors out", async () => {
-    const adapter = {
-      id: "mock",
-      features: {
-        caching: "none",
-        thinking: false,
-        multimodal: { input: false, output: false },
-      },
-      // biome-ignore lint/suspicious/noExplicitAny: minimal mock
-      stream(_params: any): AsyncIterable<any> {
-        return (async function* () {
-          throw new Error("model unavailable");
-          // biome-ignore lint/correctness/noUnreachable: keep typed as async generator
-          yield;
-        })();
-      },
-    } as unknown as ProviderAdapter;
-    const provider = new ClaudeMutationProvider({
-      adapter,
-      model: "claude-sonnet-4-5",
-    });
-    const result = await provider.next(baseState);
-    expect(result.prompt).toBe(baseState.best.prompt);
-    expect(result.rationale).toContain("claude-fallback");
-    expect(result.rationale).toContain("model unavailable");
-  });
-
-  test("rejects responses missing required schema fields", async () => {
-    const adapter = mockAdapter(`{"rewrite": "Only rewrite, no rationale"}`);
-    const provider = new ClaudeMutationProvider({
-      adapter,
-      model: "claude-sonnet-4-5",
-    });
-    const result = await provider.next(baseState);
-    expect(result.prompt).toBe(baseState.best.prompt);
-    expect(result.rationale).toContain("claude-fallback");
-  });
-
-  test("name is 'claude' for trajectory logging", () => {
-    const provider = new ClaudeMutationProvider({
-      adapter: mockAdapter(""),
-      model: "claude-sonnet-4-5",
-    });
-    expect(provider.name).toBe("claude");
-  });
-
-  // FR-003 — the provider surfaces per-call usage + exposes pricing getters.
-  test("returns per-call usage from the response on the success path", async () => {
-    const adapter = mockAdapter(`{"rewrite": "Be concise.", "rationale": "Removes ambiguity."}`, {
-      input: 1234,
-      output: 567,
-      cacheRead: 89,
-    });
-    const provider = new ClaudeMutationProvider({ adapter, model: "claude-sonnet-4-5" });
-    const result = await provider.next(baseState);
-    expect(result.prompt).toBe("Be concise.");
-    expect(result.usage).toEqual({ input: 1234, output: 567, cacheRead: 89 });
-  });
-
-  test("forwards consumed usage even when the response is unusable (JSON miss)", async () => {
-    const adapter = mockAdapter("not actually json", { input: 100, output: 20 });
-    const provider = new ClaudeMutationProvider({ adapter, model: "claude-sonnet-4-5" });
-    const result = await provider.next(baseState);
-    // Fell back to current best, but the spend the model DID incur is reported.
-    expect(result.prompt).toBe(baseState.best.prompt);
-    expect(result.rationale).toContain("claude-fallback");
-    expect(result.usage).toEqual({ input: 100, output: 20 });
-  });
-
-  test("omits usage when the model call never completes (stream error)", async () => {
-    const adapter = {
-      id: "mock",
-      features: { caching: "none", thinking: false, multimodal: { input: false, output: false } },
-      // biome-ignore lint/suspicious/noExplicitAny: minimal mock
-      stream(_params: any): AsyncIterable<any> {
-        return (async function* () {
-          throw new Error("model unavailable");
-          // biome-ignore lint/correctness/noUnreachable: keep typed as async generator
-          yield;
-        })();
-      },
-    } as unknown as ProviderAdapter;
-    const provider = new ClaudeMutationProvider({ adapter, model: "claude-sonnet-4-5" });
-    const result = await provider.next(baseState);
-    expect(result.prompt).toBe(baseState.best.prompt);
-    expect(result.usage).toBeUndefined();
-  });
-
-  test("exposes modelId + maxOutputTokens getters for the cost-gate", () => {
-    const provider = new ClaudeMutationProvider({
-      adapter: mockAdapter(""),
-      model: "claude-opus-4-7",
-      maxTokens: 4096,
-    });
-    expect(provider.modelId).toBe("claude-opus-4-7");
-    expect(provider.maxOutputTokens).toBe(4096);
-  });
-
-  test("maxOutputTokens getter defaults to 2048", () => {
-    const provider = new ClaudeMutationProvider({
-      adapter: mockAdapter(""),
-      model: "claude-sonnet-4-5",
-    });
-    expect(provider.maxOutputTokens).toBe(2048);
-  });
-
-  // FR-003 (input-estimate gap) — estimateInputChars must count the FULL
-  // serialized input (system block + rendered dev-set failure block), not
-  // just the candidate prompt, so the orchestrator's cost-gate cannot
-  // under-count input for a wide dev window.
-  test("estimateInputChars counts the system block + rendered failure block", () => {
-    const provider = new ClaudeMutationProvider({
-      adapter: mockAdapter(""),
-      model: "claude-sonnet-4-5",
-    });
-    const est = provider.estimateInputChars(baseState);
-    // Strictly greater than the prompt length alone — the deficit the naive
-    // `best.prompt.length` estimate would have missed.
-    expect(est).toBeGreaterThan(baseState.best.prompt.length);
-    // The estimate covers the rendered failure block: it exceeds the prompt
-    // length PLUS the combined dev-set input + expected_output text that
-    // next() serializes — exactly the content the naive `best.prompt.length`
-    // heuristic ignored.
-    const devTextChars = SAMPLE_DEV.reduce(
-      (acc, s) => acc + s.input.length + (s.expected_output?.length ?? 0),
-      0,
-    );
-    expect(est).toBeGreaterThan(baseState.best.prompt.length + devTextChars);
-  });
-
-  test("createClaudeMutationProvider factory yields an equivalent provider", async () => {
-    const adapter = mockAdapter(
-      `{"rewrite": "Be concise and exact.", "rationale": "Tightens the instruction."}`,
-    );
-    const provider = createClaudeMutationProvider({
-      adapter,
-      model: "claude-sonnet-4-5",
-      maxTokens: 1024,
-    });
-    expect(provider).toBeInstanceOf(ClaudeMutationProvider);
-    expect(provider.name).toBe("claude");
-    expect(provider.modelId).toBe("claude-sonnet-4-5");
-    expect(provider.maxOutputTokens).toBe(1024);
-    // Behaves like the directly-constructed provider end-to-end.
-    const result = await provider.next(baseState);
-    expect(result.prompt).toBe("Be concise and exact.");
-  });
-
-  test("estimateInputChars grows with the dev-set failure window", () => {
-    const wideDev: ReadonlyArray<Sample> = [
-      ...SAMPLE_DEV,
-      { id: "d3", input: "x".repeat(5_000), expected_output: "y".repeat(5_000) },
-      { id: "d4", input: "z".repeat(5_000), expected_output: "w".repeat(5_000) },
-    ];
-    const provider = new ClaudeMutationProvider({
-      adapter: mockAdapter(""),
-      model: "claude-sonnet-4-5",
-      maxFailuresInPrompt: 4,
-    });
-    const narrow = provider.estimateInputChars(baseState);
-    const wide = provider.estimateInputChars({ ...baseState, devSet: wideDev });
-    // A larger window serializes more failure text → a larger input estimate,
-    // exactly the case the original prompt-only heuristic under-counted.
-    expect(wide).toBeGreaterThan(narrow + 18_000);
-  });
-});
-
-describe("ClaudeMutationProviderError", () => {
-  test("carries the adapter code, stable name, and cause chain", () => {
-    const cause = new Error("stream aborted");
-    const err = new ClaudeMutationProviderError("mutation provider failed", cause);
-    expect(err).toBeInstanceOf(Error);
-    expect(err.name).toBe("ClaudeMutationProviderError");
-    expect(err.code).toBe("adapter");
-    expect(err.message).toBe("mutation provider failed");
-    expect(err.cause).toBe(cause);
-    expect(err.toJSON()).toMatchObject({
-      name: "ClaudeMutationProviderError",
-      code: "adapter",
-      message: "mutation provider failed",
-      cause: { name: "Error", message: "stream aborted" },
-    });
-  });
-
-  test("constructs without a cause", () => {
-    const err = new ClaudeMutationProviderError("no cause");
-    expect(err.cause).toBeUndefined();
-  });
-});

package/src/index.ts

@@ -1,298 +0,0 @@
-/**
- * Pillar 2 — `prompt-optimizer-claude`. The model-driven MutationProvider
- * that closes the gap flagged at
- * [packages/prompt-optimizer/src/index.ts:91](../../prompt-optimizer/src/index.ts):
- * "Real-world prompt tuners (DSPy, OPRO) use model-driven rewriting; v0
- * of this module ships rule-based mutations." This package IS that
- * model-driven rewriter.
- *
- * Behaviour:
- *   1. Receives the current best Candidate via the MutationProvider
- *      seam (state.best.prompt + state.trajectory).
- *   2. Selects a window of failing dev samples (the ones whose recent
- *      grades suggest the prompt is the blocker — heuristic: the
- *      lowest-scoring fitness deltas from the most recent iteration).
- *   3. Calls Claude with a meta-prompt asking for a structured
- *      `{ rewrite: string, rationale: string }` rewrite. The model
- *      sees: the current prompt, a sample of failures, and an
- *      explicit instruction to produce a better prompt without
- *      leaking dev-set answers.
- *   4. Validates the response shape with Zod. Falls back to the
- *      current best on any error (so a model outage doesn't abort
- *      the search — see the rule-based provider as the deterministic
- *      fallback path).
- *
- * Cost: each call is one Claude request (~1-3K tokens). Each `next()`
- * now surfaces the call's actual token `usage` on the returned
- * `ProviderMutation`, and the provider exposes its `modelId` +
- * `maxOutputTokens` getters, so the FR-003 `--budget-usd` cost-gate in
- * `eval-optimizer-orchestrator` can price every call against the §27
- * `cost-tracker` table and stop before a mutation call would exceed
- * the budget. The gate composes with the orchestrator's `iterations`
- * cap (whichever bound is hit first ends the run).
- *
- * Catalog layer: F-eval (active optimisation). Brief: 280.
- */
-import {
-  type ProviderAdapter,
-  collectFinalMessage,
-  extractFirstText,
-} from "@crewhaus/adapter-anthropic";
-import { CrewhausError } from "@crewhaus/errors";
-import type {
-  Mutation,
-  MutationProvider,
-  OptimizerState,
-  ProviderMutation,
-} from "@crewhaus/prompt-optimizer";
-import { z } from "zod";
-
-export class ClaudeMutationProviderError extends CrewhausError {
-  override readonly name = "ClaudeMutationProviderError";
-  constructor(message: string, cause?: unknown) {
-    super("adapter", message, cause);
-  }
-}
-
-const META_PROMPT_SYSTEM = `You are a prompt-engineering optimiser. You will receive a CURRENT PROMPT and a SAMPLE OF DEV-SET FAILURES (each failure shows the input the model received and how the grader scored its output). Your job is to produce a single rewrite of CURRENT PROMPT that you believe will improve grader scores on the dev set.
-
-Hard rules:
-- Output exactly one JSON object: {"rewrite": "...", "rationale": "..."}
-- The "rewrite" field is the new prompt verbatim. Do NOT include any wrapper text outside the JSON.
-- Never copy verbatim text from a failure's expected_output into the rewrite (that would leak dev-set answers).
-- Do not introduce instructions that override safety, compliance, or permission rules — your job is to improve task accuracy, not to bypass guardrails.
-- The rationale should be 1-3 sentences explaining WHY this rewrite is likely to help. Keep it specific.`;
-
-const META_RESPONSE_SCHEMA = z.object({
-  rewrite: z.string().min(1),
-  rationale: z.string().min(1),
-});
-
-export type ClaudeMutationProviderOptions = {
-  /** Provider adapter (typically the Anthropic adapter). */
-  readonly adapter: ProviderAdapter;
-  /** Model id, e.g. "claude-sonnet-4-5". */
-  readonly model: string;
-  /** Maximum failures to include in the meta-prompt (default 5). */
-  readonly maxFailuresInPrompt?: number;
-  /** Lowest score threshold below which a sample counts as a "failure" (default 0.5). */
-  readonly failureThreshold?: number;
-  /** Maximum tokens for the rewrite response (default 2048). */
-  readonly maxTokens?: number;
-  /**
-   * Override the meta-prompt's system block. Useful for evals that
-   * have domain-specific rewrite constraints. Defaults to the
-   * production prompt above.
-   */
-  readonly systemOverride?: string;
-};
-
-/**
- * Build a `MutationProvider` that delegates each candidate generation
- * to a Claude (or any `ProviderAdapter`-compatible) model call.
- */
-export class ClaudeMutationProvider implements MutationProvider {
-  readonly name = "claude";
-  private readonly adapter: ProviderAdapter;
-  private readonly model: string;
-  private readonly maxFailuresInPrompt: number;
-  private readonly failureThreshold: number;
-  private readonly maxTokens: number;
-  private readonly systemBlock: string;
-
-  constructor(opts: ClaudeMutationProviderOptions) {
-    this.adapter = opts.adapter;
-    this.model = opts.model;
-    this.maxFailuresInPrompt = opts.maxFailuresInPrompt ?? 5;
-    this.failureThreshold = opts.failureThreshold ?? 0.5;
-    this.maxTokens = opts.maxTokens ?? 2048;
-    this.systemBlock = opts.systemOverride ?? META_PROMPT_SYSTEM;
-  }
-
-  /**
-   * The model id this provider calls, exposed read-only so the FR-003
-   * cost-gate can price each call via `resolvePricing(DEFAULT_PRICING,
-   * "anthropic", modelId)`. The `MutationProvider` interface only
-   * guarantees `name` + `next()`; the orchestrator feature-detects this
-   * getter and falls back to a zero-cost meter for providers that don't
-   * expose it.
-   */
-  get modelId(): string {
-    return this.model;
-  }
-
-  /**
-   * The provider id of the injected adapter, exposed read-only so the
-   * FR-003 cost-gate prices calls against the REAL provider's pricing
-   * table instead of assuming "anthropic". Feature-detected by the
-   * orchestrator like `modelId` — providers without the getter price as
-   * Anthropic (the historical behaviour).
-   */
-  get providerId(): string {
-    return this.adapter.providerId;
-  }
-
-  /**
-   * The output-token ceiling for each call, exposed read-only so the
-   * cost-gate can compute a worst-case pre-call estimate (the gate must
-   * decide BEFORE a call whether it would exceed the budget).
-   */
-  get maxOutputTokens(): number {
-    return this.maxTokens;
-  }
-
-  /**
-   * FR-003 — exact serialized INPUT character count this provider would
-   * transmit for `state`, so the cost-gate prices the *real* meta-prompt
-   * rather than just `best.prompt.length + a fixed overhead`. The naive
-   * estimate (prompt length only) under-counts the system block AND the
-   * rendered dev-set failure block (each failure's input + expected_output)
-   * that `next()` actually sends; with a large dev window + small maxTokens
-   * that deficit could let a gate-passing call exceed the budget after the
-   * fact. Returning the full system+user char count here makes the
-   * `chars/4` token estimate cover everything the model is billed for, so
-   * the orchestrator's estimate-before guarantee holds unconditionally
-   * (input is now bounded from above, output is already the ceiling).
-   *
-   * The orchestrator feature-detects this method (it is not part of the
-   * `MutationProvider` interface); providers that omit it fall back to the
-   * `best.prompt.length + metaOverheadChars` heuristic.
-   */
-  estimateInputChars(state: OptimizerState): number {
-    const failures = this.selectFailures(state);
-    const userMessage = this.buildUserMessage(state.best.prompt, failures);
-    return this.systemBlock.length + userMessage.length;
-  }
-
-  async next(state: OptimizerState): Promise<ProviderMutation> {
-    const failures = this.selectFailures(state);
-    const userMessage = this.buildUserMessage(state.best.prompt, failures);
-
-    // The call's actual token usage, captured so the FR-003 cost-gate
-    // can fold it into a running spend total. Stays undefined until the
-    // model call round-trips; a failed/unusable response still records
-    // whatever was consumed (the model DID run) so spend is accounted.
-    let callUsage: ProviderMutation["usage"];
-    let rawText: string | undefined;
-    try {
-      const final = await collectFinalMessage(
-        this.adapter.stream({
-          model: this.model,
-          system: [{ type: "text", text: this.systemBlock }],
-          messages: [{ role: "user", content: userMessage }],
-          maxTokens: this.maxTokens,
-        }),
-      );
-      callUsage = {
-        input: final.usage.input,
-        output: final.usage.output,
-        ...(final.usage.cacheRead !== undefined ? { cacheRead: final.usage.cacheRead } : {}),
-      };
-      rawText = extractFirstText(final);
-    } catch (err) {
-      // Mutator unavailability is not fatal — fall back to current best.
-      // The orchestrator's outer loop will record a degenerate iteration
-      // (score unchanged) and the search continues. No usage is available
-      // on a stream error (the call did not complete), so spend is zero.
-      return this.fallback(state, `model error: ${(err as Error).message}`);
-    }
-    if (rawText === undefined) {
-      return this.fallback(state, "model returned no text block", callUsage);
-    }
-
-    // Extract JSON: tolerate ```json fences and leading prose. We
-    // search for the first balanced `{...}` substring.
-    const jsonMatch = rawText.match(/\{[\s\S]*\}/);
-    if (jsonMatch === null) {
-      return this.fallback(state, "model response did not contain a JSON object", callUsage);
-    }
-    let parsed: { rewrite: string; rationale: string };
-    try {
-      const raw = JSON.parse(jsonMatch[0]);
-      parsed = META_RESPONSE_SCHEMA.parse(raw);
-    } catch (err) {
-      return this.fallback(
-        state,
-        `model response failed schema validation: ${(err as Error).message}`,
-        callUsage,
-      );
-    }
-
-    const mutation: Mutation = { kind: "rephrase-instruction" };
-    // We use the rephrase-instruction kind to record the mutation in
-    // the trajectory; the actual rewrite is full-replacement, not the
-    // rule-based provider's "append a sentence" behaviour. Future work:
-    // add a distinct `{ kind: "model-rewrite"; rationale: string }`
-    // variant to the Mutation union for better trajectory metadata.
-    return {
-      prompt: parsed.rewrite,
-      mutations: [mutation],
-      rationale: parsed.rationale,
-      ...(callUsage !== undefined ? { usage: callUsage } : {}),
-    };
-  }
-
-  /** Build the user message for the meta-prompt. */
-  private buildUserMessage(
-    currentPrompt: string,
-    failures: ReadonlyArray<{ input: string; observedScore: number; expected?: string }>,
-  ): string {
-    const failureBlock =
-      failures.length === 0
-        ? "(No dev-set failures available yet — propose a refinement that improves clarity, specificity, or instruction-following.)"
-        : failures
-            .map(
-              (f, i) =>
-                `--- Failure ${i + 1} (observed score ${f.observedScore.toFixed(2)}) ---\nInput: ${f.input}\n${f.expected !== undefined ? `Expected output (do NOT copy this verbatim into the rewrite): ${f.expected}\n` : ""}`,
-            )
-            .join("\n");
-    return `CURRENT PROMPT:\n${currentPrompt}\n\nSAMPLE OF DEV-SET FAILURES:\n${failureBlock}\n\nReturn one JSON object: {"rewrite": "...", "rationale": "..."}`;
-  }
-
-  /** Identify failure samples from the trajectory. */
-  private selectFailures(
-    state: OptimizerState,
-  ): ReadonlyArray<{ input: string; observedScore: number; expected?: string }> {
-    // The trajectory records aggregate scores per candidate, not per
-    // sample. For v0 we surface the dev set as raw inputs and let
-    // the model reason about them generically; future iterations can
-    // wire a per-sample grade map through the OptimizerState. The
-    // result is still useful because the model sees the dev set
-    // distribution even without per-sample grades.
-    return state.devSet.slice(0, this.maxFailuresInPrompt).map((s) => ({
-      input: s.input,
-      // Use the trajectory's most recent score as a coarse signal.
-      observedScore: state.best.score,
-      ...(s.expected_output !== undefined ? { expected: s.expected_output } : {}),
-    }));
-  }
-
-  /**
-   * Fallback when the model can't produce a usable rewrite. Returns
-   * the current best verbatim so the search loop records a no-op
-   * iteration. The orchestrator logs the fallback reason. When the
-   * model DID round-trip (returned text/JSON that turned out unusable),
-   * the `usage` actually consumed is forwarded so the cost-gate still
-   * accounts for the spend; a stream error (no completed call) passes
-   * no usage and is therefore charged zero.
-   */
-  private fallback(
-    state: OptimizerState,
-    reason: string,
-    usage?: ProviderMutation["usage"],
-  ): ProviderMutation {
-    return {
-      prompt: state.best.prompt,
-      mutations: [],
-      rationale: `claude-fallback: ${reason}`,
-      ...(usage !== undefined ? { usage } : {}),
-    };
-  }
-}
-
-/** Convenience factory mirroring `createAnthropicAdapter` ergonomics. */
-export function createClaudeMutationProvider(
-  opts: ClaudeMutationProviderOptions,
-): ClaudeMutationProvider {
-  return new ClaudeMutationProvider(opts);
-}