@crewhaus/prompt-optimizer-claude 0.1.1 → 0.1.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@crewhaus/prompt-optimizer-claude",
-  "version": "0.1.1",
+  "version": "0.1.3",
   "type": "module",
   "description": "Pillar-2 model-driven MutationProvider — asks Claude to rewrite the prompt given dev-set failures. Closes the v0 prompt-optimizer's L91 'rule-based-only' gap.",
   "main": "src/index.ts",
@@ -13,19 +13,19 @@
   },
   "dependencies": {
     "@anthropic-ai/sdk": "^0.96.0",
-    "@crewhaus/adapter-anthropic": "0.1.1",
-    "@crewhaus/errors": "0.1.1",
-    "@crewhaus/prompt-optimizer": "0.1.1",
+    "@crewhaus/adapter-anthropic": "0.1.3",
+    "@crewhaus/errors": "0.1.3",
+    "@crewhaus/prompt-optimizer": "0.1.3",
     "zod": "^3.23.8"
   },
   "devDependencies": {
-    "@crewhaus/eval-dataset": "0.1.1"
+    "@crewhaus/eval-dataset": "0.1.3"
   },
   "license": "Apache-2.0",
   "author": {
     "name": "Max Meier",
-    "email": "max@studiomax.io",
-    "url": "https://studiomax.io"
+    "email": "max@crewhaus.ai",
+    "url": "https://crewhaus.ai"
   },
   "repository": {
     "type": "git",
@@ -37,12 +37,7 @@
     "url": "https://github.com/crewhaus/factory/issues"
   },
   "publishConfig": {
-    "access": "restricted"
-  },
-  "files": [
-    "src",
-    "README.md",
-    "LICENSE",
-    "NOTICE"
-  ]
+    "access": "public"
+  },
+  "files": ["src", "README.md", "LICENSE", "NOTICE"]
 }

package/src/index.test.ts

@@ -2,7 +2,11 @@ import { describe, expect, test } from "bun:test";
 import type { ProviderAdapter } from "@crewhaus/adapter-anthropic";
 import type { Sample } from "@crewhaus/eval-dataset";
 import type { OptimizerState } from "@crewhaus/prompt-optimizer";
-import { ClaudeMutationProvider } from "./index";
+import {
+  ClaudeMutationProvider,
+  ClaudeMutationProviderError,
+  createClaudeMutationProvider,
+} from "./index";
 
 const SAMPLE_TRAIN: ReadonlyArray<Sample> = [
   { id: "t1", input: "What is 2+2?", expected_output: "4" },
@@ -20,7 +24,10 @@ const SAMPLE_DEV: ReadonlyArray<Sample> = [
  * Matches the canonical event protocol consumed by
  * `consumeStream`/`collectFinalMessage`.
  */
-function mockAdapter(content: string): ProviderAdapter {
+function mockAdapter(
+  content: string,
+  usage: { input: number; output: number; cacheRead?: number } = { input: 0, output: 0 },
+): ProviderAdapter {
   return {
     id: "mock",
     features: {
@@ -31,7 +38,16 @@ function mockAdapter(content: string): ProviderAdapter {
     // biome-ignore lint/suspicious/noExplicitAny: minimal mock
     stream(_params: any): AsyncIterable<any> {
       return (async function* () {
-        yield { kind: "message_start", usage: { input: 0, output: 0 } };
+        // message_start carries input (+ cache) counts; message_delta
+        // carries the running output count — matches the real protocol.
+        yield {
+          kind: "message_start",
+          usage: {
+            input: usage.input,
+            output: 0,
+            ...(usage.cacheRead !== undefined ? { cacheRead: usage.cacheRead } : {}),
+          },
+        };
         yield {
           kind: "content_block_start",
           index: 0,
@@ -43,7 +59,11 @@ function mockAdapter(content: string): ProviderAdapter {
           delta: { type: "text_delta", text: content },
         };
         yield { kind: "content_block_stop", index: 0 };
-        yield { kind: "message_delta", stopReason: "end_turn" };
+        yield {
+          kind: "message_delta",
+          stopReason: "end_turn",
+          usage: { input: usage.input, output: usage.output },
+        };
         yield { kind: "message_stop" };
       })();
     },
@@ -64,6 +84,16 @@ const baseState: OptimizerState = {
 };
 
 describe("ClaudeMutationProvider", () => {
+  test("exposes the adapter's providerId for the FR-003 cost-gate (provider-agnostic pricing)", () => {
+    const adapter = {
+      ...mockAdapter("{}"),
+      providerId: "openai",
+    } as unknown as ProviderAdapter;
+    const provider = new ClaudeMutationProvider({ adapter, model: "gpt-4o-mini" });
+    expect(provider.providerId).toBe("openai");
+    expect(provider.modelId).toBe("gpt-4o-mini");
+  });
+
   test("parses a clean JSON response and returns it as the rewrite", async () => {
     const adapter = mockAdapter(
       `{"rewrite": "Think step by step before answering. Be concise.", "rationale": "Adds explicit chain-of-thought scaffolding which helps on multi-step problems."}`,
@@ -148,4 +178,147 @@ describe("ClaudeMutationProvider", () => {
     });
     expect(provider.name).toBe("claude");
   });
+
+  // FR-003 — the provider surfaces per-call usage + exposes pricing getters.
+  test("returns per-call usage from the response on the success path", async () => {
+    const adapter = mockAdapter(`{"rewrite": "Be concise.", "rationale": "Removes ambiguity."}`, {
+      input: 1234,
+      output: 567,
+      cacheRead: 89,
+    });
+    const provider = new ClaudeMutationProvider({ adapter, model: "claude-sonnet-4-5" });
+    const result = await provider.next(baseState);
+    expect(result.prompt).toBe("Be concise.");
+    expect(result.usage).toEqual({ input: 1234, output: 567, cacheRead: 89 });
+  });
+
+  test("forwards consumed usage even when the response is unusable (JSON miss)", async () => {
+    const adapter = mockAdapter("not actually json", { input: 100, output: 20 });
+    const provider = new ClaudeMutationProvider({ adapter, model: "claude-sonnet-4-5" });
+    const result = await provider.next(baseState);
+    // Fell back to current best, but the spend the model DID incur is reported.
+    expect(result.prompt).toBe(baseState.best.prompt);
+    expect(result.rationale).toContain("claude-fallback");
+    expect(result.usage).toEqual({ input: 100, output: 20 });
+  });
+
+  test("omits usage when the model call never completes (stream error)", async () => {
+    const adapter = {
+      id: "mock",
+      features: { caching: "none", thinking: false, multimodal: { input: false, output: false } },
+      // biome-ignore lint/suspicious/noExplicitAny: minimal mock
+      stream(_params: any): AsyncIterable<any> {
+        return (async function* () {
+          throw new Error("model unavailable");
+          // biome-ignore lint/correctness/noUnreachable: keep typed as async generator
+          yield;
+        })();
+      },
+    } as unknown as ProviderAdapter;
+    const provider = new ClaudeMutationProvider({ adapter, model: "claude-sonnet-4-5" });
+    const result = await provider.next(baseState);
+    expect(result.prompt).toBe(baseState.best.prompt);
+    expect(result.usage).toBeUndefined();
+  });
+
+  test("exposes modelId + maxOutputTokens getters for the cost-gate", () => {
+    const provider = new ClaudeMutationProvider({
+      adapter: mockAdapter(""),
+      model: "claude-opus-4-7",
+      maxTokens: 4096,
+    });
+    expect(provider.modelId).toBe("claude-opus-4-7");
+    expect(provider.maxOutputTokens).toBe(4096);
+  });
+
+  test("maxOutputTokens getter defaults to 2048", () => {
+    const provider = new ClaudeMutationProvider({
+      adapter: mockAdapter(""),
+      model: "claude-sonnet-4-5",
+    });
+    expect(provider.maxOutputTokens).toBe(2048);
+  });
+
+  // FR-003 (input-estimate gap) — estimateInputChars must count the FULL
+  // serialized input (system block + rendered dev-set failure block), not
+  // just the candidate prompt, so the orchestrator's cost-gate cannot
+  // under-count input for a wide dev window.
+  test("estimateInputChars counts the system block + rendered failure block", () => {
+    const provider = new ClaudeMutationProvider({
+      adapter: mockAdapter(""),
+      model: "claude-sonnet-4-5",
+    });
+    const est = provider.estimateInputChars(baseState);
+    // Strictly greater than the prompt length alone — the deficit the naive
+    // `best.prompt.length` estimate would have missed.
+    expect(est).toBeGreaterThan(baseState.best.prompt.length);
+    // The estimate covers the rendered failure block: it exceeds the prompt
+    // length PLUS the combined dev-set input + expected_output text that
+    // next() serializes — exactly the content the naive `best.prompt.length`
+    // heuristic ignored.
+    const devTextChars = SAMPLE_DEV.reduce(
+      (acc, s) => acc + s.input.length + (s.expected_output?.length ?? 0),
+      0,
+    );
+    expect(est).toBeGreaterThan(baseState.best.prompt.length + devTextChars);
+  });
+
+  test("createClaudeMutationProvider factory yields an equivalent provider", async () => {
+    const adapter = mockAdapter(
+      `{"rewrite": "Be concise and exact.", "rationale": "Tightens the instruction."}`,
+    );
+    const provider = createClaudeMutationProvider({
+      adapter,
+      model: "claude-sonnet-4-5",
+      maxTokens: 1024,
+    });
+    expect(provider).toBeInstanceOf(ClaudeMutationProvider);
+    expect(provider.name).toBe("claude");
+    expect(provider.modelId).toBe("claude-sonnet-4-5");
+    expect(provider.maxOutputTokens).toBe(1024);
+    // Behaves like the directly-constructed provider end-to-end.
+    const result = await provider.next(baseState);
+    expect(result.prompt).toBe("Be concise and exact.");
+  });
+
+  test("estimateInputChars grows with the dev-set failure window", () => {
+    const wideDev: ReadonlyArray<Sample> = [
+      ...SAMPLE_DEV,
+      { id: "d3", input: "x".repeat(5_000), expected_output: "y".repeat(5_000) },
+      { id: "d4", input: "z".repeat(5_000), expected_output: "w".repeat(5_000) },
+    ];
+    const provider = new ClaudeMutationProvider({
+      adapter: mockAdapter(""),
+      model: "claude-sonnet-4-5",
+      maxFailuresInPrompt: 4,
+    });
+    const narrow = provider.estimateInputChars(baseState);
+    const wide = provider.estimateInputChars({ ...baseState, devSet: wideDev });
+    // A larger window serializes more failure text → a larger input estimate,
+    // exactly the case the original prompt-only heuristic under-counted.
+    expect(wide).toBeGreaterThan(narrow + 18_000);
+  });
+});
+
+describe("ClaudeMutationProviderError", () => {
+  test("carries the adapter code, stable name, and cause chain", () => {
+    const cause = new Error("stream aborted");
+    const err = new ClaudeMutationProviderError("mutation provider failed", cause);
+    expect(err).toBeInstanceOf(Error);
+    expect(err.name).toBe("ClaudeMutationProviderError");
+    expect(err.code).toBe("adapter");
+    expect(err.message).toBe("mutation provider failed");
+    expect(err.cause).toBe(cause);
+    expect(err.toJSON()).toMatchObject({
+      name: "ClaudeMutationProviderError",
+      code: "adapter",
+      message: "mutation provider failed",
+      cause: { name: "Error", message: "stream aborted" },
+    });
+  });
+
+  test("constructs without a cause", () => {
+    const err = new ClaudeMutationProviderError("no cause");
+    expect(err.cause).toBeUndefined();
+  });
 });

package/src/index.ts

@@ -22,10 +22,14 @@
  *      the search — see the rule-based provider as the deterministic
  *      fallback path).
  *
- * Cost: each call is one Claude request (~1-3K tokens). A cost-gate
- * (`--budget-usd`) backed by the §27 `cost-tracker` running spend
- * total is a planned follow-up; today the only safety rail is the
- * orchestrator's `iterations` cap.
+ * Cost: each call is one Claude request (~1-3K tokens). Each `next()`
+ * now surfaces the call's actual token `usage` on the returned
+ * `ProviderMutation`, and the provider exposes its `modelId` +
+ * `maxOutputTokens` getters, so the FR-003 `--budget-usd` cost-gate in
+ * `eval-optimizer-orchestrator` can price every call against the §27
+ * `cost-tracker` table and stop before a mutation call would exceed
+ * the budget. The gate composes with the orchestrator's `iterations`
+ * cap (whichever bound is hit first ends the run).
  *
  * Catalog layer: F-eval (active optimisation). Brief: 280.
  */
@@ -105,10 +109,70 @@ export class ClaudeMutationProvider implements MutationProvider {
     this.systemBlock = opts.systemOverride ?? META_PROMPT_SYSTEM;
   }
 
+  /**
+   * The model id this provider calls, exposed read-only so the FR-003
+   * cost-gate can price each call via `resolvePricing(DEFAULT_PRICING,
+   * "anthropic", modelId)`. The `MutationProvider` interface only
+   * guarantees `name` + `next()`; the orchestrator feature-detects this
+   * getter and falls back to a zero-cost meter for providers that don't
+   * expose it.
+   */
+  get modelId(): string {
+    return this.model;
+  }
+
+  /**
+   * The provider id of the injected adapter, exposed read-only so the
+   * FR-003 cost-gate prices calls against the REAL provider's pricing
+   * table instead of assuming "anthropic". Feature-detected by the
+   * orchestrator like `modelId` — providers without the getter price as
+   * Anthropic (the historical behaviour).
+   */
+  get providerId(): string {
+    return this.adapter.providerId;
+  }
+
+  /**
+   * The output-token ceiling for each call, exposed read-only so the
+   * cost-gate can compute a worst-case pre-call estimate (the gate must
+   * decide BEFORE a call whether it would exceed the budget).
+   */
+  get maxOutputTokens(): number {
+    return this.maxTokens;
+  }
+
+  /**
+   * FR-003 — exact serialized INPUT character count this provider would
+   * transmit for `state`, so the cost-gate prices the *real* meta-prompt
+   * rather than just `best.prompt.length + a fixed overhead`. The naive
+   * estimate (prompt length only) under-counts the system block AND the
+   * rendered dev-set failure block (each failure's input + expected_output)
+   * that `next()` actually sends; with a large dev window + small maxTokens
+   * that deficit could let a gate-passing call exceed the budget after the
+   * fact. Returning the full system+user char count here makes the
+   * `chars/4` token estimate cover everything the model is billed for, so
+   * the orchestrator's estimate-before guarantee holds unconditionally
+   * (input is now bounded from above, output is already the ceiling).
+   *
+   * The orchestrator feature-detects this method (it is not part of the
+   * `MutationProvider` interface); providers that omit it fall back to the
+   * `best.prompt.length + metaOverheadChars` heuristic.
+   */
+  estimateInputChars(state: OptimizerState): number {
+    const failures = this.selectFailures(state);
+    const userMessage = this.buildUserMessage(state.best.prompt, failures);
+    return this.systemBlock.length + userMessage.length;
+  }
+
   async next(state: OptimizerState): Promise<ProviderMutation> {
     const failures = this.selectFailures(state);
     const userMessage = this.buildUserMessage(state.best.prompt, failures);
 
+    // The call's actual token usage, captured so the FR-003 cost-gate
+    // can fold it into a running spend total. Stays undefined until the
+    // model call round-trips; a failed/unusable response still records
+    // whatever was consumed (the model DID run) so spend is accounted.
+    let callUsage: ProviderMutation["usage"];
     let rawText: string | undefined;
     try {
       const final = await collectFinalMessage(
@@ -119,22 +183,28 @@ export class ClaudeMutationProvider implements MutationProvider {
           maxTokens: this.maxTokens,
         }),
       );
+      callUsage = {
+        input: final.usage.input,
+        output: final.usage.output,
+        ...(final.usage.cacheRead !== undefined ? { cacheRead: final.usage.cacheRead } : {}),
+      };
       rawText = extractFirstText(final);
     } catch (err) {
       // Mutator unavailability is not fatal — fall back to current best.
       // The orchestrator's outer loop will record a degenerate iteration
-      // (score unchanged) and the search continues.
+      // (score unchanged) and the search continues. No usage is available
+      // on a stream error (the call did not complete), so spend is zero.
       return this.fallback(state, `model error: ${(err as Error).message}`);
     }
     if (rawText === undefined) {
-      return this.fallback(state, "model returned no text block");
+      return this.fallback(state, "model returned no text block", callUsage);
     }
 
     // Extract JSON: tolerate ```json fences and leading prose. We
     // search for the first balanced `{...}` substring.
     const jsonMatch = rawText.match(/\{[\s\S]*\}/);
     if (jsonMatch === null) {
-      return this.fallback(state, "model response did not contain a JSON object");
+      return this.fallback(state, "model response did not contain a JSON object", callUsage);
     }
     let parsed: { rewrite: string; rationale: string };
     try {
@@ -144,6 +214,7 @@ export class ClaudeMutationProvider implements MutationProvider {
       return this.fallback(
         state,
         `model response failed schema validation: ${(err as Error).message}`,
+        callUsage,
       );
     }
 
@@ -157,6 +228,7 @@ export class ClaudeMutationProvider implements MutationProvider {
       prompt: parsed.rewrite,
       mutations: [mutation],
       rationale: parsed.rationale,
+      ...(callUsage !== undefined ? { usage: callUsage } : {}),
     };
   }
 
@@ -198,13 +270,22 @@ export class ClaudeMutationProvider implements MutationProvider {
   /**
    * Fallback when the model can't produce a usable rewrite. Returns
    * the current best verbatim so the search loop records a no-op
-   * iteration. The orchestrator logs the fallback reason.
+   * iteration. The orchestrator logs the fallback reason. When the
+   * model DID round-trip (returned text/JSON that turned out unusable),
+   * the `usage` actually consumed is forwarded so the cost-gate still
+   * accounts for the spend; a stream error (no completed call) passes
+   * no usage and is therefore charged zero.
    */
-  private fallback(state: OptimizerState, reason: string): ProviderMutation {
+  private fallback(
+    state: OptimizerState,
+    reason: string,
+    usage?: ProviderMutation["usage"],
+  ): ProviderMutation {
     return {
       prompt: state.best.prompt,
       mutations: [],
       rationale: `claude-fallback: ${reason}`,
+      ...(usage !== undefined ? { usage } : {}),
     };
   }
 }