@crewhaus/prompt-optimizer-claude 0.1.6 → 0.1.7

package/dist/index.d.ts

@@ -115,7 +115,17 @@ export declare class ClaudeMutationProvider implements MutationProvider {
     next(state: OptimizerState): Promise<ProviderMutation>;
     /** Build the user message for the meta-prompt. */
     private buildUserMessage;
-    /** Identify failure samples from the trajectory. */
+    /**
+     * Identify the failing samples to show the model. When the fitness
+     * function supplied per-sample grades (`state.bestGrades`, wired by the
+     * CLI's eval-runner closure), we surface the samples the current best
+     * prompt ACTUALLY fails — worst-scoring first — together with the
+     * grader's rationale, so the meta-prompt can address the named root
+     * cause. This is the signal the system prompt promises. When grades are
+     * absent (a fitness fn returning a bare number, or an all-passing dev
+     * set), we fall back to surfacing the dev-set inputs with the aggregate
+     * score — the pre-failure-signal behaviour — so the search still runs.
+     */
     private selectFailures;
     /**
      * Fallback when the model can't produce a usable rewrite. Returns

package/dist/index.js

@@ -42,11 +42,12 @@ export class ClaudeMutationProviderError extends CrewhausError {
         super("adapter", message, cause);
     }
 }
-const META_PROMPT_SYSTEM = `You are a prompt-engineering optimiser. You will receive a CURRENT PROMPT and a SAMPLE OF DEV-SET FAILURES (each failure shows the input the model received and how the grader scored its output). Your job is to produce a single rewrite of CURRENT PROMPT that you believe will improve grader scores on the dev set.
+const META_PROMPT_SYSTEM = `You are a prompt-engineering optimiser. You will receive a CURRENT PROMPT and a SAMPLE OF DEV-SET FAILURES. Each failure shows the input the model received, its observed score (0..1, lower is worse), and — when available — the grader's feedback explaining WHY the output lost points. Your job is to produce a single rewrite of CURRENT PROMPT that you believe will improve grader scores on the dev set.
 
 Hard rules:
 - Output exactly one JSON object: {"rewrite": "...", "rationale": "..."}
 - The "rewrite" field is the new prompt verbatim. Do NOT include any wrapper text outside the JSON.
+- Read the grader feedback and address the ROOT CAUSE it names (e.g. "cites no source", "too verbose", "wrong format") with a general instruction — not a fix hard-coded to these specific inputs.
 - Never copy verbatim text from a failure's expected_output into the rewrite (that would leak dev-set answers).
 - Do not introduce instructions that override safety, compliance, or permission rules — your job is to improve task accuracy, not to bypass guardrails.
 - The rationale should be 1-3 sentences explaining WHY this rewrite is likely to help. Keep it specific.`;
@@ -190,21 +191,41 @@ export class ClaudeMutationProvider {
         const failureBlock = failures.length === 0
             ? "(No dev-set failures available yet — propose a refinement that improves clarity, specificity, or instruction-following.)"
             : failures
-                .map((f, i) => `--- Failure ${i + 1} (observed score ${f.observedScore.toFixed(2)}) ---\nInput: ${f.input}\n${f.expected !== undefined ? `Expected output (do NOT copy this verbatim into the rewrite): ${f.expected}\n` : ""}`)
+                .map((f, i) => `--- Failure ${i + 1} (observed score ${f.observedScore.toFixed(2)}) ---\nInput: ${f.input}\n${f.expected !== undefined ? `Expected output (do NOT copy this verbatim into the rewrite): ${f.expected}\n` : ""}${f.rationale !== undefined ? `Grader feedback: ${f.rationale}\n` : ""}`)
                 .join("\n");
         return `CURRENT PROMPT:\n${currentPrompt}\n\nSAMPLE OF DEV-SET FAILURES:\n${failureBlock}\n\nReturn one JSON object: {"rewrite": "...", "rationale": "..."}`;
     }
-    /** Identify failure samples from the trajectory. */
+    /**
+     * Identify the failing samples to show the model. When the fitness
+     * function supplied per-sample grades (`state.bestGrades`, wired by the
+     * CLI's eval-runner closure), we surface the samples the current best
+     * prompt ACTUALLY fails — worst-scoring first — together with the
+     * grader's rationale, so the meta-prompt can address the named root
+     * cause. This is the signal the system prompt promises. When grades are
+     * absent (a fitness fn returning a bare number, or an all-passing dev
+     * set), we fall back to surfacing the dev-set inputs with the aggregate
+     * score — the pre-failure-signal behaviour — so the search still runs.
+     */
     selectFailures(state) {
-        // The trajectory records aggregate scores per candidate, not per
-        // sample. For v0 we surface the dev set as raw inputs and let
-        // the model reason about them generically; future iterations can
-        // wire a per-sample grade map through the OptimizerState. The
-        // result is still useful because the model sees the dev set
-        // distribution even without per-sample grades.
+        const graded = state.bestGrades;
+        if (graded !== undefined && graded.length > 0) {
+            const byWorst = [...graded].sort((a, b) => a.score - b.score);
+            // Prefer genuine failures (below the threshold); if the dev set is
+            // already strong, fall back to the lowest scorers so the mutator
+            // still has a concrete target to push on.
+            const belowThreshold = byWorst.filter((g) => g.score < this.failureThreshold);
+            const window = (belowThreshold.length > 0 ? belowThreshold : byWorst).slice(0, this.maxFailuresInPrompt);
+            return window.map((g) => ({
+                input: g.input,
+                observedScore: g.score,
+                ...(g.expected !== undefined ? { expected: g.expected } : {}),
+                ...(g.rationale !== undefined ? { rationale: g.rationale } : {}),
+            }));
+        }
+        // No per-sample grades wired: surface the dev set as raw inputs with
+        // the aggregate score. The model still sees the dev-set distribution.
         return state.devSet.slice(0, this.maxFailuresInPrompt).map((s) => ({
             input: s.input,
-            // Use the trajectory's most recent score as a coarse signal.
             observedScore: state.best.score,
             ...(s.expected_output !== undefined ? { expected: s.expected_output } : {}),
         }));

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@crewhaus/prompt-optimizer-claude",
-  "version": "0.1.6",
+  "version": "0.1.7",
   "type": "module",
   "description": "Pillar-2 model-driven MutationProvider — asks Claude to rewrite the prompt given dev-set failures. Closes the v0 prompt-optimizer's L91 'rule-based-only' gap.",
   "main": "dist/index.js",
@@ -16,13 +16,13 @@
   },
   "dependencies": {
     "@anthropic-ai/sdk": "^0.96.0",
-    "@crewhaus/adapter-anthropic": "0.1.6",
-    "@crewhaus/errors": "0.1.6",
-    "@crewhaus/prompt-optimizer": "0.1.6",
+    "@crewhaus/adapter-anthropic": "0.1.7",
+    "@crewhaus/errors": "0.1.7",
+    "@crewhaus/prompt-optimizer": "0.1.7",
     "zod": "^3.23.8"
   },
   "devDependencies": {
-    "@crewhaus/eval-dataset": "0.1.6"
+    "@crewhaus/eval-dataset": "0.1.7"
   },
   "license": "Apache-2.0",
   "author": {