@enricai/barnacle 1.0.0

package/dist/scripts/judge-llm-batch.d.ts

@@ -0,0 +1,98 @@
+/**
+ * LLM batch judge: reads a calls.ndjson capture file, filters by callType,
+ * scores each sample on a three-dimensional rubric (schema adherence, factual
+ * grounding, hallucination-freeness), aggregates the results, and writes a
+ * schema-valid verdict JSON to the judge-out directory.
+ *
+ * Usage:
+ *   pnpm judge:llm \
+ *     --calls-ndjson .barnacle/calls.ndjson \
+ *     --call-type act \
+ *     --batch-index 0 \
+ *     --judge-model claude-sonnet-4-6
+ *
+ * Knobs (all optional):
+ *   --out-dir <path>        default judge-out
+ *   --dry-run               stubs the scorer (CI mode, no API calls)
+ *   --batch-index <N>       default 0; disambiguates multiple verdict files
+ */
+import Anthropic from "@anthropic-ai/sdk";
+import { type JudgeAggregate, type JudgeVerdict, type JudgeVerdictEntry, type LlmCallSample } from "../api/schemas/telemetry";
+/** Raw scoring dimensions returned by the scorer for a single sample. */
+export interface SampleScore {
+    schemaOk: boolean;
+    schemaRationale: string;
+    factuallyGrounded: boolean;
+    factualRationale: string;
+    hallucinationFree: boolean;
+    hallucinationRationale: string;
+    worstOffender?: string;
+}
+/**
+ * Injectable scorer — real implementation calls the LLM; dry-run stub
+ * returns deterministic values. Mirrors recon-heal's requestPatchFn injection
+ * so the deterministic core (parse/aggregate/write) is fully unit-testable.
+ */
+export type ScorerFn = (sample: LlmCallSample) => Promise<SampleScore>;
+/**
+ * Reads an NDJSON file and returns the lines parsed as LlmCallSample objects.
+ * Lines that fail JSON parsing or Zod validation are returned with
+ * parsedOk=false so they auto-fail schema adherence in the judge pass.
+ */
+export declare function parseSamples(ndjsonContent: string): LlmCallSample[];
+/**
+ * Filters samples to only those matching the requested callType.
+ */
+export declare function filterByCallType(samples: LlmCallSample[], callType: string): LlmCallSample[];
+/**
+ * Derives a JudgeVerdictEntry from a sample and its score. Enforces the rule
+ * that parsedOk=false automatically fails schema adherence regardless of what
+ * the scorer returns.
+ */
+export declare function computeVerdict(sample: LlmCallSample, score: SampleScore): JudgeVerdictEntry;
+/**
+ * Computes aggregate counts over a set of verdict entries.
+ */
+export declare function aggregate(verdicts: JudgeVerdictEntry[]): JudgeAggregate;
+/**
+ * Validates and writes the verdict object as JSON to the judge-out directory.
+ * Returns the written file path.
+ */
+export declare function writeVerdict(params: {
+    outDir: string;
+    verdict: JudgeVerdict;
+}): string;
+/**
+ * Dry-run scorer: marks every dimension true with a placeholder rationale.
+ * Used in CI and tests to exercise the full pipeline without any API call.
+ */
+export declare function makeDryRunScorer(): ScorerFn;
+/**
+ * Real LLM scorer: sends each sample to the Anthropic API for judgment.
+ * Defaults to the configured scraper model (respects STAGEHAND_MODEL env var)
+ * unless overridden by `judgeModel`.
+ */
+export declare function makeAnthropicScorer(client: Anthropic, judgeModel?: string): ScorerFn;
+/** Parameters accepted by `runJudge`. */
+export interface JudgeParams {
+    callsNdjsonPath: string;
+    callType: string;
+    batchIndex?: number;
+    judgeModel?: string;
+    outDir?: string;
+    dryRun?: boolean;
+    /** Injectable scorer for unit tests. */
+    scorerFn?: ScorerFn;
+}
+/** Return value of `runJudge`. */
+export interface JudgeResult {
+    verdictPath: string;
+    verdict: JudgeVerdict;
+}
+/**
+ * Drives the full parse → filter → score → aggregate → write pipeline for a
+ * single batch. Accepts an injectable `scorerFn` so tests exercise the full
+ * pipeline without any live model call.
+ */
+export declare function runJudge(params: JudgeParams): Promise<JudgeResult>;
+//# sourceMappingURL=judge-llm-batch.d.ts.map

package/dist/scripts/judge-llm-batch.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"judge-llm-batch.d.ts","sourceRoot":"","sources":["../../src/scripts/judge-llm-batch.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAKH,OAAO,SAAS,MAAM,mBAAmB,CAAC;AAG1C,OAAO,EACL,KAAK,cAAc,EACnB,KAAK,YAAY,EACjB,KAAK,iBAAiB,EAEtB,KAAK,aAAa,EAEnB,MAAM,yBAAyB,CAAC;AAcjC,yEAAyE;AACzE,MAAM,WAAW,WAAW;IAC1B,QAAQ,EAAE,OAAO,CAAC;IAClB,eAAe,EAAE,MAAM,CAAC;IACxB,iBAAiB,EAAE,OAAO,CAAC;IAC3B,gBAAgB,EAAE,MAAM,CAAC;IACzB,iBAAiB,EAAE,OAAO,CAAC;IAC3B,sBAAsB,EAAE,MAAM,CAAC;IAC/B,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAED;;;;GAIG;AACH,MAAM,MAAM,QAAQ,GAAG,CAAC,MAAM,EAAE,aAAa,KAAK,OAAO,CAAC,WAAW,CAAC,CAAC;AAIvE;;;;GAIG;AACH,wBAAgB,YAAY,CAAC,aAAa,EAAE,MAAM,GAAG,aAAa,EAAE,CAyBnE;AAqBD;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,aAAa,EAAE,EAAE,QAAQ,EAAE,MAAM,GAAG,aAAa,EAAE,CAE5F;AAED;;;;GAIG;AACH,wBAAgB,cAAc,CAAC,MAAM,EAAE,aAAa,EAAE,KAAK,EAAE,WAAW,GAAG,iBAAiB,CAmB3F;AAED;;GAEG;AACH,wBAAgB,SAAS,CAAC,QAAQ,EAAE,iBAAiB,EAAE,GAAG,cAAc,CAevE;AAED;;;GAGG;AACH,wBAAgB,YAAY,CAAC,MAAM,EAAE;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,YAAY,CAAA;CAAE,GAAG,MAAM,CAatF;AAID;;;GAGG;AACH,wBAAgB,gBAAgB,IAAI,QAAQ,CAS3C;AAuBD;;;;GAIG;AACH,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,SAAS,EAAE,UAAU,CAAC,EAAE,MAAM,GAAG,QAAQ,CA4CpF;AAeD,yCAAyC;AACzC,MAAM,WAAW,WAAW;IAC1B,eAAe,EAAE,MAAM,CAAC;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,wCAAwC;IACxC,QAAQ,CAAC,EAAE,QAAQ,CAAC;CACrB;AAED,kCAAkC;AAClC,MAAM,WAAW,WAAW;IAC1B,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,YAAY,CAAC;CACvB;AAED;;;;GAIG;AACH,wBAAsB,QAAQ,CAAC,MAAM,EAAE,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC,CAqExE"}

package/dist/scripts/judge-llm-batch.js

@@ -0,0 +1,350 @@
+"use strict";
+/**
+ * LLM batch judge: reads a calls.ndjson capture file, filters by callType,
+ * scores each sample on a three-dimensional rubric (schema adherence, factual
+ * grounding, hallucination-freeness), aggregates the results, and writes a
+ * schema-valid verdict JSON to the judge-out directory.
+ *
+ * Usage:
+ *   pnpm judge:llm \
+ *     --calls-ndjson .barnacle/calls.ndjson \
+ *     --call-type act \
+ *     --batch-index 0 \
+ *     --judge-model claude-sonnet-4-6
+ *
+ * Knobs (all optional):
+ *   --out-dir <path>        default judge-out
+ *   --dry-run               stubs the scorer (CI mode, no API calls)
+ *   --batch-index <N>       default 0; disambiguates multiple verdict files
+ */
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parseSamples = parseSamples;
+exports.filterByCallType = filterByCallType;
+exports.computeVerdict = computeVerdict;
+exports.aggregate = aggregate;
+exports.writeVerdict = writeVerdict;
+exports.makeDryRunScorer = makeDryRunScorer;
+exports.makeAnthropicScorer = makeAnthropicScorer;
+exports.runJudge = runJudge;
+const node_fs_1 = require("node:fs");
+const node_path_1 = require("node:path");
+const sdk_1 = __importDefault(require("@anthropic-ai/sdk"));
+const zod_1 = require("@anthropic-ai/sdk/helpers/zod");
+const date_fns_1 = require("date-fns");
+const telemetry_1 = require("../api/schemas/telemetry");
+const config_1 = require("../config");
+const schemas_1 = require("../lib/llm/schemas");
+const logging_1 = require("../lib/logging");
+const logger = (0, logging_1.getScriptLogger)("judge-llm-batch");
+// ── defaults ──────────────────────────────────────────────────────────────────
+const DEFAULT_OUT_DIR = "judge-out";
+const DEFAULT_BATCH_INDEX = 0;
+// ── pure core ─────────────────────────────────────────────────────────────────
+/**
+ * Reads an NDJSON file and returns the lines parsed as LlmCallSample objects.
+ * Lines that fail JSON parsing or Zod validation are returned with
+ * parsedOk=false so they auto-fail schema adherence in the judge pass.
+ */
+function parseSamples(ndjsonContent) {
+    const samples = [];
+    for (const raw of ndjsonContent.split("\n")) {
+        const line = raw.trim();
+        if (line.length === 0)
+            continue;
+        let parsed;
+        try {
+            parsed = JSON.parse(line);
+        }
+        catch {
+            samples.push(makeMalformedSample(line));
+            continue;
+        }
+        const result = telemetry_1.llmCallSampleSchema.safeParse(parsed);
+        if (!result.success) {
+            samples.push(makeMalformedSample(line));
+            continue;
+        }
+        samples.push(result.data);
+    }
+    return samples;
+}
+function makeMalformedSample(raw) {
+    return {
+        callId: `malformed-${raw.slice(0, 16)}`,
+        callType: "unknown",
+        model: "unknown",
+        systemPrompt: null,
+        userContent: raw.slice(0, 256),
+        responseContent: null,
+        parsedOk: false,
+        inputTokens: null,
+        outputTokens: null,
+        latencyMs: null,
+        success: false,
+        errorMessage: null,
+        failureKind: null,
+        ts: (0, date_fns_1.formatISO)(new Date()),
+    };
+}
+/**
+ * Filters samples to only those matching the requested callType.
+ */
+function filterByCallType(samples, callType) {
+    return samples.filter((s) => s.callType === callType);
+}
+/**
+ * Derives a JudgeVerdictEntry from a sample and its score. Enforces the rule
+ * that parsedOk=false automatically fails schema adherence regardless of what
+ * the scorer returns.
+ */
+function computeVerdict(sample, score) {
+    const schemaOk = sample.parsedOk ? score.schemaOk : false;
+    const schemaRationale = sample.parsedOk
+        ? score.schemaRationale
+        : "parsedOk=false: response could not be parsed as the expected schema";
+    const pass = schemaOk && score.factuallyGrounded && score.hallucinationFree;
+    return {
+        callId: sample.callId,
+        schemaOk,
+        schemaRationale,
+        factuallyGrounded: score.factuallyGrounded,
+        factualRationale: score.factualRationale,
+        hallucinationFree: score.hallucinationFree,
+        hallucinationRationale: score.hallucinationRationale,
+        ...(score.worstOffender !== undefined ? { worstOffender: score.worstOffender } : {}),
+        pass,
+    };
+}
+/**
+ * Computes aggregate counts over a set of verdict entries.
+ */
+function aggregate(verdicts) {
+    const n = verdicts.length;
+    let schemaPass = 0;
+    let factualPass = 0;
+    let hallucinationFreePass = 0;
+    let overallPass = 0;
+    for (const v of verdicts) {
+        if (v.schemaOk)
+            schemaPass++;
+        if (v.factuallyGrounded)
+            factualPass++;
+        if (v.hallucinationFree)
+            hallucinationFreePass++;
+        if (v.pass)
+            overallPass++;
+    }
+    return { n, schemaPass, factualPass, hallucinationFreePass, overallPass };
+}
+/**
+ * Validates and writes the verdict object as JSON to the judge-out directory.
+ * Returns the written file path.
+ */
+function writeVerdict(params) {
+    const { outDir, verdict } = params;
+    const parsed = telemetry_1.judgeVerdictSchema.safeParse(verdict);
+    if (!parsed.success) {
+        throw new Error(`verdict does not satisfy judgeVerdictSchema: ${parsed.error.message}`);
+    }
+    (0, node_fs_1.mkdirSync)(outDir, { recursive: true });
+    const fileName = `verdict-${verdict.callType}-${verdict.batchIndex}.json`;
+    const filePath = (0, node_path_1.join)(outDir, fileName);
+    (0, node_fs_1.writeFileSync)(filePath, JSON.stringify(verdict, null, 2));
+    return filePath;
+}
+// ── scorer implementations ────────────────────────────────────────────────────
+/**
+ * Dry-run scorer: marks every dimension true with a placeholder rationale.
+ * Used in CI and tests to exercise the full pipeline without any API call.
+ */
+function makeDryRunScorer() {
+    return async () => ({
+        schemaOk: true,
+        schemaRationale: "dry-run: schema adherence assumed",
+        factuallyGrounded: true,
+        factualRationale: "dry-run: factual grounding assumed",
+        hallucinationFree: true,
+        hallucinationRationale: "dry-run: hallucination-free assumed",
+    });
+}
+function anthropicModelName() {
+    const raw = config_1.config.scraper.model;
+    return raw.startsWith("anthropic/") ? raw.slice("anthropic/".length) : raw;
+}
+const SCORE_SYSTEM_PROMPT = `You are a quality judge for LLM call outputs. Evaluate each sample on three dimensions:
+
+1. SCHEMA_OK: Did the model's response match the expected output structure?
+   - true if responseContent is valid JSON that matches the callType's expected schema
+   - false if malformed, missing required fields, or has unexpected structure
+
+2. FACTUALLY_GROUNDED: Is the response factually grounded given the input?
+   - true if all factual claims in the response are consistent with the userContent context
+   - false if the response contradicts or ignores facts present in the userContent
+
+3. HALLUCINATION_FREE: Does the response avoid hallucinating content?
+   - true if the response contains no fabricated facts, URLs, entities, or values
+   - false if it invents information not implied by the prompt
+
+Set worstOffender to one of "schema", "factual", or "hallucination" naming the worst-failing dimension, or null when all three pass. Each rationale is one short sentence.`;
+/**
+ * Real LLM scorer: sends each sample to the Anthropic API for judgment.
+ * Defaults to the configured scraper model (respects STAGEHAND_MODEL env var)
+ * unless overridden by `judgeModel`.
+ */
+function makeAnthropicScorer(client, judgeModel) {
+    const model = judgeModel ?? anthropicModelName();
+    return async (sample) => {
+        const userMessage = [
+            `callType: ${sample.callType}`,
+            `model: ${sample.model}`,
+            `parsedOk: ${sample.parsedOk}`,
+            `systemPrompt: ${sample.systemPrompt ?? "(none)"}`,
+            `userContent: ${sample.userContent}`,
+            `responseContent: ${sample.responseContent ?? "(null)"}`,
+        ].join("\n");
+        try {
+            const response = await client.messages.parse({
+                model,
+                max_tokens: 512,
+                system: SCORE_SYSTEM_PROMPT,
+                messages: [{ role: "user", content: userMessage }],
+                output_config: {
+                    format: (0, zod_1.zodOutputFormat)(schemas_1.JUDGE_VERDICT_SCHEMA),
+                },
+            });
+            const parsed = response.parsed_output;
+            if (parsed === null) {
+                return fallbackScore("structured-output enabled but parsed_output is null");
+            }
+            return {
+                schemaOk: parsed.schemaOk,
+                schemaRationale: parsed.schemaRationale,
+                factuallyGrounded: parsed.factuallyGrounded,
+                factualRationale: parsed.factualRationale,
+                hallucinationFree: parsed.hallucinationFree,
+                hallucinationRationale: parsed.hallucinationRationale,
+                ...(typeof parsed.worstOffender === "string"
+                    ? { worstOffender: parsed.worstOffender }
+                    : {}),
+            };
+        }
+        catch (err) {
+            return fallbackScore(`API call threw an error: ${err instanceof Error ? err.message : String(err)}`);
+        }
+    };
+}
+function fallbackScore(reason) {
+    return {
+        schemaOk: false,
+        schemaRationale: `judge error: ${reason}`,
+        factuallyGrounded: false,
+        factualRationale: `judge error: ${reason}`,
+        hallucinationFree: false,
+        hallucinationRationale: `judge error: ${reason}`,
+    };
+}
+/**
+ * Drives the full parse → filter → score → aggregate → write pipeline for a
+ * single batch. Accepts an injectable `scorerFn` so tests exercise the full
+ * pipeline without any live model call.
+ */
+async function runJudge(params) {
+    const { callsNdjsonPath, callType, batchIndex = DEFAULT_BATCH_INDEX, judgeModel, outDir = DEFAULT_OUT_DIR, dryRun = false, } = params;
+    let scorerFn = params.scorerFn;
+    if (!scorerFn) {
+        if (dryRun) {
+            scorerFn = makeDryRunScorer();
+        }
+        else {
+            if (!config_1.config.scraper.anthropicApiKey) {
+                throw new Error("judge:llm requires ANTHROPIC_API_KEY");
+            }
+            const client = new sdk_1.default({ apiKey: config_1.config.scraper.anthropicApiKey });
+            scorerFn = makeAnthropicScorer(client, judgeModel);
+        }
+    }
+    const resolvedJudgeModel = judgeModel ?? (dryRun ? "dry-run" : anthropicModelName());
+    logger.info(`judge-llm-batch: reading ${callsNdjsonPath}`);
+    let ndjsonContent;
+    try {
+        ndjsonContent = (0, node_fs_1.readFileSync)(callsNdjsonPath, "utf-8");
+    }
+    catch (err) {
+        throw new Error(`judge-llm-batch: cannot read ${callsNdjsonPath}: ${String(err)}`);
+    }
+    const allSamples = parseSamples(ndjsonContent);
+    logger.info(`judge-llm-batch: parsed ${allSamples.length} total samples`);
+    const samples = filterByCallType(allSamples, callType);
+    logger.info(`judge-llm-batch: ${samples.length} samples for callType="${callType}"`);
+    const verdicts = [];
+    for (const sample of samples) {
+        logger.info(`judge-llm-batch: scoring callId=${sample.callId}`);
+        const score = await scorerFn(sample);
+        const entry = computeVerdict(sample, score);
+        verdicts.push(entry);
+        logger.info(`judge-llm-batch: callId=${sample.callId} pass=${entry.pass} schema=${entry.schemaOk} factual=${entry.factuallyGrounded} halluc=${entry.hallucinationFree}`);
+    }
+    const agg = aggregate(verdicts);
+    logger.info(`judge-llm-batch: aggregate n=${agg.n} schemaPass=${agg.schemaPass} factualPass=${agg.factualPass} hallucinationFreePass=${agg.hallucinationFreePass} overallPass=${agg.overallPass}`);
+    const verdict = {
+        callType,
+        batchIndex,
+        judgedAt: (0, date_fns_1.formatISO)(new Date()),
+        judgeModel: resolvedJudgeModel,
+        verdicts,
+        aggregate: agg,
+    };
+    const verdictPath = writeVerdict({ outDir, verdict });
+    logger.info(`judge-llm-batch: verdict written to ${verdictPath}`);
+    return { verdictPath, verdict };
+}
+// ── CLI ───────────────────────────────────────────────────────────────────────
+/**
+ * Parses CLI args and exits with a usage message when required args are absent.
+ */
+function parseCli() {
+    const args = process.argv.slice(2);
+    let callsNdjsonPath = "";
+    let callType = "";
+    let batchIndex = DEFAULT_BATCH_INDEX;
+    let judgeModel;
+    let outDir = DEFAULT_OUT_DIR;
+    let dryRun = false;
+    for (let i = 0; i < args.length; i++) {
+        if (args[i] === "--calls-ndjson" && args[i + 1])
+            callsNdjsonPath = args[++i];
+        else if (args[i] === "--call-type" && args[i + 1])
+            callType = args[++i];
+        else if (args[i] === "--batch-index" && args[i + 1])
+            batchIndex = Number(args[++i]);
+        else if (args[i] === "--judge-model" && args[i + 1])
+            judgeModel = args[++i];
+        else if (args[i] === "--out-dir" && args[i + 1])
+            outDir = args[++i];
+        else if (args[i] === "--dry-run")
+            dryRun = true;
+    }
+    if (!callsNdjsonPath || !callType) {
+        logger.error("usage: judge-llm-batch.ts --calls-ndjson <path> --call-type <type> " +
+            "[--batch-index N] [--judge-model <model>] [--out-dir <path>] [--dry-run]");
+        process.exit(1);
+    }
+    return { callsNdjsonPath, callType, batchIndex, judgeModel, outDir, dryRun };
+}
+async function main() {
+    const cliArgs = parseCli();
+    const { verdictPath, verdict } = await runJudge(cliArgs);
+    logger.info(`judge-llm-batch complete: verdict written to ${verdictPath}`);
+    logger.info(`judge-llm-batch: n=${verdict.aggregate.n} overallPass=${verdict.aggregate.overallPass}`);
+}
+if (process.argv[1] !== undefined &&
+    (process.argv[1].endsWith("judge-llm-batch.ts") || process.argv[1].endsWith("judge-llm-batch.js"))) {
+    main().catch((err) => {
+        logger.error(`judge-llm-batch failed: ${String(err)}`);
+        process.exit(1);
+    });
+}
+//# sourceMappingURL=judge-llm-batch.js.map

package/dist/scripts/judge-llm-batch.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"judge-llm-batch.js","sourceRoot":"","sources":["../../src/scripts/judge-llm-batch.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;;;GAiBG;;;;;AAsDH,oCAyBC;AAwBD,4CAEC;AAOD,wCAmBC;AAKD,8BAeC;AAMD,oCAaC;AAQD,4CASC;AA4BD,kDA4CC;AAsCD,4BAqEC;AA5WD,qCAAiE;AACjE,yCAAiC;AAEjC,4DAA0C;AAC1C,uDAAgE;AAChE,uCAAqC;AACrC,uDAOiC;AACjC,qCAAkC;AAClC,+CAAyD;AACzD,2CAAgD;AAEhD,MAAM,MAAM,GAAG,IAAA,yBAAe,EAAC,iBAAiB,CAAC,CAAC;AAElD,iFAAiF;AAEjF,MAAM,eAAe,GAAG,WAAW,CAAC;AACpC,MAAM,mBAAmB,GAAG,CAAC,CAAC;AAsB9B,iFAAiF;AAEjF;;;;GAIG;AACH,SAAgB,YAAY,CAAC,aAAqB;IAChD,MAAM,OAAO,GAAoB,EAAE,CAAC;IAEpC,KAAK,MAAM,GAAG,IAAI,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QAC5C,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC;QACxB,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QAEhC,IAAI,MAAe,CAAC;QACpB,IAAI,CAAC;YACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC5B,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,CAAC,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAC,CAAC;YACxC,SAAS;QACX,CAAC;QAED,MAAM,MAAM,GAAG,+BAAmB,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QACrD,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,OAAO,CAAC,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAC,CAAC;YACxC,SAAS;QACX,CAAC;QAED,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAC5B,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,mBAAmB,CAAC,GAAW;IACtC,OAAO;QACL,MAAM,EAAE,aAAa,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE;QACvC,QAAQ,EAAE,SAAS;QACnB,KAAK,EAAE,SAAS;QAChB,YAAY,EAAE,IAAI;QAClB,WAAW,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;QAC9B,eAAe,EAAE,IAAI;QACrB,QAAQ,EAAE,KAAK;QACf,WAAW,EAAE,IAAI;QACjB,YAAY,EAAE,IAAI;QAClB,SAAS,EAAE,IAAI;QACf,OAAO,EAAE,KAAK;QACd,YAAY,EAAE,IAAI;QAClB,WAAW,EAAE,IAAI;QACjB,EAAE,EAAE,IAAA,oBAAS,EAAC,IAAI,IAAI,EAAE,CAAC;KAC1B,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,SAAgB,gBAAgB,CAAC,OAAwB,EAAE,QAAgB;IACzE,OAAO,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC;AACxD,CAAC;AAED;;;;GAIG;AACH,SAAgB,cAAc,CAAC,MAAqB,EAAE,KAAkB;IACtE,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC;IAC1D,MAAM,eAAe,GAAG,MAAM,CAAC,QAAQ;QACrC,CAAC,CAAC,KAAK,CAAC,eAAe;QACvB,CAAC,CAAC,qEAAqE,CAAC;IAE1E,MAAM,IAAI,GAAG,QAAQ,IAAI,KAAK,CAAC,iBAAiB,IAAI,KAAK,CAAC,iBAAiB,CAAC;IAE5E,OAAO;QACL,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,QAAQ;QACR,eAAe;QACf,iBAAiB,EAAE,KAAK,CAAC,iBAAiB;QAC1C,gBAAgB,EAAE,KAAK,CAAC,gBAAgB;QACxC,iBAAiB,EAAE,KAAK,CAAC,iBAAiB;QAC1C,sBAAsB,EAAE,KAAK,CAAC,sBAAsB;QACpD,GAAG,CAAC,KAAK,CAAC,aAAa,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,KAAK,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACpF,IAAI;KACL,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,SAAgB,SAAS,CAAC,QAA6B;IACrD,MAAM,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC;IAC1B,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,IAAI,qBAAqB,GAAG,CAAC,CAAC;IAC9B,IAAI,WAAW,GAAG,CAAC,CAAC;IAEpB,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;QACzB,IAAI,CAAC,CAAC,QAAQ;YAAE,UAAU,EAAE,CAAC;QAC7B,IAAI,CAAC,CAAC,iBAAiB;YAAE,WAAW,EAAE,CAAC;QACvC,IAAI,CAAC,CAAC,iBAAiB;YAAE,qBAAqB,EAAE,CAAC;QACjD,IAAI,CAAC,CAAC,IAAI;YAAE,WAAW,EAAE,CAAC;IAC5B,CAAC;IAED,OAAO,EAAE,CAAC,EAAE,UAAU,EAAE,WAAW,EAAE,qBAAqB,EAAE,WAAW,EAAE,CAAC;AAC5E,CAAC;AAED;;;GAGG;AACH,SAAgB,YAAY,CAAC,MAAiD;IAC5E,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,CAAC;IAEnC,MAAM,MAAM,GAAG,8BAAkB,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;IACrD,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,MAAM,IAAI,KAAK,CAAC,gDAAgD,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;IAC1F,CAAC;IAED,IAAA,mBAAS,EAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACvC,MAAM,QAAQ,GAAG,WAAW,OAAO,CAAC,QAAQ,IAAI,OAAO,CAAC,UAAU,OAAO,CAAC;IAC1E,MAAM,QAAQ,GAAG,IAAA,gBAAI,EAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;IACxC,IAAA,uBAAa,EAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IAC1D,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,iFAAiF;AAEjF;;;GAGG;AACH,SAAgB,gBAAgB;IAC9B,OAAO,KAAK,IAA0B,EAAE,CAAC,CAAC;QACxC,QAAQ,EAAE,IAAI;QACd,eAAe,EAAE,mCAAmC;QACpD,iBAAiB,EAAE,IAAI;QACvB,gBAAgB,EAAE,oCAAoC;QACtD,iBAAiB,EAAE,IAAI;QACvB,sBAAsB,EAAE,qCAAqC;KAC9D,CAAC,CAAC;AACL,CAAC;AAED,SAAS,kBAAkB;IACzB,MAAM,GAAG,GAAG,eAAM,CAAC,OAAO,CAAC,KAAK,CAAC;IACjC,OAAO,GAAG,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;AAC7E,CAAC;AAED,MAAM,mBAAmB,GAAG;;;;;;;;;;;;;;2KAc+I,CAAC;AAE5K;;;;GAIG;AACH,SAAgB,mBAAmB,CAAC,MAAiB,EAAE,UAAmB;IACxE,MAAM,KAAK,GAAG,UAAU,IAAI,kBAAkB,EAAE,CAAC;IAEjD,OAAO,KAAK,EAAE,MAAqB,EAAwB,EAAE;QAC3D,MAAM,WAAW,GAAG;YAClB,aAAa,MAAM,CAAC,QAAQ,EAAE;YAC9B,UAAU,MAAM,CAAC,KAAK,EAAE;YACxB,aAAa,MAAM,CAAC,QAAQ,EAAE;YAC9B,iBAAiB,MAAM,CAAC,YAAY,IAAI,QAAQ,EAAE;YAClD,gBAAgB,MAAM,CAAC,WAAW,EAAE;YACpC,oBAAoB,MAAM,CAAC,eAAe,IAAI,QAAQ,EAAE;SACzD,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEb,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;gBAC3C,KAAK;gBACL,UAAU,EAAE,GAAG;gBACf,MAAM,EAAE,mBAAmB;gBAC3B,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC;gBAClD,aAAa,EAAE;oBACb,MAAM,EAAE,IAAA,qBAAe,EAAC,8BAAoB,CAAC;iBAC9C;aACF,CAAC,CAAC;YACH,MAAM,MAAM,GAAG,QAAQ,CAAC,aAAa,CAAC;YACtC,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;gBACpB,OAAO,aAAa,CAAC,qDAAqD,CAAC,CAAC;YAC9E,CAAC;YACD,OAAO;gBACL,QAAQ,EAAE,MAAM,CAAC,QAAQ;gBACzB,eAAe,EAAE,MAAM,CAAC,eAAe;gBACvC,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;gBAC3C,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;gBACzC,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;gBAC3C,sBAAsB,EAAE,MAAM,CAAC,sBAAsB;gBACrD,GAAG,CAAC,OAAO,MAAM,CAAC,aAAa,KAAK,QAAQ;oBAC1C,CAAC,CAAC,EAAE,aAAa,EAAE,MAAM,CAAC,aAAa,EAAE;oBACzC,CAAC,CAAC,EAAE,CAAC;aACR,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,aAAa,CAClB,4BAA4B,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAC/E,CAAC;QACJ,CAAC;IACH,CAAC,CAAC;AACJ,CAAC;AAED,SAAS,aAAa,CAAC,MAAc;IACnC,OAAO;QACL,QAAQ,EAAE,KAAK;QACf,eAAe,EAAE,gBAAgB,MAAM,EAAE;QACzC,iBAAiB,EAAE,KAAK;QACxB,gBAAgB,EAAE,gBAAgB,MAAM,EAAE;QAC1C,iBAAiB,EAAE,KAAK;QACxB,sBAAsB,EAAE,gBAAgB,MAAM,EAAE;KACjD,CAAC;AACJ,CAAC;AAsBD;;;;GAIG;AACI,KAAK,UAAU,QAAQ,CAAC,MAAmB;IAChD,MAAM,EACJ,eAAe,EACf,QAAQ,EACR,UAAU,GAAG,mBAAmB,EAChC,UAAU,EACV,MAAM,GAAG,eAAe,EACxB,MAAM,GAAG,KAAK,GACf,GAAG,MAAM,CAAC;IAEX,IAAI,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;IAE/B,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,IAAI,MAAM,EAAE,CAAC;YACX,QAAQ,GAAG,gBAAgB,EAAE,CAAC;QAChC,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,eAAM,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;gBACpC,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;YAC1D,CAAC;YACD,MAAM,MAAM,GAAG,IAAI,aAAS,CAAC,EAAE,MAAM,EAAE,eAAM,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC,CAAC;YACzE,QAAQ,GAAG,mBAAmB,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;QACrD,CAAC;IACH,CAAC;IAED,MAAM,kBAAkB,GAAG,UAAU,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,kBAAkB,EAAE,CAAC,CAAC;IAErF,MAAM,CAAC,IAAI,CAAC,4BAA4B,eAAe,EAAE,CAAC,CAAC;IAC3D,IAAI,aAAqB,CAAC;IAC1B,IAAI,CAAC;QACH,aAAa,GAAG,IAAA,sBAAY,EAAC,eAAe,EAAE,OAAO,CAAC,CAAC;IACzD,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,IAAI,KAAK,CAAC,gCAAgC,eAAe,KAAK,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACrF,CAAC;IAED,MAAM,UAAU,GAAG,YAAY,CAAC,aAAa,CAAC,CAAC;IAC/C,MAAM,CAAC,IAAI,CAAC,2BAA2B,UAAU,CAAC,MAAM,gBAAgB,CAAC,CAAC;IAE1E,MAAM,OAAO,GAAG,gBAAgB,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;IACvD,MAAM,CAAC,IAAI,CAAC,oBAAoB,OAAO,CAAC,MAAM,0BAA0B,QAAQ,GAAG,CAAC,CAAC;IAErF,MAAM,QAAQ,GAAwB,EAAE,CAAC;IACzC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,MAAM,CAAC,IAAI,CAAC,mCAAmC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;QAChE,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,CAAC;QACrC,MAAM,KAAK,GAAG,cAAc,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QAC5C,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACrB,MAAM,CAAC,IAAI,CACT,2BAA2B,MAAM,CAAC,MAAM,SAAS,KAAK,CAAC,IAAI,WAAW,KAAK,CAAC,QAAQ,YAAY,KAAK,CAAC,iBAAiB,WAAW,KAAK,CAAC,iBAAiB,EAAE,CAC5J,CAAC;IACJ,CAAC;IAED,MAAM,GAAG,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAC;IAChC,MAAM,CAAC,IAAI,CACT,gCAAgC,GAAG,CAAC,CAAC,eAAe,GAAG,CAAC,UAAU,gBAAgB,GAAG,CAAC,WAAW,0BAA0B,GAAG,CAAC,qBAAqB,gBAAgB,GAAG,CAAC,WAAW,EAAE,CACtL,CAAC;IAEF,MAAM,OAAO,GAAiB;QAC5B,QAAQ;QACR,UAAU;QACV,QAAQ,EAAE,IAAA,oBAAS,EAAC,IAAI,IAAI,EAAE,CAAC;QAC/B,UAAU,EAAE,kBAAkB;QAC9B,QAAQ;QACR,SAAS,EAAE,GAAG;KACf,CAAC;IAEF,MAAM,WAAW,GAAG,YAAY,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC;IACtD,MAAM,CAAC,IAAI,CAAC,uCAAuC,WAAW,EAAE,CAAC,CAAC;IAElE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,CAAC;AAClC,CAAC;AAED,iFAAiF;AAEjF;;GAEG;AACH,SAAS,QAAQ;IACf,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACnC,IAAI,eAAe,GAAG,EAAE,CAAC;IACzB,IAAI,QAAQ,GAAG,EAAE,CAAC;IAClB,IAAI,UAAU,GAAG,mBAAmB,CAAC;IACrC,IAAI,UAA8B,CAAC;IACnC,IAAI,MAAM,GAAG,eAAe,CAAC;IAC7B,IAAI,MAAM,GAAG,KAAK,CAAC;IAEnB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACrC,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,gBAAgB,IAAI,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;YAAE,eAAe,GAAG,IAAI,CAAC,EAAE,CAAC,CAAE,CAAC;aACzE,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,aAAa,IAAI,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;YAAE,QAAQ,GAAG,IAAI,CAAC,EAAE,CAAC,CAAE,CAAC;aACpE,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,eAAe,IAAI,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;YAAE,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;aAC/E,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,eAAe,IAAI,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;YAAE,UAAU,GAAG,IAAI,CAAC,EAAE,CAAC,CAAE,CAAC;aACxE,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,WAAW,IAAI,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;YAAE,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC,CAAE,CAAC;aAChE,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,WAAW;YAAE,MAAM,GAAG,IAAI,CAAC;IAClD,CAAC;IAED,IAAI,CAAC,eAAe,IAAI,CAAC,QAAQ,EAAE,CAAC;QAClC,MAAM,CAAC,KAAK,CACV,qEAAqE;YACnE,0EAA0E,CAC7E,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,OAAO,EAAE,eAAe,EAAE,QAAQ,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;AAC/E,CAAC;AAED,KAAK,UAAU,IAAI;IACjB,MAAM,OAAO,GAAG,QAAQ,EAAE,CAAC;IAC3B,MAAM,EAAE,WAAW,EAAE,OAAO,EAAE,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,CAAC;IAEzD,MAAM,CAAC,IAAI,CAAC,gDAAgD,WAAW,EAAE,CAAC,CAAC;IAC3E,MAAM,CAAC,IAAI,CACT,sBAAsB,OAAO,CAAC,SAAS,CAAC,CAAC,gBAAgB,OAAO,CAAC,SAAS,CAAC,WAAW,EAAE,CACzF,CAAC;AACJ,CAAC;AAED,IACE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,SAAS;IAC7B,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,oBAAoB,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,oBAAoB,CAAC,CAAC,EAClG,CAAC;IACD,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;QACnB,MAAM,CAAC,KAAK,CAAC,2BAA2B,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACvD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,CAAC;AACL,CAAC"}

package/dist/scripts/llm-heal.d.ts

@@ -0,0 +1,165 @@
+/**
+ * LLM prompt-template self-heal harness: given a judge verdict file containing
+ * failing samples for a callType, iterate — run a baseline pass rate, ask the
+ * patch generator to propose a minimal edit to the prompt template, replay the
+ * patched prompt against the captured user_content, score, and check convergence.
+ * Writes a healing-<callType>.md report for human review and manual apply.
+ * Production prompts in src/prompts/ are NEVER modified.
+ *
+ * Usage:
+ *   pnpm heal:llm \
+ *     --verdict-path judge-out/verdict-act-0.json \
+ *     --call-type act
+ *
+ * Knobs (all optional):
+ *   --max-iterations <N>        default 5
+ *   --n-replays <N>             default 5
+ *   --success-threshold <0..1>  default 0.9
+ *   --plateau-delta <0..1>      default 0.03
+ *   --plateau-window <N>        default 3
+ *   --out-dir <path>            default llm-heal-out
+ *   --judge-model <model>       override the judge model
+ *   --dry-run                   stubs the scorer (CI mode, no API calls)
+ */
+import Anthropic from "@anthropic-ai/sdk";
+import { type LlmCallSample } from "../api/schemas/telemetry";
+import { buildAnthropicClient } from "../lib/llm/anthropic-client";
+import { type LlmCallInput } from "../lib/telemetry/call-capture";
+import { type ScorerFn } from "../scripts/judge-llm-batch";
+import { type FlowPatch, type HealVerdict, type IterationRecord } from "../scripts/recon-heal";
+/**
+ * Persistent state written after each iteration so the operator can inspect
+ * progress. Mirrors recon-heal's HealState but scoped to prompt-template arms.
+ */
+export interface LlmHealState {
+    callType: string;
+    baselinePassRate: number;
+    history: IterationRecord[];
+    bestPassRate: number;
+    bestPatch: FlowPatch | null;
+    bestIterN: number;
+}
+/** Injectable replay runner — real or dry-run stub. */
+export type ReplayFn = (params: {
+    samples: LlmCallSample[];
+    promptTemplate: string;
+    scorerFn: ScorerFn;
+    nReplays: number;
+}) => Promise<{
+    passRate: number;
+    passCount: number;
+    failCount: number;
+}>;
+/** Injectable patch-request function for unit tests. */
+export type RequestPromptPatchFn = (params: {
+    client: Anthropic;
+    callType: string;
+    promptTemplate: string;
+    failingSamples: LlmCallSample[];
+    iterN: number;
+    priorAttempts: Array<{
+        iter: number;
+        anchor: string;
+        replacement: string;
+        strategy: string;
+        outcome: "improved" | "no_change" | "regressed";
+    }>;
+    captureFn?: (input: LlmCallInput) => Promise<void>;
+}) => Promise<FlowPatch | null>;
+/** Parameters accepted by `phaseLlmHeal`. */
+export interface LlmHealParams {
+    verdictPath: string;
+    callType: string;
+    maxIterations?: number;
+    nReplays?: number;
+    successThreshold?: number;
+    plateauDelta?: number;
+    plateauWindow?: number;
+    outDir?: string;
+    judgeModel?: string;
+    dryRun?: boolean;
+    replayFn?: ReplayFn;
+    requestPatchFn?: RequestPromptPatchFn;
+    scorerFn?: ScorerFn;
+}
+/** Return value of `phaseLlmHeal`. */
+export interface LlmHealResult {
+    verdict: HealVerdict;
+    reportPath: string;
+    state: LlmHealState;
+}
+export { buildAnthropicClient };
+/**
+ * Asks the model to propose a minimal patch to the prompt template, following
+ * the same anchor/replacement discipline as recon-flow-patch-generator but
+ * applied to a natural-language system or user prompt string.
+ */
+export declare function requestPromptPatch(params: {
+    client: Anthropic;
+    callType: string;
+    promptTemplate: string;
+    failingSamples: LlmCallSample[];
+    iterN: number;
+    priorAttempts: Array<{
+        iter: number;
+        anchor: string;
+        replacement: string;
+        strategy: string;
+        outcome: "improved" | "no_change" | "regressed";
+    }>;
+    captureFn?: (input: LlmCallInput) => Promise<void>;
+}): Promise<FlowPatch | null>;
+/**
+ * Scores each failing sample with the given scorer, treating the patched
+ * prompt as the system prompt context. Repeats nReplays times per sample and
+ * averages the pass rate — accounts for LLM non-determinism on borderline cases.
+ */
+export declare function replayPromptArm(params: {
+    samples: LlmCallSample[];
+    promptTemplate: string;
+    scorerFn: ScorerFn;
+    nReplays: number;
+}): Promise<{
+    passRate: number;
+    passCount: number;
+    failCount: number;
+}>;
+/**
+ * Dry-run replay runner: marks every sample as passing without calling the
+ * scorer. Used in CI to verify the loop mechanics without any API calls.
+ */
+export declare function makeDryRunReplayFn(passRate?: number): ReplayFn;
+/**
+ * Writes the current heal state to llm-heal-out/<callType>/state.json so the
+ * operator can inspect progress across interruptions.
+ */
+export declare function writeLlmHealState(outDir: string, callType: string, state: LlmHealState): void;
+/**
+ * Persists per-iteration inputs and outputs for audit.
+ */
+export declare function writeLlmIterationArtifacts(params: {
+    outDir: string;
+    callType: string;
+    iterN: number;
+    patch: FlowPatch | null;
+    patchedPrompt: string;
+    passRate: number;
+    passCount: number;
+    failCount: number;
+}): void;
+/**
+ * Writes the healing report markdown and returns its path.
+ */
+export declare function writeLlmHealReport(params: {
+    outDir: string;
+    callType: string;
+    state: LlmHealState;
+    verdict: HealVerdict;
+}): string;
+/**
+ * Drives the full baseline → patch → replay → converge loop for a single
+ * callType's failing prompt. Accepts injectable `replayFn`, `requestPatchFn`,
+ * and `scorerFn` so tests can stub LLM calls and scoring.
+ */
+export declare function phaseLlmHeal(params: LlmHealParams): Promise<LlmHealResult>;
+//# sourceMappingURL=llm-heal.d.ts.map

package/dist/scripts/llm-heal.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"llm-heal.d.ts","sourceRoot":"","sources":["../../src/scripts/llm-heal.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAMH,OAAO,SAAS,MAAM,mBAAmB,CAAC;AAI1C,OAAO,EAAyC,KAAK,aAAa,EAAE,MAAM,yBAAyB,CAAC;AAEpG,OAAO,EAAE,oBAAoB,EAAE,MAAM,4BAA4B,CAAC;AAGlE,OAAO,EAGL,KAAK,YAAY,EAClB,MAAM,8BAA8B,CAAC;AAEtC,OAAO,EAIL,KAAK,QAAQ,EACd,MAAM,2BAA2B,CAAC;AACnC,OAAO,EAGL,KAAK,SAAS,EACd,KAAK,WAAW,EAChB,KAAK,eAAe,EACrB,MAAM,sBAAsB,CAAC;AAe9B;;;GAGG;AACH,MAAM,WAAW,YAAY;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,gBAAgB,EAAE,MAAM,CAAC;IACzB,OAAO,EAAE,eAAe,EAAE,CAAC;IAC3B,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,SAAS,GAAG,IAAI,CAAC;IAC5B,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,uDAAuD;AACvD,MAAM,MAAM,QAAQ,GAAG,CAAC,MAAM,EAAE;IAC9B,OAAO,EAAE,aAAa,EAAE,CAAC;IACzB,cAAc,EAAE,MAAM,CAAC;IACvB,QAAQ,EAAE,QAAQ,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;CAClB,KAAK,OAAO,CAAC;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,CAAC,CAAC;AAE1E,wDAAwD;AACxD,MAAM,MAAM,oBAAoB,GAAG,CAAC,MAAM,EAAE;IAC1C,MAAM,EAAE,SAAS,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,cAAc,EAAE,MAAM,CAAC;IACvB,cAAc,EAAE,aAAa,EAAE,CAAC;IAChC,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,EAAE,KAAK,CAAC;QACnB,IAAI,EAAE,MAAM,CAAC;QACb,MAAM,EAAE,MAAM,CAAC;QACf,WAAW,EAAE,MAAM,CAAC;QACpB,QAAQ,EAAE,MAAM,CAAC;QACjB,OAAO,EAAE,UAAU,GAAG,WAAW,GAAG,WAAW,CAAC;KACjD,CAAC,CAAC;IACH,SAAS,CAAC,EAAE,CAAC,KAAK,EAAE,YAAY,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CACpD,KAAK,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC,CAAC;AAEhC,6CAA6C;AAC7C,MAAM,WAAW,aAAa;IAC5B,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,cAAc,CAAC,EAAE,oBAAoB,CAAC;IACtC,QAAQ,CAAC,EAAE,QAAQ,CAAC;CACrB;AAED,sCAAsC;AACtC,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,WAAW,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,YAAY,CAAC;CACrB;AAED,OAAO,EAAE,oBAAoB,EAAE,CAAC;AAwBhC;;;;GAIG;AACH,wBAAsB,kBAAkB,CAAC,MAAM,EAAE;IAC/C,MAAM,EAAE,SAAS,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,cAAc,EAAE,MAAM,CAAC;IACvB,cAAc,EAAE,aAAa,EAAE,CAAC;IAChC,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,EAAE,KAAK,CAAC;QACnB,IAAI,EAAE,MAAM,CAAC;QACb,MAAM,EAAE,MAAM,CAAC;QACf,WAAW,EAAE,MAAM,CAAC;QACpB,QAAQ,EAAE,MAAM,CAAC;QACjB,OAAO,EAAE,UAAU,GAAG,WAAW,GAAG,WAAW,CAAC;KACjD,CAAC,CAAC;IACH,SAAS,CAAC,EAAE,CAAC,KAAK,EAAE,YAAY,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CACpD,GAAG,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC,CA4H5B;AAID;;;;GAIG;AACH,wBAAsB,eAAe,CAAC,MAAM,EAAE;IAC5C,OAAO,EAAE,aAAa,EAAE,CAAC;IACzB,cAAc,EAAE,MAAM,CAAC;IACvB,QAAQ,EAAE,QAAQ,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;CAClB,GAAG,OAAO,CAAC;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,CAAC,CAiCtE;AAED;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,QAAQ,GAAE,MAAY,GAAG,QAAQ,CAUnE;AAID;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,YAAY,GAAG,IAAI,CAI7F;AAED;;GAEG;AACH,wBAAgB,0BAA0B,CAAC,MAAM,EAAE;IACjD,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,SAAS,GAAG,IAAI,CAAC;IACxB,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;CACnB,GAAG,IAAI,CAWP;AAID;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,MAAM,EAAE;IACzC,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,YAAY,CAAC;IACpB,OAAO,EAAE,WAAW,CAAC;CACtB,GAAG,MAAM,CA0DT;AAID;;;;GAIG;AACH,wBAAsB,YAAY,CAAC,MAAM,EAAE,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC,CAuQhF"}