nemoris 0.1.0 → 0.1.2

package/src/runtime/improvement-harness.js

@@ -1,581 +1,581 @@
-import path from "node:path";
-import { ConfigLoader } from "../config/loader.js";
-import { RunStore } from "./run-store.js";
-import { listFilesRecursive, readJson } from "../utils/fs.js";
-import { classifyRuntimeFailure } from "./report-fallback.js";
-
-function _deepMerge(base, override) {
-  if (Array.isArray(base) || Array.isArray(override)) {
-    return Array.isArray(override) ? [...override] : Array.isArray(base) ? [...base] : [];
-  }
-
-  if (!base || typeof base !== "object") {
-    return override === undefined ? base : override;
-  }
-
-  if (!override || typeof override !== "object") {
-    return override === undefined ? { ...base } : override;
-  }
-
-  const merged = { ...base };
-  for (const [key, value] of Object.entries(override)) {
-    merged[key] = key in base ? _deepMerge(base[key], value) : value;
-  }
-  return merged;
-}
-
-function buildVariant(target, variantId) {
-  switch (variantId) {
-    case "baseline":
-      return {
-        id: "baseline",
-        description: "Run the lane with its current guidance and routing defaults.",
-        overrides: {
-          modelOverride: target.defaultModelOverride || null
-        }
-      };
-    case "focus_concrete":
-      return {
-        id: "focus_concrete",
-        description: "Push the report to be more concrete, operator-facing, and less generic.",
-        overrides: {
-          modelOverride: target.defaultModelOverride || null,
-          reportGuidanceOverride: {
-            focus: ["concrete evidence", "explicit operator signal", "succinct useful status"],
-            qualityChecks: ["avoid generic reassurance", "prefer named facts over abstractions", "state None explicitly"],
-            avoid: ["vague encouragement", "boilerplate filler"]
-          }
-        }
-      };
-    case "retrieval_lexical_heavy":
-      return {
-        id: "retrieval_lexical_heavy",
-        description: "Bias retrieval toward lexical match signal for clearer grounded reports.",
-        overrides: {
-          modelOverride: target.defaultModelOverride || null,
-          retrievalBlendOverride: {
-            lexicalWeight: 0.48,
-            embeddingWeight: 0.2,
-            recencyWeight: 0.14,
-            salienceWeight: 0.12,
-            typeWeight: 0.04,
-            semanticRescueBonus: 0.04
-          }
-        }
-      };
-    case "retrieval_embedding_heavy":
-      return {
-        id: "retrieval_embedding_heavy",
-        description: "Bias retrieval toward embedding similarity when vectors are fresh and available.",
-        overrides: {
-          modelOverride: target.defaultModelOverride || null,
-          retrievalBlendOverride: {
-            lexicalWeight: 0.26,
-            embeddingWeight: 0.42,
-            recencyWeight: 0.12,
-            salienceWeight: 0.12,
-            typeWeight: 0.04,
-            semanticRescueBonus: 0.08
-          }
-        }
-      };
-    case "report_model_bump":
-      return {
-        id: "report_model_bump",
-        description: "Try the manual-bump local report model for richer structure.",
-        overrides: {
-          modelOverride: "ollama/qwen3:14b"
-        }
-      };
-    default:
-      throw new Error(`Unknown improvement variant: ${variantId}`);
-  }
-}
-
-function summarizeEvaluation(evaluation) {
-  return {
-    filePath: evaluation.filePath,
-    overallScore: evaluation.rubric?.overallScore ?? null,
-    contractAdherence: evaluation.rubric?.components?.contractAdherence ?? null,
-    v2OutputQuality: evaluation.rubric?.components?.v2OutputQuality ?? null,
-    retrieval: evaluation.retrieval
-      ? {
-          memoryCount: evaluation.retrieval.memoryCount,
-          lexicalCount: evaluation.retrieval.lexicalCount,
-          semanticCount: evaluation.retrieval.semanticCount,
-          qmdCount: evaluation.retrieval.qmdCount,
-          embeddingQueryMode: evaluation.retrieval.embeddingQueryMode || null,
-          freshEmbeddingCount: evaluation.retrieval.freshEmbeddingCount || 0,
-          staleEmbeddingCount: evaluation.retrieval.staleEmbeddingCount || 0,
-          missingEmbeddingCount: evaluation.retrieval.missingEmbeddingCount || 0,
-          failedEmbeddingCount: evaluation.retrieval.failedEmbeddingCount || 0,
-          embeddingError: evaluation.retrieval.embeddingError || null
-        }
-      : null,
-    interaction: evaluation.interaction || null,
-    findings: evaluation.comparisonNotes || []
-  };
-}
-
-function normalizeTargetId(targetId) {
-  return String(targetId || "").replace(/_([a-z])/g, (_match, letter) => letter.toUpperCase());
-}
-
-function summarizeFailure(error) {
-  return {
-    message: error.message,
-    runFile: error.runFile || null,
-    classification: classifyRuntimeFailure(error?.message || ""),
-    fallback: error.fallback || null
-  };
-}
-
-function summarizeSkip(reason, details = {}) {
-  return {
-    reason,
-    ...details
-  };
-}
-
-function compareEvaluations(baseline, candidate) {
-  const baselineScore = baseline?.overallScore ?? null;
-  const candidateScore = candidate?.overallScore ?? null;
-  const baselineContract = baseline?.contractAdherence ?? null;
-  const candidateContract = candidate?.contractAdherence ?? null;
-  const baselineOutput = baseline?.v2OutputQuality ?? null;
-  const candidateOutput = candidate?.v2OutputQuality ?? null;
-  const baselineSemantic = baseline?.retrieval?.semanticCount ?? 0;
-  const candidateSemantic = candidate?.retrieval?.semanticCount ?? 0;
-
-  const findings = [];
-  if (baselineScore != null && candidateScore != null) {
-    if (candidateScore > baselineScore) findings.push("Candidate improved overall deterministic eval score.");
-    else if (candidateScore < baselineScore) findings.push("Candidate regressed on overall deterministic eval score.");
-  }
-  if (baselineContract != null && candidateContract != null) {
-    if (candidateContract > baselineContract) findings.push("Candidate improved contract adherence.");
-    else if (candidateContract < baselineContract) findings.push("Candidate regressed on contract adherence.");
-  }
-  if (baselineOutput != null && candidateOutput != null) {
-    if (candidateOutput > baselineOutput) findings.push("Candidate improved output-quality signals.");
-    else if (candidateOutput < baselineOutput) findings.push("Candidate regressed on output-quality signals.");
-  }
-  if (candidateSemantic > baselineSemantic) findings.push("Candidate surfaced more semantic retrieval hits.");
-  if ((candidate?.retrieval?.failedEmbeddingCount || 0) > (baseline?.retrieval?.failedEmbeddingCount || 0)) {
-    findings.push("Candidate increased embedding failure count.");
-  }
-
-  const improved =
-    candidateScore != null &&
-    baselineScore != null &&
-    candidateScore > baselineScore &&
-    (candidateContract ?? 0) >= (baselineContract ?? 0) &&
-    (candidate?.retrieval?.failedEmbeddingCount || 0) <= (baseline?.retrieval?.failedEmbeddingCount || 0);
-
-  return {
-    improved,
-    baseline,
-    candidate,
-    findings
-  };
-}
-
-function buildEmbeddingSummary({ readiness = null, health = null, rebuilt = false, blockedReason = null, queryMode = null } = {}) {
-  return {
-    readiness,
-    health,
-    rebuilt,
-    blockedReason,
-    queryMode
-  };
-}
-
-export class ImprovementHarness {
-  constructor({ projectRoot, stateRoot, executor, evaluator }) {
-    this.projectRoot = projectRoot;
-    this.stateRoot = stateRoot;
-    this.executor = executor;
-    this.evaluator = evaluator;
-    this.loader = new ConfigLoader({ rootDir: path.join(projectRoot, "config") });
-    this.runStore = new RunStore({ rootDir: path.join(stateRoot, "improvements") });
-  }
-
-  async findLatestVariantArtifact(targetId, variantId, options = {}) {
-    const targetDir = path.join(this.stateRoot, "improvements", normalizeTargetId(targetId));
-    const files = (await listFilesRecursive(targetDir)).filter((filePath) => filePath.endsWith(".json")).sort().reverse();
-
-    for (const filePath of files) {
-      const artifact = await readJson(filePath, null);
-      if (!artifact) continue;
-      if (artifact.targetId !== normalizeTargetId(targetId)) continue;
-      if (artifact.variant?.id !== variantId) continue;
-      if (options.successfulOnly && !artifact.ok) continue;
-      return {
-        filePath,
-        ...artifact
-      };
-    }
-
-    return null;
-  }
-
-  async listTargets() {
-    const config = await this.loader.loadAll();
-    return Object.entries(config.improvementTargets || {}).map(([targetId, target]) => ({
-      id: targetId,
-      jobId: target.jobId,
-      defaultMode: target.defaultMode || "provider",
-      defaultModelOverride: target.defaultModelOverride || null,
-      defaultTimeoutMs: target.defaultTimeoutMs || null,
-      allowedKnobs: target.allowedKnobs || [],
-      recommendedVariants: target.recommendedVariants || []
-    }));
-  }
-
-  async getTarget(targetId) {
-    const config = await this.loader.loadAll();
-    const resolvedTargetId = normalizeTargetId(targetId);
-    const target = config.improvementTargets?.[resolvedTargetId];
-    if (!target) {
-      throw new Error(`Unknown improvement target: ${targetId}`);
-    }
-    return {
-      id: resolvedTargetId,
-      ...target
-    };
-  }
-
-  async checkVariantPrerequisites(target, variant) {
-    if (variant.id !== "retrieval_embedding_heavy") {
-      return null;
-    }
-
-    const runtime = await this.executor.scheduler.loadRuntime();
-    const job = runtime.jobs?.[target.jobId];
-    const agentId = job?.agentId;
-    if (!agentId) {
-      return {
-        blocked: true,
-        reason: "Could not resolve agent for embedding-heavy variant."
-      };
-    }
-
-    const health = await this.executor.scheduler.memoryStore.getEmbeddingHealth(agentId, {
-      embeddingIndex: this.executor.scheduler.embeddingIndex,
-      probe: false
-    });
-
-    if ((health.embeddingHealth?.freshCount || 0) > 0) {
-      return null;
-    }
-
-    const embeddingService = this.executor.scheduler.embeddingIndex?.embeddingService || null;
-    if (!embeddingService) {
-      return {
-        blocked: true,
-        reason: "No embedding service is configured for this runtime.",
-        health
-      };
-    }
-
-    const readiness = await embeddingService.getReadiness();
-    if (readiness?.ready) {
-      try {
-        await this.executor.scheduler.memoryStore.rebuildEmbeddings(agentId, {
-          embeddingIndex: this.executor.scheduler.embeddingIndex
-        });
-        const refreshed = await this.executor.scheduler.memoryStore.getEmbeddingHealth(agentId, {
-          embeddingIndex: this.executor.scheduler.embeddingIndex,
-          probe: false
-        });
-        if ((refreshed.embeddingHealth?.freshCount || 0) > 0) {
-          return {
-            blocked: false,
-            rebuilt: true,
-            health: refreshed,
-            readiness
-          };
-        }
-        return {
-          blocked: true,
-          reason: "Embeddings were rebuildable but no fresh vectors were produced for this lane.",
-          health: refreshed,
-          readiness,
-          rebuilt: true
-        };
-      } catch (error) {
-        const refreshed = await this.executor.scheduler.memoryStore.getEmbeddingHealth(agentId, {
-          embeddingIndex: this.executor.scheduler.embeddingIndex,
-          probe: false
-        });
-        return {
-          blocked: true,
-          reason: error?.message || String(error),
-          health: refreshed,
-          readiness,
-          rebuilt: true
-        };
-      }
-    }
-
-    return {
-      blocked: true,
-      reason: readiness?.reason || "Embeddings are unavailable or degraded for this lane.",
-      health,
-      readiness
-    };
-  }
-
-  async runVariant(targetId, variantId = "baseline", options = {}) {
-    const target = await this.getTarget(targetId);
-    const variant = buildVariant(target, variantId);
-    const mode = target.defaultMode || "provider";
-    let embedding = buildEmbeddingSummary();
-    let artifact;
-    try {
-      const prerequisiteFailure = await this.checkVariantPrerequisites(target, variant);
-      embedding = buildEmbeddingSummary({
-        readiness: prerequisiteFailure?.readiness || null,
-        health: prerequisiteFailure?.health?.embeddingHealth || null,
-        rebuilt: Boolean(prerequisiteFailure?.rebuilt),
-        blockedReason: prerequisiteFailure?.blocked ? prerequisiteFailure.reason : null
-      });
-      if (prerequisiteFailure?.blocked) {
-        artifact = {
-          timestamp: new Date().toISOString(),
-          targetId: target.id,
-          jobId: target.jobId,
-          variant,
-          ok: false,
-          skipped: true,
-          embedding,
-          skip: summarizeSkip(prerequisiteFailure.reason, {
-            embeddingHealth: prerequisiteFailure.health?.embeddingHealth || null,
-            embeddingReadiness: prerequisiteFailure.readiness || null
-          })
-        };
-        const filePath = await this.runStore.saveRun(target.id, artifact);
-        return {
-          filePath,
-          ...artifact
-        };
-      }
-
-      const run = await this.executor.executeJob(target.jobId, {
-        mode,
-        shadowImport: true,
-        modelOverride: variant.overrides.modelOverride || null,
-        reportGuidanceOverride: variant.overrides.reportGuidanceOverride || null,
-        retrievalBlendOverride: variant.overrides.retrievalBlendOverride || null,
-        providerTimeoutMs: target.defaultTimeoutMs || null,
-        allowReportFallback: options.allowReportFallback === true
-      });
-      const evaluation = await this.evaluator.evaluateAndPersistJob(target.jobId);
-      artifact = {
-        timestamp: new Date().toISOString(),
-        targetId: target.id,
-        jobId: target.jobId,
-        variant,
-        ok: true,
-        embedding: buildEmbeddingSummary({
-          readiness: embedding.readiness,
-          health: evaluation.retrieval?.embeddingHealth || embedding.health,
-          rebuilt: embedding.rebuilt,
-          blockedReason: null,
-          queryMode: evaluation.retrieval?.embeddingQueryMode || null
-        }),
-        run: {
-          filePath: run.filePath,
-          mode: run.mode,
-          providerId: run.providerId,
-          modelId: run.modelId,
-          routingDecision: run.routingDecision,
-          preflight: run.preflight,
-          summary: run.result?.summary || null,
-          fallback: run.fallback || null
-        },
-        evaluation: summarizeEvaluation(evaluation)
-      };
-    } catch (error) {
-      artifact = {
-        timestamp: new Date().toISOString(),
-        targetId: target.id,
-        jobId: target.jobId,
-        variant,
-        ok: false,
-        embedding,
-        error: summarizeFailure(error)
-      };
-    }
-    const filePath = await this.runStore.saveRun(target.id, artifact);
-    return {
-      filePath,
-      ...artifact
-    };
-  }
-
-  async compareVariants(targetId, baselineVariantId = "baseline", candidateVariantId, options = {}) {
-    if (!candidateVariantId) {
-      throw new Error("compareVariants requires a candidate variant id.");
-    }
-    const baseline = (await this.findLatestVariantArtifact(targetId, baselineVariantId, { successfulOnly: true })) ||
-      (await this.runVariant(targetId, baselineVariantId, options));
-    const candidate = await this.runVariant(targetId, candidateVariantId, options);
-    let comparisonResult;
-    if (!baseline.ok || !candidate.ok) {
-      comparisonResult = {
-        improved: false,
-        baseline: baseline.ok ? baseline.evaluation : null,
-        candidate: candidate.ok ? candidate.evaluation : null,
-        findings: [
-          ...(!baseline.ok
-            ? [baseline.skipped ? `Baseline variant skipped: ${baseline.skip.reason}` : `Baseline variant failed: ${baseline.error.message}`]
-            : []),
-          ...(!candidate.ok
-            ? [candidate.skipped ? `Candidate variant skipped: ${candidate.skip.reason}` : `Candidate variant failed: ${candidate.error.message}`]
-            : [])
-        ]
-      };
-    } else {
-      comparisonResult = compareEvaluations(baseline.evaluation, candidate.evaluation);
-      if (
-        candidateVariantId === "retrieval_embedding_heavy" &&
-        (
-          candidate.evaluation?.retrieval?.embeddingQueryMode === "lexical_fallback" ||
-          (candidate.evaluation?.retrieval?.freshEmbeddingCount || 0) === 0
-        )
-      ) {
-        comparisonResult.improved = false;
-        comparisonResult.findings.push(
-          "Candidate could not exercise embedding-heavy retrieval because embeddings were unavailable or degraded."
-        );
-      }
-    }
-    const comparison = {
-      timestamp: new Date().toISOString(),
-      targetId: baseline.targetId,
-      baselineVariantId,
-      candidateVariantId,
-      fallback: {
-        baseline: baseline.run?.fallback || baseline.error?.fallback || null,
-        candidate: candidate.run?.fallback || candidate.error?.fallback || null
-      },
-      embedding: {
-        baseline: baseline.embedding || null,
-        candidate: candidate.embedding || null
-      },
-      comparison: comparisonResult
-    };
-    const filePath = await this.runStore.saveRun(`${baseline.targetId}-comparison`, comparison);
-    return {
-      filePath,
-      ...comparison
-    };
-  }
-
-  async repairAndCompare(targetId, candidateVariantId, baselineVariantId = "baseline", options = {}) {
-    if (!candidateVariantId) {
-      throw new Error("repairAndCompare requires a candidate variant id.");
-    }
-
-    const target = await this.getTarget(targetId);
-    const candidateVariant = buildVariant(target, candidateVariantId);
-    const prerequisite = await this.checkVariantPrerequisites(target, candidateVariant);
-
-    if (prerequisite?.blocked) {
-      const artifact = {
-        timestamp: new Date().toISOString(),
-        targetId: target.id,
-        baselineVariantId,
-        candidateVariantId,
-        repaired: false,
-        blocked: true,
-        embedding: buildEmbeddingSummary({
-          readiness: prerequisite.readiness || null,
-          health: prerequisite.health?.embeddingHealth || null,
-          rebuilt: Boolean(prerequisite.rebuilt),
-          blockedReason: prerequisite.reason
-        }),
-        findings: [`Repair blocked: ${prerequisite.reason}`]
-      };
-      const filePath = await this.runStore.saveRun(`${target.id}-repair`, artifact);
-      return {
-        filePath,
-        ...artifact
-      };
-    }
-
-    const rerunBaseline = Boolean(prerequisite?.rebuilt);
-    const baseline = rerunBaseline
-      ? await this.runVariant(targetId, baselineVariantId, options)
-      : (await this.findLatestVariantArtifact(targetId, baselineVariantId, { successfulOnly: true })) ||
-        (await this.runVariant(targetId, baselineVariantId, options));
-    const candidate = await this.runVariant(targetId, candidateVariantId, options);
-
-    let comparisonResult;
-    if (!baseline.ok || !candidate.ok) {
-      comparisonResult = {
-        improved: false,
-        baseline: baseline.ok ? baseline.evaluation : null,
-        candidate: candidate.ok ? candidate.evaluation : null,
-        findings: [
-          ...(!baseline.ok
-            ? [baseline.skipped ? `Baseline variant skipped: ${baseline.skip.reason}` : `Baseline variant failed: ${baseline.error.message}`]
-            : []),
-          ...(!candidate.ok
-            ? [candidate.skipped ? `Candidate variant skipped: ${candidate.skip.reason}` : `Candidate variant failed: ${candidate.error.message}`]
-            : [])
-        ]
-      };
-    } else {
-      comparisonResult = compareEvaluations(baseline.evaluation, candidate.evaluation);
-      if (
-        candidateVariantId === "retrieval_embedding_heavy" &&
-        (
-          candidate.evaluation?.retrieval?.embeddingQueryMode === "lexical_fallback" ||
-          (candidate.evaluation?.retrieval?.freshEmbeddingCount || 0) === 0
-        )
-      ) {
-        comparisonResult.improved = false;
-        comparisonResult.findings.push(
-          "Candidate could not exercise embedding-heavy retrieval because embeddings were unavailable or degraded."
-        );
-      }
-    }
-
-    const artifact = {
-      timestamp: new Date().toISOString(),
-      targetId: target.id,
-      baselineVariantId,
-      candidateVariantId,
-      repaired: Boolean(prerequisite?.rebuilt),
-      blocked: false,
-      fallback: {
-        baseline: baseline.run?.fallback || baseline.error?.fallback || null,
-        candidate: candidate.run?.fallback || candidate.error?.fallback || null
-      },
-      embedding: {
-        readiness: prerequisite?.readiness || null,
-        health: candidate.embedding?.health || prerequisite?.health?.embeddingHealth || null,
-        rebuilt: Boolean(prerequisite?.rebuilt),
-        blockedReason: null,
-        queryMode: candidate.embedding?.queryMode || null
-      },
-      baseline: {
-        filePath: baseline.filePath,
-        evaluationFile: baseline.evaluation?.filePath || null
-      },
-      candidate: {
-        filePath: candidate.filePath,
-        evaluationFile: candidate.evaluation?.filePath || null,
-        status: candidate.ok ? "evaluated" : candidate.skipped ? "skipped" : candidate.error?.classification || "failed"
-      },
-      comparison: comparisonResult
-    };
-    const filePath = await this.runStore.saveRun(`${target.id}-repair`, artifact);
-    return {
-      filePath,
-      ...artifact
-    };
-  }
-}
+import path from "node:path";
+import { ConfigLoader } from "../config/loader.js";
+import { RunStore } from "./run-store.js";
+import { listFilesRecursive, readJson } from "../utils/fs.js";
+import { classifyRuntimeFailure } from "./report-fallback.js";
+
+function _deepMerge(base, override) {
+  if (Array.isArray(base) || Array.isArray(override)) {
+    return Array.isArray(override) ? [...override] : Array.isArray(base) ? [...base] : [];
+  }
+
+  if (!base || typeof base !== "object") {
+    return override === undefined ? base : override;
+  }
+
+  if (!override || typeof override !== "object") {
+    return override === undefined ? { ...base } : override;
+  }
+
+  const merged = { ...base };
+  for (const [key, value] of Object.entries(override)) {
+    merged[key] = key in base ? _deepMerge(base[key], value) : value;
+  }
+  return merged;
+}
+
+function buildVariant(target, variantId) {
+  switch (variantId) {
+    case "baseline":
+      return {
+        id: "baseline",
+        description: "Run the lane with its current guidance and routing defaults.",
+        overrides: {
+          modelOverride: target.defaultModelOverride || null
+        }
+      };
+    case "focus_concrete":
+      return {
+        id: "focus_concrete",
+        description: "Push the report to be more concrete, operator-facing, and less generic.",
+        overrides: {
+          modelOverride: target.defaultModelOverride || null,
+          reportGuidanceOverride: {
+            focus: ["concrete evidence", "explicit operator signal", "succinct useful status"],
+            qualityChecks: ["avoid generic reassurance", "prefer named facts over abstractions", "state None explicitly"],
+            avoid: ["vague encouragement", "boilerplate filler"]
+          }
+        }
+      };
+    case "retrieval_lexical_heavy":
+      return {
+        id: "retrieval_lexical_heavy",
+        description: "Bias retrieval toward lexical match signal for clearer grounded reports.",
+        overrides: {
+          modelOverride: target.defaultModelOverride || null,
+          retrievalBlendOverride: {
+            lexicalWeight: 0.48,
+            embeddingWeight: 0.2,
+            recencyWeight: 0.14,
+            salienceWeight: 0.12,
+            typeWeight: 0.04,
+            semanticRescueBonus: 0.04
+          }
+        }
+      };
+    case "retrieval_embedding_heavy":
+      return {
+        id: "retrieval_embedding_heavy",
+        description: "Bias retrieval toward embedding similarity when vectors are fresh and available.",
+        overrides: {
+          modelOverride: target.defaultModelOverride || null,
+          retrievalBlendOverride: {
+            lexicalWeight: 0.26,
+            embeddingWeight: 0.42,
+            recencyWeight: 0.12,
+            salienceWeight: 0.12,
+            typeWeight: 0.04,
+            semanticRescueBonus: 0.08
+          }
+        }
+      };
+    case "report_model_bump":
+      return {
+        id: "report_model_bump",
+        description: "Try the manual-bump local report model for richer structure.",
+        overrides: {
+          modelOverride: "ollama/qwen3:14b"
+        }
+      };
+    default:
+      throw new Error(`Unknown improvement variant: ${variantId}`);
+  }
+}
+
+function summarizeEvaluation(evaluation) {
+  return {
+    filePath: evaluation.filePath,
+    overallScore: evaluation.rubric?.overallScore ?? null,
+    contractAdherence: evaluation.rubric?.components?.contractAdherence ?? null,
+    v2OutputQuality: evaluation.rubric?.components?.v2OutputQuality ?? null,
+    retrieval: evaluation.retrieval
+      ? {
+          memoryCount: evaluation.retrieval.memoryCount,
+          lexicalCount: evaluation.retrieval.lexicalCount,
+          semanticCount: evaluation.retrieval.semanticCount,
+          qmdCount: evaluation.retrieval.qmdCount,
+          embeddingQueryMode: evaluation.retrieval.embeddingQueryMode || null,
+          freshEmbeddingCount: evaluation.retrieval.freshEmbeddingCount || 0,
+          staleEmbeddingCount: evaluation.retrieval.staleEmbeddingCount || 0,
+          missingEmbeddingCount: evaluation.retrieval.missingEmbeddingCount || 0,
+          failedEmbeddingCount: evaluation.retrieval.failedEmbeddingCount || 0,
+          embeddingError: evaluation.retrieval.embeddingError || null
+        }
+      : null,
+    interaction: evaluation.interaction || null,
+    findings: evaluation.comparisonNotes || []
+  };
+}
+
+function normalizeTargetId(targetId) {
+  return String(targetId || "").replace(/_([a-z])/g, (_match, letter) => letter.toUpperCase());
+}
+
+function summarizeFailure(error) {
+  return {
+    message: error.message,
+    runFile: error.runFile || null,
+    classification: classifyRuntimeFailure(error?.message || ""),
+    fallback: error.fallback || null
+  };
+}
+
+function summarizeSkip(reason, details = {}) {
+  return {
+    reason,
+    ...details
+  };
+}
+
+function compareEvaluations(baseline, candidate) {
+  const baselineScore = baseline?.overallScore ?? null;
+  const candidateScore = candidate?.overallScore ?? null;
+  const baselineContract = baseline?.contractAdherence ?? null;
+  const candidateContract = candidate?.contractAdherence ?? null;
+  const baselineOutput = baseline?.v2OutputQuality ?? null;
+  const candidateOutput = candidate?.v2OutputQuality ?? null;
+  const baselineSemantic = baseline?.retrieval?.semanticCount ?? 0;
+  const candidateSemantic = candidate?.retrieval?.semanticCount ?? 0;
+
+  const findings = [];
+  if (baselineScore != null && candidateScore != null) {
+    if (candidateScore > baselineScore) findings.push("Candidate improved overall deterministic eval score.");
+    else if (candidateScore < baselineScore) findings.push("Candidate regressed on overall deterministic eval score.");
+  }
+  if (baselineContract != null && candidateContract != null) {
+    if (candidateContract > baselineContract) findings.push("Candidate improved contract adherence.");
+    else if (candidateContract < baselineContract) findings.push("Candidate regressed on contract adherence.");
+  }
+  if (baselineOutput != null && candidateOutput != null) {
+    if (candidateOutput > baselineOutput) findings.push("Candidate improved output-quality signals.");
+    else if (candidateOutput < baselineOutput) findings.push("Candidate regressed on output-quality signals.");
+  }
+  if (candidateSemantic > baselineSemantic) findings.push("Candidate surfaced more semantic retrieval hits.");
+  if ((candidate?.retrieval?.failedEmbeddingCount || 0) > (baseline?.retrieval?.failedEmbeddingCount || 0)) {
+    findings.push("Candidate increased embedding failure count.");
+  }
+
+  const improved =
+    candidateScore != null &&
+    baselineScore != null &&
+    candidateScore > baselineScore &&
+    (candidateContract ?? 0) >= (baselineContract ?? 0) &&
+    (candidate?.retrieval?.failedEmbeddingCount || 0) <= (baseline?.retrieval?.failedEmbeddingCount || 0);
+
+  return {
+    improved,
+    baseline,
+    candidate,
+    findings
+  };
+}
+
+function buildEmbeddingSummary({ readiness = null, health = null, rebuilt = false, blockedReason = null, queryMode = null } = {}) {
+  return {
+    readiness,
+    health,
+    rebuilt,
+    blockedReason,
+    queryMode
+  };
+}
+
+export class ImprovementHarness {
+  constructor({ projectRoot, stateRoot, executor, evaluator }) {
+    this.projectRoot = projectRoot;
+    this.stateRoot = stateRoot;
+    this.executor = executor;
+    this.evaluator = evaluator;
+    this.loader = new ConfigLoader({ rootDir: path.join(projectRoot, "config") });
+    this.runStore = new RunStore({ rootDir: path.join(stateRoot, "improvements") });
+  }
+
+  async findLatestVariantArtifact(targetId, variantId, options = {}) {
+    const targetDir = path.join(this.stateRoot, "improvements", normalizeTargetId(targetId));
+    const files = (await listFilesRecursive(targetDir)).filter((filePath) => filePath.endsWith(".json")).sort().reverse();
+
+    for (const filePath of files) {
+      const artifact = await readJson(filePath, null);
+      if (!artifact) continue;
+      if (artifact.targetId !== normalizeTargetId(targetId)) continue;
+      if (artifact.variant?.id !== variantId) continue;
+      if (options.successfulOnly && !artifact.ok) continue;
+      return {
+        filePath,
+        ...artifact
+      };
+    }
+
+    return null;
+  }
+
+  async listTargets() {
+    const config = await this.loader.loadAll();
+    return Object.entries(config.improvementTargets || {}).map(([targetId, target]) => ({
+      id: targetId,
+      jobId: target.jobId,
+      defaultMode: target.defaultMode || "provider",
+      defaultModelOverride: target.defaultModelOverride || null,
+      defaultTimeoutMs: target.defaultTimeoutMs || null,
+      allowedKnobs: target.allowedKnobs || [],
+      recommendedVariants: target.recommendedVariants || []
+    }));
+  }
+
+  async getTarget(targetId) {
+    const config = await this.loader.loadAll();
+    const resolvedTargetId = normalizeTargetId(targetId);
+    const target = config.improvementTargets?.[resolvedTargetId];
+    if (!target) {
+      throw new Error(`Unknown improvement target: ${targetId}`);
+    }
+    return {
+      id: resolvedTargetId,
+      ...target
+    };
+  }
+
+  async checkVariantPrerequisites(target, variant) {
+    if (variant.id !== "retrieval_embedding_heavy") {
+      return null;
+    }
+
+    const runtime = await this.executor.scheduler.loadRuntime();
+    const job = runtime.jobs?.[target.jobId];
+    const agentId = job?.agentId;
+    if (!agentId) {
+      return {
+        blocked: true,
+        reason: "Could not resolve agent for embedding-heavy variant."
+      };
+    }
+
+    const health = await this.executor.scheduler.memoryStore.getEmbeddingHealth(agentId, {
+      embeddingIndex: this.executor.scheduler.embeddingIndex,
+      probe: false
+    });
+
+    if ((health.embeddingHealth?.freshCount || 0) > 0) {
+      return null;
+    }
+
+    const embeddingService = this.executor.scheduler.embeddingIndex?.embeddingService || null;
+    if (!embeddingService) {
+      return {
+        blocked: true,
+        reason: "No embedding service is configured for this runtime.",
+        health
+      };
+    }
+
+    const readiness = await embeddingService.getReadiness();
+    if (readiness?.ready) {
+      try {
+        await this.executor.scheduler.memoryStore.rebuildEmbeddings(agentId, {
+          embeddingIndex: this.executor.scheduler.embeddingIndex
+        });
+        const refreshed = await this.executor.scheduler.memoryStore.getEmbeddingHealth(agentId, {
+          embeddingIndex: this.executor.scheduler.embeddingIndex,
+          probe: false
+        });
+        if ((refreshed.embeddingHealth?.freshCount || 0) > 0) {
+          return {
+            blocked: false,
+            rebuilt: true,
+            health: refreshed,
+            readiness
+          };
+        }
+        return {
+          blocked: true,
+          reason: "Embeddings were rebuildable but no fresh vectors were produced for this lane.",
+          health: refreshed,
+          readiness,
+          rebuilt: true
+        };
+      } catch (error) {
+        const refreshed = await this.executor.scheduler.memoryStore.getEmbeddingHealth(agentId, {
+          embeddingIndex: this.executor.scheduler.embeddingIndex,
+          probe: false
+        });
+        return {
+          blocked: true,
+          reason: error?.message || String(error),
+          health: refreshed,
+          readiness,
+          rebuilt: true
+        };
+      }
+    }
+
+    return {
+      blocked: true,
+      reason: readiness?.reason || "Embeddings are unavailable or degraded for this lane.",
+      health,
+      readiness
+    };
+  }
+
+  async runVariant(targetId, variantId = "baseline", options = {}) {
+    const target = await this.getTarget(targetId);
+    const variant = buildVariant(target, variantId);
+    const mode = target.defaultMode || "provider";
+    let embedding = buildEmbeddingSummary();
+    let artifact;
+    try {
+      const prerequisiteFailure = await this.checkVariantPrerequisites(target, variant);
+      embedding = buildEmbeddingSummary({
+        readiness: prerequisiteFailure?.readiness || null,
+        health: prerequisiteFailure?.health?.embeddingHealth || null,
+        rebuilt: Boolean(prerequisiteFailure?.rebuilt),
+        blockedReason: prerequisiteFailure?.blocked ? prerequisiteFailure.reason : null
+      });
+      if (prerequisiteFailure?.blocked) {
+        artifact = {
+          timestamp: new Date().toISOString(),
+          targetId: target.id,
+          jobId: target.jobId,
+          variant,
+          ok: false,
+          skipped: true,
+          embedding,
+          skip: summarizeSkip(prerequisiteFailure.reason, {
+            embeddingHealth: prerequisiteFailure.health?.embeddingHealth || null,
+            embeddingReadiness: prerequisiteFailure.readiness || null
+          })
+        };
+        const filePath = await this.runStore.saveRun(target.id, artifact);
+        return {
+          filePath,
+          ...artifact
+        };
+      }
+
+      const run = await this.executor.executeJob(target.jobId, {
+        mode,
+        shadowImport: true,
+        modelOverride: variant.overrides.modelOverride || null,
+        reportGuidanceOverride: variant.overrides.reportGuidanceOverride || null,
+        retrievalBlendOverride: variant.overrides.retrievalBlendOverride || null,
+        providerTimeoutMs: target.defaultTimeoutMs || null,
+        allowReportFallback: options.allowReportFallback === true
+      });
+      const evaluation = await this.evaluator.evaluateAndPersistJob(target.jobId);
+      artifact = {
+        timestamp: new Date().toISOString(),
+        targetId: target.id,
+        jobId: target.jobId,
+        variant,
+        ok: true,
+        embedding: buildEmbeddingSummary({
+          readiness: embedding.readiness,
+          health: evaluation.retrieval?.embeddingHealth || embedding.health,
+          rebuilt: embedding.rebuilt,
+          blockedReason: null,
+          queryMode: evaluation.retrieval?.embeddingQueryMode || null
+        }),
+        run: {
+          filePath: run.filePath,
+          mode: run.mode,
+          providerId: run.providerId,
+          modelId: run.modelId,
+          routingDecision: run.routingDecision,
+          preflight: run.preflight,
+          summary: run.result?.summary || null,
+          fallback: run.fallback || null
+        },
+        evaluation: summarizeEvaluation(evaluation)
+      };
+    } catch (error) {
+      artifact = {
+        timestamp: new Date().toISOString(),
+        targetId: target.id,
+        jobId: target.jobId,
+        variant,
+        ok: false,
+        embedding,
+        error: summarizeFailure(error)
+      };
+    }
+    const filePath = await this.runStore.saveRun(target.id, artifact);
+    return {
+      filePath,
+      ...artifact
+    };
+  }
+
+  async compareVariants(targetId, baselineVariantId = "baseline", candidateVariantId, options = {}) {
+    if (!candidateVariantId) {
+      throw new Error("compareVariants requires a candidate variant id.");
+    }
+    const baseline = (await this.findLatestVariantArtifact(targetId, baselineVariantId, { successfulOnly: true })) ||
+      (await this.runVariant(targetId, baselineVariantId, options));
+    const candidate = await this.runVariant(targetId, candidateVariantId, options);
+    let comparisonResult;
+    if (!baseline.ok || !candidate.ok) {
+      comparisonResult = {
+        improved: false,
+        baseline: baseline.ok ? baseline.evaluation : null,
+        candidate: candidate.ok ? candidate.evaluation : null,
+        findings: [
+          ...(!baseline.ok
+            ? [baseline.skipped ? `Baseline variant skipped: ${baseline.skip.reason}` : `Baseline variant failed: ${baseline.error.message}`]
+            : []),
+          ...(!candidate.ok
+            ? [candidate.skipped ? `Candidate variant skipped: ${candidate.skip.reason}` : `Candidate variant failed: ${candidate.error.message}`]
+            : [])
+        ]
+      };
+    } else {
+      comparisonResult = compareEvaluations(baseline.evaluation, candidate.evaluation);
+      if (
+        candidateVariantId === "retrieval_embedding_heavy" &&
+        (
+          candidate.evaluation?.retrieval?.embeddingQueryMode === "lexical_fallback" ||
+          (candidate.evaluation?.retrieval?.freshEmbeddingCount || 0) === 0
+        )
+      ) {
+        comparisonResult.improved = false;
+        comparisonResult.findings.push(
+          "Candidate could not exercise embedding-heavy retrieval because embeddings were unavailable or degraded."
+        );
+      }
+    }
+    const comparison = {
+      timestamp: new Date().toISOString(),
+      targetId: baseline.targetId,
+      baselineVariantId,
+      candidateVariantId,
+      fallback: {
+        baseline: baseline.run?.fallback || baseline.error?.fallback || null,
+        candidate: candidate.run?.fallback || candidate.error?.fallback || null
+      },
+      embedding: {
+        baseline: baseline.embedding || null,
+        candidate: candidate.embedding || null
+      },
+      comparison: comparisonResult
+    };
+    const filePath = await this.runStore.saveRun(`${baseline.targetId}-comparison`, comparison);
+    return {
+      filePath,
+      ...comparison
+    };
+  }
+
+  async repairAndCompare(targetId, candidateVariantId, baselineVariantId = "baseline", options = {}) {
+    if (!candidateVariantId) {
+      throw new Error("repairAndCompare requires a candidate variant id.");
+    }
+
+    const target = await this.getTarget(targetId);
+    const candidateVariant = buildVariant(target, candidateVariantId);
+    const prerequisite = await this.checkVariantPrerequisites(target, candidateVariant);
+
+    if (prerequisite?.blocked) {
+      const artifact = {
+        timestamp: new Date().toISOString(),
+        targetId: target.id,
+        baselineVariantId,
+        candidateVariantId,
+        repaired: false,
+        blocked: true,
+        embedding: buildEmbeddingSummary({
+          readiness: prerequisite.readiness || null,
+          health: prerequisite.health?.embeddingHealth || null,
+          rebuilt: Boolean(prerequisite.rebuilt),
+          blockedReason: prerequisite.reason
+        }),
+        findings: [`Repair blocked: ${prerequisite.reason}`]
+      };
+      const filePath = await this.runStore.saveRun(`${target.id}-repair`, artifact);
+      return {
+        filePath,
+        ...artifact
+      };
+    }
+
+    const rerunBaseline = Boolean(prerequisite?.rebuilt);
+    const baseline = rerunBaseline
+      ? await this.runVariant(targetId, baselineVariantId, options)
+      : (await this.findLatestVariantArtifact(targetId, baselineVariantId, { successfulOnly: true })) ||
+        (await this.runVariant(targetId, baselineVariantId, options));
+    const candidate = await this.runVariant(targetId, candidateVariantId, options);
+
+    let comparisonResult;
+    if (!baseline.ok || !candidate.ok) {
+      comparisonResult = {
+        improved: false,
+        baseline: baseline.ok ? baseline.evaluation : null,
+        candidate: candidate.ok ? candidate.evaluation : null,
+        findings: [
+          ...(!baseline.ok
+            ? [baseline.skipped ? `Baseline variant skipped: ${baseline.skip.reason}` : `Baseline variant failed: ${baseline.error.message}`]
+            : []),
+          ...(!candidate.ok
+            ? [candidate.skipped ? `Candidate variant skipped: ${candidate.skip.reason}` : `Candidate variant failed: ${candidate.error.message}`]
+            : [])
+        ]
+      };
+    } else {
+      comparisonResult = compareEvaluations(baseline.evaluation, candidate.evaluation);
+      if (
+        candidateVariantId === "retrieval_embedding_heavy" &&
+        (
+          candidate.evaluation?.retrieval?.embeddingQueryMode === "lexical_fallback" ||
+          (candidate.evaluation?.retrieval?.freshEmbeddingCount || 0) === 0
+        )
+      ) {
+        comparisonResult.improved = false;
+        comparisonResult.findings.push(
+          "Candidate could not exercise embedding-heavy retrieval because embeddings were unavailable or degraded."
+        );
+      }
+    }
+
+    const artifact = {
+      timestamp: new Date().toISOString(),
+      targetId: target.id,
+      baselineVariantId,
+      candidateVariantId,
+      repaired: Boolean(prerequisite?.rebuilt),
+      blocked: false,
+      fallback: {
+        baseline: baseline.run?.fallback || baseline.error?.fallback || null,
+        candidate: candidate.run?.fallback || candidate.error?.fallback || null
+      },
+      embedding: {
+        readiness: prerequisite?.readiness || null,
+        health: candidate.embedding?.health || prerequisite?.health?.embeddingHealth || null,
+        rebuilt: Boolean(prerequisite?.rebuilt),
+        blockedReason: null,
+        queryMode: candidate.embedding?.queryMode || null
+      },
+      baseline: {
+        filePath: baseline.filePath,
+        evaluationFile: baseline.evaluation?.filePath || null
+      },
+      candidate: {
+        filePath: candidate.filePath,
+        evaluationFile: candidate.evaluation?.filePath || null,
+        status: candidate.ok ? "evaluated" : candidate.skipped ? "skipped" : candidate.error?.classification || "failed"
+      },
+      comparison: comparisonResult
+    };
+    const filePath = await this.runStore.saveRun(`${target.id}-repair`, artifact);
+    return {
+      filePath,
+      ...artifact
+    };
+  }
+}