@tangle-network/agent-eval 0.7.2 → 0.11.1

package/dist/index.js

@@ -410,22 +410,22 @@ function confidenceInterval(scores, confidence = 0.95) {
   if (scores.length === 0) return { mean: 0, lower: 0, upper: 0 };
   if (scores.length === 1) return { mean: scores[0], lower: scores[0], upper: scores[0] };
   const n = scores.length;
-  const mean4 = scores.reduce((a, b) => a + b, 0) / n;
+  const mean5 = scores.reduce((a, b) => a + b, 0) / n;
   const B = 1e3;
   const bootstrapMeans = [];
   for (let i = 0; i < B; i++) {
-    let sum = 0;
+    let sum2 = 0;
     for (let j = 0; j < n; j++) {
-      sum += scores[Math.floor(Math.random() * n)];
+      sum2 += scores[Math.floor(Math.random() * n)];
     }
-    bootstrapMeans.push(sum / n);
+    bootstrapMeans.push(sum2 / n);
   }
   bootstrapMeans.sort((a, b) => a - b);
   const alpha = 1 - confidence;
   const lowerIdx = Math.floor(alpha / 2 * B);
   const upperIdx = Math.floor((1 - alpha / 2) * B) - 1;
   return {
-    mean: mean4,
+    mean: mean5,
     lower: bootstrapMeans[lowerIdx],
     upper: bootstrapMeans[Math.min(upperIdx, B - 1)]
   };
@@ -513,11 +513,11 @@ function pairedTTest(before, after) {
   const n = before.length;
   if (n < 2) return { t: 0, df: 0, p: 1 };
   const diffs = before.map((b, i) => after[i] - b);
-  const mean4 = diffs.reduce((a, b) => a + b, 0) / n;
-  const variance2 = diffs.reduce((acc, d) => acc + (d - mean4) ** 2, 0) / (n - 1);
+  const mean5 = diffs.reduce((a, b) => a + b, 0) / n;
+  const variance2 = diffs.reduce((acc, d) => acc + (d - mean5) ** 2, 0) / (n - 1);
   const se = Math.sqrt(variance2 / n);
-  if (se === 0) return { t: mean4 === 0 ? 0 : Infinity, df: n - 1, p: mean4 === 0 ? 1 : 0 };
-  const t = mean4 / se;
+  if (se === 0) return { t: mean5 === 0 ? 0 : Infinity, df: n - 1, p: mean5 === 0 ? 1 : 0 };
+  const t = mean5 / se;
   const df = n - 1;
   const p = 2 * (1 - studentTCdf(Math.abs(t), df));
   return { t, df, p };
@@ -541,9 +541,9 @@ function wilcoxonSignedRank(before, after) {
   }
   let wPlus = 0;
   for (let k = 0; k < n; k++) if (diffs[k] > 0) wPlus += ranks3[k];
-  const mean4 = n * (n + 1) / 4;
+  const mean5 = n * (n + 1) / 4;
   const variance2 = n * (n + 1) * (2 * n + 1) / 24;
-  const z = (wPlus - mean4) / Math.sqrt(variance2);
+  const z = (wPlus - mean5) / Math.sqrt(variance2);
   const p = 2 * (1 - normalCdf(Math.abs(z)));
   return { w: wPlus, p };
 }
@@ -1562,12 +1562,12 @@ function analyzeAntiSlop(outputs, config) {
       }
       let dupes = 0;
       for (const n of seen.values()) if (n > 1) dupes += n - 1;
-      const ratio = dupes / sentences.length;
-      if (ratio > config.repetitionThreshold) {
+      const ratio2 = dupes / sentences.length;
+      if (ratio2 > config.repetitionThreshold) {
         counts.repetition += 1;
         issues.push({
           category: "repetition",
-          detail: `${(ratio * 100).toFixed(0)}% duplicated (threshold ${(config.repetitionThreshold * 100).toFixed(0)}%)`
+          detail: `${(ratio2 * 100).toFixed(0)}% duplicated (threshold ${(config.repetitionThreshold * 100).toFixed(0)}%)`
         });
       }
     }
@@ -2187,7 +2187,7 @@ var RunCritic = class {
     );
     const success = trace.run.outcome?.pass === true ? 1 : trace.run.status === "completed" ? 0.5 : 0;
     if (!success) notes.push("run did not complete with pass=true");
-    const judgeAverage = judgeSpans2.length ? judgeSpans2.reduce((sum, span) => sum + normalizeJudgeScore(span.score), 0) / judgeSpans2.length : void 0;
+    const judgeAverage = judgeSpans2.length ? judgeSpans2.reduce((sum2, span) => sum2 + normalizeJudgeScore(span.score), 0) / judgeSpans2.length : void 0;
     const outcomeScore = typeof trace.run.outcome?.score === "number" ? clamp01(trace.run.outcome.score > 1 ? trace.run.outcome.score / 100 : trace.run.outcome.score) : void 0;
     const goalProgress = outcomeScore ?? judgeAverage ?? success;
     const successfulTools = toolSpans2.filter((span) => span.status !== "error").length;
@@ -2197,7 +2197,7 @@ var RunCritic = class {
     const patchQuality = patchEvidence > 0 ? clamp01(patchEvidence / 4) : 0;
     if (!patchQuality) notes.push("no artifact or edit evidence recorded");
     const sandboxTests = sandboxSpans.filter((span) => typeof span.testsTotal === "number" && span.testsTotal > 0);
-    const testReality = sandboxTests.length ? sandboxTests.reduce((sum, span) => sum + (span.testsPassed ?? 0) / Math.max(1, span.testsTotal ?? 1), 0) / sandboxTests.length : toolSpans2.some((span) => /\btest|vitest|pytest|jest|build|tsc\b/i.test(JSON.stringify(span.args))) ? 0.4 : 0;
+    const testReality = sandboxTests.length ? sandboxTests.reduce((sum2, span) => sum2 + (span.testsPassed ?? 0) / Math.max(1, span.testsTotal ?? 1), 0) / sandboxTests.length : toolSpans2.some((span) => /\btest|vitest|pytest|jest|build|tsc\b/i.test(JSON.stringify(span.args))) ? 0.4 : 0;
     if (!testReality) notes.push("no real test/build evidence recorded");
     const blockerSpans = judgeSpans2.filter(
       (span) => isBlockingJudge(span)
@@ -2213,7 +2213,7 @@ var RunCritic = class {
     const repoGroundedness = positiveGroundingSignals + driftSignals === 0 ? 0 : positiveGroundingSignals / (positiveGroundingSignals + driftSignals);
     const driftPenalty = positiveGroundingSignals + driftSignals === 0 ? 0 : driftSignals / (positiveGroundingSignals + driftSignals);
     if (driftSignals > 0) notes.push(`detected ${driftSignals} drift signal(s)`);
-    const costUsd = trace.budget.length ? Math.max(...trace.budget.filter((entry) => entry.dimension === "usd").map((entry) => entry.consumed), 0) : llmSpans2.reduce((sum, span) => sum + (span.costUsd ?? 0), 0);
+    const costUsd = trace.budget.length ? Math.max(...trace.budget.filter((entry) => entry.dimension === "usd").map((entry) => entry.consumed), 0) : llmSpans2.reduce((sum2, span) => sum2 + (span.costUsd ?? 0), 0);
     const wallSeconds = trace.run.endedAt && trace.run.startedAt ? Math.max(0, (trace.run.endedAt - trace.run.startedAt) / 1e3) : 0;
     return {
       success,
@@ -2423,7 +2423,7 @@ function rankRows(rows, weights) {
   }
   return [...buckets.entries()].map(([variantId, values]) => ({
     variantId,
-    mean: values.reduce((sum, value) => sum + value, 0) / values.length,
+    mean: values.reduce((sum2, value) => sum2 + value, 0) / values.length,
     runs: values.length
   })).sort((a, b) => b.mean - a.mean);
 }
@@ -2571,7 +2571,7 @@ async function mapLimit(items, limit, fn) {
   return results;
 }
 function mean(values) {
-  return values.length ? values.reduce((sum, value) => sum + value, 0) / values.length : 0;
+  return values.length ? values.reduce((sum2, value) => sum2 + value, 0) / values.length : 0;
 }
 function meanRunScore(scores) {
   return {
@@ -5095,10 +5095,10 @@ function analyzeSeries(values, options = {}) {
     return { state: "insufficient-data", windowMean: 0, windowCv: 0, tailRun: 0, stable: false };
   }
   const tail = values.slice(-window);
-  const mean4 = tail.reduce((a, b) => a + b, 0) / tail.length;
-  const variance2 = tail.reduce((acc, v) => acc + (v - mean4) ** 2, 0) / tail.length;
+  const mean5 = tail.reduce((a, b) => a + b, 0) / tail.length;
+  const variance2 = tail.reduce((acc, v) => acc + (v - mean5) ** 2, 0) / tail.length;
   const stdDev = Math.sqrt(variance2);
-  const refMean = Math.abs(mean4) > 1e-9 ? Math.abs(mean4) : 1;
+  const refMean = Math.abs(mean5) > 1e-9 ? Math.abs(mean5) : 1;
   const cv = stdDev / refMean;
   const stable = tail.length >= window && cv <= stableCv;
   let tailRun = 0;
@@ -5119,7 +5119,7 @@ function analyzeSeries(values, options = {}) {
   } else {
     state = "noisy";
   }
-  return { state, windowMean: mean4, windowCv: cv, tailRun, stable };
+  return { state, windowMean: mean5, windowCv: cv, tailRun, stable };
 }
 
 // src/state-continuity.ts
@@ -5712,8 +5712,8 @@ function calibrateJudge(golden, candidate) {
   const kappa = weightedKappa(humans.map(Math.round), judges.map(Math.round));
   const absDiffs = common.map((c) => Math.abs(c.j - c.h));
   const mae = absDiffs.reduce((a, b) => a + b, 0) / n;
-  const worst = [...map.entries()].filter(([, v]) => Number.isFinite(v.j)).map(([itemId, v]) => ({ itemId, judge: v.j, human: v.h, delta: Math.abs(v.j - v.h) })).sort((a, b) => b.delta - a.delta).slice(0, 5);
-  return { n, pearson: pearson2, kappa, mae, worstItems: worst };
+  const worst2 = [...map.entries()].filter(([, v]) => Number.isFinite(v.j)).map(([itemId, v]) => ({ itemId, judge: v.j, human: v.h, delta: Math.abs(v.j - v.h) })).sort((a, b) => b.delta - a.delta).slice(0, 5);
+  return { n, pearson: pearson2, kappa, mae, worstItems: worst2 };
 }
 function positionalBias(scores) {
   const pairs = /* @__PURE__ */ new Map();
@@ -6047,12 +6047,12 @@ async function paraphraseRobustness(prompt, mutators, scoreFn, options = {}) {
     variantScores.push({ mutator: id, score, mutated });
     all.push(score);
   }
-  const mean4 = all.reduce((a, b) => a + b, 0) / all.length;
-  const variance2 = all.reduce((a, v) => a + (v - mean4) ** 2, 0) / all.length;
+  const mean5 = all.reduce((a, b) => a + b, 0) / all.length;
+  const variance2 = all.reduce((a, v) => a + (v - mean5) ** 2, 0) / all.length;
   const stdDev = Math.sqrt(variance2);
-  const ref = Math.abs(mean4) > 1e-9 ? Math.abs(mean4) : 1;
+  const ref = Math.abs(mean5) > 1e-9 ? Math.abs(mean5) : 1;
   const robustness = Math.max(0, 1 - stdDev / ref);
-  return { originalScore, variantScores, meanScore: mean4, stdDev, robustness };
+  return { originalScore, variantScores, meanScore: mean5, stdDev, robustness };
 }
 var lowercaseMutator = (p) => p.toLowerCase();
 var sentenceReorderMutator = (p, seed) => {
@@ -6114,9 +6114,9 @@ function visualDiff(a, b, options = {}) {
     const dg = Math.abs(a.data[i + 1] - b.data[i + 1]);
     const db = Math.abs(a.data[i + 2] - b.data[i + 2]);
     const da = Math.abs(a.data[i + 3] - b.data[i + 3]);
-    const worst = Math.max(dr, dg, db, da);
-    if (worst > maxDelta) maxDelta = worst;
-    if (worst > tolerance) differing++;
+    const worst2 = Math.max(dr, dg, db, da);
+    if (worst2 > maxDelta) maxDelta = worst2;
+    if (worst2 > tolerance) differing++;
   }
   const diffRatio = totalPixels > 0 ? differing / totalPixels : 0;
   const status = diffRatio < 1e-3 ? "unchanged" : diffRatio > 0.05 ? "severely-changed" : "changed";
@@ -6973,8 +6973,8 @@ async function prmBestOfN(store, grader, runIds) {
   if (runIds.length === 0) throw new Error("prmBestOfN: at least 1 candidate required");
   const graded = await Promise.all(runIds.map((id) => grader.grade(store, id)));
   const ranked = [...graded].sort((a, b) => b.aggregateScore - a.aggregateScore);
-  const mean4 = graded.reduce((a, g) => a + g.aggregateScore, 0) / graded.length;
-  const variance2 = graded.reduce((a, g) => a + (g.aggregateScore - mean4) ** 2, 0) / graded.length;
+  const mean5 = graded.reduce((a, g) => a + g.aggregateScore, 0) / graded.length;
+  const variance2 = graded.reduce((a, g) => a + (g.aggregateScore - mean5) ** 2, 0) / graded.length;
   return { winner: ranked[0], ranked, stdDev: Math.sqrt(variance2) };
 }
 async function prmEnsembleBestOfN(store, graders, runIds) {
@@ -6996,8 +6996,8 @@ async function prmEnsembleBestOfN(store, graders, runIds) {
   const ranked = [...byRun.values()].sort(
     (a, b) => (bordaScores.get(b.runId) ?? 0) - (bordaScores.get(a.runId) ?? 0)
   );
-  const mean4 = ranked.reduce((a, g) => a + g.aggregateScore, 0) / ranked.length;
-  const variance2 = ranked.reduce((a, g) => a + (g.aggregateScore - mean4) ** 2, 0) / ranked.length;
+  const mean5 = ranked.reduce((a, g) => a + g.aggregateScore, 0) / ranked.length;
+  const variance2 = ranked.reduce((a, g) => a + (g.aggregateScore - mean5) ** 2, 0) / ranked.length;
   return { winner: ranked[0], ranked, stdDev: Math.sqrt(variance2) };
 }
 
@@ -7055,7 +7055,7 @@ async function commitBisect(options) {
 }
 async function promptBisect(options) {
   const split = options.paragraphSplitter ?? ((p) => p.split(/\n\s*\n/));
-  const join2 = (paragraphs) => paragraphs.join("\n\n");
+  const join3 = (paragraphs) => paragraphs.join("\n\n");
   const goodParas = split(options.good);
   const badParas = split(options.bad);
   if (goodParas.length !== badParas.length) {
@@ -7073,7 +7073,7 @@ async function promptBisect(options) {
   const result = await bisect({
     good: goodMask,
     bad: badMask,
-    runEval: (mask) => options.runEval(join2(paragraphsFor(mask))),
+    runEval: (mask) => options.runEval(join3(paragraphsFor(mask))),
     maxIterations: options.maxIterations ?? n + 5,
     halfway: (g, b) => {
       for (let i = 0; i < g.length; i++) {
@@ -7104,12 +7104,12 @@ async function promptBisect(options) {
     }
   }
   const materializedPath = result.path.map((s) => ({
-    state: join2(paragraphsFor(s.state)),
+    state: join3(paragraphsFor(s.state)),
     score: s.score,
     pass: s.pass
   }));
   return {
-    culprit: join2(paragraphsFor(culprit)),
+    culprit: join3(paragraphsFor(culprit)),
     path: materializedPath,
     converged: result.converged,
     inputInconsistent: result.inputInconsistent,
@@ -7527,8 +7527,8 @@ async function proposeSynthesisTargets(dataset, traceStore, options = {}) {
     const sRuns = runs.filter((r) => r.scenarioId === s.id);
     const scores = sRuns.map((r) => r.outcome?.score).filter((x) => typeof x === "number");
     if (scores.length < 3) continue;
-    const mean4 = scores.reduce((a, b) => a + b, 0) / scores.length;
-    const variance2 = scores.reduce((a, b) => a + (b - mean4) ** 2, 0) / scores.length;
+    const mean5 = scores.reduce((a, b) => a + b, 0) / scores.length;
+    const variance2 = scores.reduce((a, b) => a + (b - mean5) ** 2, 0) / scores.length;
     if (variance2 > varianceThreshold) {
       targets.push({
         reason: "high-variance",
@@ -8007,6 +8007,1975 @@ async function euAiActReport(ctx, signals) {
     generatedAt: (/* @__PURE__ */ new Date()).toISOString()
   };
 }
+
+// src/llm-client.ts
+var LlmCallError = class extends Error {
+  constructor(message, status, body, model) {
+    super(message);
+    this.status = status;
+    this.body = body;
+    this.model = model;
+    this.name = "LlmCallError";
+  }
+  status;
+  body;
+  model;
+};
+var DEFAULT_BASE_URL = "https://router.tangle.tools/v1";
+var DEFAULT_TIMEOUT_MS = 6e4;
+var DEFAULT_MAX_RETRIES = 3;
+var RETRYABLE_STATUS = /* @__PURE__ */ new Set([429, 502, 503, 504]);
+function isRetryableError(err) {
+  if (err instanceof LlmCallError) return RETRYABLE_STATUS.has(err.status);
+  if (err instanceof Error) {
+    return err.name === "AbortError" || err.name === "TimeoutError" || /fetch failed|ECONNRESET|ETIMEDOUT|EAI_AGAIN/i.test(err.message);
+  }
+  return false;
+}
+function parseRetryAfter(headers) {
+  const h = headers.get("retry-after");
+  if (!h) return null;
+  const asNumber = Number(h);
+  if (Number.isFinite(asNumber) && asNumber > 0) return asNumber * 1e3;
+  const asDate = Date.parse(h);
+  if (Number.isFinite(asDate)) return Math.max(0, asDate - Date.now());
+  return null;
+}
+function backoffMs(attempt) {
+  return Math.min(500 * Math.pow(2, attempt), 16e3);
+}
+function buildHeaders(opts) {
+  const headers = {
+    "Content-Type": "application/json",
+    Accept: "application/json"
+  };
+  if (opts.authHeader) {
+    headers[opts.authHeader.name] = opts.authHeader.value;
+  } else if (opts.bearer || opts.apiKey) {
+    headers.Authorization = `Bearer ${opts.bearer ?? opts.apiKey}`;
+  }
+  return headers;
+}
+function isSchemaRejection(status, body) {
+  if (status !== 400) return false;
+  const lower = body.toLowerCase();
+  return lower.includes("response_format") || lower.includes("json_schema") || lower.includes("is unavailable") || lower.includes("not supported");
+}
+function buildBody(req, forceJsonObject) {
+  const body = {
+    model: req.model,
+    messages: req.messages,
+    temperature: req.temperature ?? 0
+  };
+  if (req.maxTokens != null) body.max_tokens = req.maxTokens;
+  if (req.jsonSchema && !forceJsonObject) {
+    body.response_format = {
+      type: "json_schema",
+      json_schema: { name: req.jsonSchema.name, schema: req.jsonSchema.schema, strict: true }
+    };
+  } else if (req.jsonMode || req.jsonSchema) {
+    body.response_format = { type: "json_object" };
+  }
+  return body;
+}
+async function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+function stripFencedJson(raw) {
+  const trimmed = raw.trim();
+  const m = trimmed.match(/^```(?:json)?\s*\n?([\s\S]*?)\n?```\s*$/);
+  return m ? m[1].trim() : trimmed;
+}
+async function callLlm(req, opts = {}) {
+  const baseUrl = (opts.baseUrl ?? DEFAULT_BASE_URL).replace(/\/+$/, "");
+  const url = `${baseUrl}/chat/completions`;
+  const timeoutMs = req.timeoutMs ?? opts.defaultTimeoutMs ?? DEFAULT_TIMEOUT_MS;
+  const maxRetries = opts.maxRetries ?? DEFAULT_MAX_RETRIES;
+  const fetchFn = opts.fetch ?? globalThis.fetch;
+  const headers = buildHeaders(opts);
+  let lastErr;
+  for (let attempt = 0; attempt < maxRetries; attempt++) {
+    const controller = new AbortController();
+    const timeoutHandle = setTimeout(() => controller.abort(), timeoutMs);
+    const started = Date.now();
+    try {
+      const res = await fetchFn(url, {
+        method: "POST",
+        headers,
+        body: JSON.stringify(buildBody(req, false)),
+        signal: controller.signal
+      });
+      clearTimeout(timeoutHandle);
+      if (!res.ok) {
+        const body = await res.text();
+        const err = new LlmCallError(
+          `LLM call ${res.status}: ${body.slice(0, 300)}`,
+          res.status,
+          body,
+          req.model
+        );
+        if (RETRYABLE_STATUS.has(res.status) && attempt < maxRetries - 1) {
+          lastErr = err;
+          const retryAfter = parseRetryAfter(res.headers);
+          await sleep(retryAfter ?? backoffMs(attempt));
+          continue;
+        }
+        throw err;
+      }
+      const json = await res.json();
+      const choice = json.choices?.[0];
+      const usageRaw = json.usage ?? {};
+      const costFromProxy = json._response_cost ?? json.cost_usd;
+      return {
+        content: choice?.message?.content ?? "",
+        usage: {
+          promptTokens: Number(usageRaw.prompt_tokens ?? 0),
+          completionTokens: Number(usageRaw.completion_tokens ?? 0),
+          totalTokens: Number(usageRaw.total_tokens ?? 0),
+          cachedPromptTokens: usageRaw.prompt_tokens_details && typeof usageRaw.prompt_tokens_details === "object" ? Number(
+            usageRaw.prompt_tokens_details.cached_tokens ?? 0
+          ) : void 0
+        },
+        costUsd: typeof costFromProxy === "number" ? costFromProxy : null,
+        model: json.model ?? req.model,
+        durationMs: Date.now() - started,
+        raw: json
+      };
+    } catch (err) {
+      clearTimeout(timeoutHandle);
+      lastErr = err;
+      if (attempt < maxRetries - 1 && isRetryableError(err)) {
+        await sleep(backoffMs(attempt));
+        continue;
+      }
+      throw err;
+    }
+  }
+  throw lastErr instanceof Error ? lastErr : new Error(String(lastErr));
+}
+async function callLlmJson(req, opts = {}) {
+  try {
+    const result = await callLlm({ ...req, jsonMode: req.jsonMode ?? !req.jsonSchema }, opts);
+    const value = parseJsonSafely(result.content, result.model);
+    return { value, result };
+  } catch (err) {
+    if (err instanceof LlmCallError && isSchemaRejection(err.status, err.body) && req.jsonSchema) {
+      const degradedReq = { ...req, jsonMode: true, jsonSchema: void 0 };
+      const result = await callLlm(degradedReq, opts);
+      const value = parseJsonSafely(result.content, result.model);
+      return { value, result };
+    }
+    throw err;
+  }
+}
+function parseJsonSafely(content, model) {
+  const stripped = stripFencedJson(content);
+  try {
+    return JSON.parse(stripped);
+  } catch (err) {
+    throw new Error(
+      `LLM returned non-JSON content (model=${model}): ${err instanceof Error ? err.message : String(err)}
+--- raw content ---
+${content.slice(0, 800)}`
+    );
+  }
+}
+async function probeLlm(model, opts = {}) {
+  const start = Date.now();
+  try {
+    await callLlm(
+      {
+        model,
+        messages: [{ role: "user", content: "ping" }],
+        maxTokens: 64,
+        timeoutMs: opts.timeoutMs ?? 3e4
+      },
+      opts
+    );
+    return { ok: true, latencyMs: Date.now() - start, error: null };
+  } catch (err) {
+    return {
+      ok: false,
+      latencyMs: Date.now() - start,
+      error: err instanceof Error ? err.message : String(err)
+    };
+  }
+}
+var LlmClient = class {
+  constructor(opts = {}) {
+    this.opts = opts;
+  }
+  opts;
+  call(req, per) {
+    return callLlm(req, { ...this.opts, ...per });
+  }
+  callJson(req, per) {
+    return callLlmJson(req, { ...this.opts, ...per });
+  }
+};
+
+// src/multi-layer-verifier.ts
+function gradeSemanticStatus(input) {
+  if (!input.available) return "error";
+  const threshold = input.threshold ?? 0.7;
+  const criticalGaps = input.findings.filter(
+    (f) => f.severity === "critical" && (f.present === false || (f.score ?? 0) < 7)
+  );
+  return input.score >= threshold && criticalGaps.length === 0 ? "pass" : "fail";
+}
+var MultiLayerVerifier = class {
+  constructor(layers) {
+    this.layers = layers;
+    const seen = /* @__PURE__ */ new Set();
+    for (const l of layers) {
+      if (seen.has(l.name)) throw new Error(`MultiLayerVerifier: duplicate layer name "${l.name}"`);
+      seen.add(l.name);
+    }
+    for (const l of layers) {
+      for (const dep of l.dependsOn ?? []) {
+        if (!seen.has(dep)) {
+          throw new Error(
+            `MultiLayerVerifier: layer "${l.name}" depends on "${dep}" which is not registered`
+          );
+        }
+      }
+    }
+  }
+  layers;
+  async run(opts) {
+    const startedAtMs = Date.now();
+    const startedAt = new Date(startedAtMs).toISOString();
+    const controller = new AbortController();
+    const overallCap = opts.overallCapMs;
+    const overallTimer = overallCap != null ? setTimeout(() => controller.abort(new Error("overall cap exceeded")), overallCap) : null;
+    const results = [];
+    const byName = {};
+    try {
+      for (const layer of this.layers) {
+        const unmet = (layer.dependsOn ?? []).filter((d) => byName[d]?.status !== "pass");
+        if (unmet.length > 0) {
+          const skipped = {
+            layer: layer.name,
+            status: "skipped",
+            durationMs: 0,
+            findings: [],
+            reason: `skipped \u2014 upstream not passing: ${unmet.join(", ")}`
+          };
+          results.push(skipped);
+          byName[layer.name] = skipped;
+          opts.onLayer?.(skipped);
+          continue;
+        }
+        const perLayerController = new AbortController();
+        const mergedSignal = mergeSignals(controller.signal, perLayerController.signal);
+        const layerTimer = layer.capMs != null ? setTimeout(() => perLayerController.abort(new Error(`layer ${layer.name} cap`)), layer.capMs) : null;
+        const layerStart = Date.now();
+        let result;
+        try {
+          result = await layer.run({ env: opts.env, prior: { ...byName }, signal: mergedSignal });
+        } catch (err) {
+          const aborted = mergedSignal.aborted;
+          result = {
+            layer: layer.name,
+            status: aborted ? "timeout" : "error",
+            durationMs: Date.now() - layerStart,
+            findings: [
+              {
+                severity: "major",
+                message: err instanceof Error ? err.message : String(err),
+                layer: layer.name
+              }
+            ],
+            reason: err instanceof Error ? err.message : String(err)
+          };
+        } finally {
+          if (layerTimer) clearTimeout(layerTimer);
+        }
+        result.findings = result.findings.map((f) => ({ ...f, layer: f.layer ?? layer.name }));
+        results.push(result);
+        byName[layer.name] = result;
+        opts.onLayer?.(result);
+        if (controller.signal.aborted) break;
+      }
+      const report = aggregate(this.layers, results, startedAt, startedAtMs);
+      return report;
+    } finally {
+      if (overallTimer) clearTimeout(overallTimer);
+    }
+  }
+};
+function aggregate(layers, results, startedAt, startedAtMs) {
+  const weightByName = /* @__PURE__ */ new Map();
+  const failContribByName = /* @__PURE__ */ new Map();
+  for (const l of layers) {
+    weightByName.set(l.name, l.weight ?? 1);
+    failContribByName.set(l.name, l.failContributesToScore ?? false);
+  }
+  let passCount = 0;
+  let failCount = 0;
+  let skippedCount = 0;
+  let errorCount = 0;
+  let scoredWeightSum = 0;
+  let scoredWeightedTotal = 0;
+  let ranAnyScoredLayer = false;
+  let anyScoredLayerFailed = false;
+  for (const r of results) {
+    const weight = weightByName.get(r.layer) ?? 1;
+    const failContrib = failContribByName.get(r.layer) ?? false;
+    if (r.status === "pass") passCount++;
+    else if (r.status === "fail") failCount++;
+    else if (r.status === "skipped") skippedCount++;
+    else errorCount++;
+    if (r.score != null && weight > 0) {
+      if (r.status === "pass") {
+        ranAnyScoredLayer = true;
+        scoredWeightSum += weight;
+        scoredWeightedTotal += weight * r.score;
+      } else if (r.status === "fail") {
+        if (failContrib) {
+          ranAnyScoredLayer = true;
+          scoredWeightSum += weight;
+          scoredWeightedTotal += weight * r.score;
+        }
+        anyScoredLayerFailed = true;
+      }
+    } else if (r.status === "fail") {
+      anyScoredLayerFailed = true;
+    }
+  }
+  const finishedAtMs = Date.now();
+  return {
+    layers: results,
+    passCount,
+    failCount,
+    skippedCount,
+    errorCount,
+    allPass: ranAnyScoredLayer && !anyScoredLayerFailed && failCount === 0 && errorCount === 0,
+    blendedScore: scoredWeightSum > 0 ? scoredWeightedTotal / scoredWeightSum : 0,
+    durationMs: finishedAtMs - startedAtMs,
+    startedAt,
+    finishedAt: new Date(finishedAtMs).toISOString()
+  };
+}
+function mergeSignals(a, b) {
+  if (a.aborted) return a;
+  if (b.aborted) return b;
+  const c = new AbortController();
+  const onAbort = (signal) => () => c.abort(signal.reason);
+  a.addEventListener("abort", onAbort(a), { once: true });
+  b.addEventListener("abort", onAbort(b), { once: true });
+  return c.signal;
+}
+
+// src/command-runner.ts
+import { spawnSync } from "child_process";
+import { existsSync as existsSync3, readFileSync as readFileSync3, readdirSync as readdirSync2, statSync as statSync2 } from "fs";
+import { join as join2 } from "path";
+var localCommandRunner = {
+  name: "local",
+  async run(input) {
+    const start = Date.now();
+    const res = spawnSync(input.cmd, input.argv, {
+      cwd: input.cwd,
+      encoding: "utf8",
+      timeout: input.capMs,
+      env: { ...process.env, CI: "1", ...input.env ?? {} },
+      input: input.stdin
+    });
+    const durationMs = Date.now() - start;
+    const timedOut = !!(res.error && "code" in res.error && res.error.code === "ETIMEDOUT");
+    return {
+      status: res.status ?? null,
+      stdout: (res.stdout ?? "").toString(),
+      stderr: (res.stderr ?? "").toString(),
+      durationMs,
+      timedOut,
+      runnerError: res.error && !timedOut ? String(res.error.message ?? res.error) : void 0
+    };
+  },
+  async hasBin(name) {
+    const r = spawnSync("which", [name], { encoding: "utf8", timeout: 2e3 });
+    return r.status === 0 && (r.stdout ?? "").trim().length > 0;
+  },
+  async fileExists(path) {
+    return existsSync3(path);
+  },
+  async readFile(path) {
+    try {
+      return readFileSync3(path, "utf8");
+    } catch {
+      return null;
+    }
+  },
+  async readDir(path) {
+    let entries;
+    try {
+      entries = readdirSync2(path);
+    } catch {
+      return [];
+    }
+    const out = [];
+    for (const name of entries) {
+      try {
+        const st = statSync2(join2(path, name));
+        out.push({
+          name,
+          isDirectory: st.isDirectory(),
+          isFile: st.isFile(),
+          sizeBytes: st.isFile() ? st.size : null
+        });
+      } catch {
+      }
+    }
+    return out;
+  }
+};
+
+// src/multi-toolchain-layer.ts
+var STATUS_RANK = {
+  pass: 0,
+  skipped: 1,
+  fail: 2,
+  timeout: 3,
+  error: 4
+};
+function worst(a, b) {
+  return (STATUS_RANK[a] ?? 0) >= (STATUS_RANK[b] ?? 0) ? a : b;
+}
+var SEVERITY_RANK = {
+  info: 0,
+  minor: 1,
+  major: 2,
+  critical: 3
+};
+function maxSeverity(findings) {
+  let best = "info";
+  for (const f of findings) {
+    if (SEVERITY_RANK[f.severity] > SEVERITY_RANK[best]) best = f.severity;
+  }
+  return best;
+}
+function mergeLayerResults(name, perAdapter, options = {}) {
+  const mergeDuration = options.mergeDuration ?? "max";
+  const mergeDiagnostics = options.mergeDiagnostics ?? "max";
+  const prefix = options.messagePrefixer;
+  if (perAdapter.length === 0) {
+    return {
+      layer: name,
+      status: "skipped",
+      durationMs: 0,
+      findings: [],
+      reason: "no adapters"
+    };
+  }
+  if (perAdapter.length === 1) {
+    const only = perAdapter[0];
+    return {
+      ...only.result,
+      layer: name,
+      findings: only.result.findings.map((f) => ({
+        ...f,
+        layer: name,
+        message: prefix ? `${prefix(only.adapter)} ${f.message}` : f.message,
+        detail: { ...f.detail ?? {}, adapter: only.adapter }
+      })),
+      reason: only.result.reason ?? `${only.adapter}: ${only.result.status}`
+    };
+  }
+  let status = "pass";
+  let weightedScoreSum = 0;
+  let weightCount = 0;
+  const findings = [];
+  let durationMs = 0;
+  const reasonParts = [];
+  const diagnostics = {};
+  for (const { adapter, result } of perAdapter) {
+    status = worst(status, result.status);
+    if (typeof result.score === "number") {
+      weightedScoreSum += result.score;
+      weightCount += 1;
+    }
+    durationMs = mergeDuration === "sum" ? durationMs + result.durationMs : Math.max(durationMs, result.durationMs);
+    reasonParts.push(`${adapter}: ${result.status}`);
+    for (const f of result.findings) {
+      findings.push({
+        ...f,
+        layer: name,
+        message: prefix ? `${prefix(adapter)} ${f.message}` : f.message,
+        detail: { ...f.detail ?? {}, adapter }
+      });
+    }
+    for (const [k, v] of Object.entries(result.diagnostics ?? {})) {
+      if (typeof v !== "number" || !Number.isFinite(v)) continue;
+      const prev = diagnostics[k];
+      if (prev == null) diagnostics[k] = v;
+      else diagnostics[k] = mergeDiagnostics === "sum" ? prev + v : Math.max(prev, v);
+    }
+  }
+  return {
+    layer: name,
+    status,
+    score: weightCount > 0 ? weightedScoreSum / weightCount : void 0,
+    durationMs,
+    findings,
+    reason: reasonParts.join(" \xB7 "),
+    diagnostics: Object.keys(diagnostics).length > 0 ? diagnostics : void 0,
+    detail: {
+      adapters: perAdapter.map(({ adapter, result }) => ({
+        adapter,
+        status: result.status,
+        score: result.score ?? null
+      })),
+      worstSeverity: maxSeverity(findings)
+    }
+  };
+}
+function multiToolchainLayer(config) {
+  const maxParallel = Math.max(1, config.maxParallel ?? 8);
+  return {
+    name: config.name,
+    dependsOn: config.dependsOn,
+    weight: config.weight,
+    failContributesToScore: config.failContributesToScore,
+    capMs: config.capMs,
+    async run(ctx) {
+      if (config.adapters.length === 0) {
+        return {
+          layer: config.name,
+          status: "skipped",
+          durationMs: 0,
+          findings: [],
+          reason: "no adapters detected"
+        };
+      }
+      const runOne = async (adapter) => {
+        const adapterName = config.adapterName(adapter);
+        try {
+          const r = await config.run(adapter, ctx);
+          return { adapter: adapterName, result: r };
+        } catch (err) {
+          return {
+            adapter: adapterName,
+            result: {
+              layer: config.name,
+              status: "error",
+              durationMs: 0,
+              findings: [
+                {
+                  severity: "major",
+                  layer: config.name,
+                  message: err instanceof Error ? err.message : String(err),
+                  detail: { adapter: adapterName }
+                }
+              ],
+              reason: err instanceof Error ? err.message : String(err)
+            }
+          };
+        }
+      };
+      const results = [];
+      for (let i = 0; i < config.adapters.length; i += maxParallel) {
+        const chunk = config.adapters.slice(i, i + maxParallel);
+        const chunkResults = await Promise.all(chunk.map(runOne));
+        results.push(...chunkResults);
+      }
+      return mergeLayerResults(config.name, results);
+    }
+  };
+}
+
+// src/reviewer.ts
+var REVIEWER_SCHEMA = {
+  type: "object",
+  additionalProperties: false,
+  required: ["observations", "diagnosis", "nextShotInstruction", "shouldContinue", "confidence"],
+  properties: {
+    observations: { type: "string", minLength: 20, maxLength: 2e3 },
+    diagnosis: { type: "string", minLength: 20, maxLength: 1500 },
+    nextShotInstruction: { type: "string", minLength: 40, maxLength: 3e3 },
+    shouldContinue: { type: "boolean" },
+    confidence: { type: "number", minimum: 0, maximum: 1 }
+  }
+};
+function summarizeMemory(memory) {
+  if (memory.length === 0) return "(no prior shots)";
+  return memory.map((m) => {
+    const header = `shot ${m.shot} \u2014 confidence=${(m.confidence ?? 0).toFixed(2)} shouldContinue=${m.shouldContinue ?? "?"}`;
+    const obs = m.observations ? `  observations: ${m.observations.slice(0, 400)}` : "";
+    const diag = m.diagnosis ? `  diagnosis: ${m.diagnosis.slice(0, 400)}` : "";
+    const instr = m.nextShotInstruction ? `  instruction given: ${m.nextShotInstruction.slice(0, 400)}` : "";
+    return [header, obs, diag, instr].filter(Boolean).join("\n");
+  }).join("\n\n");
+}
+function buildReviewerPrompt(input) {
+  const system = "You are a senior-engineer-grade reviewer directing an agent through a multi-shot build. Your job is NOT to grade; your job IS to direct the worker's next shot using the trace, verification result, prior memory, and user request. Return STRICT JSON. No prose outside the JSON.";
+  const failingLayersBlock = input.verification.failingLayers && input.verification.failingLayers.length > 0 ? `failing layers: ${input.verification.failingLayers.join(", ")}` : "no layers failing";
+  const user = `=== SHOT NUMBER ===
+shot ${input.shot} of the review loop
+
+=== USER REQUEST ===
+${input.userRequest}
+
+=== WORKER TRACE (shot ${input.shot}) ===
+${input.traceSummary}
+${input.extraContext ? `
+=== EXTRA CONTEXT ===
+${input.extraContext}
+` : ""}
+=== VERIFICATION (shot ${input.shot}) ===
+blendedScore: ${input.verification.blendedScore.toFixed(2)}
+allPass: ${input.verification.allPass}
+failCount: ${input.verification.failCount}
+${failingLayersBlock}
+
+=== REVIEWER MEMORY ===
+${summarizeMemory(input.memory)}
+${input.trailingContext ? `
+=== TRAILING CONTEXT ===
+${input.trailingContext}
+` : ""}
+=== YOUR TASK ===
+Return STRICT JSON:
+
+1. observations (20-2000 chars): first-person worker behavior from the trace (tool call counts, errors, loops).
+2. diagnosis (20-1500 chars): root cause of current failures, not a restatement of verification.
+3. nextShotInstruction (40-3000 chars): concrete "FIX THESE:" directive for the worker's next shot. Reference memory when instructions repeat.
+4. shouldContinue (boolean): FALSE if verification.allPass=true, if worker is thrashing, if confidence < 0.3, or if the request looks unachievable. TRUE otherwise.
+5. confidence (0-1): self-assessment.
+
+RULES:
+- If verification.allPass is true, shouldContinue MUST be false.
+- If memory shows the same failing layer for 2 shots, reduce confidence \u2014 strategy isn't working.
+- If the trace shows zero tool calls, the worker didn't run \u2014 surface that.
+- Do NOT re-grade. Direct.`;
+  return { system, user };
+}
+var DEFAULT_SOFT_FAIL = {
+  observations: "reviewer soft-failed \u2014 no observations captured",
+  diagnosis: "reviewer soft-failed \u2014 inspect verification findings and retry",
+  nextShotInstruction: "Inspect the verification findings above and address the highest-severity failing layer first. If install failed, start there; otherwise work from the first failing gate and address compilation/build errors before layout/semantic issues.",
+  shouldContinue: true,
+  confidence: 0.3
+};
+function createDefaultReviewer(options) {
+  const softFail2 = {
+    ...DEFAULT_SOFT_FAIL,
+    ...options.softFailDefaults ?? {}
+  };
+  const promptBuilder = options.promptBuilder ?? buildReviewerPrompt;
+  const timeoutMs = options.timeoutMs ?? 18e4;
+  return async (input) => {
+    const start = Date.now();
+    const { system, user } = promptBuilder(input);
+    try {
+      const { value, result } = await callLlmJson(
+        {
+          model: options.model,
+          messages: [
+            { role: "system", content: system },
+            { role: "user", content: user }
+          ],
+          jsonSchema: { name: "reviewer_output", schema: REVIEWER_SCHEMA },
+          temperature: 0,
+          timeoutMs
+        },
+        options.llm ?? {}
+      );
+      return {
+        shot: input.shot,
+        observations: String(value.observations ?? softFail2.observations),
+        diagnosis: String(value.diagnosis ?? softFail2.diagnosis),
+        nextShotInstruction: String(value.nextShotInstruction ?? softFail2.nextShotInstruction),
+        shouldContinue: Boolean(value.shouldContinue),
+        confidence: Math.max(0, Math.min(1, Number(value.confidence ?? softFail2.confidence))),
+        costUsd: result.costUsd ?? null,
+        durationMs: Date.now() - start,
+        available: true
+      };
+    } catch (err) {
+      return {
+        shot: input.shot,
+        observations: softFail2.observations,
+        diagnosis: softFail2.diagnosis,
+        nextShotInstruction: softFail2.nextShotInstruction,
+        shouldContinue: softFail2.shouldContinue,
+        confidence: softFail2.confidence,
+        costUsd: null,
+        durationMs: Date.now() - start,
+        available: false,
+        error: err instanceof Error ? err.message : String(err)
+      };
+    }
+  };
+}
+
+// src/semantic-concept-judge.ts
+var DEFAULT_COMPLEXITY_WEIGHTS = {
+  render: 1,
+  integrate: 2,
+  compute: 2.5
+};
+var SEMANTIC_CONCEPT_JUDGE_VERSION = "semantic-concept-judge-v1-2026-04-24";
+var DEFAULT_MAX_SOURCE = 45e3;
+var DEFAULT_MAX_HTML = 3e4;
+var DEFAULT_MAX_PER_FILE = 2e4;
+var DEFAULT_TIMEOUT = 18e4;
+var DEFAULT_MODEL = "claude-sonnet-4-6";
+var SEMANTIC_SCHEMA = {
+  type: "object",
+  additionalProperties: false,
+  required: ["summary", "concepts"],
+  properties: {
+    summary: { type: "string", minLength: 20, maxLength: 600 },
+    concepts: {
+      type: "array",
+      minItems: 1,
+      items: {
+        type: "object",
+        additionalProperties: false,
+        required: ["concept", "present", "score", "evidence", "severity"],
+        properties: {
+          concept: { type: "string", minLength: 1, maxLength: 120 },
+          present: { type: "boolean" },
+          score: { type: "number", minimum: 0, maximum: 10 },
+          evidence: { type: "string", minLength: 5, maxLength: 400 },
+          severity: { type: "string", enum: ["critical", "major", "minor", "info"] }
+        }
+      }
+    }
+  }
+};
+function truncate(body, cap, label) {
+  if (body.length <= cap) return body;
+  return body.slice(0, cap) + `
+\u2026 [truncated ${body.length - cap} chars of ${label}]`;
+}
+function buildPrompt(input, opts) {
+  const sourceBlob = input.sourceFiles.filter((f) => f.content.length <= opts.maxPerFileChars).map((f) => `--- FILE: ${f.path} ---
+${f.content}`).join("\n\n");
+  const html = input.servedHtml ?? "";
+  return `You are a strict code-review judge evaluating whether an agent's 0-to-1 build actually implements the features the user asked for.
+
+You MUST distinguish:
+  (a) WORKING code that implements the concept (rendered UI, wired handler, real API call),
+  (b) KEYWORD-PRESENT stub (comments mentioning the concept, variable names, TODOs),
+  (c) ABSENT (concept nowhere).
+
+A comment like "// TODO: add mint button" is NOT present \u2014 score 2-3. Only count a concept as present if there is real functional code: a rendered component, a call handler wired to state or a network call, a computed value actually used.
+
+USER REQUEST (what the agent was asked to build):
+${input.userRequest}
+
+${input.artifactLabel ? `ARTIFACT METADATA:
+  name: ${input.artifactLabel}
+  description: ${input.artifactDescription ?? ""}
+
+` : ""}EXPECTED CONCEPTS (each must be graded independently):
+${input.expectedConcepts.map((c, i) => `  ${i + 1}. "${c.name}"${c.keywords?.length ? ` \u2014 hints: [${c.keywords.slice(0, 6).join(" | ")}]` : ""}`).join("\n")}
+
+${html ? `SERVED HTML (what the preview returns when hit):
+${truncate(html, opts.maxHtmlChars, "HTML")}
+
+` : ""}SOURCE FILES (the agent's workdir):
+${truncate(sourceBlob, opts.maxSourceChars, "source")}
+
+For EACH concept, return:
+  - concept: the concept name as given (match exactly)
+  - present: boolean \u2014 does a working implementation exist?
+  - score: 0-10 \u2014 10 = production-ready; 7 = functional but thin; 4 = partial/stubbed; 2 = keyword-only comment; 0 = absent
+  - evidence: cite "<file>:<line>" or "served-html:<selector>" pointing at the strongest supporting code. If the concept is absent or stubbed, explain what's missing.
+  - severity:
+      "info" when present: true AND score >= 7
+      "minor" when present: true AND 4 <= score < 7
+      "major" when present: false OR score < 4
+      "critical" when the concept is not only absent but a core user flow depends on it
+
+Also produce a "summary" (one sentence, 20-600 chars): overall verdict on whether this is a shippable implementation of the user request vs a keyword-dense placeholder.
+
+BE SKEPTICAL. Keyword matching already passed \u2014 your job is to catch what keyword matching misses. If the agent shipped a working build, say so. If it shipped a stub, say so. Don't grade on effort.
+
+Return STRICT JSON. No prose outside the JSON.`;
+}
+async function runSemanticConceptJudge(input, options = {}) {
+  const start = Date.now();
+  const totalCount = input.expectedConcepts.length;
+  if (totalCount === 0) {
+    return {
+      kind: "semantic-concept",
+      version: SEMANTIC_CONCEPT_JUDGE_VERSION,
+      score: 0,
+      presentCount: 0,
+      totalCount: 0,
+      findings: [],
+      summary: "no expected concepts declared",
+      durationMs: 0,
+      costUsd: null,
+      available: false,
+      error: "no expected concepts declared"
+    };
+  }
+  const opts = {
+    model: options.model ?? DEFAULT_MODEL,
+    timeoutMs: options.timeoutMs ?? DEFAULT_TIMEOUT,
+    maxSourceChars: options.maxSourceChars ?? DEFAULT_MAX_SOURCE,
+    maxPerFileChars: options.maxPerFileChars ?? DEFAULT_MAX_PER_FILE,
+    maxHtmlChars: options.maxHtmlChars ?? DEFAULT_MAX_HTML,
+    llm: options.llm ?? {},
+    weightConcepts: options.weightConcepts ?? "mean",
+    complexityWeights: { ...DEFAULT_COMPLEXITY_WEIGHTS, ...options.complexityWeights ?? {} }
+  };
+  const weightForConcept = (spec) => {
+    if (opts.weightConcepts === "mean") return 1;
+    if (spec.weight != null) return spec.weight;
+    if (opts.weightConcepts === "complexity") {
+      return opts.complexityWeights[spec.complexity ?? "render"] ?? 1;
+    }
+    return 1;
+  };
+  const weightByName = new Map(
+    input.expectedConcepts.map((c) => [c.name, weightForConcept(c)])
+  );
+  try {
+    const { value, result } = await callLlmJson(
+      {
+        model: opts.model,
+        messages: [
+          {
+            role: "system",
+            content: "You are a strict code-review judge. Return strict JSON only. No prose outside the JSON. A keyword in a comment is NOT a working implementation."
+          },
+          { role: "user", content: buildPrompt(input, opts) }
+        ],
+        jsonSchema: { name: "semantic_concept_judge", schema: SEMANTIC_SCHEMA },
+        temperature: 0,
+        timeoutMs: opts.timeoutMs
+      },
+      opts.llm
+    );
+    if (!value?.concepts || !Array.isArray(value.concepts)) {
+      throw new Error('judge returned malformed response \u2014 expected array under "concepts"');
+    }
+    const findings = value.concepts.map((c) => ({
+      concept: String(c.concept),
+      present: Boolean(c.present),
+      score: Math.max(0, Math.min(10, Number(c.score ?? 0))),
+      evidence: String(c.evidence ?? ""),
+      severity: ["critical", "major", "minor", "info"].includes(c.severity) ? c.severity : "info"
+    }));
+    const presentCount = findings.filter((f) => f.present && f.score >= 7).length;
+    let weightSum = 0;
+    let weightedScoreSum = 0;
+    for (const f of findings) {
+      const w = weightByName.get(f.concept) ?? 1;
+      weightSum += w;
+      weightedScoreSum += w * f.score;
+    }
+    const scoreAvg = weightSum > 0 ? weightedScoreSum / weightSum : findings.reduce((a, f) => a + f.score, 0) / Math.max(1, findings.length);
+    return {
+      kind: "semantic-concept",
+      version: SEMANTIC_CONCEPT_JUDGE_VERSION,
+      score: Number((scoreAvg / 10).toFixed(3)),
+      presentCount,
+      totalCount,
+      findings,
+      summary: String(value.summary ?? ""),
+      durationMs: Date.now() - start,
+      costUsd: result.costUsd ?? null,
+      available: true
+    };
+  } catch (err) {
+    return {
+      kind: "semantic-concept",
+      version: SEMANTIC_CONCEPT_JUDGE_VERSION,
+      score: 0,
+      presentCount: 0,
+      totalCount,
+      findings: [],
+      summary: "",
+      durationMs: Date.now() - start,
+      costUsd: null,
+      available: false,
+      error: err instanceof Error ? err.message : String(err)
+    };
+  }
+}
+function createSemanticConceptJudge(options = {}) {
+  return (input) => runSemanticConceptJudge(input, options);
+}
+
+// src/intent-match-judge.ts
+var INTENT_MATCH_JUDGE_VERSION = "intent-match-judge-v1-2026-04-24";
+var DEFAULT_MODEL2 = "claude-sonnet-4-6";
+var DEFAULT_TIMEOUT2 = 9e4;
+var DEFAULT_MAX_SOURCE2 = 25e3;
+var DEFAULT_MAX_PER_FILE2 = 12e3;
+var DEFAULT_MAX_HTML2 = 2e4;
+var INTENT_SCHEMA = {
+  type: "object",
+  additionalProperties: false,
+  required: ["score", "evidence"],
+  properties: {
+    score: { type: "number", minimum: 0, maximum: 1 },
+    evidence: { type: "string", minLength: 10, maxLength: 400 }
+  }
+};
+function truncate2(body, cap, label) {
+  if (body.length <= cap) return body;
+  return body.slice(0, cap) + `
+\u2026 [truncated ${body.length - cap} chars of ${label}]`;
+}
+function buildPrompt2(input, opts) {
+  const sourceBlob = input.sourceFiles.filter((f) => f.content.length <= opts.maxPerFileChars).map((f) => `--- FILE: ${f.path} ---
+${f.content}`).join("\n\n");
+  const html = input.servedHtml ?? "";
+  return `You are evaluating whether an agent built THE RIGHT APP for a user request.
+
+You are NOT scoring feature coverage. You are NOT scoring code quality.
+You are answering ONE question: when a person looks at this work, do they
+see an honest attempt at the user's request \u2014 or do they see the wrong
+project entirely?
+
+USER REQUEST:
+${input.userRequest}
+
+${input.artifactLabel ? `ARTIFACT METADATA:
+  name: ${input.artifactLabel}
+  description: ${input.artifactDescription ?? ""}
+
+` : ""}${html ? `SERVED HTML (what the preview returns):
+${truncate2(html, opts.maxHtmlChars, "HTML")}
+
+` : ""}SOURCE FILES (the agent's workdir):
+${truncate2(sourceBlob, opts.maxSourceChars, "source")}
+
+Score 0\u20131:
+  1.0 \u2014 unmistakably the right app. Even with bugs, gaps, or missing
+        features, a reviewer would say "yes, this is what was asked for."
+  0.7 \u2014 recognizable. Domain matches; some required surface areas exist.
+        A reviewer would say "right direction, lots of work needed."
+  0.4 \u2014 partially related. Wrong framing or wrong product entirely but
+        with some shared keywords or a tangential overlap.
+  0.0 \u2014 wrong project. The agent shipped something unrelated to the
+        request (e.g. asked for an NFT mint page, shipped a generic
+        landing page with zero NFT-related code).
+
+Evidence: one sentence citing the strongest concrete signal \u2014 a file
+name, a route, a rendered headline, a missing core surface. Don't
+restate the request.
+
+Examples:
+  - "src/App.tsx renders <MintWidget /> with mint-1/mint-5 buttons and
+     wagmi imports \u2014 clearly the requested NFT mint page." \u2192 0.95
+  - "src/App.tsx is the default Vite React template; no
+     mint/wallet/contract code in any file under src/." \u2192 0.05
+
+Return STRICT JSON. No prose outside.`;
+}
+async function runIntentMatchJudge(input, options = {}) {
+  const start = Date.now();
+  const opts = {
+    model: options.model ?? DEFAULT_MODEL2,
+    timeoutMs: options.timeoutMs ?? DEFAULT_TIMEOUT2,
+    maxSourceChars: options.maxSourceChars ?? DEFAULT_MAX_SOURCE2,
+    maxPerFileChars: options.maxPerFileChars ?? DEFAULT_MAX_PER_FILE2,
+    maxHtmlChars: options.maxHtmlChars ?? DEFAULT_MAX_HTML2,
+    llm: options.llm ?? {}
+  };
+  if (input.sourceFiles.length === 0 && !input.servedHtml) {
+    return {
+      kind: "intent-match",
+      version: INTENT_MATCH_JUDGE_VERSION,
+      score: 0,
+      evidence: "no source files and no served HTML \u2014 nothing to evaluate",
+      durationMs: 0,
+      costUsd: null,
+      available: false,
+      error: "no input artifact"
+    };
+  }
+  try {
+    const { value, result } = await callLlmJson(
+      {
+        model: opts.model,
+        messages: [
+          {
+            role: "system",
+            content: "You are a holistic code reviewer answering one question: did the agent build the right app for the user. Return strict JSON. No prose outside."
+          },
+          { role: "user", content: buildPrompt2(input, opts) }
+        ],
+        jsonSchema: { name: "intent_match_judge", schema: INTENT_SCHEMA },
+        temperature: 0,
+        timeoutMs: opts.timeoutMs
+      },
+      opts.llm
+    );
+    const score = Math.max(0, Math.min(1, Number(value?.score ?? 0)));
+    return {
+      kind: "intent-match",
+      version: INTENT_MATCH_JUDGE_VERSION,
+      score: Number(score.toFixed(3)),
+      evidence: String(value?.evidence ?? "").slice(0, 400),
+      durationMs: Date.now() - start,
+      costUsd: result.costUsd ?? null,
+      available: true
+    };
+  } catch (err) {
+    return {
+      kind: "intent-match",
+      version: INTENT_MATCH_JUDGE_VERSION,
+      score: 0,
+      evidence: "",
+      durationMs: Date.now() - start,
+      costUsd: null,
+      available: false,
+      error: err instanceof Error ? err.message : String(err)
+    };
+  }
+}
+function createIntentMatchJudge(options = {}) {
+  return (input) => runIntentMatchJudge(input, options);
+}
+
+// src/flow-layer.ts
+function flowLayer(input) {
+  return {
+    name: input.name ?? "flow",
+    dependsOn: input.dependsOn ?? ["serve"],
+    weight: input.weight ?? 1,
+    capMs: input.capMs ?? 6e4,
+    run: async (ctx) => {
+      const start = Date.now();
+      const spec = ctx.env?.flowSpec ?? input.flowSpec;
+      if (!spec) {
+        return {
+          layer: input.name ?? "flow",
+          status: "skipped",
+          durationMs: 0,
+          findings: [],
+          reason: "no flowSpec supplied"
+        };
+      }
+      let runner;
+      try {
+        runner = await input.runner();
+      } catch (err) {
+        return {
+          layer: input.name ?? "flow",
+          status: "error",
+          durationMs: Date.now() - start,
+          findings: [
+            {
+              severity: "major",
+              message: `flow runner failed to start: ${err instanceof Error ? err.message : String(err)}`
+            }
+          ],
+          reason: "runner-init-failed"
+        };
+      }
+      const findings = [];
+      const stepResults = [];
+      let openOk = false;
+      try {
+        const opened = await runner.open(spec.url);
+        openOk = opened.ok;
+        if (!opened.ok) {
+          findings.push({
+            severity: "major",
+            message: `flow.open(${spec.url}) failed${opened.evidence ? `: ${opened.evidence}` : ""}`
+          });
+        }
+        if (openOk || spec.continueOnFail) {
+          for (let i = 0; i < spec.steps.length; i++) {
+            const step = spec.steps[i];
+            if (ctx.signal.aborted) break;
+            const stepStart = Date.now();
+            let result;
+            try {
+              result = await runner.step(step);
+            } catch (err) {
+              result = {
+                ok: false,
+                evidence: err instanceof Error ? err.message : String(err),
+                durationMs: Date.now() - stepStart
+              };
+            }
+            stepResults.push({ step, result, index: i });
+            if (!result.ok) {
+              findings.push({
+                severity: step.severity ?? "major",
+                message: `step[${i}] ${step.action}${step.target ? `(${step.target})` : ""} failed${result.evidence ? `: ${result.evidence}` : ""}`
+              });
+              if (!spec.continueOnFail) break;
+            }
+          }
+        }
+      } finally {
+        try {
+          await runner.close();
+        } catch {
+        }
+      }
+      const totalSteps = spec.steps.length;
+      const ranSteps = stepResults.length;
+      const passedSteps = stepResults.filter((s) => s.result.ok).length;
+      const status = !openOk ? "fail" : passedSteps === totalSteps ? "pass" : "fail";
+      const denominator = 1 + totalSteps;
+      const numerator = (openOk ? 1 : 0) + passedSteps;
+      const score = denominator > 0 ? Number((numerator / denominator).toFixed(3)) : 0;
+      return {
+        layer: input.name ?? "flow",
+        status,
+        score,
+        durationMs: Date.now() - start,
+        findings,
+        reason: status === "pass" ? `${totalSteps}/${totalSteps} steps passed` : `${passedSteps}/${totalSteps} steps passed${ranSteps < totalSteps ? ` (stopped at step ${ranSteps})` : ""}`,
+        diagnostics: {
+          flowOpenOk: openOk ? 1 : 0,
+          flowStepsTotal: totalSteps,
+          flowStepsPassed: passedSteps,
+          flowStepsRan: ranSteps
+        }
+      };
+    }
+  };
+}
+
+// src/deploy-gate-layer.ts
+function deployGateLayer(input) {
+  const requireArtifact = input.requireArtifact ?? true;
+  return {
+    name: input.name ?? "deploy",
+    dependsOn: input.dependsOn ?? ["build"],
+    weight: input.weight ?? 1,
+    capMs: input.capMs ?? 12e4,
+    run: async (ctx) => {
+      const start = Date.now();
+      let runner;
+      try {
+        runner = await input.runner();
+      } catch (err) {
+        return {
+          layer: input.name ?? "deploy",
+          status: "error",
+          durationMs: Date.now() - start,
+          findings: [
+            {
+              severity: "major",
+              message: `deploy runner failed to start: ${err instanceof Error ? err.message : String(err)}`
+            }
+          ],
+          reason: "runner-init-failed"
+        };
+      }
+      let result;
+      try {
+        result = await runner.run();
+      } catch (err) {
+        return {
+          layer: input.name ?? "deploy",
+          status: "error",
+          durationMs: Date.now() - start,
+          findings: [
+            {
+              severity: "major",
+              message: `deploy command threw: ${err instanceof Error ? err.message : String(err)}`
+            }
+          ],
+          reason: "runner-throw"
+        };
+      }
+      if (ctx.signal.aborted) {
+        return {
+          layer: input.name ?? "deploy",
+          status: "timeout",
+          durationMs: Date.now() - start,
+          findings: [],
+          reason: "aborted by overall cap"
+        };
+      }
+      const artifactOk = !requireArtifact || result.artifactValid === true;
+      const pass = result.ok && artifactOk;
+      const findings = [];
+      if (!result.ok) {
+        findings.push({
+          severity: "critical",
+          message: "deploy build exited non-zero",
+          evidence: (result.output ?? "").slice(-1200)
+        });
+      }
+      if (result.ok && requireArtifact && !result.artifactValid) {
+        findings.push({
+          severity: "major",
+          message: `deploy build succeeded but artifact ${result.artifactDir ?? "(unknown)"} is invalid or empty`
+        });
+      }
+      return {
+        layer: input.name ?? "deploy",
+        status: pass ? "pass" : "fail",
+        score: pass ? 1 : 0,
+        durationMs: Date.now() - start,
+        findings,
+        reason: pass ? `deploy build OK${input.family ? ` (${input.family})` : ""}${result.artifactDir ? ` \u2192 ${result.artifactDir}` : ""}` : !result.ok ? "build command failed" : "artifact missing or invalid",
+        diagnostics: {
+          deployBuildOk: result.ok ? 1 : 0,
+          deployArtifactOk: result.artifactValid === true ? 1 : 0,
+          deployBuildMs: result.durationMs ?? null
+        }
+      };
+    }
+  };
+}
+function viteDeployRunner(input) {
+  return {
+    run: async () => {
+      const start = Date.now();
+      const cmd = input.buildCommand ?? "npm run build";
+      const artifactDir = input.artifactDir ?? "dist";
+      const artifactEntry = input.artifactEntry ?? "index.html";
+      const timeoutMs = input.timeoutMs ?? 9e4;
+      const result = await input.exec(cmd, { cwd: input.workdir, timeoutMs });
+      const ok = result.exitCode === 0;
+      let artifactValid = false;
+      try {
+        const entryExists = await input.exists(`${artifactDir}/${artifactEntry}`);
+        artifactValid = ok && Boolean(entryExists);
+      } catch {
+        artifactValid = false;
+      }
+      const tail = ((result.stderr || result.stdout) ?? "").slice(-1500);
+      return {
+        ok,
+        output: tail,
+        durationMs: Date.now() - start,
+        artifactDir,
+        artifactValid
+      };
+    }
+  };
+}
+
+// src/keyword-coverage-judge.ts
+function htmlContainsElement(html, selector) {
+  const tagOnly = /^([a-zA-Z][\w-]*)$/.exec(selector);
+  if (tagOnly) {
+    const re = new RegExp(`<${tagOnly[1]}\\b`, "i");
+    return re.test(html);
+  }
+  const tagAttrEq = /^([a-zA-Z][\w-]*)\[([\w-]+)\s*=\s*["']?([^"'\]]+)["']?\]$/.exec(selector);
+  if (tagAttrEq) {
+    const [, tag, attr, value] = tagAttrEq;
+    const re = new RegExp(
+      `<${tag}\\b[^>]*\\b${attr}\\s*=\\s*["']${value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}["']`,
+      "i"
+    );
+    return re.test(html);
+  }
+  const tagAttrPresence = /^([a-zA-Z][\w-]*)\[([\w-]+)\]$/.exec(selector);
+  if (tagAttrPresence) {
+    const [, tag, attr] = tagAttrPresence;
+    const re = new RegExp(`<${tag}\\b[^>]*\\b${attr}\\b`, "i");
+    return re.test(html);
+  }
+  return null;
+}
+function extractAssetUrls(html, baseUrl) {
+  const urls = /* @__PURE__ */ new Set();
+  const linkRe = /<link\b[^>]*\bhref\s*=\s*["']([^"']+)["'][^>]*>/gi;
+  const scriptRe = /<script\b[^>]*\bsrc\s*=\s*["']([^"']+)["'][^>]*>/gi;
+  for (const re of [linkRe, scriptRe]) {
+    let match;
+    while ((match = re.exec(html)) !== null) {
+      const raw = match[1];
+      try {
+        urls.add(new URL(raw, baseUrl).toString());
+      } catch {
+      }
+    }
+  }
+  return Array.from(urls);
+}
+function runKeywordCoverageJudge(html, expectedConcepts, assets = []) {
+  const start = Date.now();
+  if (expectedConcepts.length === 0) {
+    return {
+      score: 0,
+      presentCount: 0,
+      totalCount: 0,
+      findings: [],
+      durationMs: 0,
+      totalAssembledBytes: 0
+    };
+  }
+  const haystack = (html + "\n" + assets.join("\n")).toLowerCase();
+  const findings = expectedConcepts.map((concept) => {
+    const matchedKeywords = [];
+    for (const kw of concept.keywords) {
+      if (haystack.includes(kw.toLowerCase())) matchedKeywords.push(kw);
+    }
+    const requiredElementPresent = concept.requiredElement ? htmlContainsElement(html, concept.requiredElement) : null;
+    const passesElementGate = requiredElementPresent === null || requiredElementPresent === true;
+    const found = matchedKeywords.length > 0 && passesElementGate;
+    return { concept: concept.name, found, matchedKeywords, requiredElementPresent };
+  });
+  const presentCount = findings.filter((f) => f.found).length;
+  return {
+    score: presentCount / expectedConcepts.length,
+    presentCount,
+    totalCount: expectedConcepts.length,
+    findings,
+    durationMs: Date.now() - start,
+    totalAssembledBytes: haystack.length
+  };
+}
+async function runKeywordCoverageJudgeUrl(previewUrl, expectedConcepts, options = {}) {
+  const start = Date.now();
+  const fetchFn = options.fetch ?? globalThis.fetch;
+  const htmlTimeout = options.htmlTimeoutMs ?? 5e3;
+  const assetTimeout = options.assetTimeoutMs ?? 3e3;
+  if (expectedConcepts.length === 0) {
+    return {
+      score: 0,
+      presentCount: 0,
+      totalCount: 0,
+      findings: [],
+      durationMs: 0,
+      totalAssembledBytes: 0
+    };
+  }
+  let html = "";
+  try {
+    const resp = await fetchFn(previewUrl, {
+      redirect: "follow",
+      signal: AbortSignal.timeout(htmlTimeout)
+    });
+    if (!resp.ok) {
+      return softFail(expectedConcepts, start, `preview HTTP ${resp.status}`);
+    }
+    html = await resp.text();
+  } catch (err) {
+    return softFail(expectedConcepts, start, err instanceof Error ? err.message : String(err));
+  }
+  const assetUrls = extractAssetUrls(html, previewUrl);
+  const assetBodies = await Promise.all(
+    assetUrls.map(async (u) => {
+      try {
+        const r = await fetchFn(u, {
+          redirect: "follow",
+          signal: AbortSignal.timeout(assetTimeout)
+        });
+        if (!r.ok) return "";
+        return await r.text();
+      } catch {
+        return "";
+      }
+    })
+  );
+  return runKeywordCoverageJudge(html, expectedConcepts, assetBodies);
+}
+function softFail(expectedConcepts, start, error) {
+  return {
+    score: 0,
+    presentCount: 0,
+    totalCount: expectedConcepts.length,
+    findings: expectedConcepts.map((c) => ({
+      concept: c.name,
+      found: false,
+      matchedKeywords: [],
+      requiredElementPresent: null
+    })),
+    durationMs: Date.now() - start,
+    totalAssembledBytes: 0,
+    error
+  };
+}
+
+// src/error-count-extractor.ts
+var ERROR_COUNT_PATTERNS = [
+  {
+    // tsc / ts-node: `src/foo.ts(12,3): error TS1234: ...`
+    name: "typescript-tsc",
+    regex: /[\w./-]+\(\d+,\d+\): error TS\d+:/g
+  },
+  {
+    // pytest: `FAILED tests/test_foo.py::test_bar`
+    name: "pytest-failed",
+    regex: /^FAILED\s+\S+/gm
+  },
+  {
+    // rustc: `error[E0308]: ...` or `error: ...`
+    name: "rustc",
+    regex: /^error(?:\[[A-Z]\d+\])?:/gm
+  },
+  {
+    // go build: `./foo.go:12:3: ...` — any file:line:col: is an error line
+    name: "golang",
+    regex: /^\.\/[\w./-]+\.go:\d+:\d+:/gm
+  },
+  {
+    // eslint default formatter per-line: `  12:34  error  message  rule-id`
+    name: "eslint",
+    regex: /^\s+\d+:\d+\s+error\s+/gm
+  },
+  {
+    // eslint summary line: `✖ 17 problems (12 errors, 5 warnings)`
+    // Use this only when the per-line formatter isn't present; transform
+    // reads the errors count directly.
+    name: "eslint-summary",
+    regex: /✖\s+\d+\s+problems?\s+\((\d+)\s+errors?/gm,
+    transform: (m) => Number(m[1] ?? 0)
+  }
+];
+function extractErrorCount(text, opts = {}) {
+  if (!text) return { count: null, matched: null, samples: [] };
+  const patterns = [...opts.extra ?? [], ...ERROR_COUNT_PATTERNS].filter(
+    (p) => !opts.only || opts.only.includes(p.name)
+  );
+  for (const p of patterns) {
+    const matches2 = Array.from(text.matchAll(p.regex));
+    if (matches2.length === 0) continue;
+    const count = p.transform ? matches2.reduce((sum2, m) => sum2 + p.transform(m), 0) : matches2.length;
+    return {
+      count,
+      matched: p.name,
+      samples: matches2.slice(0, 5).map((m) => m[0])
+    };
+  }
+  return { count: null, matched: null, samples: [] };
+}
+
+// src/reference-replay.ts
+import { appendFileSync as appendFileSync2, existsSync as existsSync4, mkdirSync as mkdirSync2, readFileSync as readFileSync4 } from "fs";
+import { dirname as dirname2 } from "path";
+var DEFAULT_MATCH_THRESHOLD = 0.55;
+var ALL_SPLITS = ["train", "dev", "test", "holdout"];
+async function runReferenceReplay(cases, options) {
+  const now = options.now ?? Date.now;
+  const startedAt = now();
+  const runId = options.runId ?? `${options.variantId ?? "reference-replay"}-${startedAt}`;
+  const allowedSplits = new Set(options.splits ?? ALL_SPLITS);
+  const caseRuns = [];
+  for (const [caseIndex, replayCase] of cases.entries()) {
+    const split = replayCase.split ?? "train";
+    if (split === "holdout" && !options.includeHoldout) continue;
+    if (!allowedSplits.has(split)) continue;
+    const caseStart = now();
+    const executionScenario = {
+      id: replayCase.id,
+      split,
+      input: replayCase.input,
+      ...replayCase.metadata !== void 0 ? { metadata: replayCase.metadata } : {}
+    };
+    let candidates = [];
+    let error;
+    try {
+      throwIfAborted(options.abortSignal);
+      candidates = await runAdapter(options.adapter, executionScenario, {
+        runId,
+        caseIndex,
+        abortSignal: options.abortSignal
+      });
+      throwIfAborted(options.abortSignal);
+    } catch (cause) {
+      if (options.abortSignal?.aborted) throw cause;
+      if (!options.continueOnError) throw cause;
+      error = cause instanceof Error ? cause.message : String(cause);
+    }
+    const scenario = {
+      id: replayCase.id,
+      split,
+      references: replayCase.references,
+      candidates,
+      ...replayCase.metadata !== void 0 ? { metadata: replayCase.metadata } : {}
+    };
+    const scoreOptions2 = {
+      matcher: options.matcher,
+      matchThreshold: options.matchThreshold,
+      matchStrategy: options.matchStrategy,
+      includeHoldout: true
+    };
+    const scenarioScore = scoreReferenceReplay([scenario], scoreOptions2).scenarios[0];
+    caseRuns.push({
+      caseId: replayCase.id,
+      split,
+      input: replayCase.input,
+      references: replayCase.references,
+      candidates,
+      score: scenarioScore,
+      durationMs: Math.max(0, now() - caseStart),
+      ...replayCase.metadata !== void 0 ? { metadata: replayCase.metadata } : {},
+      ...error !== void 0 ? { error } : {}
+    });
+  }
+  const completedAt = now();
+  const scoreOptions = {
+    matcher: options.matcher,
+    matchThreshold: options.matchThreshold,
+    matchStrategy: options.matchStrategy,
+    includeHoldout: true
+  };
+  const run = {
+    id: runId,
+    startedAt,
+    completedAt,
+    durationMs: Math.max(0, completedAt - startedAt),
+    cases: caseRuns,
+    score: scoreReferenceReplay(caseRuns.map((caseRun) => ({
+      id: caseRun.caseId,
+      split: caseRun.split,
+      references: caseRun.references,
+      candidates: caseRun.candidates,
+      ...caseRun.metadata !== void 0 ? { metadata: caseRun.metadata } : {}
+    })), scoreOptions),
+    ...options.variantId !== void 0 ? { variantId: options.variantId } : {},
+    ...options.metadata !== void 0 ? { metadata: options.metadata } : {}
+  };
+  await options.store?.save(run);
+  return run;
+}
+function decideReferenceReplayRunPromotion(baseline, candidate, policy = {}) {
+  return decideReferenceReplayPromotion(baseline.score, candidate.score, policy);
+}
+function inMemoryReferenceReplayStore(initial = []) {
+  const runs = [...initial];
+  return {
+    async save(run) {
+      runs.push(run);
+    },
+    async list() {
+      return [...runs];
+    }
+  };
+}
+function jsonlReferenceReplayStore(path) {
+  return {
+    async save(run) {
+      mkdirSync2(dirname2(path), { recursive: true });
+      appendFileSync2(path, JSON.stringify(run) + "\n");
+    },
+    async list() {
+      if (!existsSync4(path)) return [];
+      return readJsonl(path);
+    }
+  };
+}
+function scoreReferenceReplay(scenarios, options = {}) {
+  const matcher = options.matcher ?? defaultReferenceReplayMatcher;
+  const threshold = options.matchThreshold ?? DEFAULT_MATCH_THRESHOLD;
+  const matchStrategy = options.matchStrategy ?? "reference-order";
+  const allowedSplits = new Set(options.splits ?? ALL_SPLITS);
+  const scores = scenarios.filter((scenario) => {
+    const split = scenario.split ?? "train";
+    if (split === "holdout" && !options.includeHoldout) return false;
+    return allowedSplits.has(split);
+  }).map((scenario) => scoreScenario(scenario, matcher, threshold, matchStrategy));
+  return {
+    scenarios: scores,
+    aggregate: aggregateScenarioScores(scores),
+    bySplit: aggregateBySplit(scores)
+  };
+}
+function compareReferenceReplay(baseline, candidate) {
+  const splits = /* @__PURE__ */ new Set([
+    ...Object.keys(baseline.bySplit),
+    ...Object.keys(candidate.bySplit)
+  ]);
+  return [...splits].sort(bySplitOrder).map((split) => {
+    const before = baseline.bySplit[split] ?? emptyAggregate();
+    const after = candidate.bySplit[split] ?? emptyAggregate();
+    return {
+      split,
+      baselineF1: before.f1,
+      candidateF1: after.f1,
+      f1Delta: after.f1 - before.f1,
+      baselineRecall: before.recall,
+      candidateRecall: after.recall,
+      recallDelta: after.recall - before.recall
+    };
+  });
+}
+function decideReferenceReplayPromotion(baseline, candidate, policy = {}) {
+  const requiredSplits = policy.requiredSplits ?? ["dev", "test"];
+  const minF1Delta = policy.minF1Delta ?? 0;
+  const maxRegression = policy.maxRegression ?? 0;
+  const requireHoldout = policy.requireHoldoutNonRegression ?? true;
+  const comparisons = compareReferenceReplay(baseline, candidate);
+  const missingRequiredSplits = requiredSplits.filter((split) => !hasSplit(baseline, split) || !hasSplit(candidate, split));
+  const compared = comparisons.filter((item) => requiredSplits.includes(item.split));
+  const regressions = comparisons.filter((item) => item.f1Delta < -maxRegression);
+  const aggregateDelta = candidate.aggregate.f1 - baseline.aggregate.f1;
+  if (missingRequiredSplits.length > 0) {
+    return {
+      promote: false,
+      reason: `Required split missing from baseline or candidate: ${missingRequiredSplits.join(", ")}`,
+      aggregateDelta,
+      comparisons,
+      regressions
+    };
+  }
+  if (compared.length === 0) {
+    return {
+      promote: false,
+      reason: `No required split scores found: ${requiredSplits.join(", ")}`,
+      aggregateDelta,
+      comparisons,
+      regressions
+    };
+  }
+  if (regressions.length > 0) {
+    return {
+      promote: false,
+      reason: `Regression in ${regressions.map((r) => r.split).join(", ")}`,
+      aggregateDelta,
+      comparisons,
+      regressions
+    };
+  }
+  if (requireHoldout && (!hasSplit(baseline, "holdout") || !hasSplit(candidate, "holdout"))) {
+    return {
+      promote: false,
+      reason: "Holdout split is required for promotion",
+      aggregateDelta,
+      comparisons,
+      regressions
+    };
+  }
+  const requiredMeanDelta = mean4(compared.map((item) => item.f1Delta));
+  if (requiredMeanDelta < minF1Delta) {
+    return {
+      promote: false,
+      reason: `Required split F1 delta ${formatPct(requiredMeanDelta)} below ${formatPct(minF1Delta)}`,
+      aggregateDelta,
+      comparisons,
+      regressions
+    };
+  }
+  return {
+    promote: true,
+    reason: `Required splits improved by ${formatPct(requiredMeanDelta)} with no regressions`,
+    aggregateDelta,
+    comparisons,
+    regressions
+  };
+}
+function defaultReferenceReplayMatcher(reference, candidate) {
+  const referenceText = `${reference.title} ${reference.description ?? ""}`;
+  const candidateText = `${candidate.title} ${candidate.description ?? ""}`;
+  const textScore = tokenJaccard(referenceText, candidateText);
+  const severityScore = reference.severity && candidate.severity ? normalize(reference.severity) === normalize(candidate.severity) ? 0.1 : -0.05 : 0;
+  const tagScore = tagOverlap(reference.tags, candidate.tags) * 0.15;
+  const score = clamp012(textScore * 0.85 + tagScore + severityScore);
+  return { score, reason: `token=${textScore.toFixed(2)} tags=${tagScore.toFixed(2)} severity=${severityScore.toFixed(2)}` };
+}
+function scoreScenario(scenario, matcher, threshold, matchStrategy) {
+  return matchStrategy === "global-greedy" ? scoreScenarioGlobalGreedy(scenario, matcher, threshold) : scoreScenarioReferenceOrder(scenario, matcher, threshold);
+}
+function scoreScenarioReferenceOrder(scenario, matcher, threshold) {
+  const candidatesLeft = scenario.candidates.map((candidate, index) => ({ candidate, index }));
+  const matches2 = [];
+  for (const reference of scenario.references) {
+    let best = null;
+    for (const item of candidatesLeft) {
+      const result = scorePair(scenario, matcher, reference, item.candidate);
+      if (!best || result.score > best.score) {
+        best = { ...item, ...result };
+      }
+    }
+    const weight = reference.weight ?? 1;
+    if (best && best.score >= threshold) {
+      const matchIndex = candidatesLeft.findIndex((item) => item.index === best.index);
+      if (matchIndex >= 0) candidatesLeft.splice(matchIndex, 1);
+      matches2.push({
+        scenarioId: scenario.id,
+        referenceId: reference.id,
+        candidateId: best.candidate.id,
+        score: best.score,
+        matched: true,
+        weight,
+        reason: best.reason
+      });
+    } else {
+      matches2.push({
+        scenarioId: scenario.id,
+        referenceId: reference.id,
+        candidateId: best?.candidate.id ?? null,
+        score: best?.score ?? 0,
+        matched: false,
+        weight,
+        reason: best?.reason ?? "no candidates"
+      });
+    }
+  }
+  return buildScenarioScore(scenario, matches2, candidatesLeft.length);
+}
+function scoreScenarioGlobalGreedy(scenario, matcher, threshold) {
+  const pairs = [];
+  for (const [referenceIndex, reference] of scenario.references.entries()) {
+    for (const [candidateIndex, candidate] of scenario.candidates.entries()) {
+      pairs.push({
+        referenceIndex,
+        candidateIndex,
+        reference,
+        candidate,
+        ...scorePair(scenario, matcher, reference, candidate)
+      });
+    }
+  }
+  pairs.sort(
+    (a, b) => b.score - a.score || a.referenceIndex - b.referenceIndex || a.candidateIndex - b.candidateIndex
+  );
+  const selectedByReference = /* @__PURE__ */ new Map();
+  const selectedCandidates = /* @__PURE__ */ new Set();
+  for (const pair of pairs) {
+    if (pair.score < threshold) break;
+    if (selectedByReference.has(pair.referenceIndex) || selectedCandidates.has(pair.candidateIndex)) continue;
+    selectedByReference.set(pair.referenceIndex, pair);
+    selectedCandidates.add(pair.candidateIndex);
+  }
+  const matches2 = scenario.references.map((reference, referenceIndex) => {
+    const weight = reference.weight ?? 1;
+    const selected = selectedByReference.get(referenceIndex);
+    if (selected) {
+      return {
+        scenarioId: scenario.id,
+        referenceId: reference.id,
+        candidateId: selected.candidate.id,
+        score: selected.score,
+        matched: true,
+        weight,
+        reason: selected.reason
+      };
+    }
+    const bestRejected = pairs.find((pair) => pair.referenceIndex === referenceIndex);
+    return {
+      scenarioId: scenario.id,
+      referenceId: reference.id,
+      candidateId: bestRejected?.candidate.id ?? null,
+      score: bestRejected?.score ?? 0,
+      matched: false,
+      weight,
+      reason: bestRejected?.reason ?? "no candidates"
+    };
+  });
+  return buildScenarioScore(scenario, matches2, scenario.candidates.length - selectedCandidates.size);
+}
+function scorePair(scenario, matcher, reference, candidate) {
+  const result = matcher(reference, candidate, scenario);
+  if (!Number.isFinite(result.score)) {
+    throw new Error(`reference replay matcher returned non-finite score for ${scenario.id}:${reference.id}:${candidate.id}`);
+  }
+  return { score: clamp012(result.score), reason: result.reason ?? "" };
+}
+function buildScenarioScore(scenario, matches2, falsePositives) {
+  const matched = matches2.filter((match) => match.matched).length;
+  const total = scenario.references.length;
+  const matchedWeight = matches2.filter((match) => match.matched).reduce((sum2, match) => sum2 + match.weight, 0);
+  const totalWeight = matches2.reduce((sum2, match) => sum2 + match.weight, 0);
+  const precision = ratio(matched, matched + falsePositives);
+  const recall = ratio(matched, total);
+  return {
+    scenarioId: scenario.id,
+    split: scenario.split ?? "train",
+    matched,
+    total,
+    falsePositives,
+    matchedWeight,
+    totalWeight,
+    precision,
+    recall,
+    f1: f1(precision, recall),
+    matches: matches2
+  };
+}
+function aggregateBySplit(scores) {
+  const out = {};
+  for (const split of ALL_SPLITS) {
+    const scoped = scores.filter((score) => score.split === split);
+    if (scoped.length > 0) out[split] = aggregateScenarioScores(scoped);
+  }
+  return out;
+}
+function aggregateScenarioScores(scores) {
+  const matched = sum(scores.map((score) => score.matched));
+  const total = sum(scores.map((score) => score.total));
+  const falsePositives = sum(scores.map((score) => score.falsePositives));
+  const matchedWeight = sum(scores.map((score) => score.matchedWeight));
+  const totalWeight = sum(scores.map((score) => score.totalWeight));
+  const precision = ratio(matched, matched + falsePositives);
+  const recall = ratio(matched, total);
+  return {
+    matched,
+    total,
+    falsePositives,
+    matchedWeight,
+    totalWeight,
+    precision,
+    recall,
+    f1: f1(precision, recall),
+    weightedRecall: ratio(matchedWeight, totalWeight)
+  };
+}
+function emptyAggregate() {
+  return {
+    matched: 0,
+    total: 0,
+    falsePositives: 0,
+    matchedWeight: 0,
+    totalWeight: 0,
+    precision: 0,
+    recall: 0,
+    f1: 0,
+    weightedRecall: 0
+  };
+}
+function hasSplit(score, split) {
+  return score.bySplit[split] !== void 0;
+}
+function f1(precision, recall) {
+  return precision + recall === 0 ? 0 : 2 * precision * recall / (precision + recall);
+}
+function ratio(numerator, denominator) {
+  return denominator > 0 ? numerator / denominator : 0;
+}
+function tokenJaccard(a, b) {
+  const left = new Set(tokens(a));
+  const right = new Set(tokens(b));
+  if (left.size === 0 || right.size === 0) return 0;
+  let intersection = 0;
+  for (const token of left) {
+    if (right.has(token)) intersection++;
+  }
+  return intersection / (left.size + right.size - intersection);
+}
+function tagOverlap(a, b) {
+  if (!a?.length || !b?.length) return 0;
+  const left = new Set(a.map(normalize));
+  const right = new Set(b.map(normalize));
+  let intersection = 0;
+  for (const tag of left) {
+    if (right.has(tag)) intersection++;
+  }
+  return intersection / Math.max(left.size, right.size);
+}
+function tokens(text) {
+  return normalize(text).split(/\s+/).filter((token) => token.length >= 3 && !STOP_WORDS.has(token));
+}
+function normalize(text) {
+  return text.toLowerCase().replace(/[^a-z0-9]+/g, " ").trim();
+}
+function clamp012(value) {
+  if (!Number.isFinite(value)) return 0;
+  return Math.max(0, Math.min(1, value));
+}
+function sum(values) {
+  return values.reduce((acc, value) => acc + value, 0);
+}
+function mean4(values) {
+  return values.length ? sum(values) / values.length : 0;
+}
+function formatPct(value) {
+  return `${(value * 100).toFixed(1)}%`;
+}
+function bySplitOrder(a, b) {
+  return ALL_SPLITS.indexOf(a) - ALL_SPLITS.indexOf(b);
+}
+function runAdapter(adapter, scenario, context) {
+  return typeof adapter === "function" ? adapter(scenario, context) : adapter.run(scenario, context);
+}
+function throwIfAborted(signal) {
+  if (!signal?.aborted) return;
+  if (signal.reason instanceof Error) throw signal.reason;
+  throw new Error(signal.reason ? String(signal.reason) : "reference replay aborted");
+}
+function readJsonl(path) {
+  const raw = readFileSync4(path, "utf8");
+  const out = [];
+  for (const line of raw.split("\n")) {
+    const trimmed = line.trim();
+    if (!trimmed) continue;
+    out.push(JSON.parse(trimmed));
+  }
+  return out;
+}
+var STOP_WORDS = /* @__PURE__ */ new Set([
+  "the",
+  "and",
+  "for",
+  "with",
+  "that",
+  "this",
+  "from",
+  "into",
+  "onto",
+  "are",
+  "can",
+  "will",
+  "should",
+  "could",
+  "would",
+  "when",
+  "where",
+  "which"
+]);
+
+// src/reference-replay-steering.ts
+function referenceReplayRunsToSteeringRows(runs, options = {}) {
+  const rows = [];
+  for (const run of runs) {
+    const variantId = run.variantId ?? run.id;
+    const bundle = options.bundleForRun?.(run) ?? {
+      id: variantId,
+      metadata: run.metadata
+    };
+    for (const caseRun of run.cases) {
+      rows.push({
+        variantId,
+        scenarioId: caseRun.caseId,
+        bundle,
+        score: options.scoreForCase?.(caseRun, run) ?? referenceReplayScenarioToRunScore(caseRun.score, caseRun.durationMs),
+        metadata: {
+          runId: run.id,
+          split: caseRun.split,
+          task: caseRun.metadata?.task ?? caseRun.metadata?.repo ?? caseRun.caseId,
+          referenceCount: caseRun.references.length,
+          candidateCount: caseRun.candidates.length,
+          matched: caseRun.score.matched,
+          total: caseRun.score.total,
+          falsePositives: caseRun.score.falsePositives,
+          precision: caseRun.score.precision,
+          recall: caseRun.score.recall,
+          f1: caseRun.score.f1,
+          error: caseRun.error,
+          ...caseRun.metadata ?? {}
+        }
+      });
+    }
+  }
+  return rows;
+}
+function referenceReplayScenarioToRunScore(scenarioScore, durationMs = 0) {
+  const success = scenarioScore.f1;
+  const recall = scenarioScore.recall;
+  const precision = scenarioScore.precision;
+  const failed = scenarioScore.total > 0 && scenarioScore.matched === 0;
+  return {
+    success,
+    goalProgress: recall,
+    repoGroundedness: precision,
+    driftPenalty: 1 - precision,
+    toolUseQuality: precision,
+    patchQuality: 0,
+    testReality: scenarioScore.total > 0 ? 1 : 0,
+    finalGate: success,
+    reviewerBlockers: failed ? 1 : 0,
+    costUsd: 0,
+    wallSeconds: Math.max(0, durationMs / 1e3),
+    notes: [
+      `reference-replay matched ${scenarioScore.matched}/${scenarioScore.total}`,
+      `precision=${precision.toFixed(3)} recall=${recall.toFixed(3)} f1=${success.toFixed(3)}`
+    ]
+  };
+}
 export {
   AgentDriver,
   AxGepaSteeringOptimizer,
@@ -8017,6 +9986,7 @@ export {
   ConvergenceTracker,
   CostTracker,
   DEFAULT_AGENT_SLOS,
+  DEFAULT_COMPLEXITY_WEIGHTS,
   DEFAULT_RULES as DEFAULT_FAILURE_RULES,
   DEFAULT_FINDERS,
   DEFAULT_HARNESS_OBJECTIVES,
@@ -8027,19 +9997,24 @@ export {
   Dataset,
   DockerSandboxDriver,
   DualAgentBench,
+  ERROR_COUNT_PATTERNS,
   ExperimentTracker,
   FAILURE_CLASSES,
   FileSystemOutcomeStore,
   FileSystemTraceStore,
   HoldoutAuditor,
   HoldoutLockedError,
+  INTENT_MATCH_JUDGE_VERSION,
   InMemoryExperimentStore,
   InMemoryOutcomeStore,
   InMemoryTraceStore,
   InMemoryWorkspaceInspector,
   JudgeRunner,
+  LlmCallError,
+  LlmClient,
   MODEL_PRICING,
   MetricsCollector,
+  MultiLayerVerifier,
   OTEL_AGENT_EVAL_SCOPE,
   OptimizationLoop,
   PairwiseSteeringOptimizer,
@@ -8050,6 +10025,7 @@ export {
   PromptRegistry,
   REDACTION_VERSION,
   RunCritic,
+  SEMANTIC_CONCEPT_JUDGE_VERSION,
   SandboxHarness,
   ScenarioRegistry,
   SubprocessSandboxDriver,
@@ -8068,10 +10044,13 @@ export {
   bisect,
   bonferroni,
   budgetBreachView,
+  buildReviewerPrompt,
   buildTrajectory,
   byteLengthRange,
   calibrateJudge,
   calibrationCurve,
+  callLlm,
+  callLlmJson,
   canaryLeakView,
   causalAttribution,
   checkCanaries,
@@ -8084,6 +10063,7 @@ export {
   coherenceJudge,
   collectionPreserved,
   commitBisect,
+  compareReferenceReplay,
   compareToBaseline,
   compilerJudge,
   composeParsers,
@@ -8095,10 +10075,17 @@ export {
   correlationStudy,
   createAntiSlopJudge,
   createCustomJudge,
+  createDefaultReviewer,
   createDomainExpertJudge,
+  createIntentMatchJudge,
   createLlmReviewer,
+  createSemanticConceptJudge,
   crossTraceDiff,
+  decideReferenceReplayPromotion,
+  decideReferenceReplayRunPromotion,
   defaultJudges,
+  defaultReferenceReplayMatcher,
+  deployGateLayer,
   distillPlaybook,
   dominates,
   estimateCost,
@@ -8112,6 +10099,8 @@ export {
   exportRewardModel,
   exportRunAsOtlp,
   exportTrainingData,
+  extractAssetUrls,
+  extractErrorCount,
   failureClusterView,
   fileContains,
   fileExists,
@@ -8121,12 +10110,16 @@ export {
   findLiteralTruePass,
   findSkipCountsAsPass,
   firstDivergenceView,
+  flowLayer,
   formatBenchmarkReport,
   formatDriverReport,
   formatFindings,
+  gradeSemanticStatus,
   groupBy,
   hashContent,
   hashScenarios,
+  htmlContainsElement,
+  inMemoryReferenceReplayStore,
   inMemoryReviewStore,
   interRaterReliability,
   iqr,
@@ -8139,6 +10132,7 @@ export {
   jestTestParser,
   jsonHasKeys,
   jsonShape,
+  jsonlReferenceReplayStore,
   jsonlReviewStore,
   judgeAgreementView,
   judgeSpans,
@@ -8147,9 +10141,12 @@ export {
   llmSpanFromProvider,
   llmSpans,
   loadScorerFromGrader,
+  localCommandRunner,
   lowercaseMutator,
   mannWhitneyU,
+  mergeLayerResults,
   mergeSteeringBundle,
+  multiToolchainLayer,
   nistAiRmfReport,
   nonRefusalRubric,
   normalizeScores,
@@ -8165,6 +10162,7 @@ export {
   printDriverSummary,
   prmBestOfN,
   prmEnsembleBestOfN,
+  probeLlm,
   promptBisect,
   proposeSynthesisTargets,
   pytestTestParser,
@@ -8172,6 +10170,8 @@ export {
   redTeamReport,
   redactString,
   redactValue,
+  referenceReplayRunsToSteeringRows,
+  referenceReplayScenarioToRunScore,
   regexMatch,
   regexMatches,
   regressionView,
@@ -8191,9 +10191,14 @@ export {
   runExpectations,
   runFailureClass,
   runHarnessExperiment,
+  runIntentMatchJudge,
   runJudgeFleet,
+  runKeywordCoverageJudge,
+  runKeywordCoverageJudgeUrl,
   runProposeReview,
+  runReferenceReplay,
   runSelfPlay,
+  runSemanticConceptJudge,
   runTestGradedScenario,
   runsForScenario,
   scanForMuffledGates,
@@ -8201,6 +10206,7 @@ export {
   scoreContinuity,
   scoreProject,
   scoreRedTeamOutput,
+  scoreReferenceReplay,
   securityJudge,
   selectHarnessVariant,
   selfPreference,
@@ -8208,6 +10214,7 @@ export {
   signManifest,
   soc2Report,
   statusAdvanced,
+  stripFencedJson,
   stuckLoopView,
   summarize,
   summarizeHarnessResults,
@@ -8227,6 +10234,7 @@ export {
   verbosityBias,
   verifyManifest,
   visualDiff,
+  viteDeployRunner,
   vitestTestParser,
   weightedMean,
   welchsTTest,