nemoris 0.1.0 → 0.1.2

package/src/runtime/evaluation.js

@@ -1,1089 +1,1089 @@
-import path from "node:path";
-import { RunReviewer } from "./review.js";
-import { Scheduler } from "./scheduler.js";
-import { RunStore } from "./run-store.js";
-import { NotificationStore } from "./notification-store.js";
-import { DeliveryStore } from "./delivery-store.js";
-import { validateOutputContract } from "./output-contract-validator.js";
-
-function groupBy(items, keyFn) {
-  const map = new Map();
-  for (const item of items) {
-    const key = keyFn(item);
-    if (!map.has(key)) map.set(key, []);
-    map.get(key).push(item);
-  }
-  return map;
-}
-
-function clamp(value, min = 0, max = 1) {
-  return Math.max(min, Math.min(max, value));
-}
-
-function average(values) {
-  if (values.length === 0) return null;
-  return values.reduce((sum, value) => sum + value, 0) / values.length;
-}
-
-function choosePrimaryV2Run(runs) {
-  if (!runs.length) return null;
-  return runs.find((run) => run.mode === "provider") || runs[0];
-}
-
-function pickBestLiveMatch(liveMatches) {
-  return liveMatches.find((match) => match.latestRun) || liveMatches[0] || null;
-}
-
-function normalizeText(value) {
-  return String(value || "")
-    .replace(/\s+/g, " ")
-    .trim();
-}
-
-function stripOuterFence(text) {
-  const raw = String(text || "").trim();
-  const fencedMatch = raw.match(/^```(?:json|markdown|md|text)?\s*([\s\S]*?)\s*```$/i);
-  return fencedMatch ? fencedMatch[1].trim() : raw;
-}
-
-function parseJsonCandidate(text) {
-  const candidate = stripOuterFence(text);
-  if (!(candidate.startsWith("{") && candidate.endsWith("}"))) {
-    return null;
-  }
-
-  try {
-    return JSON.parse(candidate);
-  } catch {
-    return null;
-  }
-}
-
-function titleCase(value) {
-  return String(value || "")
-    .replace(/[_-]+/g, " ")
-    .replace(/^#+\s*/, "")
-    .replace(/\b\w/g, (char) => char.toUpperCase())
-    .trim();
-}
-
-function normalizeInline(value) {
-  return normalizeText(value) || "None";
-}
-
-function normalizeSectionKey(value) {
-  return String(value || "")
-    .toLowerCase()
-    .replace(/^#+\s*/, "")
-    .replace(/[*:_-]+/g, " ")
-    .replace(/\s+/g, " ")
-    .trim();
-}
-
-function renderValue(value, { contract = null, summary = null } = {}) {
-  if (value == null) return "";
-
-  if (typeof value === "string") {
-    const parsed = parseJsonCandidate(value);
-    if (parsed && typeof parsed === "object") {
-      if (parsed.output !== undefined) {
-        return renderValue(parsed.output, {
-          contract,
-          summary: parsed.summary || summary
-        });
-      }
-
-      return renderValue(parsed, { contract, summary });
-    }
-
-    return stripOuterFence(value);
-  }
-
-  if (Array.isArray(value)) {
-    return value
-      .map((item) => renderValue(item, { contract, summary }))
-      .filter(Boolean)
-      .join("\n");
-  }
-
-  if (typeof value === "object") {
-    const entries = Object.entries(value);
-    const sectionOrder = contract?.requiredSections?.length
-      ? contract.requiredSections
-      : entries.map(([key]) => key);
-    const valueMap = new Map(entries.map(([key, entryValue]) => [normalizeSectionKey(key), entryValue]));
-    const sectionStyle = contract?.profile?.sectionStyle || (contract?.format === "structured_rollup" ? "headings" : "bullets");
-    const requireStatus = contract?.profile?.requireStatus ?? (contract?.format === "bulleted_briefing");
-
-    if (sectionStyle === "bullets") {
-      const lines = [];
-      if (requireStatus && summary) lines.push(`Status: ${normalizeInline(summary)}`);
-      for (const section of sectionOrder) {
-        lines.push(`- ${titleCase(section)}: ${normalizeInline(valueMap.get(normalizeSectionKey(section)))}`);
-      }
-      return lines.join("\n");
-    }
-
-    if (sectionStyle === "headings") {
-      const lines = [];
-      for (const section of sectionOrder) {
-        lines.push(`## ${titleCase(section)}`);
-        lines.push(`- ${normalizeInline(valueMap.get(normalizeSectionKey(section)))}`);
-        lines.push("");
-      }
-      return lines.join("\n").trim();
-    }
-
-    return entries
-      .map(([key, entryValue]) => `${titleCase(key)}: ${normalizeInline(entryValue)}`)
-      .join("\n");
-  }
-
-  return String(value);
-}
-
-function extractRunText(run, contract = null) {
-  if (!run) return "";
-  const summary = run?.result?.summary || run?.summary || "";
-  const value = run?.result?.output ?? run?.output ?? summary;
-  return renderValue(value, { contract, summary });
-}
-
-function extractLiveText(run) {
-  if (!run) return "";
-  return renderValue(run.summary || run.error || "");
-}
-
-function qualitySignals(text) {
-  const raw = String(text || "");
-  const normalized = normalizeText(text);
-  const lower = normalized.toLowerCase();
-  const findings = [];
-  let score = 1;
-
-  if (!normalized) {
-    findings.push("Output is empty.");
-    return { score: 0, findings };
-  }
-
-  if (normalized.length < 40) {
-    findings.push("Output is very short and may not carry enough signal.");
-    score -= 0.18;
-  }
-
-  if (/\[(title|link|summary|sub)\]/i.test(normalized)) {
-    findings.push("Output still contains placeholder text.");
-    score -= 0.4;
-  }
-
-  if (/i('| a)?m sorry|does not support|cannot complete|failed to/i.test(lower)) {
-    findings.push("Output contains apology or tool-failure language.");
-    score -= 0.28;
-  }
-
-  if (/```/.test(raw)) {
-    findings.push("Output is wrapped in a code block instead of a direct report.");
-    score -= 0.12;
-  }
-
-  if (/timed out|timeout/i.test(lower)) {
-    findings.push("Output references a timeout or stalled execution.");
-    score -= 0.35;
-  }
-
-  const repeatedLinePenalty = detectRepeatedLinePenalty(normalized);
-  if (repeatedLinePenalty > 0) {
-    findings.push("Output repeats nearly identical numbered lines.");
-    score -= repeatedLinePenalty;
-  }
-
-  return {
-    score: clamp(Number(score.toFixed(4))),
-    findings
-  };
-}
-
-function detectRepeatedLinePenalty(text) {
-  const lines = text
-    .split("\n")
-    .map((line) => line.trim())
-    .filter(Boolean)
-    .filter((line) => /^\d+\./.test(line));
-
-  if (lines.length < 3) return 0;
-
-  const normalized = lines.map((line) => line.replace(/^\d+\.\s*/, "").toLowerCase());
-  const unique = new Set(normalized);
-  const duplicateRatio = 1 - unique.size / normalized.length;
-  if (duplicateRatio < 0.5) return 0;
-  return Number((duplicateRatio * 0.25).toFixed(4));
-}
-
-function expandRelatedNotificationFiles(baseFiles, notifications) {
-  const related = new Set(baseFiles);
-  let changed = true;
-  while (changed) {
-    changed = false;
-    for (const item of notifications) {
-      if (!related.has(item.filePath)) continue;
-      for (const generated of item.generatedNotificationFiles || []) {
-        if (related.has(generated)) continue;
-        related.add(generated);
-        changed = true;
-      }
-    }
-  }
-  return related;
-}
-
-function evaluateLiveHistory(liveRuns) {
-  if (liveRuns.length === 0) {
-    return {
-      runCount: 0,
-      okCount: 0,
-      errorCount: 0,
-      skippedCount: 0,
-      failureRate: null,
-      avgDurationMs: null,
-      avgTotalTokens: null,
-      avgQualityScore: null,
-      findings: ["No recent live history available."]
-    };
-  }
-
-  const okCount = liveRuns.filter((run) => run.status === "ok").length;
-  const errorCount = liveRuns.filter((run) => run.status === "error").length;
-  const skippedCount = liveRuns.filter((run) => run.status === "skipped").length;
-  const failureRate = errorCount / liveRuns.length;
-  const durations = liveRuns.map((run) => run.durationMs).filter((value) => Number.isFinite(value));
-  const tokens = liveRuns.map((run) => run.usage?.total_tokens).filter((value) => Number.isFinite(value));
-  const qualityScores = liveRuns
-    .map((run) => qualitySignals(run.summary || run.error || "").score)
-    .filter((value) => Number.isFinite(value));
-
-  const findings = [];
-  if (failureRate >= 0.34) findings.push("Live cron lane is failing often.");
-  if (skippedCount > 0) findings.push("Live cron history includes intentional skips.");
-  if (tokens.some((value) => value > 12000)) findings.push("Live cron lane shows high token usage spikes.");
-
-  return {
-    runCount: liveRuns.length,
-    okCount,
-    errorCount,
-    skippedCount,
-    failureRate: Number(failureRate.toFixed(4)),
-    avgDurationMs: durations.length ? Math.round(average(durations)) : null,
-    avgTotalTokens: tokens.length ? Math.round(average(tokens)) : null,
-    avgQualityScore: qualityScores.length ? Number(average(qualityScores).toFixed(4)) : null,
-    findings
-  };
-}
-
-function scoreStatus(status) {
-  if (status === "ok") return 1;
-  if (status === "skipped") return 0.6;
-  if (status === "error") return 0;
-  return 0.5;
-}
-
-function buildRubric({ v2Run, liveHistory, matchedLiveJob, outputContract = null, contractCheck = null }) {
-  const findings = [];
-  const v2Text = extractRunText(v2Run, outputContract);
-  const v2Quality = qualitySignals(v2Text);
-  const latestLive = liveHistory[0] || null;
-  const latestLiveQuality = qualitySignals(extractLiveText(latestLive));
-  const history = evaluateLiveHistory(liveHistory);
-  const isDryRun = v2Run != null && v2Run.mode !== "provider";
-
-  const comparisonReadiness = v2Run
-    ? v2Run.mode === "provider"
-      ? 1
-      : 0.45
-    : 0;
-  const comparisonCoverage = matchedLiveJob ? 1 : 0.2;
-  const contractAdherence = contractCheck ? contractCheck.satisfiedRatio : null;
-  if (!v2Run) findings.push("No V2 run exists for this lane yet.");
-  else if (isDryRun) findings.push("Only a dry-run V2 artifact exists; output quality and contract adherence are excluded from the rubric score.");
-  if (!matchedLiveJob) findings.push("Comparison coverage is low because no live cron analogue is mapped.");
-
-  if (!isDryRun && v2Quality.findings.length) findings.push(...v2Quality.findings.map((item) => `V2: ${item}`));
-  if (latestLiveQuality.findings.length) findings.push(...latestLiveQuality.findings.map((item) => `Live: ${item}`));
-  if (history.findings.length) findings.push(...history.findings);
-
-  const fieldScores = contractCheck?.fieldScores || null;
-  if (!isDryRun && fieldScores) {
-    if (fieldScores.weakFields.length) {
-      findings.push(`Field-level quality is low for: ${fieldScores.weakFields.join(", ")}`);
-    }
-    if (fieldScores.placeholderFields.length) {
-      findings.push(`Placeholder text detected in: ${fieldScores.placeholderFields.join(", ")}`);
-    }
-  }
-
-  if (!matchedLiveJob) {
-    findings.push("No live cron analogue matched for this V2 job.");
-  }
-
-  const statusAlignment =
-    latestLive && v2Run
-      ? Number((scoreStatus(v2Run.result ? "ok" : v2Run.status) * scoreStatus(latestLive.status)).toFixed(4))
-      : null;
-  const liveReliability = history.failureRate == null ? null : Number((1 - history.failureRate).toFixed(4));
-
-  const fieldLevelQuality = !isDryRun && fieldScores ? fieldScores.averageScore : null;
-
-  const components = [
-    isDryRun ? null : v2Quality.score,
-    comparisonReadiness,
-    comparisonCoverage,
-    isDryRun ? null : contractAdherence,
-    fieldLevelQuality,
-    liveReliability,
-    statusAlignment
-  ].filter((value) => value != null);
-  const overallScore = components.length ? Number(average(components).toFixed(4)) : 0;
-
-  return {
-    overallScore,
-    dryRunExcluded: isDryRun,
-    components: {
-      v2OutputQuality: isDryRun ? null : v2Quality.score,
-      comparisonReadiness,
-      comparisonCoverage,
-      contractAdherence: isDryRun ? null : contractAdherence,
-      fieldLevelQuality,
-      liveReliability,
-      statusAlignment
-    },
-    findings: uniqueStrings(findings),
-    metrics: {
-      liveRunCount: history.runCount,
-      liveOkCount: history.okCount,
-      liveErrorCount: history.errorCount,
-      liveSkippedCount: history.skippedCount,
-      liveFailureRate: history.failureRate,
-      liveAvgDurationMs: history.avgDurationMs,
-      liveAvgTotalTokens: history.avgTotalTokens,
-      liveAvgQualityScore: history.avgQualityScore,
-      latestLiveQualityScore: latestLive ? latestLiveQuality.score : null
-    }
-  };
-}
-
-function uniqueStrings(items) {
-  return [...new Set(items.filter(Boolean))];
-}
-
-function summarizeV2Run(run) {
-  if (!run) return null;
-  return {
-    timestamp: run.timestamp,
-    mode: run.mode || null,
-    providerId: run.providerId || null,
-    modelId: run.modelId || null,
-    summary: run.result?.summary || run.summary || null,
-    output: run.result?.output || run.output || null,
-    retrievalMeta: run.retrievalMeta || null,
-    retrievedMemoryCount: run.retrievedMemory?.length || 0,
-    interaction: summarizeInteraction(run),
-    fallback: run.fallback || null
-  };
-}
-
-function summarizeInteraction(run) {
-  const interaction = run?.interaction || null;
-  if (!interaction) return null;
-  return {
-    ackRequired: interaction.ack?.required ?? false,
-    completionRequired: interaction.completion?.required ?? false,
-    handoffRequired: interaction.handoff?.required ?? false,
-    yielded: run?.yielded || false,
-    yieldSignal: run?.yieldSignal || null,
-    followUpState: run?.followUpState || null,
-    followUpQueued: run?.followUpQueued || false,
-    followUpConsumed: run?.followUpConsumed || false,
-    followUpTarget: run?.followUpTarget || null,
-    followUpCompleted: run?.followUpCompleted || false,
-    followUpExpired: run?.followUpExpired || false,
-    followUpEscalated: run?.followUpEscalated || false,
-    followUpEscalationFilePath: run?.followUpEscalationFilePath || null,
-    handoffTarget: interaction.handoff?.target || null,
-    suggestedPeerCount: interaction.handoff?.suggestions?.length || 0,
-    handoffChosenPeerId: run?.handoffChosenPeerId || null,
-    handoffChosenBy: run?.handoffChosenBy || null,
-    handoffDelivered: run?.handoffDelivered || false,
-    handoffDeliveryState: run?.handoffDeliveryState || null,
-    deliveryDeduped: run?.deliveryDeduped || false,
-    deliveryRetried: run?.deliveryRetried || false,
-    deliveryUncertain: run?.deliveryUncertain || false
-  };
-}
-
-function analyzeRetrieval(run) {
-  const retrievalMeta = run?.retrievalMeta || run?.plan?.packet?.layers?.retrievalMeta || null;
-  if (!run?.retrievedMemory?.length) {
-    return {
-      memoryCount: 0,
-      lexicalCount: 0,
-      semanticCount: 0,
-      qmdCount: 0,
-      freshEmbeddingCount: 0,
-      staleEmbeddingCount: 0,
-      missingEmbeddingCount: 0,
-      failedEmbeddingCount: retrievalMeta?.embeddingHealth?.failedCount || 0,
-      embeddingQueryMode: retrievalMeta?.embeddingQueryMode || "lexical_only",
-      embeddingError: retrievalMeta?.embeddingError || retrievalMeta?.embeddingHealth?.lastError || null,
-      findings: ["No retrieved memory was captured for this run."],
-      items: []
-    };
-  }
-
-  const items = run.retrievedMemory.map((item) => ({
-    entryId: item.entryId || null,
-    title: item.title || null,
-    sourceBackend: item.sourceBackend || "file",
-    candidateSource: item.candidateSource || "indexed",
-    lexicalScore: item.lexicalScore ?? 0,
-    embeddingSimilarity: item.embeddingSimilarity ?? 0,
-    embeddingFreshness: item.embeddingFreshness || "missing",
-    retrievalSources: item.retrievalSources || []
-  }));
-
-  const lexicalCount = items.filter((item) => item.retrievalSources.includes("lexical")).length;
-  const semanticCount = items.filter((item) => item.retrievalSources.includes("semantic")).length;
-  const qmdCount = items.filter((item) => item.retrievalSources.includes("qmd") || item.sourceBackend === "qmd").length;
-  const freshEmbeddingCount = items.filter((item) => item.embeddingFreshness === "fresh").length;
-  const staleEmbeddingCount = items.filter((item) => item.embeddingFreshness === "stale").length;
-  const missingEmbeddingCount = items.filter((item) => item.embeddingFreshness === "missing").length;
-  const failedEmbeddingCount = items.filter((item) => item.embeddingFreshness === "failed").length;
-  const findings = [];
-
-  if (semanticCount === 0) findings.push("No semantic retrieval candidates reached the final packet.");
-  if (staleEmbeddingCount > 0) findings.push("Some retrieved items have stale embeddings.");
-  if (freshEmbeddingCount === 0 && missingEmbeddingCount > 0) findings.push("Retrieved file memory is falling back to non-embedded retrieval.");
-  if ((retrievalMeta?.embeddingError || retrievalMeta?.embeddingHealth?.lastError) && !findings.includes("Embedding query failed and retrieval fell back to lexical mode.")) {
-    findings.push("Embedding query failed and retrieval fell back to lexical mode.");
-  }
-
-  return {
-    memoryCount: items.length,
-    lexicalCount,
-    semanticCount,
-    qmdCount,
-    freshEmbeddingCount,
-    staleEmbeddingCount,
-    missingEmbeddingCount,
-    failedEmbeddingCount,
-    embeddingQueryMode: retrievalMeta?.embeddingQueryMode || "lexical_only",
-    embeddingError: retrievalMeta?.embeddingError || retrievalMeta?.embeddingHealth?.lastError || null,
-    embeddingHealth: retrievalMeta?.embeddingHealth || null,
-    findings,
-    items
-  };
-}
-
-function summarizeInteractionDiagnosis({
-  ackRequired,
-  ackQueued,
-  completionRequired,
-  completionQueued,
-  handoffRequired,
-  handoffQueued,
-  yielded,
-  followUpQueued,
-  followUpConsumed,
-  handoffDelivered,
-  deliveryEvidenceRequired,
-  deliveryEvidenceHealthy,
-  deliveryUncertain,
-  findings
-}) {
-  const missingStages = [];
-  if (ackRequired && !ackQueued) missingStages.push("ack");
-  if (completionRequired && !completionQueued && !yielded) missingStages.push("completion");
-  if (handoffRequired && !handoffQueued && !yielded) missingStages.push("handoff");
-
-  if (!findings?.length) {
-    return {
-      status: "healthy",
-      code: null,
-      summary: "Interaction lifecycle evidence is complete.",
-      missingStages,
-      findings: []
-    };
-  }
-
-  if (missingStages.length) {
-    return {
-      status: "action_required",
-      code: "missing_required_notifications",
-      summary: `Required interaction notifications are missing: ${missingStages.join(", ")}.`,
-      missingStages,
-      findings
-    };
-  }
-
-  if (yielded && followUpQueued && !followUpConsumed) {
-    return {
-      status: "action_required",
-      code: "follow_up_incomplete",
-      summary: "Yielded follow-up is still pending or incomplete.",
-      missingStages,
-      findings
-    };
-  }
-
-  if (deliveryEvidenceRequired && !deliveryEvidenceHealthy) {
-    return {
-      status: "action_required",
-      code: "missing_delivery_evidence",
-      summary: "Required pingback notifications do not have delivery receipts yet.",
-      missingStages,
-      findings
-    };
-  }
-
-  if (handoffRequired && handoffQueued && !handoffDelivered) {
-    return {
-      status: "action_required",
-      code: "handoff_incomplete",
-      summary: "Handoff was queued but not fully delivered.",
-      missingStages,
-      findings
-    };
-  }
-
-  if (deliveryUncertain) {
-    return {
-      status: "warning",
-      code: "delivery_uncertain",
-      summary: "Interaction lifecycle completed with uncertain delivery evidence.",
-      missingStages,
-      findings
-    };
-  }
-
-  return {
-    status: "warning",
-    code: "interaction_findings_present",
-    summary: findings[0] || "Interaction lifecycle has unresolved findings.",
-    missingStages,
-    findings
-  };
-}
-
-function analyzeInteraction(run, notifications = [], deliveries = []) {
-  const interaction = run?.interaction || null;
-  const findings = [];
-
-  const ackRequired = interaction?.ack?.required ?? false;
-  const completionRequired = interaction?.completion?.required ?? false;
-  const handoffRequired = interaction?.handoff?.required ?? false;
-  const ackQueued = notifications.some((item) => item.stage === "ack");
-  const completionQueued = notifications.some((item) => item.stage === "completion");
-  const handoffQueued = notifications.some((item) => item.stage === "handoff");
-  const followUpQueued = notifications.some((item) => item.stage === "follow_up");
-  const followUpNotification = notifications.find((item) => item.stage === "follow_up") || null;
-  const handoffNotification = notifications.find((item) => item.stage === "handoff") || null;
-  const handoffPendingChoice = handoffNotification?.status === "awaiting_choice";
-  const handoffState = handoffNotification?.handoffState || (handoffPendingChoice ? "pending" : null);
-  const handoffChosen = Boolean(handoffNotification?.chosenPeer?.peerId);
-  const handoffChosenPeerId = handoffNotification?.chosenPeer?.peerId || null;
-  const handoffChosenBy = handoffNotification?.chosenBy || null;
-  const handoffDelivery = handoffNotification
-    ? deliveries.find((item) => item.notificationFilePath === handoffNotification.filePath) || null
-    : null;
-  const handoffDelivered = Boolean(handoffDelivery);
-  const handoffDeliveryState = handoffDelivery?.delivery?.status || null;
-  const deliveryStates = deliveries.map((item) => item.delivery?.status || item.stage || "unknown");
-  const yielded = interaction?.yield?.required ?? false;
-  const yieldSignal = interaction?.yield?.signal || null;
-  const followUpState = followUpNotification?.followUpState || followUpNotification?.yieldState || null;
-  const followUpConsumed = followUpNotification?.status === "consumed" || followUpNotification?.yieldState === "consumed" || followUpNotification?.followUpState === "consumed";
-  const followUpTarget = followUpNotification?.targetSurface || interaction?.yield?.targetSurface || null;
-  const generatedFiles = new Set(followUpNotification?.generatedNotificationFiles || []);
-  const followUpCompleted = deliveries.some((item) => generatedFiles.has(item.notificationFilePath));
-  const followUpExpired = followUpState === "expired";
-  const followUpEscalated = followUpState === "escalated";
-  const deliveryDeduped = deliveryStates.includes("duplicate_prevented");
-  const deliveryRetried = deliveries.some((item) => Number(item.attempt) > 1);
-  const deliveryUncertain = deliveryStates.includes("delivery_uncertain") || deliveryStates.includes("uncertain");
-  const yieldedCompletionHandled = yielded && followUpQueued && Boolean(followUpNotification?.payload?.completion);
-  const yieldedHandoffHandled = yielded && followUpQueued && Boolean(followUpNotification?.payload?.handoff);
-  const visibleDeliveryCount = yielded ? deliveries.filter((item) => !generatedFiles.has(item.notificationFilePath)).length : deliveries.length;
-  const deliveryEvidenceRequired = !yielded && Boolean(ackQueued || completionQueued || handoffQueued);
-  const deliveryEvidenceHealthy = !deliveryEvidenceRequired || visibleDeliveryCount > 0;
-
-  if (ackRequired && !ackQueued) findings.push("Required ack notification was not queued.");
-  if (completionRequired && !completionQueued && !yieldedCompletionHandled) {
-    findings.push("Required completion notification was not queued.");
-  }
-  if (handoffRequired && !handoffQueued && !yieldedHandoffHandled) {
-    findings.push("Configured handoff was not queued.");
-  }
-  if (yielded && !followUpQueued) findings.push("Run yielded but no follow-up payload was persisted.");
-  if (followUpQueued && !followUpConsumed) findings.push("Follow-up payload was persisted but never consumed.");
-  if (followUpExpired) findings.push("Follow-up expired before it was consumed.");
-  if (followUpEscalated) findings.push("Follow-up expired and was escalated to the operator.");
-  if (handoffPendingChoice) findings.push("Handoff is awaiting an explicit peer choice.");
-  if (handoffState === "expired") findings.push("Handoff expired without an operator choice.");
-  if (handoffState === "escalated") findings.push("Handoff expired and was escalated to the operator.");
-  if (handoffState === "blocked") findings.push(`Handoff is blocked${handoffNotification?.blockedReason ? `: ${handoffNotification.blockedReason}` : "."}`);
-  if (handoffQueued && !handoffPendingChoice && !handoffChosen) findings.push("Handoff was queued but no peer choice was recorded.");
-  if (handoffChosen && !handoffDelivered) findings.push("Handoff was promoted to a peer but not delivered yet.");
-  if (handoffDeliveryState && /error|blocked/i.test(handoffDeliveryState)) findings.push(`Handoff delivery did not complete cleanly: ${handoffDeliveryState}.`);
-  if (!deliveryEvidenceHealthy) {
-    findings.push("No delivery receipts exist yet for queued interaction notifications.");
-  }
-  if (deliveryDeduped) findings.push("A duplicate delivery attempt was prevented by dedupe policy.");
-  if (deliveryUncertain) findings.push("A delivery attempt was marked uncertain and will not be blindly retried.");
-
-  const diagnosis = summarizeInteractionDiagnosis({
-    ackRequired,
-    ackQueued,
-    completionRequired,
-    completionQueued,
-    handoffRequired,
-    handoffQueued,
-    yielded,
-    followUpQueued,
-    followUpConsumed,
-    handoffDelivered,
-    deliveryEvidenceRequired,
-    deliveryEvidenceHealthy,
-    deliveryUncertain,
-    findings
-  });
-
-  return {
-    ackRequired,
-    ackQueued,
-    completionRequired,
-    completionQueued,
-    handoffRequired,
-    handoffQueued,
-    yielded,
-    yieldSignal,
-    followUpState,
-    followUpQueued,
-    followUpConsumed,
-    followUpTarget,
-    followUpCompleted,
-    followUpExpired,
-    followUpEscalated,
-    followUpEscalationFilePath: followUpNotification?.escalationNotificationFilePath || null,
-    handoffPendingChoice,
-    handoffState,
-    handoffExpired: handoffState === "expired",
-    handoffEscalated: handoffState === "escalated",
-    handoffBlocked: handoffState === "blocked",
-    handoffChosen,
-    handoffChosenPeerId,
-    handoffChosenBy,
-    handoffDelivered,
-    handoffDeliveryState,
-    handoffBlockedReason: handoffNotification?.blockedReason || null,
-    deliveryEvidenceRequired,
-    deliveryEvidenceHealthy,
-    deliveryReceiptCount: visibleDeliveryCount,
-    deliveryStates,
-    deliveryDeduped,
-    deliveryRetried,
-    deliveryUncertain,
-    diagnosis,
-    findings
-  };
-}
-
-function summarizeLiveRun(run) {
-  if (!run) return null;
-  return {
-    ts: run.ts || null,
-    status: run.status || null,
-    summary: run.summary || null,
-    error: run.error || null,
-    durationMs: run.durationMs ?? null,
-    provider: run.provider || null,
-    model: run.model || null,
-    usage: run.usage || null
-  };
-}
-
-const SECTION_HINTS = [
-  "calendar",
-  "weather",
-  "projects",
-  "project",
-  "inbox",
-  "issues",
-  "backlog",
-  "update",
-  "summary",
-  "next actions",
-  "alerts",
-  "status"
-];
-
-const SECTION_ALIASES = new Map([
-  ["project", "projects"],
-  ["next action", "next actions"]
-]);
-
-function canonicalSectionKey(value) {
-  const normalized = normalizeSectionKey(value);
-  return SECTION_ALIASES.get(normalized) || normalized;
-}
-
-function extractSectionKeys(nonEmptyLines) {
-  const sectionKeys = [];
-  const seen = new Set();
-  const patterns = [
-    /^#{1,6}\s+(.+)$/,
-    /^\*\*([^*]+)\*\*$/,
-    /^[-*]\s+\**([^:*]+?)\**:\s+.+$/,
-    /^([^:]{2,40}):\s+.+$/
-  ];
-
-  for (const line of nonEmptyLines) {
-    for (const pattern of patterns) {
-      const match = line.match(pattern);
-      if (!match) continue;
-      const key = canonicalSectionKey(match[1]);
-      if (!key || key.length < 2) continue;
-      if (seen.has(key)) break;
-      seen.add(key);
-      sectionKeys.push(key);
-      break;
-    }
-  }
-
-  return sectionKeys;
-}
-
-function comparableSections(structure) {
-  return structure.sectionKeys.length ? structure.sectionKeys : structure.sectionHints;
-}
-
-function analyzeOutputStructure(text) {
-  const raw = String(text || "");
-  const trimmed = raw.trim();
-  const lines = raw.split("\n").map((line) => line.trim());
-  const nonEmptyLines = lines.filter(Boolean);
-  const lower = trimmed.toLowerCase();
-  const sectionKeys = extractSectionKeys(nonEmptyLines);
-
-  return {
-    charCount: trimmed.length,
-    lineCount: nonEmptyLines.length,
-    headingCount: nonEmptyLines.filter((line) => /^#{1,6}\s/.test(line) || /^\*\*[^*]+\*\*/.test(line)).length,
-    bulletCount: nonEmptyLines.filter((line) => /^[-*]\s/.test(line)).length,
-    numberedCount: nonEmptyLines.filter((line) => /^\d+\.\s/.test(line)).length,
-    codeFenceCount: (raw.match(/```/g) || []).length / 2,
-    sectionKeys,
-    sectionHints: SECTION_HINTS.filter((hint) => lower.includes(hint)),
-    hasMarkdownTable: /\|.+\|/.test(raw),
-    hasEmoji: /\p{Extended_Pictographic}/u.test(raw)
-  };
-}
-
-function buildOutputDiff(v2Run, liveRun, contract = null) {
-  const v2Text = extractRunText(v2Run, contract);
-  const liveText = extractLiveText(liveRun);
-  const v2 = analyzeOutputStructure(v2Text);
-  const live = analyzeOutputStructure(liveText);
-  const v2Sections = comparableSections(v2);
-  const liveSections = comparableSections(live);
-  const sharedSections = v2Sections.filter((hint) => liveSections.includes(hint));
-  const missingFromV2 = liveSections.filter((hint) => !v2Sections.includes(hint));
-  const extraInV2 = v2Sections.filter((hint) => !liveSections.includes(hint));
-  const findings = [];
-
-  if (!liveText) {
-    findings.push("No live output is available for structural comparison.");
-  } else {
-    if (missingFromV2.length) findings.push(`V2 is missing live sections: ${missingFromV2.join(", ")}.`);
-    if (v2.bulletCount + v2.numberedCount < live.bulletCount + live.numberedCount) {
-      findings.push("V2 output is less structured than the live report.");
-    }
-    if (v2.charCount > 0 && live.charCount > 0) {
-      const verbosityRatio = Number((v2.charCount / live.charCount).toFixed(4));
-      if (verbosityRatio < 0.55) findings.push("V2 output is much shorter than the live report.");
-      if (verbosityRatio > 1.8) findings.push("V2 output is much longer than the live report.");
-    }
-  }
-
-  return {
-    v2,
-    live,
-    alignment: {
-      sharedSections,
-      missingFromV2,
-      extraInV2,
-      bulletDelta: (v2.bulletCount + v2.numberedCount) - (live.bulletCount + live.numberedCount),
-      headingDelta: v2.headingCount - live.headingCount,
-      charDelta: v2.charCount - live.charCount
-    },
-    findings
-  };
-}
-
-function assessOutputContract(contract, v2Run, liveRun) {
-  if (!contract) return null;
-
-  const v2Validation = validateOutputContract(contract, v2Run?.result?.output ?? v2Run?.output ?? "");
-  const liveValidation = validateOutputContract(contract, liveRun?.summary || liveRun?.error || "");
-  const missingFromV2 = v2Validation?.missingSections || [];
-  const missingFromLive = liveValidation?.missingSections || [];
-  const findings = [
-    ...(v2Validation?.findings.map((item) => `V2: ${item}`) || []),
-    ...(liveValidation?.findings.map((item) => `Live: ${item}`) || [])
-  ];
-
-  return {
-    format: contract.format || null,
-    requiredSections: contract.requiredSections || [],
-    styleHints: contract.styleHints || [],
-    satisfiedRatio: v2Validation?.satisfiedRatio ?? 0,
-    missingFromV2,
-    missingFromLive,
-    emptyInV2: v2Validation?.emptySections || [],
-    emptyInLive: liveValidation?.emptySections || [],
-    parsedV2: v2Validation?.parsed || null,
-    parsedLive: liveValidation?.parsed || null,
-    fieldScores: v2Validation?.fieldScores || null,
-    findings
-  };
-}
-
-export { buildRubric };
-
-export class Evaluator {
-  constructor({ projectRoot, liveRoot, stateRoot }) {
-    this.liveRoot = liveRoot;
-    this.stateRoot = stateRoot;
-    this.reviewer = new RunReviewer({ stateRoot });
-    this.scheduler = new Scheduler({ projectRoot, liveRoot, stateRoot });
-    this.evalStore = new RunStore({ rootDir: path.join(stateRoot, "evaluations") });
-    this.notificationStore = new NotificationStore({ rootDir: path.join(stateRoot, "notifications") });
-    this.deliveryStore = new DeliveryStore({ rootDir: path.join(stateRoot, "deliveries") });
-  }
-
-  async resolveInteractionArtifacts(run, review = null) {
-    const reviewNotifications = review?.recentNotifications || [];
-    const _reviewDeliveries = review?.recentDeliveries || [];
-    const baseFiles = run?.notificationFiles || [];
-    const relatedNotificationFiles = expandRelatedNotificationFiles(
-      baseFiles,
-      reviewNotifications
-    );
-    const knownNotifications = reviewNotifications.filter((item) => relatedNotificationFiles.has(item.filePath));
-    const missingNotificationFiles = [...relatedNotificationFiles].filter(
-      (filePath) => !knownNotifications.some((item) => item.filePath === filePath)
-    );
-    const loadedNotifications = await this.notificationStore.getNotifications(missingNotificationFiles);
-    const notifications = [...knownNotifications, ...loadedNotifications].sort((a, b) =>
-      String(a.timestamp || "").localeCompare(String(b.timestamp || ""))
-    );
-    const expandedNotificationFiles = expandRelatedNotificationFiles(baseFiles, notifications);
-
-    // Load all delivery receipts once for deterministic interaction evidence.
-    const allDeliveries = await this.deliveryStore.listAll();
-    const deliveries = allDeliveries.filter((item) => expandedNotificationFiles.has(item.notificationFilePath));
-    return {
-      notificationFiles: expandedNotificationFiles,
-      notifications,
-      deliveries
-    };
-  }
-
-  async evaluate(limit = 20) {
-    const [review, comparisons] = await Promise.all([
-      this.reviewer.review(limit),
-      this.scheduler.compareJobs()
-    ]);
-
-    const recentRuns = review.recentRuns.filter((run) => !run.jobId.endsWith("-comparison"));
-    const grouped = groupBy(recentRuns, (run) => run.jobId);
-
-    const jobs = await Promise.all(
-      Array.from(grouped.entries()).map(async ([jobId, runs]) => this.evaluateJob(jobId, {
-        runs,
-        review,
-        comparisons
-      }))
-    );
-
-    return {
-      jobs: [...jobs].sort((a, b) => a.rubric.overallScore - b.rubric.overallScore),
-      recentRuns,
-      scheduler: review.scheduler
-    };
-  }
-
-  async evaluateJob(jobId, options = {}) {
-    const runtime = await this.scheduler.loadRuntime();
-    const jobConfig = runtime.jobs[jobId] || null;
-    const review = options.review || (await this.reviewer.review(options.limit ?? 20));
-    const comparisons = options.comparisons || (await this.scheduler.compareJobs());
-    const runs = options.runs || review.recentRuns.filter((run) => run.jobId === jobId && !run.jobId.endsWith("-comparison"));
-    const schedulerState = review.scheduler.find((item) => item.jobId === jobId) || null;
-    const comparison = comparisons.find((item) => item.v2JobId === jobId) || null;
-    const liveMatches = await Promise.all(
-      (comparison?.closestLiveJobs || []).map(async (match) => ({
-        ...match,
-        recentRuns: await this.scheduler.bridge.loadCronRunHistory(match.id, 5),
-        latestRun: (await this.scheduler.bridge.loadCronRunHistory(match.id, 1))[0] || null
-      }))
-    );
-
-    const selectedRun = choosePrimaryV2Run(runs);
-    const interactionRun = runs[0] || selectedRun;
-    const selectedLiveMatch = pickBestLiveMatch(liveMatches);
-    const liveHistory = selectedLiveMatch?.recentRuns || [];
-    const outputContract = normalizeOutputContract(jobConfig?.outputContract || null);
-    const outputDiff = buildOutputDiff(selectedRun, selectedLiveMatch?.latestRun || null, outputContract);
-    const contractCheck = assessOutputContract(outputContract, selectedRun, selectedLiveMatch?.latestRun || null);
-    const retrieval = analyzeRetrieval(selectedRun);
-    const interactionArtifacts = await this.resolveInteractionArtifacts(interactionRun, review);
-    const interaction = analyzeInteraction(interactionRun, interactionArtifacts.notifications, interactionArtifacts.deliveries);
-    const rubric = buildRubric({
-      v2Run: selectedRun,
-      liveHistory,
-      matchedLiveJob: selectedLiveMatch,
-      outputContract,
-      contractCheck
-    });
-
-    return {
-      jobId,
-      schedulerState,
-      maintenance: review?.maintenance || null,
-      latestRun: runs[0] || null,
-      primaryRun: summarizeV2Run(selectedRun),
-      interactionRunTimestamp: interactionRun?.timestamp || null,
-      runCount: runs.length,
-      modeCounts: countModes(runs),
-      liveMatches: liveMatches.map((match) => ({
-        id: match.id,
-        name: match.name,
-        description: match.description || null,
-        enabled: match.enabled,
-        schedule: match.schedule,
-        kind: match.kind || null,
-        lastStatus: match.lastStatus || null,
-        latestRun: summarizeLiveRun(match.latestRun)
-      })),
-      selectedLiveMatch: selectedLiveMatch
-        ? {
-            id: selectedLiveMatch.id,
-            name: selectedLiveMatch.name,
-            latestRun: summarizeLiveRun(selectedLiveMatch.latestRun)
-          }
-        : null,
-      retrieval,
-      interaction,
-      outputContract,
-      contractCheck,
-      outputDiff,
-      rubric,
-      comparisonNotes: buildComparisonNotes({
-        jobId,
-        selectedRun,
-        selectedLiveMatch,
-        schedulerState,
-        maintenance: review?.maintenance || null,
-        rubric,
-        outputDiff,
-        contractCheck,
-        retrieval,
-        interaction
-      }),
-      interactionArtifacts: {
-        notificationCount: interactionArtifacts.notifications.length,
-        deliveryCount: interactionArtifacts.deliveries.length,
-        notificationFileCount: interactionArtifacts.notificationFiles.size
-      }
-    };
-  }
-
-  async evaluateAndPersistJob(jobId, options = {}) {
-    const report = await this.evaluateJob(jobId, options);
-    const artifact = {
-      timestamp: new Date().toISOString(),
-      kind: "job-evaluation",
-      ...report
-    };
-    const filePath = await this.evalStore.saveRun(jobId, artifact);
-    return {
-      filePath,
-      ...artifact
-    };
-  }
-}
-
-function countModes(runs) {
-  return runs.reduce((acc, run) => {
-    const mode = run.mode || "unknown";
-    acc[mode] = (acc[mode] || 0) + 1;
-    return acc;
-  }, {});
-}
-
-function buildComparisonNotes({
-  jobId,
-  selectedRun,
-  selectedLiveMatch,
-  schedulerState,
-  maintenance,
-  rubric,
-  outputDiff,
-  contractCheck,
-  retrieval = null,
-  interaction = null
-}) {
-  const notes = [];
-  if (!selectedRun) notes.push("No V2 runs recorded.");
-  if (schedulerState?.lastStatus === "ok") notes.push("Latest V2 scheduler state is healthy.");
-  if (selectedRun?.fallback?.attempted && selectedRun?.fallback?.success) {
-    notes.push(
-      `Run used report fallback from ${selectedRun.fallback.sourceLane || "local"} to ${selectedRun.fallback.finalSourceLane || "remote"} after ${selectedRun.fallback.trigger || "failure"}.`
-    );
-  } else if (selectedRun?.fallback?.attempted && !selectedRun?.fallback?.success) {
-    notes.push(`Report fallback was attempted but did not succeed${selectedRun.fallback?.fallbackError ? `: ${selectedRun.fallback.fallbackError}` : "."}`);
-  } else if (selectedRun?.fallback?.allowed === false && selectedRun?.fallback?.trigger) {
-    notes.push(`Report fallback was blocked after ${selectedRun.fallback.trigger}${selectedRun.fallback?.blockedReason ? `: ${selectedRun.fallback.blockedReason}` : "."}`);
-  }
-  if (maintenance?.wal?.some((item) => item.action && item.action !== "none" && item.action !== "error")) {
-    notes.push("Daemon maintenance recently performed WAL checkpoint work.");
-  }
-  if (maintenance?.handoffs?.expiredCount > 0) {
-    notes.push(`Daemon maintenance expired or escalated ${maintenance.handoffs.expiredCount} pending handoff(s).`);
-  }
-  if (!selectedLiveMatch) {
-    notes.push("No live cron analogue matched.");
-    return uniqueStrings([...notes, ...rubric.findings, ...outputDiff.findings, ...(contractCheck?.findings || []), ...(interaction?.findings || [])]);
-  }
-
-  const latestLive = selectedLiveMatch.latestRun;
-  notes.push(`Closest live cron match: ${selectedLiveMatch.name || selectedLiveMatch.id}.`);
-  if (latestLive?.status) notes.push(`Latest live status: ${latestLive.status}.`);
-  if (latestLive?.provider) notes.push(`Latest live provider: ${latestLive.provider}.`);
-  if (latestLive?.durationMs != null) notes.push(`Latest live duration: ${latestLive.durationMs}ms.`);
-  const retrievalFindings = retrieval?.findings || [];
-  const interactionFindings = interaction?.findings || [];
-  const outputDiffFindings =
-    jobId === "memory-rollup" && Number(contractCheck?.satisfiedRatio || 0) >= 0.99
-      ? (outputDiff.findings || []).filter((item) => !/less structured|much shorter/i.test(item))
-      : outputDiff.findings || [];
-  return uniqueStrings([...notes, ...rubric.findings, ...outputDiffFindings, ...(contractCheck?.findings || []), ...retrievalFindings, ...interactionFindings]);
-}
-
-function normalizeOutputContract(contract) {
-  if (!contract) return null;
-  return {
-    format: contract.format || null,
-    requiredSections: contract.requiredSections || [],
-    styleHints: contract.styleHints || [],
-    profile: contract.profile || null
-  };
-}
+import path from "node:path";
+import { RunReviewer } from "./review.js";
+import { Scheduler } from "./scheduler.js";
+import { RunStore } from "./run-store.js";
+import { NotificationStore } from "./notification-store.js";
+import { DeliveryStore } from "./delivery-store.js";
+import { validateOutputContract } from "./output-contract-validator.js";
+
+function groupBy(items, keyFn) {
+  const map = new Map();
+  for (const item of items) {
+    const key = keyFn(item);
+    if (!map.has(key)) map.set(key, []);
+    map.get(key).push(item);
+  }
+  return map;
+}
+
+function clamp(value, min = 0, max = 1) {
+  return Math.max(min, Math.min(max, value));
+}
+
+function average(values) {
+  if (values.length === 0) return null;
+  return values.reduce((sum, value) => sum + value, 0) / values.length;
+}
+
+function choosePrimaryV2Run(runs) {
+  if (!runs.length) return null;
+  return runs.find((run) => run.mode === "provider") || runs[0];
+}
+
+function pickBestLiveMatch(liveMatches) {
+  return liveMatches.find((match) => match.latestRun) || liveMatches[0] || null;
+}
+
+function normalizeText(value) {
+  return String(value || "")
+    .replace(/\s+/g, " ")
+    .trim();
+}
+
+function stripOuterFence(text) {
+  const raw = String(text || "").trim();
+  const fencedMatch = raw.match(/^```(?:json|markdown|md|text)?\s*([\s\S]*?)\s*```$/i);
+  return fencedMatch ? fencedMatch[1].trim() : raw;
+}
+
+function parseJsonCandidate(text) {
+  const candidate = stripOuterFence(text);
+  if (!(candidate.startsWith("{") && candidate.endsWith("}"))) {
+    return null;
+  }
+
+  try {
+    return JSON.parse(candidate);
+  } catch {
+    return null;
+  }
+}
+
+function titleCase(value) {
+  return String(value || "")
+    .replace(/[_-]+/g, " ")
+    .replace(/^#+\s*/, "")
+    .replace(/\b\w/g, (char) => char.toUpperCase())
+    .trim();
+}
+
+function normalizeInline(value) {
+  return normalizeText(value) || "None";
+}
+
+function normalizeSectionKey(value) {
+  return String(value || "")
+    .toLowerCase()
+    .replace(/^#+\s*/, "")
+    .replace(/[*:_-]+/g, " ")
+    .replace(/\s+/g, " ")
+    .trim();
+}
+
+function renderValue(value, { contract = null, summary = null } = {}) {
+  if (value == null) return "";
+
+  if (typeof value === "string") {
+    const parsed = parseJsonCandidate(value);
+    if (parsed && typeof parsed === "object") {
+      if (parsed.output !== undefined) {
+        return renderValue(parsed.output, {
+          contract,
+          summary: parsed.summary || summary
+        });
+      }
+
+      return renderValue(parsed, { contract, summary });
+    }
+
+    return stripOuterFence(value);
+  }
+
+  if (Array.isArray(value)) {
+    return value
+      .map((item) => renderValue(item, { contract, summary }))
+      .filter(Boolean)
+      .join("\n");
+  }
+
+  if (typeof value === "object") {
+    const entries = Object.entries(value);
+    const sectionOrder = contract?.requiredSections?.length
+      ? contract.requiredSections
+      : entries.map(([key]) => key);
+    const valueMap = new Map(entries.map(([key, entryValue]) => [normalizeSectionKey(key), entryValue]));
+    const sectionStyle = contract?.profile?.sectionStyle || (contract?.format === "structured_rollup" ? "headings" : "bullets");
+    const requireStatus = contract?.profile?.requireStatus ?? (contract?.format === "bulleted_briefing");
+
+    if (sectionStyle === "bullets") {
+      const lines = [];
+      if (requireStatus && summary) lines.push(`Status: ${normalizeInline(summary)}`);
+      for (const section of sectionOrder) {
+        lines.push(`- ${titleCase(section)}: ${normalizeInline(valueMap.get(normalizeSectionKey(section)))}`);
+      }
+      return lines.join("\n");
+    }
+
+    if (sectionStyle === "headings") {
+      const lines = [];
+      for (const section of sectionOrder) {
+        lines.push(`## ${titleCase(section)}`);
+        lines.push(`- ${normalizeInline(valueMap.get(normalizeSectionKey(section)))}`);
+        lines.push("");
+      }
+      return lines.join("\n").trim();
+    }
+
+    return entries
+      .map(([key, entryValue]) => `${titleCase(key)}: ${normalizeInline(entryValue)}`)
+      .join("\n");
+  }
+
+  return String(value);
+}
+
+function extractRunText(run, contract = null) {
+  if (!run) return "";
+  const summary = run?.result?.summary || run?.summary || "";
+  const value = run?.result?.output ?? run?.output ?? summary;
+  return renderValue(value, { contract, summary });
+}
+
+function extractLiveText(run) {
+  if (!run) return "";
+  return renderValue(run.summary || run.error || "");
+}
+
+function qualitySignals(text) {
+  const raw = String(text || "");
+  const normalized = normalizeText(text);
+  const lower = normalized.toLowerCase();
+  const findings = [];
+  let score = 1;
+
+  if (!normalized) {
+    findings.push("Output is empty.");
+    return { score: 0, findings };
+  }
+
+  if (normalized.length < 40) {
+    findings.push("Output is very short and may not carry enough signal.");
+    score -= 0.18;
+  }
+
+  if (/\[(title|link|summary|sub)\]/i.test(normalized)) {
+    findings.push("Output still contains placeholder text.");
+    score -= 0.4;
+  }
+
+  if (/i('| a)?m sorry|does not support|cannot complete|failed to/i.test(lower)) {
+    findings.push("Output contains apology or tool-failure language.");
+    score -= 0.28;
+  }
+
+  if (/```/.test(raw)) {
+    findings.push("Output is wrapped in a code block instead of a direct report.");
+    score -= 0.12;
+  }
+
+  if (/timed out|timeout/i.test(lower)) {
+    findings.push("Output references a timeout or stalled execution.");
+    score -= 0.35;
+  }
+
+  const repeatedLinePenalty = detectRepeatedLinePenalty(normalized);
+  if (repeatedLinePenalty > 0) {
+    findings.push("Output repeats nearly identical numbered lines.");
+    score -= repeatedLinePenalty;
+  }
+
+  return {
+    score: clamp(Number(score.toFixed(4))),
+    findings
+  };
+}
+
+function detectRepeatedLinePenalty(text) {
+  const lines = text
+    .split("\n")
+    .map((line) => line.trim())
+    .filter(Boolean)
+    .filter((line) => /^\d+\./.test(line));
+
+  if (lines.length < 3) return 0;
+
+  const normalized = lines.map((line) => line.replace(/^\d+\.\s*/, "").toLowerCase());
+  const unique = new Set(normalized);
+  const duplicateRatio = 1 - unique.size / normalized.length;
+  if (duplicateRatio < 0.5) return 0;
+  return Number((duplicateRatio * 0.25).toFixed(4));
+}
+
+function expandRelatedNotificationFiles(baseFiles, notifications) {
+  const related = new Set(baseFiles);
+  let changed = true;
+  while (changed) {
+    changed = false;
+    for (const item of notifications) {
+      if (!related.has(item.filePath)) continue;
+      for (const generated of item.generatedNotificationFiles || []) {
+        if (related.has(generated)) continue;
+        related.add(generated);
+        changed = true;
+      }
+    }
+  }
+  return related;
+}
+
+function evaluateLiveHistory(liveRuns) {
+  if (liveRuns.length === 0) {
+    return {
+      runCount: 0,
+      okCount: 0,
+      errorCount: 0,
+      skippedCount: 0,
+      failureRate: null,
+      avgDurationMs: null,
+      avgTotalTokens: null,
+      avgQualityScore: null,
+      findings: ["No recent live history available."]
+    };
+  }
+
+  const okCount = liveRuns.filter((run) => run.status === "ok").length;
+  const errorCount = liveRuns.filter((run) => run.status === "error").length;
+  const skippedCount = liveRuns.filter((run) => run.status === "skipped").length;
+  const failureRate = errorCount / liveRuns.length;
+  const durations = liveRuns.map((run) => run.durationMs).filter((value) => Number.isFinite(value));
+  const tokens = liveRuns.map((run) => run.usage?.total_tokens).filter((value) => Number.isFinite(value));
+  const qualityScores = liveRuns
+    .map((run) => qualitySignals(run.summary || run.error || "").score)
+    .filter((value) => Number.isFinite(value));
+
+  const findings = [];
+  if (failureRate >= 0.34) findings.push("Live cron lane is failing often.");
+  if (skippedCount > 0) findings.push("Live cron history includes intentional skips.");
+  if (tokens.some((value) => value > 12000)) findings.push("Live cron lane shows high token usage spikes.");
+
+  return {
+    runCount: liveRuns.length,
+    okCount,
+    errorCount,
+    skippedCount,
+    failureRate: Number(failureRate.toFixed(4)),
+    avgDurationMs: durations.length ? Math.round(average(durations)) : null,
+    avgTotalTokens: tokens.length ? Math.round(average(tokens)) : null,
+    avgQualityScore: qualityScores.length ? Number(average(qualityScores).toFixed(4)) : null,
+    findings
+  };
+}
+
+function scoreStatus(status) {
+  if (status === "ok") return 1;
+  if (status === "skipped") return 0.6;
+  if (status === "error") return 0;
+  return 0.5;
+}
+
+function buildRubric({ v2Run, liveHistory, matchedLiveJob, outputContract = null, contractCheck = null }) {
+  const findings = [];
+  const v2Text = extractRunText(v2Run, outputContract);
+  const v2Quality = qualitySignals(v2Text);
+  const latestLive = liveHistory[0] || null;
+  const latestLiveQuality = qualitySignals(extractLiveText(latestLive));
+  const history = evaluateLiveHistory(liveHistory);
+  const isDryRun = v2Run != null && v2Run.mode !== "provider";
+
+  const comparisonReadiness = v2Run
+    ? v2Run.mode === "provider"
+      ? 1
+      : 0.45
+    : 0;
+  const comparisonCoverage = matchedLiveJob ? 1 : 0.2;
+  const contractAdherence = contractCheck ? contractCheck.satisfiedRatio : null;
+  if (!v2Run) findings.push("No V2 run exists for this lane yet.");
+  else if (isDryRun) findings.push("Only a dry-run V2 artifact exists; output quality and contract adherence are excluded from the rubric score.");
+  if (!matchedLiveJob) findings.push("Comparison coverage is low because no live cron analogue is mapped.");
+
+  if (!isDryRun && v2Quality.findings.length) findings.push(...v2Quality.findings.map((item) => `V2: ${item}`));
+  if (latestLiveQuality.findings.length) findings.push(...latestLiveQuality.findings.map((item) => `Live: ${item}`));
+  if (history.findings.length) findings.push(...history.findings);
+
+  const fieldScores = contractCheck?.fieldScores || null;
+  if (!isDryRun && fieldScores) {
+    if (fieldScores.weakFields.length) {
+      findings.push(`Field-level quality is low for: ${fieldScores.weakFields.join(", ")}`);
+    }
+    if (fieldScores.placeholderFields.length) {
+      findings.push(`Placeholder text detected in: ${fieldScores.placeholderFields.join(", ")}`);
+    }
+  }
+
+  if (!matchedLiveJob) {
+    findings.push("No live cron analogue matched for this V2 job.");
+  }
+
+  const statusAlignment =
+    latestLive && v2Run
+      ? Number((scoreStatus(v2Run.result ? "ok" : v2Run.status) * scoreStatus(latestLive.status)).toFixed(4))
+      : null;
+  const liveReliability = history.failureRate == null ? null : Number((1 - history.failureRate).toFixed(4));
+
+  const fieldLevelQuality = !isDryRun && fieldScores ? fieldScores.averageScore : null;
+
+  const components = [
+    isDryRun ? null : v2Quality.score,
+    comparisonReadiness,
+    comparisonCoverage,
+    isDryRun ? null : contractAdherence,
+    fieldLevelQuality,
+    liveReliability,
+    statusAlignment
+  ].filter((value) => value != null);
+  const overallScore = components.length ? Number(average(components).toFixed(4)) : 0;
+
+  return {
+    overallScore,
+    dryRunExcluded: isDryRun,
+    components: {
+      v2OutputQuality: isDryRun ? null : v2Quality.score,
+      comparisonReadiness,
+      comparisonCoverage,
+      contractAdherence: isDryRun ? null : contractAdherence,
+      fieldLevelQuality,
+      liveReliability,
+      statusAlignment
+    },
+    findings: uniqueStrings(findings),
+    metrics: {
+      liveRunCount: history.runCount,
+      liveOkCount: history.okCount,
+      liveErrorCount: history.errorCount,
+      liveSkippedCount: history.skippedCount,
+      liveFailureRate: history.failureRate,
+      liveAvgDurationMs: history.avgDurationMs,
+      liveAvgTotalTokens: history.avgTotalTokens,
+      liveAvgQualityScore: history.avgQualityScore,
+      latestLiveQualityScore: latestLive ? latestLiveQuality.score : null
+    }
+  };
+}
+
+function uniqueStrings(items) {
+  return [...new Set(items.filter(Boolean))];
+}
+
+function summarizeV2Run(run) {
+  if (!run) return null;
+  return {
+    timestamp: run.timestamp,
+    mode: run.mode || null,
+    providerId: run.providerId || null,
+    modelId: run.modelId || null,
+    summary: run.result?.summary || run.summary || null,
+    output: run.result?.output || run.output || null,
+    retrievalMeta: run.retrievalMeta || null,
+    retrievedMemoryCount: run.retrievedMemory?.length || 0,
+    interaction: summarizeInteraction(run),
+    fallback: run.fallback || null
+  };
+}
+
+function summarizeInteraction(run) {
+  const interaction = run?.interaction || null;
+  if (!interaction) return null;
+  return {
+    ackRequired: interaction.ack?.required ?? false,
+    completionRequired: interaction.completion?.required ?? false,
+    handoffRequired: interaction.handoff?.required ?? false,
+    yielded: run?.yielded || false,
+    yieldSignal: run?.yieldSignal || null,
+    followUpState: run?.followUpState || null,
+    followUpQueued: run?.followUpQueued || false,
+    followUpConsumed: run?.followUpConsumed || false,
+    followUpTarget: run?.followUpTarget || null,
+    followUpCompleted: run?.followUpCompleted || false,
+    followUpExpired: run?.followUpExpired || false,
+    followUpEscalated: run?.followUpEscalated || false,
+    followUpEscalationFilePath: run?.followUpEscalationFilePath || null,
+    handoffTarget: interaction.handoff?.target || null,
+    suggestedPeerCount: interaction.handoff?.suggestions?.length || 0,
+    handoffChosenPeerId: run?.handoffChosenPeerId || null,
+    handoffChosenBy: run?.handoffChosenBy || null,
+    handoffDelivered: run?.handoffDelivered || false,
+    handoffDeliveryState: run?.handoffDeliveryState || null,
+    deliveryDeduped: run?.deliveryDeduped || false,
+    deliveryRetried: run?.deliveryRetried || false,
+    deliveryUncertain: run?.deliveryUncertain || false
+  };
+}
+
+function analyzeRetrieval(run) {
+  const retrievalMeta = run?.retrievalMeta || run?.plan?.packet?.layers?.retrievalMeta || null;
+  if (!run?.retrievedMemory?.length) {
+    return {
+      memoryCount: 0,
+      lexicalCount: 0,
+      semanticCount: 0,
+      qmdCount: 0,
+      freshEmbeddingCount: 0,
+      staleEmbeddingCount: 0,
+      missingEmbeddingCount: 0,
+      failedEmbeddingCount: retrievalMeta?.embeddingHealth?.failedCount || 0,
+      embeddingQueryMode: retrievalMeta?.embeddingQueryMode || "lexical_only",
+      embeddingError: retrievalMeta?.embeddingError || retrievalMeta?.embeddingHealth?.lastError || null,
+      findings: ["No retrieved memory was captured for this run."],
+      items: []
+    };
+  }
+
+  const items = run.retrievedMemory.map((item) => ({
+    entryId: item.entryId || null,
+    title: item.title || null,
+    sourceBackend: item.sourceBackend || "file",
+    candidateSource: item.candidateSource || "indexed",
+    lexicalScore: item.lexicalScore ?? 0,
+    embeddingSimilarity: item.embeddingSimilarity ?? 0,
+    embeddingFreshness: item.embeddingFreshness || "missing",
+    retrievalSources: item.retrievalSources || []
+  }));
+
+  const lexicalCount = items.filter((item) => item.retrievalSources.includes("lexical")).length;
+  const semanticCount = items.filter((item) => item.retrievalSources.includes("semantic")).length;
+  const qmdCount = items.filter((item) => item.retrievalSources.includes("qmd") || item.sourceBackend === "qmd").length;
+  const freshEmbeddingCount = items.filter((item) => item.embeddingFreshness === "fresh").length;
+  const staleEmbeddingCount = items.filter((item) => item.embeddingFreshness === "stale").length;
+  const missingEmbeddingCount = items.filter((item) => item.embeddingFreshness === "missing").length;
+  const failedEmbeddingCount = items.filter((item) => item.embeddingFreshness === "failed").length;
+  const findings = [];
+
+  if (semanticCount === 0) findings.push("No semantic retrieval candidates reached the final packet.");
+  if (staleEmbeddingCount > 0) findings.push("Some retrieved items have stale embeddings.");
+  if (freshEmbeddingCount === 0 && missingEmbeddingCount > 0) findings.push("Retrieved file memory is falling back to non-embedded retrieval.");
+  if ((retrievalMeta?.embeddingError || retrievalMeta?.embeddingHealth?.lastError) && !findings.includes("Embedding query failed and retrieval fell back to lexical mode.")) {
+    findings.push("Embedding query failed and retrieval fell back to lexical mode.");
+  }
+
+  return {
+    memoryCount: items.length,
+    lexicalCount,
+    semanticCount,
+    qmdCount,
+    freshEmbeddingCount,
+    staleEmbeddingCount,
+    missingEmbeddingCount,
+    failedEmbeddingCount,
+    embeddingQueryMode: retrievalMeta?.embeddingQueryMode || "lexical_only",
+    embeddingError: retrievalMeta?.embeddingError || retrievalMeta?.embeddingHealth?.lastError || null,
+    embeddingHealth: retrievalMeta?.embeddingHealth || null,
+    findings,
+    items
+  };
+}
+
+function summarizeInteractionDiagnosis({
+  ackRequired,
+  ackQueued,
+  completionRequired,
+  completionQueued,
+  handoffRequired,
+  handoffQueued,
+  yielded,
+  followUpQueued,
+  followUpConsumed,
+  handoffDelivered,
+  deliveryEvidenceRequired,
+  deliveryEvidenceHealthy,
+  deliveryUncertain,
+  findings
+}) {
+  const missingStages = [];
+  if (ackRequired && !ackQueued) missingStages.push("ack");
+  if (completionRequired && !completionQueued && !yielded) missingStages.push("completion");
+  if (handoffRequired && !handoffQueued && !yielded) missingStages.push("handoff");
+
+  if (!findings?.length) {
+    return {
+      status: "healthy",
+      code: null,
+      summary: "Interaction lifecycle evidence is complete.",
+      missingStages,
+      findings: []
+    };
+  }
+
+  if (missingStages.length) {
+    return {
+      status: "action_required",
+      code: "missing_required_notifications",
+      summary: `Required interaction notifications are missing: ${missingStages.join(", ")}.`,
+      missingStages,
+      findings
+    };
+  }
+
+  if (yielded && followUpQueued && !followUpConsumed) {
+    return {
+      status: "action_required",
+      code: "follow_up_incomplete",
+      summary: "Yielded follow-up is still pending or incomplete.",
+      missingStages,
+      findings
+    };
+  }
+
+  if (deliveryEvidenceRequired && !deliveryEvidenceHealthy) {
+    return {
+      status: "action_required",
+      code: "missing_delivery_evidence",
+      summary: "Required pingback notifications do not have delivery receipts yet.",
+      missingStages,
+      findings
+    };
+  }
+
+  if (handoffRequired && handoffQueued && !handoffDelivered) {
+    return {
+      status: "action_required",
+      code: "handoff_incomplete",
+      summary: "Handoff was queued but not fully delivered.",
+      missingStages,
+      findings
+    };
+  }
+
+  if (deliveryUncertain) {
+    return {
+      status: "warning",
+      code: "delivery_uncertain",
+      summary: "Interaction lifecycle completed with uncertain delivery evidence.",
+      missingStages,
+      findings
+    };
+  }
+
+  return {
+    status: "warning",
+    code: "interaction_findings_present",
+    summary: findings[0] || "Interaction lifecycle has unresolved findings.",
+    missingStages,
+    findings
+  };
+}
+
+function analyzeInteraction(run, notifications = [], deliveries = []) {
+  const interaction = run?.interaction || null;
+  const findings = [];
+
+  const ackRequired = interaction?.ack?.required ?? false;
+  const completionRequired = interaction?.completion?.required ?? false;
+  const handoffRequired = interaction?.handoff?.required ?? false;
+  const ackQueued = notifications.some((item) => item.stage === "ack");
+  const completionQueued = notifications.some((item) => item.stage === "completion");
+  const handoffQueued = notifications.some((item) => item.stage === "handoff");
+  const followUpQueued = notifications.some((item) => item.stage === "follow_up");
+  const followUpNotification = notifications.find((item) => item.stage === "follow_up") || null;
+  const handoffNotification = notifications.find((item) => item.stage === "handoff") || null;
+  const handoffPendingChoice = handoffNotification?.status === "awaiting_choice";
+  const handoffState = handoffNotification?.handoffState || (handoffPendingChoice ? "pending" : null);
+  const handoffChosen = Boolean(handoffNotification?.chosenPeer?.peerId);
+  const handoffChosenPeerId = handoffNotification?.chosenPeer?.peerId || null;
+  const handoffChosenBy = handoffNotification?.chosenBy || null;
+  const handoffDelivery = handoffNotification
+    ? deliveries.find((item) => item.notificationFilePath === handoffNotification.filePath) || null
+    : null;
+  const handoffDelivered = Boolean(handoffDelivery);
+  const handoffDeliveryState = handoffDelivery?.delivery?.status || null;
+  const deliveryStates = deliveries.map((item) => item.delivery?.status || item.stage || "unknown");
+  const yielded = interaction?.yield?.required ?? false;
+  const yieldSignal = interaction?.yield?.signal || null;
+  const followUpState = followUpNotification?.followUpState || followUpNotification?.yieldState || null;
+  const followUpConsumed = followUpNotification?.status === "consumed" || followUpNotification?.yieldState === "consumed" || followUpNotification?.followUpState === "consumed";
+  const followUpTarget = followUpNotification?.targetSurface || interaction?.yield?.targetSurface || null;
+  const generatedFiles = new Set(followUpNotification?.generatedNotificationFiles || []);
+  const followUpCompleted = deliveries.some((item) => generatedFiles.has(item.notificationFilePath));
+  const followUpExpired = followUpState === "expired";
+  const followUpEscalated = followUpState === "escalated";
+  const deliveryDeduped = deliveryStates.includes("duplicate_prevented");
+  const deliveryRetried = deliveries.some((item) => Number(item.attempt) > 1);
+  const deliveryUncertain = deliveryStates.includes("delivery_uncertain") || deliveryStates.includes("uncertain");
+  const yieldedCompletionHandled = yielded && followUpQueued && Boolean(followUpNotification?.payload?.completion);
+  const yieldedHandoffHandled = yielded && followUpQueued && Boolean(followUpNotification?.payload?.handoff);
+  const visibleDeliveryCount = yielded ? deliveries.filter((item) => !generatedFiles.has(item.notificationFilePath)).length : deliveries.length;
+  const deliveryEvidenceRequired = !yielded && Boolean(ackQueued || completionQueued || handoffQueued);
+  const deliveryEvidenceHealthy = !deliveryEvidenceRequired || visibleDeliveryCount > 0;
+
+  if (ackRequired && !ackQueued) findings.push("Required ack notification was not queued.");
+  if (completionRequired && !completionQueued && !yieldedCompletionHandled) {
+    findings.push("Required completion notification was not queued.");
+  }
+  if (handoffRequired && !handoffQueued && !yieldedHandoffHandled) {
+    findings.push("Configured handoff was not queued.");
+  }
+  if (yielded && !followUpQueued) findings.push("Run yielded but no follow-up payload was persisted.");
+  if (followUpQueued && !followUpConsumed) findings.push("Follow-up payload was persisted but never consumed.");
+  if (followUpExpired) findings.push("Follow-up expired before it was consumed.");
+  if (followUpEscalated) findings.push("Follow-up expired and was escalated to the operator.");
+  if (handoffPendingChoice) findings.push("Handoff is awaiting an explicit peer choice.");
+  if (handoffState === "expired") findings.push("Handoff expired without an operator choice.");
+  if (handoffState === "escalated") findings.push("Handoff expired and was escalated to the operator.");
+  if (handoffState === "blocked") findings.push(`Handoff is blocked${handoffNotification?.blockedReason ? `: ${handoffNotification.blockedReason}` : "."}`);
+  if (handoffQueued && !handoffPendingChoice && !handoffChosen) findings.push("Handoff was queued but no peer choice was recorded.");
+  if (handoffChosen && !handoffDelivered) findings.push("Handoff was promoted to a peer but not delivered yet.");
+  if (handoffDeliveryState && /error|blocked/i.test(handoffDeliveryState)) findings.push(`Handoff delivery did not complete cleanly: ${handoffDeliveryState}.`);
+  if (!deliveryEvidenceHealthy) {
+    findings.push("No delivery receipts exist yet for queued interaction notifications.");
+  }
+  if (deliveryDeduped) findings.push("A duplicate delivery attempt was prevented by dedupe policy.");
+  if (deliveryUncertain) findings.push("A delivery attempt was marked uncertain and will not be blindly retried.");
+
+  const diagnosis = summarizeInteractionDiagnosis({
+    ackRequired,
+    ackQueued,
+    completionRequired,
+    completionQueued,
+    handoffRequired,
+    handoffQueued,
+    yielded,
+    followUpQueued,
+    followUpConsumed,
+    handoffDelivered,
+    deliveryEvidenceRequired,
+    deliveryEvidenceHealthy,
+    deliveryUncertain,
+    findings
+  });
+
+  return {
+    ackRequired,
+    ackQueued,
+    completionRequired,
+    completionQueued,
+    handoffRequired,
+    handoffQueued,
+    yielded,
+    yieldSignal,
+    followUpState,
+    followUpQueued,
+    followUpConsumed,
+    followUpTarget,
+    followUpCompleted,
+    followUpExpired,
+    followUpEscalated,
+    followUpEscalationFilePath: followUpNotification?.escalationNotificationFilePath || null,
+    handoffPendingChoice,
+    handoffState,
+    handoffExpired: handoffState === "expired",
+    handoffEscalated: handoffState === "escalated",
+    handoffBlocked: handoffState === "blocked",
+    handoffChosen,
+    handoffChosenPeerId,
+    handoffChosenBy,
+    handoffDelivered,
+    handoffDeliveryState,
+    handoffBlockedReason: handoffNotification?.blockedReason || null,
+    deliveryEvidenceRequired,
+    deliveryEvidenceHealthy,
+    deliveryReceiptCount: visibleDeliveryCount,
+    deliveryStates,
+    deliveryDeduped,
+    deliveryRetried,
+    deliveryUncertain,
+    diagnosis,
+    findings
+  };
+}
+
+function summarizeLiveRun(run) {
+  if (!run) return null;
+  return {
+    ts: run.ts || null,
+    status: run.status || null,
+    summary: run.summary || null,
+    error: run.error || null,
+    durationMs: run.durationMs ?? null,
+    provider: run.provider || null,
+    model: run.model || null,
+    usage: run.usage || null
+  };
+}
+
+const SECTION_HINTS = [
+  "calendar",
+  "weather",
+  "projects",
+  "project",
+  "inbox",
+  "issues",
+  "backlog",
+  "update",
+  "summary",
+  "next actions",
+  "alerts",
+  "status"
+];
+
+const SECTION_ALIASES = new Map([
+  ["project", "projects"],
+  ["next action", "next actions"]
+]);
+
+function canonicalSectionKey(value) {
+  const normalized = normalizeSectionKey(value);
+  return SECTION_ALIASES.get(normalized) || normalized;
+}
+
+function extractSectionKeys(nonEmptyLines) {
+  const sectionKeys = [];
+  const seen = new Set();
+  const patterns = [
+    /^#{1,6}\s+(.+)$/,
+    /^\*\*([^*]+)\*\*$/,
+    /^[-*]\s+\**([^:*]+?)\**:\s+.+$/,
+    /^([^:]{2,40}):\s+.+$/
+  ];
+
+  for (const line of nonEmptyLines) {
+    for (const pattern of patterns) {
+      const match = line.match(pattern);
+      if (!match) continue;
+      const key = canonicalSectionKey(match[1]);
+      if (!key || key.length < 2) continue;
+      if (seen.has(key)) break;
+      seen.add(key);
+      sectionKeys.push(key);
+      break;
+    }
+  }
+
+  return sectionKeys;
+}
+
+function comparableSections(structure) {
+  return structure.sectionKeys.length ? structure.sectionKeys : structure.sectionHints;
+}
+
+function analyzeOutputStructure(text) {
+  const raw = String(text || "");
+  const trimmed = raw.trim();
+  const lines = raw.split("\n").map((line) => line.trim());
+  const nonEmptyLines = lines.filter(Boolean);
+  const lower = trimmed.toLowerCase();
+  const sectionKeys = extractSectionKeys(nonEmptyLines);
+
+  return {
+    charCount: trimmed.length,
+    lineCount: nonEmptyLines.length,
+    headingCount: nonEmptyLines.filter((line) => /^#{1,6}\s/.test(line) || /^\*\*[^*]+\*\*/.test(line)).length,
+    bulletCount: nonEmptyLines.filter((line) => /^[-*]\s/.test(line)).length,
+    numberedCount: nonEmptyLines.filter((line) => /^\d+\.\s/.test(line)).length,
+    codeFenceCount: (raw.match(/```/g) || []).length / 2,
+    sectionKeys,
+    sectionHints: SECTION_HINTS.filter((hint) => lower.includes(hint)),
+    hasMarkdownTable: /\|.+\|/.test(raw),
+    hasEmoji: /\p{Extended_Pictographic}/u.test(raw)
+  };
+}
+
+function buildOutputDiff(v2Run, liveRun, contract = null) {
+  const v2Text = extractRunText(v2Run, contract);
+  const liveText = extractLiveText(liveRun);
+  const v2 = analyzeOutputStructure(v2Text);
+  const live = analyzeOutputStructure(liveText);
+  const v2Sections = comparableSections(v2);
+  const liveSections = comparableSections(live);
+  const sharedSections = v2Sections.filter((hint) => liveSections.includes(hint));
+  const missingFromV2 = liveSections.filter((hint) => !v2Sections.includes(hint));
+  const extraInV2 = v2Sections.filter((hint) => !liveSections.includes(hint));
+  const findings = [];
+
+  if (!liveText) {
+    findings.push("No live output is available for structural comparison.");
+  } else {
+    if (missingFromV2.length) findings.push(`V2 is missing live sections: ${missingFromV2.join(", ")}.`);
+    if (v2.bulletCount + v2.numberedCount < live.bulletCount + live.numberedCount) {
+      findings.push("V2 output is less structured than the live report.");
+    }
+    if (v2.charCount > 0 && live.charCount > 0) {
+      const verbosityRatio = Number((v2.charCount / live.charCount).toFixed(4));
+      if (verbosityRatio < 0.55) findings.push("V2 output is much shorter than the live report.");
+      if (verbosityRatio > 1.8) findings.push("V2 output is much longer than the live report.");
+    }
+  }
+
+  return {
+    v2,
+    live,
+    alignment: {
+      sharedSections,
+      missingFromV2,
+      extraInV2,
+      bulletDelta: (v2.bulletCount + v2.numberedCount) - (live.bulletCount + live.numberedCount),
+      headingDelta: v2.headingCount - live.headingCount,
+      charDelta: v2.charCount - live.charCount
+    },
+    findings
+  };
+}
+
+function assessOutputContract(contract, v2Run, liveRun) {
+  if (!contract) return null;
+
+  const v2Validation = validateOutputContract(contract, v2Run?.result?.output ?? v2Run?.output ?? "");
+  const liveValidation = validateOutputContract(contract, liveRun?.summary || liveRun?.error || "");
+  const missingFromV2 = v2Validation?.missingSections || [];
+  const missingFromLive = liveValidation?.missingSections || [];
+  const findings = [
+    ...(v2Validation?.findings.map((item) => `V2: ${item}`) || []),
+    ...(liveValidation?.findings.map((item) => `Live: ${item}`) || [])
+  ];
+
+  return {
+    format: contract.format || null,
+    requiredSections: contract.requiredSections || [],
+    styleHints: contract.styleHints || [],
+    satisfiedRatio: v2Validation?.satisfiedRatio ?? 0,
+    missingFromV2,
+    missingFromLive,
+    emptyInV2: v2Validation?.emptySections || [],
+    emptyInLive: liveValidation?.emptySections || [],
+    parsedV2: v2Validation?.parsed || null,
+    parsedLive: liveValidation?.parsed || null,
+    fieldScores: v2Validation?.fieldScores || null,
+    findings
+  };
+}
+
+export { buildRubric };
+
+export class Evaluator {
+  constructor({ projectRoot, liveRoot, stateRoot }) {
+    this.liveRoot = liveRoot;
+    this.stateRoot = stateRoot;
+    this.reviewer = new RunReviewer({ stateRoot });
+    this.scheduler = new Scheduler({ projectRoot, liveRoot, stateRoot });
+    this.evalStore = new RunStore({ rootDir: path.join(stateRoot, "evaluations") });
+    this.notificationStore = new NotificationStore({ rootDir: path.join(stateRoot, "notifications") });
+    this.deliveryStore = new DeliveryStore({ rootDir: path.join(stateRoot, "deliveries") });
+  }
+
+  async resolveInteractionArtifacts(run, review = null) {
+    const reviewNotifications = review?.recentNotifications || [];
+    const _reviewDeliveries = review?.recentDeliveries || [];
+    const baseFiles = run?.notificationFiles || [];
+    const relatedNotificationFiles = expandRelatedNotificationFiles(
+      baseFiles,
+      reviewNotifications
+    );
+    const knownNotifications = reviewNotifications.filter((item) => relatedNotificationFiles.has(item.filePath));
+    const missingNotificationFiles = [...relatedNotificationFiles].filter(
+      (filePath) => !knownNotifications.some((item) => item.filePath === filePath)
+    );
+    const loadedNotifications = await this.notificationStore.getNotifications(missingNotificationFiles);
+    const notifications = [...knownNotifications, ...loadedNotifications].sort((a, b) =>
+      String(a.timestamp || "").localeCompare(String(b.timestamp || ""))
+    );
+    const expandedNotificationFiles = expandRelatedNotificationFiles(baseFiles, notifications);
+
+    // Load all delivery receipts once for deterministic interaction evidence.
+    const allDeliveries = await this.deliveryStore.listAll();
+    const deliveries = allDeliveries.filter((item) => expandedNotificationFiles.has(item.notificationFilePath));
+    return {
+      notificationFiles: expandedNotificationFiles,
+      notifications,
+      deliveries
+    };
+  }
+
+  async evaluate(limit = 20) {
+    const [review, comparisons] = await Promise.all([
+      this.reviewer.review(limit),
+      this.scheduler.compareJobs()
+    ]);
+
+    const recentRuns = review.recentRuns.filter((run) => !run.jobId.endsWith("-comparison"));
+    const grouped = groupBy(recentRuns, (run) => run.jobId);
+
+    const jobs = await Promise.all(
+      Array.from(grouped.entries()).map(async ([jobId, runs]) => this.evaluateJob(jobId, {
+        runs,
+        review,
+        comparisons
+      }))
+    );
+
+    return {
+      jobs: [...jobs].sort((a, b) => a.rubric.overallScore - b.rubric.overallScore),
+      recentRuns,
+      scheduler: review.scheduler
+    };
+  }
+
+  async evaluateJob(jobId, options = {}) {
+    const runtime = await this.scheduler.loadRuntime();
+    const jobConfig = runtime.jobs[jobId] || null;
+    const review = options.review || (await this.reviewer.review(options.limit ?? 20));
+    const comparisons = options.comparisons || (await this.scheduler.compareJobs());
+    const runs = options.runs || review.recentRuns.filter((run) => run.jobId === jobId && !run.jobId.endsWith("-comparison"));
+    const schedulerState = review.scheduler.find((item) => item.jobId === jobId) || null;
+    const comparison = comparisons.find((item) => item.v2JobId === jobId) || null;
+    const liveMatches = await Promise.all(
+      (comparison?.closestLiveJobs || []).map(async (match) => ({
+        ...match,
+        recentRuns: await this.scheduler.bridge.loadCronRunHistory(match.id, 5),
+        latestRun: (await this.scheduler.bridge.loadCronRunHistory(match.id, 1))[0] || null
+      }))
+    );
+
+    const selectedRun = choosePrimaryV2Run(runs);
+    const interactionRun = runs[0] || selectedRun;
+    const selectedLiveMatch = pickBestLiveMatch(liveMatches);
+    const liveHistory = selectedLiveMatch?.recentRuns || [];
+    const outputContract = normalizeOutputContract(jobConfig?.outputContract || null);
+    const outputDiff = buildOutputDiff(selectedRun, selectedLiveMatch?.latestRun || null, outputContract);
+    const contractCheck = assessOutputContract(outputContract, selectedRun, selectedLiveMatch?.latestRun || null);
+    const retrieval = analyzeRetrieval(selectedRun);
+    const interactionArtifacts = await this.resolveInteractionArtifacts(interactionRun, review);
+    const interaction = analyzeInteraction(interactionRun, interactionArtifacts.notifications, interactionArtifacts.deliveries);
+    const rubric = buildRubric({
+      v2Run: selectedRun,
+      liveHistory,
+      matchedLiveJob: selectedLiveMatch,
+      outputContract,
+      contractCheck
+    });
+
+    return {
+      jobId,
+      schedulerState,
+      maintenance: review?.maintenance || null,
+      latestRun: runs[0] || null,
+      primaryRun: summarizeV2Run(selectedRun),
+      interactionRunTimestamp: interactionRun?.timestamp || null,
+      runCount: runs.length,
+      modeCounts: countModes(runs),
+      liveMatches: liveMatches.map((match) => ({
+        id: match.id,
+        name: match.name,
+        description: match.description || null,
+        enabled: match.enabled,
+        schedule: match.schedule,
+        kind: match.kind || null,
+        lastStatus: match.lastStatus || null,
+        latestRun: summarizeLiveRun(match.latestRun)
+      })),
+      selectedLiveMatch: selectedLiveMatch
+        ? {
+            id: selectedLiveMatch.id,
+            name: selectedLiveMatch.name,
+            latestRun: summarizeLiveRun(selectedLiveMatch.latestRun)
+          }
+        : null,
+      retrieval,
+      interaction,
+      outputContract,
+      contractCheck,
+      outputDiff,
+      rubric,
+      comparisonNotes: buildComparisonNotes({
+        jobId,
+        selectedRun,
+        selectedLiveMatch,
+        schedulerState,
+        maintenance: review?.maintenance || null,
+        rubric,
+        outputDiff,
+        contractCheck,
+        retrieval,
+        interaction
+      }),
+      interactionArtifacts: {
+        notificationCount: interactionArtifacts.notifications.length,
+        deliveryCount: interactionArtifacts.deliveries.length,
+        notificationFileCount: interactionArtifacts.notificationFiles.size
+      }
+    };
+  }
+
+  async evaluateAndPersistJob(jobId, options = {}) {
+    const report = await this.evaluateJob(jobId, options);
+    const artifact = {
+      timestamp: new Date().toISOString(),
+      kind: "job-evaluation",
+      ...report
+    };
+    const filePath = await this.evalStore.saveRun(jobId, artifact);
+    return {
+      filePath,
+      ...artifact
+    };
+  }
+}
+
+function countModes(runs) {
+  return runs.reduce((acc, run) => {
+    const mode = run.mode || "unknown";
+    acc[mode] = (acc[mode] || 0) + 1;
+    return acc;
+  }, {});
+}
+
+function buildComparisonNotes({
+  jobId,
+  selectedRun,
+  selectedLiveMatch,
+  schedulerState,
+  maintenance,
+  rubric,
+  outputDiff,
+  contractCheck,
+  retrieval = null,
+  interaction = null
+}) {
+  const notes = [];
+  if (!selectedRun) notes.push("No V2 runs recorded.");
+  if (schedulerState?.lastStatus === "ok") notes.push("Latest V2 scheduler state is healthy.");
+  if (selectedRun?.fallback?.attempted && selectedRun?.fallback?.success) {
+    notes.push(
+      `Run used report fallback from ${selectedRun.fallback.sourceLane || "local"} to ${selectedRun.fallback.finalSourceLane || "remote"} after ${selectedRun.fallback.trigger || "failure"}.`
+    );
+  } else if (selectedRun?.fallback?.attempted && !selectedRun?.fallback?.success) {
+    notes.push(`Report fallback was attempted but did not succeed${selectedRun.fallback?.fallbackError ? `: ${selectedRun.fallback.fallbackError}` : "."}`);
+  } else if (selectedRun?.fallback?.allowed === false && selectedRun?.fallback?.trigger) {
+    notes.push(`Report fallback was blocked after ${selectedRun.fallback.trigger}${selectedRun.fallback?.blockedReason ? `: ${selectedRun.fallback.blockedReason}` : "."}`);
+  }
+  if (maintenance?.wal?.some((item) => item.action && item.action !== "none" && item.action !== "error")) {
+    notes.push("Daemon maintenance recently performed WAL checkpoint work.");
+  }
+  if (maintenance?.handoffs?.expiredCount > 0) {
+    notes.push(`Daemon maintenance expired or escalated ${maintenance.handoffs.expiredCount} pending handoff(s).`);
+  }
+  if (!selectedLiveMatch) {
+    notes.push("No live cron analogue matched.");
+    return uniqueStrings([...notes, ...rubric.findings, ...outputDiff.findings, ...(contractCheck?.findings || []), ...(interaction?.findings || [])]);
+  }
+
+  const latestLive = selectedLiveMatch.latestRun;
+  notes.push(`Closest live cron match: ${selectedLiveMatch.name || selectedLiveMatch.id}.`);
+  if (latestLive?.status) notes.push(`Latest live status: ${latestLive.status}.`);
+  if (latestLive?.provider) notes.push(`Latest live provider: ${latestLive.provider}.`);
+  if (latestLive?.durationMs != null) notes.push(`Latest live duration: ${latestLive.durationMs}ms.`);
+  const retrievalFindings = retrieval?.findings || [];
+  const interactionFindings = interaction?.findings || [];
+  const outputDiffFindings =
+    jobId === "memory-rollup" && Number(contractCheck?.satisfiedRatio || 0) >= 0.99
+      ? (outputDiff.findings || []).filter((item) => !/less structured|much shorter/i.test(item))
+      : outputDiff.findings || [];
+  return uniqueStrings([...notes, ...rubric.findings, ...outputDiffFindings, ...(contractCheck?.findings || []), ...retrievalFindings, ...interactionFindings]);
+}
+
+function normalizeOutputContract(contract) {
+  if (!contract) return null;
+  return {
+    format: contract.format || null,
+    requiredSections: contract.requiredSections || [],
+    styleHints: contract.styleHints || [],
+    profile: contract.profile || null
+  };
+}