nemoris 0.1.0

package/src/runtime/evaluation.js

@@ -0,0 +1,1089 @@
+import path from "node:path";
+import { RunReviewer } from "./review.js";
+import { Scheduler } from "./scheduler.js";
+import { RunStore } from "./run-store.js";
+import { NotificationStore } from "./notification-store.js";
+import { DeliveryStore } from "./delivery-store.js";
+import { validateOutputContract } from "./output-contract-validator.js";
+
+function groupBy(items, keyFn) {
+  const map = new Map();
+  for (const item of items) {
+    const key = keyFn(item);
+    if (!map.has(key)) map.set(key, []);
+    map.get(key).push(item);
+  }
+  return map;
+}
+
+function clamp(value, min = 0, max = 1) {
+  return Math.max(min, Math.min(max, value));
+}
+
+function average(values) {
+  if (values.length === 0) return null;
+  return values.reduce((sum, value) => sum + value, 0) / values.length;
+}
+
+function choosePrimaryV2Run(runs) {
+  if (!runs.length) return null;
+  return runs.find((run) => run.mode === "provider") || runs[0];
+}
+
+function pickBestLiveMatch(liveMatches) {
+  return liveMatches.find((match) => match.latestRun) || liveMatches[0] || null;
+}
+
+function normalizeText(value) {
+  return String(value || "")
+    .replace(/\s+/g, " ")
+    .trim();
+}
+
+function stripOuterFence(text) {
+  const raw = String(text || "").trim();
+  const fencedMatch = raw.match(/^```(?:json|markdown|md|text)?\s*([\s\S]*?)\s*```$/i);
+  return fencedMatch ? fencedMatch[1].trim() : raw;
+}
+
+function parseJsonCandidate(text) {
+  const candidate = stripOuterFence(text);
+  if (!(candidate.startsWith("{") && candidate.endsWith("}"))) {
+    return null;
+  }
+
+  try {
+    return JSON.parse(candidate);
+  } catch {
+    return null;
+  }
+}
+
+function titleCase(value) {
+  return String(value || "")
+    .replace(/[_-]+/g, " ")
+    .replace(/^#+\s*/, "")
+    .replace(/\b\w/g, (char) => char.toUpperCase())
+    .trim();
+}
+
+function normalizeInline(value) {
+  return normalizeText(value) || "None";
+}
+
+function normalizeSectionKey(value) {
+  return String(value || "")
+    .toLowerCase()
+    .replace(/^#+\s*/, "")
+    .replace(/[*:_-]+/g, " ")
+    .replace(/\s+/g, " ")
+    .trim();
+}
+
+function renderValue(value, { contract = null, summary = null } = {}) {
+  if (value == null) return "";
+
+  if (typeof value === "string") {
+    const parsed = parseJsonCandidate(value);
+    if (parsed && typeof parsed === "object") {
+      if (parsed.output !== undefined) {
+        return renderValue(parsed.output, {
+          contract,
+          summary: parsed.summary || summary
+        });
+      }
+
+      return renderValue(parsed, { contract, summary });
+    }
+
+    return stripOuterFence(value);
+  }
+
+  if (Array.isArray(value)) {
+    return value
+      .map((item) => renderValue(item, { contract, summary }))
+      .filter(Boolean)
+      .join("\n");
+  }
+
+  if (typeof value === "object") {
+    const entries = Object.entries(value);
+    const sectionOrder = contract?.requiredSections?.length
+      ? contract.requiredSections
+      : entries.map(([key]) => key);
+    const valueMap = new Map(entries.map(([key, entryValue]) => [normalizeSectionKey(key), entryValue]));
+    const sectionStyle = contract?.profile?.sectionStyle || (contract?.format === "structured_rollup" ? "headings" : "bullets");
+    const requireStatus = contract?.profile?.requireStatus ?? (contract?.format === "bulleted_briefing");
+
+    if (sectionStyle === "bullets") {
+      const lines = [];
+      if (requireStatus && summary) lines.push(`Status: ${normalizeInline(summary)}`);
+      for (const section of sectionOrder) {
+        lines.push(`- ${titleCase(section)}: ${normalizeInline(valueMap.get(normalizeSectionKey(section)))}`);
+      }
+      return lines.join("\n");
+    }
+
+    if (sectionStyle === "headings") {
+      const lines = [];
+      for (const section of sectionOrder) {
+        lines.push(`## ${titleCase(section)}`);
+        lines.push(`- ${normalizeInline(valueMap.get(normalizeSectionKey(section)))}`);
+        lines.push("");
+      }
+      return lines.join("\n").trim();
+    }
+
+    return entries
+      .map(([key, entryValue]) => `${titleCase(key)}: ${normalizeInline(entryValue)}`)
+      .join("\n");
+  }
+
+  return String(value);
+}
+
+function extractRunText(run, contract = null) {
+  if (!run) return "";
+  const summary = run?.result?.summary || run?.summary || "";
+  const value = run?.result?.output ?? run?.output ?? summary;
+  return renderValue(value, { contract, summary });
+}
+
+function extractLiveText(run) {
+  if (!run) return "";
+  return renderValue(run.summary || run.error || "");
+}
+
+function qualitySignals(text) {
+  const raw = String(text || "");
+  const normalized = normalizeText(text);
+  const lower = normalized.toLowerCase();
+  const findings = [];
+  let score = 1;
+
+  if (!normalized) {
+    findings.push("Output is empty.");
+    return { score: 0, findings };
+  }
+
+  if (normalized.length < 40) {
+    findings.push("Output is very short and may not carry enough signal.");
+    score -= 0.18;
+  }
+
+  if (/\[(title|link|summary|sub)\]/i.test(normalized)) {
+    findings.push("Output still contains placeholder text.");
+    score -= 0.4;
+  }
+
+  if (/i('| a)?m sorry|does not support|cannot complete|failed to/i.test(lower)) {
+    findings.push("Output contains apology or tool-failure language.");
+    score -= 0.28;
+  }
+
+  if (/```/.test(raw)) {
+    findings.push("Output is wrapped in a code block instead of a direct report.");
+    score -= 0.12;
+  }
+
+  if (/timed out|timeout/i.test(lower)) {
+    findings.push("Output references a timeout or stalled execution.");
+    score -= 0.35;
+  }
+
+  const repeatedLinePenalty = detectRepeatedLinePenalty(normalized);
+  if (repeatedLinePenalty > 0) {
+    findings.push("Output repeats nearly identical numbered lines.");
+    score -= repeatedLinePenalty;
+  }
+
+  return {
+    score: clamp(Number(score.toFixed(4))),
+    findings
+  };
+}
+
+function detectRepeatedLinePenalty(text) {
+  const lines = text
+    .split("\n")
+    .map((line) => line.trim())
+    .filter(Boolean)
+    .filter((line) => /^\d+\./.test(line));
+
+  if (lines.length < 3) return 0;
+
+  const normalized = lines.map((line) => line.replace(/^\d+\.\s*/, "").toLowerCase());
+  const unique = new Set(normalized);
+  const duplicateRatio = 1 - unique.size / normalized.length;
+  if (duplicateRatio < 0.5) return 0;
+  return Number((duplicateRatio * 0.25).toFixed(4));
+}
+
+function expandRelatedNotificationFiles(baseFiles, notifications) {
+  const related = new Set(baseFiles);
+  let changed = true;
+  while (changed) {
+    changed = false;
+    for (const item of notifications) {
+      if (!related.has(item.filePath)) continue;
+      for (const generated of item.generatedNotificationFiles || []) {
+        if (related.has(generated)) continue;
+        related.add(generated);
+        changed = true;
+      }
+    }
+  }
+  return related;
+}
+
+function evaluateLiveHistory(liveRuns) {
+  if (liveRuns.length === 0) {
+    return {
+      runCount: 0,
+      okCount: 0,
+      errorCount: 0,
+      skippedCount: 0,
+      failureRate: null,
+      avgDurationMs: null,
+      avgTotalTokens: null,
+      avgQualityScore: null,
+      findings: ["No recent live history available."]
+    };
+  }
+
+  const okCount = liveRuns.filter((run) => run.status === "ok").length;
+  const errorCount = liveRuns.filter((run) => run.status === "error").length;
+  const skippedCount = liveRuns.filter((run) => run.status === "skipped").length;
+  const failureRate = errorCount / liveRuns.length;
+  const durations = liveRuns.map((run) => run.durationMs).filter((value) => Number.isFinite(value));
+  const tokens = liveRuns.map((run) => run.usage?.total_tokens).filter((value) => Number.isFinite(value));
+  const qualityScores = liveRuns
+    .map((run) => qualitySignals(run.summary || run.error || "").score)
+    .filter((value) => Number.isFinite(value));
+
+  const findings = [];
+  if (failureRate >= 0.34) findings.push("Live cron lane is failing often.");
+  if (skippedCount > 0) findings.push("Live cron history includes intentional skips.");
+  if (tokens.some((value) => value > 12000)) findings.push("Live cron lane shows high token usage spikes.");
+
+  return {
+    runCount: liveRuns.length,
+    okCount,
+    errorCount,
+    skippedCount,
+    failureRate: Number(failureRate.toFixed(4)),
+    avgDurationMs: durations.length ? Math.round(average(durations)) : null,
+    avgTotalTokens: tokens.length ? Math.round(average(tokens)) : null,
+    avgQualityScore: qualityScores.length ? Number(average(qualityScores).toFixed(4)) : null,
+    findings
+  };
+}
+
+function scoreStatus(status) {
+  if (status === "ok") return 1;
+  if (status === "skipped") return 0.6;
+  if (status === "error") return 0;
+  return 0.5;
+}
+
+function buildRubric({ v2Run, liveHistory, matchedLiveJob, outputContract = null, contractCheck = null }) {
+  const findings = [];
+  const v2Text = extractRunText(v2Run, outputContract);
+  const v2Quality = qualitySignals(v2Text);
+  const latestLive = liveHistory[0] || null;
+  const latestLiveQuality = qualitySignals(extractLiveText(latestLive));
+  const history = evaluateLiveHistory(liveHistory);
+  const isDryRun = v2Run != null && v2Run.mode !== "provider";
+
+  const comparisonReadiness = v2Run
+    ? v2Run.mode === "provider"
+      ? 1
+      : 0.45
+    : 0;
+  const comparisonCoverage = matchedLiveJob ? 1 : 0.2;
+  const contractAdherence = contractCheck ? contractCheck.satisfiedRatio : null;
+  if (!v2Run) findings.push("No V2 run exists for this lane yet.");
+  else if (isDryRun) findings.push("Only a dry-run V2 artifact exists; output quality and contract adherence are excluded from the rubric score.");
+  if (!matchedLiveJob) findings.push("Comparison coverage is low because no live cron analogue is mapped.");
+
+  if (!isDryRun && v2Quality.findings.length) findings.push(...v2Quality.findings.map((item) => `V2: ${item}`));
+  if (latestLiveQuality.findings.length) findings.push(...latestLiveQuality.findings.map((item) => `Live: ${item}`));
+  if (history.findings.length) findings.push(...history.findings);
+
+  const fieldScores = contractCheck?.fieldScores || null;
+  if (!isDryRun && fieldScores) {
+    if (fieldScores.weakFields.length) {
+      findings.push(`Field-level quality is low for: ${fieldScores.weakFields.join(", ")}`);
+    }
+    if (fieldScores.placeholderFields.length) {
+      findings.push(`Placeholder text detected in: ${fieldScores.placeholderFields.join(", ")}`);
+    }
+  }
+
+  if (!matchedLiveJob) {
+    findings.push("No live cron analogue matched for this V2 job.");
+  }
+
+  const statusAlignment =
+    latestLive && v2Run
+      ? Number((scoreStatus(v2Run.result ? "ok" : v2Run.status) * scoreStatus(latestLive.status)).toFixed(4))
+      : null;
+  const liveReliability = history.failureRate == null ? null : Number((1 - history.failureRate).toFixed(4));
+
+  const fieldLevelQuality = !isDryRun && fieldScores ? fieldScores.averageScore : null;
+
+  const components = [
+    isDryRun ? null : v2Quality.score,
+    comparisonReadiness,
+    comparisonCoverage,
+    isDryRun ? null : contractAdherence,
+    fieldLevelQuality,
+    liveReliability,
+    statusAlignment
+  ].filter((value) => value != null);
+  const overallScore = components.length ? Number(average(components).toFixed(4)) : 0;
+
+  return {
+    overallScore,
+    dryRunExcluded: isDryRun,
+    components: {
+      v2OutputQuality: isDryRun ? null : v2Quality.score,
+      comparisonReadiness,
+      comparisonCoverage,
+      contractAdherence: isDryRun ? null : contractAdherence,
+      fieldLevelQuality,
+      liveReliability,
+      statusAlignment
+    },
+    findings: uniqueStrings(findings),
+    metrics: {
+      liveRunCount: history.runCount,
+      liveOkCount: history.okCount,
+      liveErrorCount: history.errorCount,
+      liveSkippedCount: history.skippedCount,
+      liveFailureRate: history.failureRate,
+      liveAvgDurationMs: history.avgDurationMs,
+      liveAvgTotalTokens: history.avgTotalTokens,
+      liveAvgQualityScore: history.avgQualityScore,
+      latestLiveQualityScore: latestLive ? latestLiveQuality.score : null
+    }
+  };
+}
+
+function uniqueStrings(items) {
+  return [...new Set(items.filter(Boolean))];
+}
+
+function summarizeV2Run(run) {
+  if (!run) return null;
+  return {
+    timestamp: run.timestamp,
+    mode: run.mode || null,
+    providerId: run.providerId || null,
+    modelId: run.modelId || null,
+    summary: run.result?.summary || run.summary || null,
+    output: run.result?.output || run.output || null,
+    retrievalMeta: run.retrievalMeta || null,
+    retrievedMemoryCount: run.retrievedMemory?.length || 0,
+    interaction: summarizeInteraction(run),
+    fallback: run.fallback || null
+  };
+}
+
+function summarizeInteraction(run) {
+  const interaction = run?.interaction || null;
+  if (!interaction) return null;
+  return {
+    ackRequired: interaction.ack?.required ?? false,
+    completionRequired: interaction.completion?.required ?? false,
+    handoffRequired: interaction.handoff?.required ?? false,
+    yielded: run?.yielded || false,
+    yieldSignal: run?.yieldSignal || null,
+    followUpState: run?.followUpState || null,
+    followUpQueued: run?.followUpQueued || false,
+    followUpConsumed: run?.followUpConsumed || false,
+    followUpTarget: run?.followUpTarget || null,
+    followUpCompleted: run?.followUpCompleted || false,
+    followUpExpired: run?.followUpExpired || false,
+    followUpEscalated: run?.followUpEscalated || false,
+    followUpEscalationFilePath: run?.followUpEscalationFilePath || null,
+    handoffTarget: interaction.handoff?.target || null,
+    suggestedPeerCount: interaction.handoff?.suggestions?.length || 0,
+    handoffChosenPeerId: run?.handoffChosenPeerId || null,
+    handoffChosenBy: run?.handoffChosenBy || null,
+    handoffDelivered: run?.handoffDelivered || false,
+    handoffDeliveryState: run?.handoffDeliveryState || null,
+    deliveryDeduped: run?.deliveryDeduped || false,
+    deliveryRetried: run?.deliveryRetried || false,
+    deliveryUncertain: run?.deliveryUncertain || false
+  };
+}
+
+function analyzeRetrieval(run) {
+  const retrievalMeta = run?.retrievalMeta || run?.plan?.packet?.layers?.retrievalMeta || null;
+  if (!run?.retrievedMemory?.length) {
+    return {
+      memoryCount: 0,
+      lexicalCount: 0,
+      semanticCount: 0,
+      qmdCount: 0,
+      freshEmbeddingCount: 0,
+      staleEmbeddingCount: 0,
+      missingEmbeddingCount: 0,
+      failedEmbeddingCount: retrievalMeta?.embeddingHealth?.failedCount || 0,
+      embeddingQueryMode: retrievalMeta?.embeddingQueryMode || "lexical_only",
+      embeddingError: retrievalMeta?.embeddingError || retrievalMeta?.embeddingHealth?.lastError || null,
+      findings: ["No retrieved memory was captured for this run."],
+      items: []
+    };
+  }
+
+  const items = run.retrievedMemory.map((item) => ({
+    entryId: item.entryId || null,
+    title: item.title || null,
+    sourceBackend: item.sourceBackend || "file",
+    candidateSource: item.candidateSource || "indexed",
+    lexicalScore: item.lexicalScore ?? 0,
+    embeddingSimilarity: item.embeddingSimilarity ?? 0,
+    embeddingFreshness: item.embeddingFreshness || "missing",
+    retrievalSources: item.retrievalSources || []
+  }));
+
+  const lexicalCount = items.filter((item) => item.retrievalSources.includes("lexical")).length;
+  const semanticCount = items.filter((item) => item.retrievalSources.includes("semantic")).length;
+  const qmdCount = items.filter((item) => item.retrievalSources.includes("qmd") || item.sourceBackend === "qmd").length;
+  const freshEmbeddingCount = items.filter((item) => item.embeddingFreshness === "fresh").length;
+  const staleEmbeddingCount = items.filter((item) => item.embeddingFreshness === "stale").length;
+  const missingEmbeddingCount = items.filter((item) => item.embeddingFreshness === "missing").length;
+  const failedEmbeddingCount = items.filter((item) => item.embeddingFreshness === "failed").length;
+  const findings = [];
+
+  if (semanticCount === 0) findings.push("No semantic retrieval candidates reached the final packet.");
+  if (staleEmbeddingCount > 0) findings.push("Some retrieved items have stale embeddings.");
+  if (freshEmbeddingCount === 0 && missingEmbeddingCount > 0) findings.push("Retrieved file memory is falling back to non-embedded retrieval.");
+  if ((retrievalMeta?.embeddingError || retrievalMeta?.embeddingHealth?.lastError) && !findings.includes("Embedding query failed and retrieval fell back to lexical mode.")) {
+    findings.push("Embedding query failed and retrieval fell back to lexical mode.");
+  }
+
+  return {
+    memoryCount: items.length,
+    lexicalCount,
+    semanticCount,
+    qmdCount,
+    freshEmbeddingCount,
+    staleEmbeddingCount,
+    missingEmbeddingCount,
+    failedEmbeddingCount,
+    embeddingQueryMode: retrievalMeta?.embeddingQueryMode || "lexical_only",
+    embeddingError: retrievalMeta?.embeddingError || retrievalMeta?.embeddingHealth?.lastError || null,
+    embeddingHealth: retrievalMeta?.embeddingHealth || null,
+    findings,
+    items
+  };
+}
+
+function summarizeInteractionDiagnosis({
+  ackRequired,
+  ackQueued,
+  completionRequired,
+  completionQueued,
+  handoffRequired,
+  handoffQueued,
+  yielded,
+  followUpQueued,
+  followUpConsumed,
+  handoffDelivered,
+  deliveryEvidenceRequired,
+  deliveryEvidenceHealthy,
+  deliveryUncertain,
+  findings
+}) {
+  const missingStages = [];
+  if (ackRequired && !ackQueued) missingStages.push("ack");
+  if (completionRequired && !completionQueued && !yielded) missingStages.push("completion");
+  if (handoffRequired && !handoffQueued && !yielded) missingStages.push("handoff");
+
+  if (!findings?.length) {
+    return {
+      status: "healthy",
+      code: null,
+      summary: "Interaction lifecycle evidence is complete.",
+      missingStages,
+      findings: []
+    };
+  }
+
+  if (missingStages.length) {
+    return {
+      status: "action_required",
+      code: "missing_required_notifications",
+      summary: `Required interaction notifications are missing: ${missingStages.join(", ")}.`,
+      missingStages,
+      findings
+    };
+  }
+
+  if (yielded && followUpQueued && !followUpConsumed) {
+    return {
+      status: "action_required",
+      code: "follow_up_incomplete",
+      summary: "Yielded follow-up is still pending or incomplete.",
+      missingStages,
+      findings
+    };
+  }
+
+  if (deliveryEvidenceRequired && !deliveryEvidenceHealthy) {
+    return {
+      status: "action_required",
+      code: "missing_delivery_evidence",
+      summary: "Required pingback notifications do not have delivery receipts yet.",
+      missingStages,
+      findings
+    };
+  }
+
+  if (handoffRequired && handoffQueued && !handoffDelivered) {
+    return {
+      status: "action_required",
+      code: "handoff_incomplete",
+      summary: "Handoff was queued but not fully delivered.",
+      missingStages,
+      findings
+    };
+  }
+
+  if (deliveryUncertain) {
+    return {
+      status: "warning",
+      code: "delivery_uncertain",
+      summary: "Interaction lifecycle completed with uncertain delivery evidence.",
+      missingStages,
+      findings
+    };
+  }
+
+  return {
+    status: "warning",
+    code: "interaction_findings_present",
+    summary: findings[0] || "Interaction lifecycle has unresolved findings.",
+    missingStages,
+    findings
+  };
+}
+
+function analyzeInteraction(run, notifications = [], deliveries = []) {
+  const interaction = run?.interaction || null;
+  const findings = [];
+
+  const ackRequired = interaction?.ack?.required ?? false;
+  const completionRequired = interaction?.completion?.required ?? false;
+  const handoffRequired = interaction?.handoff?.required ?? false;
+  const ackQueued = notifications.some((item) => item.stage === "ack");
+  const completionQueued = notifications.some((item) => item.stage === "completion");
+  const handoffQueued = notifications.some((item) => item.stage === "handoff");
+  const followUpQueued = notifications.some((item) => item.stage === "follow_up");
+  const followUpNotification = notifications.find((item) => item.stage === "follow_up") || null;
+  const handoffNotification = notifications.find((item) => item.stage === "handoff") || null;
+  const handoffPendingChoice = handoffNotification?.status === "awaiting_choice";
+  const handoffState = handoffNotification?.handoffState || (handoffPendingChoice ? "pending" : null);
+  const handoffChosen = Boolean(handoffNotification?.chosenPeer?.peerId);
+  const handoffChosenPeerId = handoffNotification?.chosenPeer?.peerId || null;
+  const handoffChosenBy = handoffNotification?.chosenBy || null;
+  const handoffDelivery = handoffNotification
+    ? deliveries.find((item) => item.notificationFilePath === handoffNotification.filePath) || null
+    : null;
+  const handoffDelivered = Boolean(handoffDelivery);
+  const handoffDeliveryState = handoffDelivery?.delivery?.status || null;
+  const deliveryStates = deliveries.map((item) => item.delivery?.status || item.stage || "unknown");
+  const yielded = interaction?.yield?.required ?? false;
+  const yieldSignal = interaction?.yield?.signal || null;
+  const followUpState = followUpNotification?.followUpState || followUpNotification?.yieldState || null;
+  const followUpConsumed = followUpNotification?.status === "consumed" || followUpNotification?.yieldState === "consumed" || followUpNotification?.followUpState === "consumed";
+  const followUpTarget = followUpNotification?.targetSurface || interaction?.yield?.targetSurface || null;
+  const generatedFiles = new Set(followUpNotification?.generatedNotificationFiles || []);
+  const followUpCompleted = deliveries.some((item) => generatedFiles.has(item.notificationFilePath));
+  const followUpExpired = followUpState === "expired";
+  const followUpEscalated = followUpState === "escalated";
+  const deliveryDeduped = deliveryStates.includes("duplicate_prevented");
+  const deliveryRetried = deliveries.some((item) => Number(item.attempt) > 1);
+  const deliveryUncertain = deliveryStates.includes("delivery_uncertain") || deliveryStates.includes("uncertain");
+  const yieldedCompletionHandled = yielded && followUpQueued && Boolean(followUpNotification?.payload?.completion);
+  const yieldedHandoffHandled = yielded && followUpQueued && Boolean(followUpNotification?.payload?.handoff);
+  const visibleDeliveryCount = yielded ? deliveries.filter((item) => !generatedFiles.has(item.notificationFilePath)).length : deliveries.length;
+  const deliveryEvidenceRequired = !yielded && Boolean(ackQueued || completionQueued || handoffQueued);
+  const deliveryEvidenceHealthy = !deliveryEvidenceRequired || visibleDeliveryCount > 0;
+
+  if (ackRequired && !ackQueued) findings.push("Required ack notification was not queued.");
+  if (completionRequired && !completionQueued && !yieldedCompletionHandled) {
+    findings.push("Required completion notification was not queued.");
+  }
+  if (handoffRequired && !handoffQueued && !yieldedHandoffHandled) {
+    findings.push("Configured handoff was not queued.");
+  }
+  if (yielded && !followUpQueued) findings.push("Run yielded but no follow-up payload was persisted.");
+  if (followUpQueued && !followUpConsumed) findings.push("Follow-up payload was persisted but never consumed.");
+  if (followUpExpired) findings.push("Follow-up expired before it was consumed.");
+  if (followUpEscalated) findings.push("Follow-up expired and was escalated to the operator.");
+  if (handoffPendingChoice) findings.push("Handoff is awaiting an explicit peer choice.");
+  if (handoffState === "expired") findings.push("Handoff expired without an operator choice.");
+  if (handoffState === "escalated") findings.push("Handoff expired and was escalated to the operator.");
+  if (handoffState === "blocked") findings.push(`Handoff is blocked${handoffNotification?.blockedReason ? `: ${handoffNotification.blockedReason}` : "."}`);
+  if (handoffQueued && !handoffPendingChoice && !handoffChosen) findings.push("Handoff was queued but no peer choice was recorded.");
+  if (handoffChosen && !handoffDelivered) findings.push("Handoff was promoted to a peer but not delivered yet.");
+  if (handoffDeliveryState && /error|blocked/i.test(handoffDeliveryState)) findings.push(`Handoff delivery did not complete cleanly: ${handoffDeliveryState}.`);
+  if (!deliveryEvidenceHealthy) {
+    findings.push("No delivery receipts exist yet for queued interaction notifications.");
+  }
+  if (deliveryDeduped) findings.push("A duplicate delivery attempt was prevented by dedupe policy.");
+  if (deliveryUncertain) findings.push("A delivery attempt was marked uncertain and will not be blindly retried.");
+
+  const diagnosis = summarizeInteractionDiagnosis({
+    ackRequired,
+    ackQueued,
+    completionRequired,
+    completionQueued,
+    handoffRequired,
+    handoffQueued,
+    yielded,
+    followUpQueued,
+    followUpConsumed,
+    handoffDelivered,
+    deliveryEvidenceRequired,
+    deliveryEvidenceHealthy,
+    deliveryUncertain,
+    findings
+  });
+
+  return {
+    ackRequired,
+    ackQueued,
+    completionRequired,
+    completionQueued,
+    handoffRequired,
+    handoffQueued,
+    yielded,
+    yieldSignal,
+    followUpState,
+    followUpQueued,
+    followUpConsumed,
+    followUpTarget,
+    followUpCompleted,
+    followUpExpired,
+    followUpEscalated,
+    followUpEscalationFilePath: followUpNotification?.escalationNotificationFilePath || null,
+    handoffPendingChoice,
+    handoffState,
+    handoffExpired: handoffState === "expired",
+    handoffEscalated: handoffState === "escalated",
+    handoffBlocked: handoffState === "blocked",
+    handoffChosen,
+    handoffChosenPeerId,
+    handoffChosenBy,
+    handoffDelivered,
+    handoffDeliveryState,
+    handoffBlockedReason: handoffNotification?.blockedReason || null,
+    deliveryEvidenceRequired,
+    deliveryEvidenceHealthy,
+    deliveryReceiptCount: visibleDeliveryCount,
+    deliveryStates,
+    deliveryDeduped,
+    deliveryRetried,
+    deliveryUncertain,
+    diagnosis,
+    findings
+  };
+}
+
+function summarizeLiveRun(run) {
+  if (!run) return null;
+  return {
+    ts: run.ts || null,
+    status: run.status || null,
+    summary: run.summary || null,
+    error: run.error || null,
+    durationMs: run.durationMs ?? null,
+    provider: run.provider || null,
+    model: run.model || null,
+    usage: run.usage || null
+  };
+}
+
+const SECTION_HINTS = [
+  "calendar",
+  "weather",
+  "projects",
+  "project",
+  "inbox",
+  "issues",
+  "backlog",
+  "update",
+  "summary",
+  "next actions",
+  "alerts",
+  "status"
+];
+
+const SECTION_ALIASES = new Map([
+  ["project", "projects"],
+  ["next action", "next actions"]
+]);
+
+function canonicalSectionKey(value) {
+  const normalized = normalizeSectionKey(value);
+  return SECTION_ALIASES.get(normalized) || normalized;
+}
+
+function extractSectionKeys(nonEmptyLines) {
+  const sectionKeys = [];
+  const seen = new Set();
+  const patterns = [
+    /^#{1,6}\s+(.+)$/,
+    /^\*\*([^*]+)\*\*$/,
+    /^[-*]\s+\**([^:*]+?)\**:\s+.+$/,
+    /^([^:]{2,40}):\s+.+$/
+  ];
+
+  for (const line of nonEmptyLines) {
+    for (const pattern of patterns) {
+      const match = line.match(pattern);
+      if (!match) continue;
+      const key = canonicalSectionKey(match[1]);
+      if (!key || key.length < 2) continue;
+      if (seen.has(key)) break;
+      seen.add(key);
+      sectionKeys.push(key);
+      break;
+    }
+  }
+
+  return sectionKeys;
+}
+
+function comparableSections(structure) {
+  return structure.sectionKeys.length ? structure.sectionKeys : structure.sectionHints;
+}
+
+function analyzeOutputStructure(text) {
+  const raw = String(text || "");
+  const trimmed = raw.trim();
+  const lines = raw.split("\n").map((line) => line.trim());
+  const nonEmptyLines = lines.filter(Boolean);
+  const lower = trimmed.toLowerCase();
+  const sectionKeys = extractSectionKeys(nonEmptyLines);
+
+  return {
+    charCount: trimmed.length,
+    lineCount: nonEmptyLines.length,
+    headingCount: nonEmptyLines.filter((line) => /^#{1,6}\s/.test(line) || /^\*\*[^*]+\*\*/.test(line)).length,
+    bulletCount: nonEmptyLines.filter((line) => /^[-*]\s/.test(line)).length,
+    numberedCount: nonEmptyLines.filter((line) => /^\d+\.\s/.test(line)).length,
+    codeFenceCount: (raw.match(/```/g) || []).length / 2,
+    sectionKeys,
+    sectionHints: SECTION_HINTS.filter((hint) => lower.includes(hint)),
+    hasMarkdownTable: /\|.+\|/.test(raw),
+    hasEmoji: /\p{Extended_Pictographic}/u.test(raw)
+  };
+}
+
+function buildOutputDiff(v2Run, liveRun, contract = null) {
+  const v2Text = extractRunText(v2Run, contract);
+  const liveText = extractLiveText(liveRun);
+  const v2 = analyzeOutputStructure(v2Text);
+  const live = analyzeOutputStructure(liveText);
+  const v2Sections = comparableSections(v2);
+  const liveSections = comparableSections(live);
+  const sharedSections = v2Sections.filter((hint) => liveSections.includes(hint));
+  const missingFromV2 = liveSections.filter((hint) => !v2Sections.includes(hint));
+  const extraInV2 = v2Sections.filter((hint) => !liveSections.includes(hint));
+  const findings = [];
+
+  if (!liveText) {
+    findings.push("No live output is available for structural comparison.");
+  } else {
+    if (missingFromV2.length) findings.push(`V2 is missing live sections: ${missingFromV2.join(", ")}.`);
+    if (v2.bulletCount + v2.numberedCount < live.bulletCount + live.numberedCount) {
+      findings.push("V2 output is less structured than the live report.");
+    }
+    if (v2.charCount > 0 && live.charCount > 0) {
+      const verbosityRatio = Number((v2.charCount / live.charCount).toFixed(4));
+      if (verbosityRatio < 0.55) findings.push("V2 output is much shorter than the live report.");
+      if (verbosityRatio > 1.8) findings.push("V2 output is much longer than the live report.");
+    }
+  }
+
+  return {
+    v2,
+    live,
+    alignment: {
+      sharedSections,
+      missingFromV2,
+      extraInV2,
+      bulletDelta: (v2.bulletCount + v2.numberedCount) - (live.bulletCount + live.numberedCount),
+      headingDelta: v2.headingCount - live.headingCount,
+      charDelta: v2.charCount - live.charCount
+    },
+    findings
+  };
+}
+
+function assessOutputContract(contract, v2Run, liveRun) {
+  if (!contract) return null;
+
+  const v2Validation = validateOutputContract(contract, v2Run?.result?.output ?? v2Run?.output ?? "");
+  const liveValidation = validateOutputContract(contract, liveRun?.summary || liveRun?.error || "");
+  const missingFromV2 = v2Validation?.missingSections || [];
+  const missingFromLive = liveValidation?.missingSections || [];
+  const findings = [
+    ...(v2Validation?.findings.map((item) => `V2: ${item}`) || []),
+    ...(liveValidation?.findings.map((item) => `Live: ${item}`) || [])
+  ];
+
+  return {
+    format: contract.format || null,
+    requiredSections: contract.requiredSections || [],
+    styleHints: contract.styleHints || [],
+    satisfiedRatio: v2Validation?.satisfiedRatio ?? 0,
+    missingFromV2,
+    missingFromLive,
+    emptyInV2: v2Validation?.emptySections || [],
+    emptyInLive: liveValidation?.emptySections || [],
+    parsedV2: v2Validation?.parsed || null,
+    parsedLive: liveValidation?.parsed || null,
+    fieldScores: v2Validation?.fieldScores || null,
+    findings
+  };
+}
+
+export { buildRubric };
+
+export class Evaluator {
+  constructor({ projectRoot, liveRoot, stateRoot }) {
+    this.liveRoot = liveRoot;
+    this.stateRoot = stateRoot;
+    this.reviewer = new RunReviewer({ stateRoot });
+    this.scheduler = new Scheduler({ projectRoot, liveRoot, stateRoot });
+    this.evalStore = new RunStore({ rootDir: path.join(stateRoot, "evaluations") });
+    this.notificationStore = new NotificationStore({ rootDir: path.join(stateRoot, "notifications") });
+    this.deliveryStore = new DeliveryStore({ rootDir: path.join(stateRoot, "deliveries") });
+  }
+
+  async resolveInteractionArtifacts(run, review = null) {
+    const reviewNotifications = review?.recentNotifications || [];
+    const _reviewDeliveries = review?.recentDeliveries || [];
+    const baseFiles = run?.notificationFiles || [];
+    const relatedNotificationFiles = expandRelatedNotificationFiles(
+      baseFiles,
+      reviewNotifications
+    );
+    const knownNotifications = reviewNotifications.filter((item) => relatedNotificationFiles.has(item.filePath));
+    const missingNotificationFiles = [...relatedNotificationFiles].filter(
+      (filePath) => !knownNotifications.some((item) => item.filePath === filePath)
+    );
+    const loadedNotifications = await this.notificationStore.getNotifications(missingNotificationFiles);
+    const notifications = [...knownNotifications, ...loadedNotifications].sort((a, b) =>
+      String(a.timestamp || "").localeCompare(String(b.timestamp || ""))
+    );
+    const expandedNotificationFiles = expandRelatedNotificationFiles(baseFiles, notifications);
+
+    // Load all delivery receipts once for deterministic interaction evidence.
+    const allDeliveries = await this.deliveryStore.listAll();
+    const deliveries = allDeliveries.filter((item) => expandedNotificationFiles.has(item.notificationFilePath));
+    return {
+      notificationFiles: expandedNotificationFiles,
+      notifications,
+      deliveries
+    };
+  }
+
+  async evaluate(limit = 20) {
+    const [review, comparisons] = await Promise.all([
+      this.reviewer.review(limit),
+      this.scheduler.compareJobs()
+    ]);
+
+    const recentRuns = review.recentRuns.filter((run) => !run.jobId.endsWith("-comparison"));
+    const grouped = groupBy(recentRuns, (run) => run.jobId);
+
+    const jobs = await Promise.all(
+      Array.from(grouped.entries()).map(async ([jobId, runs]) => this.evaluateJob(jobId, {
+        runs,
+        review,
+        comparisons
+      }))
+    );
+
+    return {
+      jobs: [...jobs].sort((a, b) => a.rubric.overallScore - b.rubric.overallScore),
+      recentRuns,
+      scheduler: review.scheduler
+    };
+  }
+
+  async evaluateJob(jobId, options = {}) {
+    const runtime = await this.scheduler.loadRuntime();
+    const jobConfig = runtime.jobs[jobId] || null;
+    const review = options.review || (await this.reviewer.review(options.limit ?? 20));
+    const comparisons = options.comparisons || (await this.scheduler.compareJobs());
+    const runs = options.runs || review.recentRuns.filter((run) => run.jobId === jobId && !run.jobId.endsWith("-comparison"));
+    const schedulerState = review.scheduler.find((item) => item.jobId === jobId) || null;
+    const comparison = comparisons.find((item) => item.v2JobId === jobId) || null;
+    const liveMatches = await Promise.all(
+      (comparison?.closestLiveJobs || []).map(async (match) => ({
+        ...match,
+        recentRuns: await this.scheduler.bridge.loadCronRunHistory(match.id, 5),
+        latestRun: (await this.scheduler.bridge.loadCronRunHistory(match.id, 1))[0] || null
+      }))
+    );
+
+    const selectedRun = choosePrimaryV2Run(runs);
+    const interactionRun = runs[0] || selectedRun;
+    const selectedLiveMatch = pickBestLiveMatch(liveMatches);
+    const liveHistory = selectedLiveMatch?.recentRuns || [];
+    const outputContract = normalizeOutputContract(jobConfig?.outputContract || null);
+    const outputDiff = buildOutputDiff(selectedRun, selectedLiveMatch?.latestRun || null, outputContract);
+    const contractCheck = assessOutputContract(outputContract, selectedRun, selectedLiveMatch?.latestRun || null);
+    const retrieval = analyzeRetrieval(selectedRun);
+    const interactionArtifacts = await this.resolveInteractionArtifacts(interactionRun, review);
+    const interaction = analyzeInteraction(interactionRun, interactionArtifacts.notifications, interactionArtifacts.deliveries);
+    const rubric = buildRubric({
+      v2Run: selectedRun,
+      liveHistory,
+      matchedLiveJob: selectedLiveMatch,
+      outputContract,
+      contractCheck
+    });
+
+    return {
+      jobId,
+      schedulerState,
+      maintenance: review?.maintenance || null,
+      latestRun: runs[0] || null,
+      primaryRun: summarizeV2Run(selectedRun),
+      interactionRunTimestamp: interactionRun?.timestamp || null,
+      runCount: runs.length,
+      modeCounts: countModes(runs),
+      liveMatches: liveMatches.map((match) => ({
+        id: match.id,
+        name: match.name,
+        description: match.description || null,
+        enabled: match.enabled,
+        schedule: match.schedule,
+        kind: match.kind || null,
+        lastStatus: match.lastStatus || null,
+        latestRun: summarizeLiveRun(match.latestRun)
+      })),
+      selectedLiveMatch: selectedLiveMatch
+        ? {
+            id: selectedLiveMatch.id,
+            name: selectedLiveMatch.name,
+            latestRun: summarizeLiveRun(selectedLiveMatch.latestRun)
+          }
+        : null,
+      retrieval,
+      interaction,
+      outputContract,
+      contractCheck,
+      outputDiff,
+      rubric,
+      comparisonNotes: buildComparisonNotes({
+        jobId,
+        selectedRun,
+        selectedLiveMatch,
+        schedulerState,
+        maintenance: review?.maintenance || null,
+        rubric,
+        outputDiff,
+        contractCheck,
+        retrieval,
+        interaction
+      }),
+      interactionArtifacts: {
+        notificationCount: interactionArtifacts.notifications.length,
+        deliveryCount: interactionArtifacts.deliveries.length,
+        notificationFileCount: interactionArtifacts.notificationFiles.size
+      }
+    };
+  }
+
+  async evaluateAndPersistJob(jobId, options = {}) {
+    const report = await this.evaluateJob(jobId, options);
+    const artifact = {
+      timestamp: new Date().toISOString(),
+      kind: "job-evaluation",
+      ...report
+    };
+    const filePath = await this.evalStore.saveRun(jobId, artifact);
+    return {
+      filePath,
+      ...artifact
+    };
+  }
+}
+
+function countModes(runs) {
+  return runs.reduce((acc, run) => {
+    const mode = run.mode || "unknown";
+    acc[mode] = (acc[mode] || 0) + 1;
+    return acc;
+  }, {});
+}
+
+function buildComparisonNotes({
+  jobId,
+  selectedRun,
+  selectedLiveMatch,
+  schedulerState,
+  maintenance,
+  rubric,
+  outputDiff,
+  contractCheck,
+  retrieval = null,
+  interaction = null
+}) {
+  const notes = [];
+  if (!selectedRun) notes.push("No V2 runs recorded.");
+  if (schedulerState?.lastStatus === "ok") notes.push("Latest V2 scheduler state is healthy.");
+  if (selectedRun?.fallback?.attempted && selectedRun?.fallback?.success) {
+    notes.push(
+      `Run used report fallback from ${selectedRun.fallback.sourceLane || "local"} to ${selectedRun.fallback.finalSourceLane || "remote"} after ${selectedRun.fallback.trigger || "failure"}.`
+    );
+  } else if (selectedRun?.fallback?.attempted && !selectedRun?.fallback?.success) {
+    notes.push(`Report fallback was attempted but did not succeed${selectedRun.fallback?.fallbackError ? `: ${selectedRun.fallback.fallbackError}` : "."}`);
+  } else if (selectedRun?.fallback?.allowed === false && selectedRun?.fallback?.trigger) {
+    notes.push(`Report fallback was blocked after ${selectedRun.fallback.trigger}${selectedRun.fallback?.blockedReason ? `: ${selectedRun.fallback.blockedReason}` : "."}`);
+  }
+  if (maintenance?.wal?.some((item) => item.action && item.action !== "none" && item.action !== "error")) {
+    notes.push("Daemon maintenance recently performed WAL checkpoint work.");
+  }
+  if (maintenance?.handoffs?.expiredCount > 0) {
+    notes.push(`Daemon maintenance expired or escalated ${maintenance.handoffs.expiredCount} pending handoff(s).`);
+  }
+  if (!selectedLiveMatch) {
+    notes.push("No live cron analogue matched.");
+    return uniqueStrings([...notes, ...rubric.findings, ...outputDiff.findings, ...(contractCheck?.findings || []), ...(interaction?.findings || [])]);
+  }
+
+  const latestLive = selectedLiveMatch.latestRun;
+  notes.push(`Closest live cron match: ${selectedLiveMatch.name || selectedLiveMatch.id}.`);
+  if (latestLive?.status) notes.push(`Latest live status: ${latestLive.status}.`);
+  if (latestLive?.provider) notes.push(`Latest live provider: ${latestLive.provider}.`);
+  if (latestLive?.durationMs != null) notes.push(`Latest live duration: ${latestLive.durationMs}ms.`);
+  const retrievalFindings = retrieval?.findings || [];
+  const interactionFindings = interaction?.findings || [];
+  const outputDiffFindings =
+    jobId === "memory-rollup" && Number(contractCheck?.satisfiedRatio || 0) >= 0.99
+      ? (outputDiff.findings || []).filter((item) => !/less structured|much shorter/i.test(item))
+      : outputDiff.findings || [];
+  return uniqueStrings([...notes, ...rubric.findings, ...outputDiffFindings, ...(contractCheck?.findings || []), ...retrievalFindings, ...interactionFindings]);
+}
+
+function normalizeOutputContract(contract) {
+  if (!contract) return null;
+  return {
+    format: contract.format || null,
+    requiredSections: contract.requiredSections || [],
+    styleHints: contract.styleHints || [],
+    profile: contract.profile || null
+  };
+}