@bendyline/docblocks-cli 2.1.1 → 2.2.0

package/dist/eval/cli.js

@@ -0,0 +1,1584 @@
+// src/eval/cli.ts
+import path5 from "path";
+import { Command } from "commander";
+
+// src/eval/cases.ts
+var EVAL_PROMPT_PROFILES = Object.freeze({
+  baseline: `Use the DocBlocks MCP server for authoring, validation, conversion, and saving. Do not create the Office artifact with shell commands or another document library. Return the exact Squisq-flavored Markdown supplied to the final convert_document call.`,
+  "content-first": `Use the DocBlocks MCP server for the complete workflow. Start with get_authoring_context, author a lossless content-first draft, validate it, then use compatible visual templates selectively and preview before the final conversion. Repair actionable diagnostics. Do not create the Office artifact with shell commands or another document library. Return the exact Squisq-flavored Markdown supplied to the final convert_document call.`
+});
+var MCP_CONTENT_EVAL_CASES = Object.freeze([
+  {
+    id: "quarterly-product-review-pptx",
+    title: "Quarterly product review deck",
+    suites: ["quick", "full"],
+    targetFormat: "pptx",
+    artifactFilename: "quarterly-product-review.pptx",
+    brief: `Create an executive-ready quarterly product review presentation for Northstar, a fictional B2B workflow product. The audience is the leadership team. Use 8 slides and make the story decision-oriented, not a data dump.
+
+Use these facts exactly:
+- Q2 active teams: 18,420, up 14% quarter over quarter.
+- Weekly retained teams: 71%, up from 66% in Q1.
+- Enterprise expansion revenue: $3.8M, 92% of the $4.1M target.
+- Activation improved from 42% to 49% after guided setup shipped.
+- Mobile weekly usage is 23%, below the 30% goal.
+- Top support driver is permissions setup, representing 31% of tickets.
+- The proposed Q3 choices are: fund permissions redesign, protect activation gains, and defer advanced mobile editing.
+
+Include a clear opening thesis, a compact scorecard, insights rather than repeated metrics, the three Q3 choices with tradeoffs, and a final decision/owner slide.`,
+    rubric: [
+      "Builds a coherent executive narrative from performance to causes to decisions.",
+      "Preserves every supplied metric and does not invent unsupported business facts.",
+      "Makes the three Q3 choices, tradeoffs, owners, and decisions easy to scan.",
+      "Uses slide-appropriate density and varied Squisq templates without sacrificing content."
+    ],
+    expectation: {
+      minItems: 8,
+      maxItems: 8,
+      minWords: 220,
+      maxWords: 650,
+      maxWordsPerSection: 95,
+      minVisualTemplates: 2,
+      requiredPhrases: ["18,420", "71%", "$3.8M", "49%", "23%", "31%", "permissions"]
+    }
+  },
+  {
+    id: "responsible-ai-training-pptx",
+    title: "Responsible AI manager training deck",
+    suites: ["full"],
+    targetFormat: "pptx",
+    artifactFilename: "responsible-ai-training.pptx",
+    brief: `Create a 9-slide manager training presentation called "Responsible AI in Everyday Work." The audience is nontechnical people managers. Teach a usable decision process, not generic inspiration.
+
+Cover: a plain-language definition, an opening scenario, a four-question risk screen, examples of acceptable and unacceptable use, handling confidential data, reviewing model output, escalation triggers, a short practice scenario with answer, and a final one-week action plan. State clearly that humans remain accountable for decisions and that confidential customer or employee data must not be pasted into unapproved tools.`,
+    rubric: [
+      "Uses plain language and a practical teaching progression.",
+      "Contains an actionable four-question screen and realistic examples.",
+      "Makes confidentiality, human accountability, and escalation memorable.",
+      "Fits a live training deck with concise slides and useful visual variation."
+    ],
+    expectation: {
+      minItems: 9,
+      maxItems: 9,
+      minWords: 240,
+      maxWords: 700,
+      maxWordsPerSection: 90,
+      minVisualTemplates: 2,
+      requiredPhrases: ["accountable", "confidential", "escalat", "four", "one-week"]
+    }
+  },
+  {
+    id: "build-vs-buy-decision-memo-docx",
+    title: "Build-versus-buy decision memo",
+    suites: ["quick", "full"],
+    targetFormat: "docx",
+    artifactFilename: "build-vs-buy-decision-memo.docx",
+    brief: `Write a professional decision memo for the COO recommending whether a fictional 420-person company should build or buy a contract-lifecycle-management system.
+
+Use these facts exactly:
+- Current manual work costs about 1,900 staff hours per quarter.
+- A vendor solution costs $310,000 in year one and $190,000 annually afterward.
+- An internal build is estimated at 8 engineer-months plus 2 ongoing engineer-months per year.
+- Security review found no blocking issue for the vendor, but data residency needs a contractual addendum.
+- Procurement can complete negotiation in six weeks.
+- The operations team needs the first workflow live within ten weeks.
+
+Recommend one path, explain the decision criteria and quantified tradeoffs, name risks and mitigations, give an implementation sequence, and end with the exact decision requested from the COO. Aim for roughly 1,000-1,400 words.`,
+    rubric: [
+      "Makes a clear recommendation supported by the supplied time, cost, and delivery facts.",
+      "Separates facts, assumptions, risks, mitigations, and decision request.",
+      "Provides a credible implementation sequence and accountable next steps.",
+      "Reads as an executive memo rather than a slide deck pasted into Word."
+    ],
+    expectation: {
+      minItems: 6,
+      maxItems: 14,
+      minWords: 850,
+      maxWords: 1550,
+      maxWordsPerSection: 360,
+      minVisualTemplates: 0,
+      requiredPhrases: [
+        "1,900",
+        "$310,000",
+        "$190,000",
+        "8 engineer-months",
+        "six weeks",
+        "ten weeks"
+      ]
+    }
+  },
+  {
+    id: "incident-response-playbook-docx",
+    title: "Incident response playbook",
+    suites: ["full"],
+    targetFormat: "docx",
+    artifactFilename: "incident-response-playbook.docx",
+    brief: `Create a concise operational playbook for responding to a suspected customer-data exposure at a SaaS company. It must be usable during an incident.
+
+Include: purpose and scope, severity triggers, the first 15/30/60 minutes, roles for incident commander/security/legal/support, an evidence-preservation checklist, communication rules, a decision log template, containment and recovery gates, customer-notification decision criteria, and a post-incident checklist. Explicitly say not to delete or alter evidence and not to make unverified claims to customers. Use tables or checklists where they improve scanability. Aim for 1,200-1,800 words.`,
+    rubric: [
+      "Works as an executable playbook under time pressure.",
+      "Distinguishes containment, investigation, recovery, and communication decisions.",
+      "Protects evidence and prevents unverified external statements.",
+      "Uses document-native structure, tables, and checklists for rapid scanning."
+    ],
+    expectation: {
+      minItems: 8,
+      maxItems: 20,
+      minWords: 1e3,
+      maxWords: 2e3,
+      maxWordsPerSection: 400,
+      minVisualTemplates: 0,
+      requiredPhrases: ["15", "30", "60", "incident commander", "evidence", "unverified"]
+    }
+  }
+]);
+function selectEvalCases(suiteOrIds) {
+  if (suiteOrIds === "quick" || suiteOrIds === "full") {
+    const suite = suiteOrIds;
+    return MCP_CONTENT_EVAL_CASES.filter((testCase) => testCase.suites.includes(suite));
+  }
+  const ids = new Set(
+    suiteOrIds.split(",").map((id) => id.trim()).filter(Boolean)
+  );
+  if (ids.size === 0) throw new Error("At least one eval case id is required");
+  const selected = MCP_CONTENT_EVAL_CASES.filter((testCase) => ids.has(testCase.id));
+  const missing = [...ids].filter((id) => !selected.some((testCase) => testCase.id === id));
+  if (missing.length > 0) throw new Error(`Unknown eval case ids: ${missing.join(", ")}`);
+  return selected;
+}
+
+// src/eval/report.ts
+import { readFile, writeFile } from "fs/promises";
+import path from "path";
+function compareEvalRuns(baseline, candidate) {
+  assertPairedCases(baseline, candidate);
+  if (baseline.provenance.judgeMode !== candidate.provenance.judgeMode) {
+    throw new Error("Paired comparison requires the same judge mode");
+  }
+  const candidateById = new Map(candidate.cases.map((result) => [result.caseId, result]));
+  const cases = baseline.cases.map((baselineCase) => {
+    const candidateCase = candidateById.get(baselineCase.caseId);
+    if (!candidateCase) {
+      throw new Error(`Candidate run is missing baseline case ${baselineCase.caseId}`);
+    }
+    return {
+      caseId: baselineCase.caseId,
+      baselineScore: baselineCase.score,
+      candidateScore: candidateCase.score,
+      delta: round(candidateCase.score - baselineCase.score),
+      baselinePassed: baselineCase.passed,
+      candidatePassed: candidateCase.passed
+    };
+  });
+  const warnings = comparisonWarnings(baseline, candidate);
+  return {
+    baselineRunId: baseline.provenance.runId,
+    candidateRunId: candidate.provenance.runId,
+    baselineScore: baseline.score,
+    candidateScore: candidate.score,
+    delta: round(candidate.score - baseline.score),
+    regressions: cases.filter(
+      (result) => result.delta < 0 || result.baselinePassed && !result.candidatePassed
+    ).length,
+    improvements: cases.filter(
+      (result) => result.delta > 0 || !result.baselinePassed && result.candidatePassed
+    ).length,
+    warnings,
+    baselineMetrics: summarizeRunMetrics(baseline),
+    candidateMetrics: summarizeRunMetrics(candidate),
+    cases
+  };
+}
+function summarizeRunMetrics(run) {
+  return run.cases.reduce((metrics, result) => {
+    const generation = result.generationUsage;
+    const judge = result.judgeUsage;
+    const trace = result.mcpTraceMetrics;
+    return {
+      generationDurationMs: metrics.generationDurationMs + result.generationDurationMs,
+      judgeDurationMs: metrics.judgeDurationMs + (result.judgeDurationMs ?? 0),
+      generationInputTokens: metrics.generationInputTokens + (generation?.inputTokens ?? 0),
+      generationCachedInputTokens: metrics.generationCachedInputTokens + (generation?.cachedInputTokens ?? 0),
+      generationUncachedInputTokens: metrics.generationUncachedInputTokens + Math.max(0, (generation?.inputTokens ?? 0) - (generation?.cachedInputTokens ?? 0)),
+      generationOutputTokens: metrics.generationOutputTokens + (generation?.outputTokens ?? 0),
+      generationReasoningTokens: metrics.generationReasoningTokens + (generation?.reasoningOutputTokens ?? 0),
+      judgeInputTokens: metrics.judgeInputTokens + (judge?.inputTokens ?? 0),
+      judgeOutputTokens: metrics.judgeOutputTokens + (judge?.outputTokens ?? 0),
+      toolCallCount: metrics.toolCallCount + (trace?.toolCallCount ?? 0),
+      toolArgumentCharacters: metrics.toolArgumentCharacters + (trace?.toolArgumentCharacters ?? 0),
+      toolResultCharacters: metrics.toolResultCharacters + (trace?.toolResultCharacters ?? 0),
+      authoringContextResultCharacters: metrics.authoringContextResultCharacters + (trace?.authoringContextResultCharacters ?? 0),
+      authoringContextTextCharacters: metrics.authoringContextTextCharacters + (trace?.authoringContextTextCharacters ?? 0),
+      authoringContextStructuredCharacters: metrics.authoringContextStructuredCharacters + (trace?.authoringContextStructuredCharacters ?? 0)
+    };
+  }, emptyRunMetrics());
+}
+async function readEvalRun(runOrDirectory) {
+  const file = path.extname(runOrDirectory) === ".json" ? runOrDirectory : path.join(runOrDirectory, "run.json");
+  const parsed = JSON.parse(await readFile(file, "utf8"));
+  if (!isRecord(parsed) || !isRecord(parsed.provenance) || !Array.isArray(parsed.cases)) {
+    throw new Error(`${file} is not a DocBlocks MCP eval run`);
+  }
+  return parsed;
+}
+async function writeRunReport(directory, run) {
+  await Promise.all([
+    writeFile(path.join(directory, "run.json"), `${JSON.stringify(run, null, 2)}
+`),
+    writeFile(path.join(directory, "summary.md"), runMarkdown(run))
+  ]);
+}
+async function writeComparisonReport(outputDirectory, comparison) {
+  await Promise.all([
+    writeFile(
+      path.join(outputDirectory, "comparison.json"),
+      `${JSON.stringify(comparison, null, 2)}
+`
+    ),
+    writeFile(path.join(outputDirectory, "comparison.md"), comparisonMarkdown(comparison))
+  ]);
+}
+function runMarkdown(run) {
+  const metrics = summarizeRunMetrics(run);
+  const rows = run.cases.map(
+    (result) => `| ${result.caseId} | ${result.targetFormat} | ${result.score.toFixed(1)} | ${result.artifactScore.toFixed(1)} | ${result.markdownScore.toFixed(1)} | ${result.llmScore?.toFixed(1) ?? "n/a"} | ${result.passed ? "pass" : "fail"} |`
+  );
+  const failures = run.cases.flatMap((result) => {
+    const failedChecks = [...result.artifactJudge.checks, ...result.markdownJudge.checks].filter(
+      (check) => check.required && !check.passed
+    );
+    return [
+      ...result.failure ? [`- **${result.caseId}:** ${result.failure}`] : [],
+      ...failedChecks.map((check) => `- **${result.caseId} / ${check.id}:** ${check.message}`),
+      ...result.llmJudge?.weaknesses.map(
+        (weakness) => `- **${result.caseId} / LLM:** ${weakness}`
+      ) ?? [],
+      ...result.llmJudge?.unsupportedClaims.map(
+        (claim) => `- **${result.caseId} / unsupported claim:** ${claim}`
+      ) ?? [],
+      ...result.llmJudge?.missingRequestedElements.map(
+        (element) => `- **${result.caseId} / missing element:** ${element}`
+      ) ?? []
+    ];
+  });
+  return `# DocBlocks MCP content eval: ${run.provenance.label}
+
+- Run: \`${run.provenance.runId}\`
+- Score: **${run.score.toFixed(1)}**
+- Result: **${run.passed ? "pass" : "fail"}**
+- Suite/profile: \`${run.provenance.suite}\` / \`${run.provenance.profile}\`
+- Judge mode/model: \`${run.provenance.judgeMode}\` / \`${run.provenance.model ?? "Codex default"}\`
+- Git: \`${run.provenance.gitCommit ?? "unknown"}\`${run.provenance.gitDirty ? " (dirty)" : ""}
+- CLI SHA-256: \`${run.provenance.cliSha256}\`
+- Judge contract SHA-256: \`${run.provenance.judgeContractSha256}\`
+
+## Efficiency
+
+- Generation time: **${formatDuration(metrics.generationDurationMs)}**
+- Generation input tokens: **${formatInteger(metrics.generationInputTokens)}** total / **${formatInteger(metrics.generationUncachedInputTokens)}** uncached
+- Generation output tokens: **${formatInteger(metrics.generationOutputTokens)}**
+- MCP calls/results: **${formatInteger(metrics.toolCallCount)}** / **${formatInteger(metrics.toolResultCharacters)} characters**
+- Authoring-context result: **${formatInteger(metrics.authoringContextResultCharacters)} characters** (**${formatInteger(metrics.authoringContextTextCharacters)} text**, **${formatInteger(metrics.authoringContextStructuredCharacters)} structured**)
+
+| Case | Format | Overall | OOXML | Markdown | LLM | Result |
+| --- | --- | ---: | ---: | ---: | ---: | --- |
+${rows.join("\n")}
+
+## Findings
+
+${failures.length > 0 ? failures.join("\n") : "- No required static failures or LLM weaknesses were reported."}
+`;
+}
+function comparisonMarkdown(comparison) {
+  const rows = comparison.cases.map(
+    (result) => `| ${result.caseId} | ${result.baselineScore.toFixed(1)} | ${result.candidateScore.toFixed(1)} | ${formatDelta(result.delta)} | ${result.baselinePassed ? "pass" : "fail"} \u2192 ${result.candidatePassed ? "pass" : "fail"} |`
+  );
+  return `# DocBlocks MCP content eval comparison
+
+- Baseline: \`${comparison.baselineRunId}\` \u2014 ${comparison.baselineScore.toFixed(1)}
+- Candidate: \`${comparison.candidateRunId}\` \u2014 ${comparison.candidateScore.toFixed(1)}
+- Aggregate delta: **${formatDelta(comparison.delta)}**
+- Improved/regressed cases: **${comparison.improvements} / ${comparison.regressions}**
+
+## Comparability
+
+${comparison.warnings.length > 0 ? comparison.warnings.map((warning) => `- Warning: ${warning}`).join("\n") : "- Paired cases, judge mode, model, prompt profile, threshold, and Codex version match."}
+
+## Efficiency
+
+| Metric | Baseline | Candidate | Delta |
+| --- | ---: | ---: | ---: |
+| Generation time | ${formatDuration(comparison.baselineMetrics.generationDurationMs)} | ${formatDuration(comparison.candidateMetrics.generationDurationMs)} | ${formatSignedDuration(comparison.candidateMetrics.generationDurationMs - comparison.baselineMetrics.generationDurationMs)} |
+| Input tokens | ${formatInteger(comparison.baselineMetrics.generationInputTokens)} | ${formatInteger(comparison.candidateMetrics.generationInputTokens)} | ${formatSignedInteger(comparison.candidateMetrics.generationInputTokens - comparison.baselineMetrics.generationInputTokens)} |
+| Uncached input tokens | ${formatInteger(comparison.baselineMetrics.generationUncachedInputTokens)} | ${formatInteger(comparison.candidateMetrics.generationUncachedInputTokens)} | ${formatSignedInteger(comparison.candidateMetrics.generationUncachedInputTokens - comparison.baselineMetrics.generationUncachedInputTokens)} |
+| Output tokens | ${formatInteger(comparison.baselineMetrics.generationOutputTokens)} | ${formatInteger(comparison.candidateMetrics.generationOutputTokens)} | ${formatSignedInteger(comparison.candidateMetrics.generationOutputTokens - comparison.baselineMetrics.generationOutputTokens)} |
+| MCP result characters | ${formatInteger(comparison.baselineMetrics.toolResultCharacters)} | ${formatInteger(comparison.candidateMetrics.toolResultCharacters)} | ${formatSignedInteger(comparison.candidateMetrics.toolResultCharacters - comparison.baselineMetrics.toolResultCharacters)} |
+| Authoring-context characters | ${formatInteger(comparison.baselineMetrics.authoringContextResultCharacters)} | ${formatInteger(comparison.candidateMetrics.authoringContextResultCharacters)} | ${formatSignedInteger(comparison.candidateMetrics.authoringContextResultCharacters - comparison.baselineMetrics.authoringContextResultCharacters)} |
+
+| Case | Baseline | Candidate | Delta | Result |
+| --- | ---: | ---: | ---: | --- |
+${rows.join("\n")}
+`;
+}
+function formatDelta(value) {
+  return `${value >= 0 ? "+" : ""}${value.toFixed(1)}`;
+}
+function assertPairedCases(baseline, candidate) {
+  const baselineIds = baseline.cases.map(({ caseId }) => caseId).sort();
+  const candidateIds = candidate.cases.map(({ caseId }) => caseId).sort();
+  if (JSON.stringify(baselineIds) !== JSON.stringify(candidateIds)) {
+    throw new Error("Paired comparison requires exactly the same case ids");
+  }
+  const candidateById = new Map(candidate.cases.map((result) => [result.caseId, result]));
+  for (const baselineCase of baseline.cases) {
+    if (candidateById.get(baselineCase.caseId)?.targetFormat !== baselineCase.targetFormat) {
+      throw new Error(`Paired case ${baselineCase.caseId} changed target format`);
+    }
+  }
+}
+function comparisonWarnings(baseline, candidate) {
+  const warnings = [];
+  const pairs = [
+    ["prompt profile", baseline.provenance.profile, candidate.provenance.profile],
+    [
+      "prompt-profile hash",
+      baseline.provenance.promptProfileSha256,
+      candidate.provenance.promptProfileSha256
+    ],
+    [
+      "judge-contract hash",
+      baseline.provenance.judgeContractSha256,
+      candidate.provenance.judgeContractSha256
+    ],
+    ["model", baseline.provenance.model, candidate.provenance.model],
+    ["threshold", baseline.provenance.threshold, candidate.provenance.threshold],
+    ["Codex version", baseline.provenance.codexVersion, candidate.provenance.codexVersion]
+  ];
+  for (const [label, baselineValue, candidateValue] of pairs) {
+    if (baselineValue !== candidateValue) {
+      warnings.push(
+        `${label} differs: ${JSON.stringify(baselineValue)} versus ${JSON.stringify(candidateValue)}`
+      );
+    }
+  }
+  if (baseline.provenance.model === null && candidate.provenance.model === null) {
+    warnings.push(
+      "both runs use the mutable Codex default model; pin --model for launch decisions"
+    );
+  }
+  return warnings;
+}
+function emptyRunMetrics() {
+  return {
+    generationDurationMs: 0,
+    judgeDurationMs: 0,
+    generationInputTokens: 0,
+    generationCachedInputTokens: 0,
+    generationUncachedInputTokens: 0,
+    generationOutputTokens: 0,
+    generationReasoningTokens: 0,
+    judgeInputTokens: 0,
+    judgeOutputTokens: 0,
+    toolCallCount: 0,
+    toolArgumentCharacters: 0,
+    toolResultCharacters: 0,
+    authoringContextResultCharacters: 0,
+    authoringContextTextCharacters: 0,
+    authoringContextStructuredCharacters: 0
+  };
+}
+function formatInteger(value) {
+  return Math.round(value).toLocaleString("en-US");
+}
+function formatSignedInteger(value) {
+  return `${value >= 0 ? "+" : ""}${formatInteger(value)}`;
+}
+function formatDuration(value) {
+  return `${(value / 1e3).toFixed(1)}s`;
+}
+function formatSignedDuration(value) {
+  return `${value >= 0 ? "+" : ""}${(value / 1e3).toFixed(1)}s`;
+}
+function round(value) {
+  return Math.round(value * 10) / 10;
+}
+function isRecord(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+// src/eval/runner.ts
+import { createHash } from "crypto";
+import { mkdir, readFile as readFile4, writeFile as writeFile3 } from "fs/promises";
+import path4 from "path";
+
+// src/eval/artifact-judge.ts
+import { readFile as readFile2 } from "fs/promises";
+import JSZip from "jszip";
+
+// src/eval/markdown-judge.ts
+var PLACEHOLDER_PATTERN = /\b(?:todo|tbd|lorem ipsum|insert (?:text|chart|image)|placeholder)\b/i;
+var HEADING_PATTERN = /^(#{1,6})\s+(.+)$/gm;
+var TEMPLATE_PATTERN = /\{\[([A-Za-z0-9_-]+)/g;
+function judgeMarkdown(markdown, testCase) {
+  const words = countWords(markdown);
+  const sections = markdownSections(markdown);
+  const headings = [...markdown.matchAll(HEADING_PATTERN)];
+  const templates = [...markdown.matchAll(TEMPLATE_PATTERN)].map((match) => match[1]);
+  const visualTemplates = templates.filter((template) => template !== "content");
+  const denseSections = sections.filter(
+    (section) => countWords(section.body) > testCase.expectation.maxWordsPerSection
+  );
+  const checks = [
+    rangeCheck(
+      "word-count",
+      words,
+      testCase.expectation.minWords,
+      testCase.expectation.maxWords,
+      true,
+      "Markdown word count"
+    ),
+    rangeCheck(
+      "section-count",
+      headings.length,
+      testCase.expectation.minItems,
+      testCase.expectation.maxItems,
+      true,
+      testCase.targetFormat === "pptx" ? "Authored slide sections" : "Document sections"
+    ),
+    {
+      id: "section-density",
+      passed: denseSections.length === 0,
+      required: true,
+      message: denseSections.length === 0 ? "Every section stays within the format-specific density budget" : `${denseSections.length} sections exceed ${testCase.expectation.maxWordsPerSection} words`,
+      observed: denseSections.length,
+      expected: 0
+    },
+    {
+      id: "visual-template-use",
+      passed: visualTemplates.length >= testCase.expectation.minVisualTemplates,
+      required: testCase.expectation.minVisualTemplates > 0,
+      message: `${visualTemplates.length} non-content Squisq template annotations found`,
+      observed: visualTemplates.length,
+      expected: testCase.expectation.minVisualTemplates
+    },
+    {
+      id: "no-placeholders",
+      passed: !PLACEHOLDER_PATTERN.test(markdown),
+      required: true,
+      message: PLACEHOLDER_PATTERN.test(markdown) ? "Draft contains placeholder language" : "Draft contains no placeholder language",
+      observed: PLACEHOLDER_PATTERN.test(markdown),
+      expected: false
+    },
+    ...testCase.expectation.requiredPhrases.map((phrase) => ({
+      id: `required-phrase:${phrase.toLowerCase()}`,
+      passed: markdown.toLowerCase().includes(phrase.toLowerCase()),
+      required: true,
+      message: `Required brief fact ${JSON.stringify(phrase)} is ${markdown.toLowerCase().includes(phrase.toLowerCase()) ? "present" : "missing"}`,
+      observed: markdown.toLowerCase().includes(phrase.toLowerCase()),
+      expected: true
+    }))
+  ];
+  return summarizeChecks(checks, {
+    wordCount: words,
+    headingCount: headings.length,
+    templateAnnotationCount: templates.length,
+    visualTemplateCount: visualTemplates.length,
+    denseSectionCount: denseSections.length
+  });
+}
+function countWords(value) {
+  return value.trim().match(/[\p{L}\p{N}][\p{L}\p{N}'’.-]*/gu)?.length ?? 0;
+}
+function markdownSections(markdown) {
+  const matches = [...markdown.matchAll(HEADING_PATTERN)];
+  return matches.map((match, index) => ({
+    heading: match[2],
+    body: markdown.slice((match.index ?? 0) + match[0].length, matches[index + 1]?.index)
+  }));
+}
+function rangeCheck(id, observed, minimum, maximum, required, label) {
+  const passed = observed >= minimum && observed <= maximum;
+  return {
+    id,
+    passed,
+    required,
+    message: `${label}: ${observed}; expected ${minimum}-${maximum}`,
+    observed,
+    expected: `${minimum}-${maximum}`
+  };
+}
+function summarizeChecks(checks, metrics) {
+  const score = checks.length === 0 ? 0 : Math.round(checks.filter((check) => check.passed).length / checks.length * 1e3) / 10;
+  return {
+    score,
+    passed: checks.every((check) => !check.required || check.passed),
+    checks,
+    metrics
+  };
+}
+
+// src/eval/artifact-judge.ts
+async function judgeArtifact(artifactPath, testCase) {
+  let bytes;
+  try {
+    bytes = await readFile2(artifactPath);
+  } catch (caught) {
+    return missingArtifact(caught);
+  }
+  let zip;
+  try {
+    zip = await JSZip.loadAsync(bytes, { checkCRC32: true });
+  } catch (caught) {
+    const message = caught instanceof Error ? caught.message : String(caught);
+    return summarizeChecks(
+      [
+        {
+          id: "valid-zip",
+          passed: false,
+          required: true,
+          message: `Office artifact is not a valid ZIP package: ${message}`,
+          observed: false,
+          expected: true
+        }
+      ],
+      { byteSize: bytes.byteLength }
+    );
+  }
+  return testCase.targetFormat === "pptx" ? judgePptx(zip, bytes.byteLength, testCase) : judgeDocx(zip, bytes.byteLength, testCase);
+}
+async function judgePptx(zip, byteSize, testCase) {
+  const slidePaths = Object.keys(zip.files).filter((name) => /^ppt\/slides\/slide\d+\.xml$/.test(name)).sort(numericOfficePartSort);
+  const slideTexts = await Promise.all(
+    slidePaths.map(async (path6) => extractXmlText(await requiredZipText(zip, path6), "a:t"))
+  );
+  const emptySlides = slideTexts.filter((text) => countWords(text) < 2).length;
+  const overfullSlides = slideTexts.filter((text) => countWords(text) > 120).length;
+  const allText = slideTexts.join(" ");
+  const checks = [
+    packagePartCheck(zip, "[Content_Types].xml"),
+    packagePartCheck(zip, "ppt/presentation.xml"),
+    packagePartCheck(zip, "ppt/_rels/presentation.xml.rels"),
+    rangeCheck2(
+      "slide-count",
+      slidePaths.length,
+      testCase.expectation.minItems,
+      testCase.expectation.maxItems
+    ),
+    {
+      id: "nonempty-slides",
+      passed: emptySlides === 0,
+      required: true,
+      message: `${emptySlides} slides contain fewer than two extractable words`,
+      observed: emptySlides,
+      expected: 0
+    },
+    {
+      id: "slide-density",
+      passed: overfullSlides === 0,
+      required: false,
+      message: `${overfullSlides} slides contain more than 120 extractable words`,
+      observed: overfullSlides,
+      expected: 0
+    },
+    {
+      id: "theme-part",
+      passed: Object.keys(zip.files).some((name) => /^ppt\/theme\/theme\d+\.xml$/.test(name)),
+      required: true,
+      message: "Presentation contains an Office theme part",
+      observed: Object.keys(zip.files).some((name) => /^ppt\/theme\/theme\d+\.xml$/.test(name)),
+      expected: true
+    }
+  ];
+  return summarizeChecks(checks, {
+    byteSize,
+    packageEntries: Object.keys(zip.files).length,
+    slideCount: slidePaths.length,
+    emptySlides,
+    overfullSlides,
+    extractableWordCount: countWords(allText)
+  });
+}
+async function judgeDocx(zip, byteSize, testCase) {
+  const documentXml = await optionalZipText(zip, "word/document.xml");
+  const text = documentXml ? extractXmlText(documentXml, "w:t") : "";
+  const paragraphCount = documentXml?.match(/<w:p(?:\s|>)/g)?.length ?? 0;
+  const headingCount = documentXml?.match(/<w:pStyle\s+[^>]*w:val="Heading[1-9]"/g)?.length ?? 0;
+  const tableCount = documentXml?.match(/<w:tbl(?:\s|>)/g)?.length ?? 0;
+  const checks = [
+    packagePartCheck(zip, "[Content_Types].xml"),
+    packagePartCheck(zip, "word/document.xml"),
+    packagePartCheck(zip, "word/styles.xml"),
+    {
+      id: "extractable-content",
+      passed: countWords(text) >= Math.round(testCase.expectation.minWords * 0.75),
+      required: true,
+      message: `DOCX contains ${countWords(text)} extractable words`,
+      observed: countWords(text),
+      expected: `>=${Math.round(testCase.expectation.minWords * 0.75)}`
+    },
+    {
+      id: "paragraph-structure",
+      passed: paragraphCount >= testCase.expectation.minItems,
+      required: true,
+      message: `DOCX contains ${paragraphCount} paragraphs`,
+      observed: paragraphCount,
+      expected: `>=${testCase.expectation.minItems}`
+    },
+    {
+      id: "heading-structure",
+      passed: headingCount >= Math.min(3, testCase.expectation.minItems),
+      required: false,
+      message: `DOCX contains ${headingCount} native heading-style paragraphs`,
+      observed: headingCount,
+      expected: `>=${Math.min(3, testCase.expectation.minItems)}`
+    }
+  ];
+  return summarizeChecks(checks, {
+    byteSize,
+    packageEntries: Object.keys(zip.files).length,
+    extractableWordCount: countWords(text),
+    paragraphCount,
+    headingCount,
+    tableCount
+  });
+}
+function packagePartCheck(zip, path6) {
+  const present = Boolean(zip.file(path6));
+  return {
+    id: `package-part:${path6}`,
+    passed: present,
+    required: true,
+    message: `${path6} is ${present ? "present" : "missing"}`,
+    observed: present,
+    expected: true
+  };
+}
+function rangeCheck2(id, observed, minimum, maximum) {
+  return {
+    id,
+    passed: observed >= minimum && observed <= maximum,
+    required: true,
+    message: `${id}: ${observed}; expected ${minimum}-${maximum}`,
+    observed,
+    expected: `${minimum}-${maximum}`
+  };
+}
+function numericOfficePartSort(left, right) {
+  return Number(left.match(/(\d+)\.xml$/)?.[1] ?? 0) - Number(right.match(/(\d+)\.xml$/)?.[1] ?? 0);
+}
+function extractXmlText(xml, tagName) {
+  const escaped = tagName.replace(":", "\\:");
+  const matches = xml.matchAll(
+    new RegExp(`<${escaped}(?:\\s[^>]*)?>([\\s\\S]*?)<\\/${escaped}>`, "g")
+  );
+  return [...matches].map((match) => decodeXml(match[1])).join(" ").replace(/\s+/g, " ").trim();
+}
+function decodeXml(value) {
+  return value.replace(/&lt;/g, "<").replace(/&gt;/g, ">").replace(/&quot;/g, '"').replace(/&apos;/g, "'").replace(/&amp;/g, "&");
+}
+async function requiredZipText(zip, path6) {
+  const value = await optionalZipText(zip, path6);
+  if (value === null) throw new Error(`Missing Office package part: ${path6}`);
+  return value;
+}
+async function optionalZipText(zip, path6) {
+  const file = zip.file(path6);
+  return file ? file.async("string") : null;
+}
+function missingArtifact(caught) {
+  const message = caught instanceof Error ? caught.message : String(caught);
+  return summarizeChecks(
+    [
+      {
+        id: "artifact-exists",
+        passed: false,
+        required: true,
+        message: `Artifact could not be read: ${message}`,
+        observed: false,
+        expected: true
+      }
+    ],
+    { byteSize: 0 }
+  );
+}
+
+// src/eval/codex-host.ts
+import { readFile as readFile3, writeFile as writeFile2 } from "fs/promises";
+import path3 from "path";
+import { z } from "zod";
+
+// src/eval/schemas.ts
+function generationOutputSchema(testCase) {
+  return {
+    type: "object",
+    properties: {
+      artifactPath: { type: "string", const: testCase.artifactFilename },
+      targetFormat: { type: "string", const: testCase.targetFormat },
+      fidelity: { type: "string", minLength: 1, maxLength: 64 },
+      markdown: { type: "string", minLength: 1, maxLength: 5e5 },
+      summary: { type: "string", minLength: 1, maxLength: 2e3 },
+      validationRepairs: {
+        type: "array",
+        items: { type: "string", minLength: 1, maxLength: 1e3 },
+        maxItems: 50
+      }
+    },
+    required: [
+      "artifactPath",
+      "targetFormat",
+      "fidelity",
+      "markdown",
+      "summary",
+      "validationRepairs"
+    ],
+    additionalProperties: false
+  };
+}
+var LLM_JUDGE_OUTPUT_SCHEMA = Object.freeze({
+  type: "object",
+  properties: {
+    scores: {
+      type: "object",
+      properties: {
+        briefAdherence: { type: "integer", minimum: 1, maximum: 5 },
+        contentQuality: { type: "integer", minimum: 1, maximum: 5 },
+        structure: { type: "integer", minimum: 1, maximum: 5 },
+        formatFitness: { type: "integer", minimum: 1, maximum: 5 },
+        squisqUsage: { type: "integer", minimum: 1, maximum: 5 },
+        overall: { type: "integer", minimum: 1, maximum: 5 }
+      },
+      required: [
+        "briefAdherence",
+        "contentQuality",
+        "structure",
+        "formatFitness",
+        "squisqUsage",
+        "overall"
+      ],
+      additionalProperties: false
+    },
+    strengths: {
+      type: "array",
+      items: { type: "string", minLength: 1, maxLength: 1e3 },
+      minItems: 1,
+      maxItems: 8
+    },
+    weaknesses: {
+      type: "array",
+      items: { type: "string", minLength: 1, maxLength: 1e3 },
+      minItems: 1,
+      maxItems: 8
+    },
+    unsupportedClaims: {
+      type: "array",
+      items: { type: "string", minLength: 1, maxLength: 1e3 },
+      maxItems: 12
+    },
+    missingRequestedElements: {
+      type: "array",
+      items: { type: "string", minLength: 1, maxLength: 1e3 },
+      maxItems: 12
+    },
+    recommendation: { type: "string", minLength: 1, maxLength: 2e3 }
+  },
+  required: [
+    "scores",
+    "strengths",
+    "weaknesses",
+    "unsupportedClaims",
+    "missingRequestedElements",
+    "recommendation"
+  ],
+  additionalProperties: false
+});
+
+// src/eval/subprocess.ts
+import { spawn } from "child_process";
+import { access } from "fs/promises";
+import path2 from "path";
+var MAX_CAPTURE_BYTES = 64 * 1024 * 1024;
+async function resolveCodexCommand(command) {
+  if (process.platform !== "win32") return { command, prefixArgs: [] };
+  const normalized = command.toLowerCase();
+  if (normalized !== "codex" && normalized !== "codex.cmd" && !normalized.endsWith("codex.cmd")) {
+    return { command, prefixArgs: [] };
+  }
+  const npmRoot = path2.isAbsolute(command) ? path2.dirname(command) : process.env.APPDATA ? path2.join(process.env.APPDATA, "npm") : null;
+  if (!npmRoot) return { command, prefixArgs: [] };
+  const script = path2.join(npmRoot, "node_modules", "@openai", "codex", "bin", "codex.js");
+  try {
+    await access(script);
+    return { command: process.execPath, prefixArgs: [script] };
+  } catch {
+    return { command, prefixArgs: [] };
+  }
+}
+async function runCapturedProcess(options) {
+  const started = Date.now();
+  return new Promise((resolve, reject) => {
+    const child = spawn(options.command, [...options.args], {
+      cwd: options.cwd,
+      windowsHide: true,
+      stdio: ["pipe", "pipe", "pipe"]
+    });
+    const stdout = [];
+    const stderr = [];
+    let stdoutBytes = 0;
+    let stderrBytes = 0;
+    let captureError = null;
+    const timer = setTimeout(() => {
+      captureError = new Error(`Process timed out after ${options.timeoutMs}ms`);
+      child.kill();
+    }, options.timeoutMs);
+    child.stdout.on("data", (chunk) => {
+      stdoutBytes += chunk.byteLength;
+      if (stdoutBytes > MAX_CAPTURE_BYTES) {
+        captureError = new Error(`Process stdout exceeded ${MAX_CAPTURE_BYTES} bytes`);
+        child.kill();
+        return;
+      }
+      stdout.push(chunk);
+    });
+    child.stderr.on("data", (chunk) => {
+      stderrBytes += chunk.byteLength;
+      if (stderrBytes > MAX_CAPTURE_BYTES) {
+        captureError = new Error(`Process stderr exceeded ${MAX_CAPTURE_BYTES} bytes`);
+        child.kill();
+        return;
+      }
+      stderr.push(chunk);
+    });
+    child.once("error", (error) => {
+      clearTimeout(timer);
+      reject(error);
+    });
+    child.once("close", (code) => {
+      clearTimeout(timer);
+      if (captureError) {
+        reject(captureError);
+        return;
+      }
+      resolve({
+        command: options.command,
+        args: options.args,
+        exitCode: code ?? -1,
+        stdout: Buffer.concat(stdout).toString("utf8"),
+        stderr: Buffer.concat(stderr).toString("utf8"),
+        durationMs: Date.now() - started
+      });
+    });
+    child.stdin.end(options.stdin ?? "");
+  });
+}
+
+// src/eval/codex-host.ts
+var LLM_JUDGE_CONTRACT_VERSION = 3;
+var generationResultSchema = z.object({
+  artifactPath: z.string().min(1),
+  targetFormat: z.enum(["pptx", "docx"]),
+  fidelity: z.string().min(1),
+  markdown: z.string().min(1),
+  summary: z.string().min(1),
+  validationRepairs: z.array(z.string())
+}).strict();
+var llmJudgeResultSchema = z.object({
+  scores: z.object({
+    briefAdherence: z.number().int().min(1).max(5),
+    contentQuality: z.number().int().min(1).max(5),
+    structure: z.number().int().min(1).max(5),
+    formatFitness: z.number().int().min(1).max(5),
+    squisqUsage: z.number().int().min(1).max(5),
+    overall: z.number().int().min(1).max(5)
+  }).strict(),
+  strengths: z.array(z.string()).min(1),
+  weaknesses: z.array(z.string()).min(1),
+  unsupportedClaims: z.array(z.string()),
+  missingRequestedElements: z.array(z.string()),
+  recommendation: z.string().min(1)
+}).strict();
+async function runGeneration(options, testCase, caseDirectory) {
+  const outputSchemaPath = path3.join(caseDirectory, "generation-output.schema.json");
+  const finalPath = path3.join(caseDirectory, "generation-final.json");
+  await writeFile2(
+    outputSchemaPath,
+    `${JSON.stringify(generationOutputSchema(testCase), null, 2)}
+`
+  );
+  const prompt = generationPrompt(testCase, options.profile);
+  await writeFile2(path3.join(caseDirectory, "generation-prompt.txt"), prompt);
+  const resolved = await resolveCodexCommand(options.codexCommand);
+  const args = [
+    ...resolved.prefixArgs,
+    ...buildCodexGenerationArgs(options, testCase, caseDirectory, outputSchemaPath, finalPath)
+  ];
+  const capture = await runCapturedProcess({
+    command: resolved.command,
+    args,
+    cwd: options.repoRoot,
+    stdin: prompt,
+    timeoutMs: 15 * 60 * 1e3
+  });
+  await Promise.all([
+    writeFile2(path3.join(caseDirectory, "generation-trace.jsonl"), capture.stdout),
+    writeFile2(path3.join(caseDirectory, "generation-stderr.log"), capture.stderr)
+  ]);
+  requireSuccessfulCodex(capture, "generation");
+  const parsed = generationResultSchema.parse(JSON.parse(await readFile3(finalPath, "utf8")));
+  if (parsed.artifactPath !== testCase.artifactFilename) {
+    throw new Error(
+      `Generation returned artifactPath ${JSON.stringify(parsed.artifactPath)}; expected ${JSON.stringify(testCase.artifactFilename)}`
+    );
+  }
+  if (parsed.targetFormat !== testCase.targetFormat) {
+    throw new Error(
+      `Generation returned targetFormat ${parsed.targetFormat}; expected ${testCase.targetFormat}`
+    );
+  }
+  return {
+    result: parsed,
+    process: capture,
+    traceMarkdown: extractTraceMarkdown(capture.stdout),
+    usage: extractTokenUsage(capture.stdout),
+    traceMetrics: extractMcpTraceMetrics(capture.stdout)
+  };
+}
+async function runLlmJudge(options, testCase, markdown, caseDirectory) {
+  const outputSchemaPath = path3.join(caseDirectory, "judge-output.schema.json");
+  const finalPath = path3.join(caseDirectory, "judge-final.json");
+  await writeFile2(outputSchemaPath, `${JSON.stringify(LLM_JUDGE_OUTPUT_SCHEMA, null, 2)}
+`);
+  const prompt = judgePrompt(testCase, markdown);
+  await writeFile2(path3.join(caseDirectory, "judge-prompt.txt"), prompt);
+  const resolved = await resolveCodexCommand(options.codexCommand);
+  const args = [
+    ...resolved.prefixArgs,
+    "exec",
+    "--ephemeral",
+    "--json",
+    "--sandbox",
+    "read-only",
+    "--skip-git-repo-check",
+    "--ignore-user-config",
+    "--ignore-rules",
+    "--color",
+    "never",
+    "--cd",
+    caseDirectory,
+    "--output-schema",
+    outputSchemaPath,
+    "--output-last-message",
+    finalPath,
+    ...options.model ? ["--model", options.model] : [],
+    "-"
+  ];
+  const capture = await runCapturedProcess({
+    command: resolved.command,
+    args,
+    cwd: options.repoRoot,
+    stdin: prompt,
+    timeoutMs: 10 * 60 * 1e3
+  });
+  await Promise.all([
+    writeFile2(path3.join(caseDirectory, "judge-trace.jsonl"), capture.stdout),
+    writeFile2(path3.join(caseDirectory, "judge-stderr.log"), capture.stderr)
+  ]);
+  requireSuccessfulCodex(capture, "judge");
+  return {
+    result: llmJudgeResultSchema.parse(JSON.parse(await readFile3(finalPath, "utf8"))),
+    process: capture,
+    usage: extractTokenUsage(capture.stdout)
+  };
+}
+function buildCodexGenerationArgs(options, testCase, caseDirectory, outputSchemaPath, finalPath) {
+  const mcpArgs = [
+    options.cliEntry,
+    "mcp",
+    "--allow-write",
+    caseDirectory,
+    "--operation-timeout-ms",
+    "180000",
+    "--artifact-ttl-ms",
+    "3600000"
+  ];
+  return [
+    "exec",
+    "--ephemeral",
+    "--json",
+    "--sandbox",
+    "read-only",
+    "--skip-git-repo-check",
+    "--ignore-user-config",
+    "--ignore-rules",
+    "--color",
+    "never",
+    "--cd",
+    caseDirectory,
+    "-c",
+    `mcp_servers.docblocks.command=${tomlString(process.execPath)}`,
+    "-c",
+    `mcp_servers.docblocks.args=${tomlArray(mcpArgs)}`,
+    "-c",
+    `mcp_servers.docblocks.cwd=${tomlString(options.repoRoot)}`,
+    "-c",
+    "mcp_servers.docblocks.required=true",
+    "-c",
+    'mcp_servers.docblocks.default_tools_approval_mode="approve"',
+    "-c",
+    "mcp_servers.docblocks.startup_timeout_sec=30",
+    "-c",
+    "mcp_servers.docblocks.tool_timeout_sec=240",
+    "--output-schema",
+    outputSchemaPath,
+    "--output-last-message",
+    finalPath,
+    ...options.model ? ["--model", options.model] : [],
+    "-"
+  ];
+}
+function generationPrompt(testCase, profile) {
+  return `You are the authoring agent in a controlled DocBlocks MCP content evaluation.
+
+${EVAL_PROMPT_PROFILES[profile]}
+
+Required execution contract:
+1. Use the configured DocBlocks MCP server and its current instructions. Call list_roots before save_artifact and use the returned writable root id.
+2. Create one ${testCase.targetFormat.toUpperCase()} artifact using editable-native fidelity and save it as exactly ${JSON.stringify(testCase.artifactFilename)} at the writable root. Do not replace an existing file.
+3. Validate the authored source for ${testCase.targetFormat} before final conversion and address actionable errors.
+4. Your final structured response must include the exact Markdown used as the source of the final convert_document call. Do not paraphrase or truncate it.
+5. Complete the task autonomously. Do not ask questions.
+
+Content brief:
+${testCase.brief}`;
+}
+function judgePrompt(testCase, markdown) {
+  const deterministic = judgeMarkdown(markdown, testCase);
+  return `Act as a strict, independent document-content evaluator. Judge only the supplied authored Markdown against the brief and rubric. Do not assume the Office artifact fixes missing, weak, or invented content. Treat Squisq template annotations as layout intent, not as evidence of quality by themselves.
+
+Score each dimension from 1 (unusable) to 5 (excellent). The overall score must reflect the lowest material weakness, not merely average the other values. Be concise and actionable.
+
+List every material claim that is not supported by the supplied brief in unsupportedClaims, quoting or closely identifying the claim. Calculations transparently derived from supplied numbers are supported; causal explanations, capabilities, commitments, owners, dates, benchmarks, and operational details are not facts unless supplied or clearly labeled as proposals or assumptions. List requested elements that are absent or materially incomplete in missingRequestedElements. Use an empty array when none are found.
+
+The deterministic observations below are authoritative. Do not recount them or claim that an observed value falls outside a range when the numbers show otherwise. The ranges are the harness acceptance envelope; continue to apply any narrower explicit target in the brief.
+
+Deterministic observations:
+- Word count: ${String(deterministic.metrics.wordCount)} (acceptance envelope ${testCase.expectation.minWords}-${testCase.expectation.maxWords})
+- Heading/section count: ${String(deterministic.metrics.headingCount)} (acceptance envelope ${testCase.expectation.minItems}-${testCase.expectation.maxItems})
+- Sections over ${testCase.expectation.maxWordsPerSection} words: ${String(deterministic.metrics.denseSectionCount)}
+- Non-content Squisq template annotations: ${String(deterministic.metrics.visualTemplateCount)} (minimum ${testCase.expectation.minVisualTemplates})
+- Deterministic acceptance: ${deterministic.passed ? "pass" : "fail"}
+
+Target format: ${testCase.targetFormat}
+
+Brief:
+${testCase.brief}
+
+Rubric:
+${testCase.rubric.map((item, index) => `${index + 1}. ${item}`).join("\n")}
+
+Authored Markdown:
+<document>
+${markdown}
+</document>`;
+}
+function extractTraceMarkdown(jsonl) {
+  const candidates = [];
+  for (const line of jsonl.split(/\r?\n/)) {
+    if (!line.trim()) continue;
+    let event;
+    try {
+      event = JSON.parse(line);
+    } catch {
+      continue;
+    }
+    collectToolMarkdown(event, candidates);
+  }
+  let selected = null;
+  for (const candidate of candidates) {
+    if (!selected || candidate.priority >= selected.priority) selected = candidate;
+  }
+  return selected?.markdown ?? null;
+}
+function extractTokenUsage(jsonl) {
+  let usage = null;
+  for (const line of jsonl.split(/\r?\n/)) {
+    if (!line.trim()) continue;
+    let value;
+    try {
+      value = JSON.parse(line);
+    } catch {
+      continue;
+    }
+    if (!isRecord2(value) || value.type !== "turn.completed" || !isRecord2(value.usage)) continue;
+    const candidate = value.usage;
+    if (typeof candidate.input_tokens === "number" && typeof candidate.cached_input_tokens === "number" && typeof candidate.output_tokens === "number" && typeof candidate.reasoning_output_tokens === "number") {
+      usage = {
+        inputTokens: candidate.input_tokens,
+        cachedInputTokens: candidate.cached_input_tokens,
+        outputTokens: candidate.output_tokens,
+        reasoningOutputTokens: candidate.reasoning_output_tokens
+      };
+    }
+  }
+  return usage;
+}
+function extractMcpTraceMetrics(jsonl) {
+  let toolCallCount = 0;
+  let toolArgumentCharacters = 0;
+  let toolResultCharacters = 0;
+  let authoringContextResultCharacters = 0;
+  let authoringContextTextCharacters = 0;
+  let authoringContextStructuredCharacters = 0;
+  for (const line of jsonl.split(/\r?\n/)) {
+    if (!line.trim()) continue;
+    let value;
+    try {
+      value = JSON.parse(line);
+    } catch {
+      continue;
+    }
+    if (!isRecord2(value) || value.type !== "item.completed" || !isRecord2(value.item)) continue;
+    const item = value.item;
+    if (item.type !== "mcp_tool_call" || typeof item.tool !== "string") continue;
+    toolCallCount += 1;
+    toolArgumentCharacters += serializedCharacters(item.arguments);
+    const resultCharacters = serializedCharacters(item.result);
+    toolResultCharacters += resultCharacters;
+    if (!item.tool.endsWith("get_authoring_context")) continue;
+    authoringContextResultCharacters += resultCharacters;
+    if (isRecord2(item.result)) {
+      authoringContextTextCharacters += serializedCharacters(item.result.content);
+      authoringContextStructuredCharacters += serializedCharacters(
+        item.result.structured_content ?? item.result.structuredContent
+      );
+    }
+  }
+  return {
+    toolCallCount,
+    toolArgumentCharacters,
+    toolResultCharacters,
+    authoringContextResultCharacters,
+    authoringContextTextCharacters,
+    authoringContextStructuredCharacters
+  };
+}
+function collectToolMarkdown(value, candidates) {
+  if (Array.isArray(value)) {
+    for (const item of value) collectToolMarkdown(item, candidates);
+    return;
+  }
+  if (!isRecord2(value)) return;
+  const toolName = [value.tool, value.name, value.tool_name].find((candidate) => typeof candidate === "string")?.toString();
+  if (toolName) {
+    const priority = toolName.endsWith("convert_document") ? 3 : toolName.endsWith("create_document_bundle") ? 2 : toolName.endsWith("validate_document") ? 1 : 0;
+    if (priority > 0) {
+      for (const args of [value.arguments, value.args, value.input]) {
+        const parsed = parsePossibleJson(args);
+        const markdown = findMarkdown(parsed);
+        if (markdown) candidates.push({ priority, markdown });
+      }
+    }
+  }
+  for (const child of Object.values(value)) collectToolMarkdown(child, candidates);
+}
+function findMarkdown(value) {
+  if (Array.isArray(value)) {
+    for (const item of value) {
+      const found = findMarkdown(item);
+      if (found) return found;
+    }
+    return null;
+  }
+  if (!isRecord2(value)) return null;
+  if (typeof value.markdown === "string" && value.markdown.trim()) return value.markdown;
+  for (const child of Object.values(value)) {
+    const found = findMarkdown(child);
+    if (found) return found;
+  }
+  return null;
+}
+function parsePossibleJson(value) {
+  if (typeof value !== "string") return value;
+  try {
+    return JSON.parse(value);
+  } catch {
+    return value;
+  }
+}
+function serializedCharacters(value) {
+  if (value === void 0) return 0;
+  const normalized = parsePossibleJson(value);
+  const serialized = JSON.stringify(normalized);
+  return serialized?.length ?? 0;
+}
+function tomlString(value) {
+  return JSON.stringify(value);
+}
+function tomlArray(values) {
+  return `[${values.map(tomlString).join(",")}]`;
+}
+function requireSuccessfulCodex(capture, stage) {
+  if (capture.exitCode === 0) return;
+  const tail = capture.stderr.slice(-4e3);
+  throw new Error(`Codex ${stage} exited with code ${capture.exitCode}: ${tail}`);
+}
+function isRecord2(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+// src/eval/runner.ts
+async function runEval(options) {
+  const startedAt = /* @__PURE__ */ new Date();
+  const runId = `${timestampId(startedAt)}-${sanitizeLabel(options.label)}`;
+  const directory = path4.join(options.reportsDirectory, runId);
+  await mkdir(options.reportsDirectory, { recursive: true });
+  await mkdir(directory, { recursive: false });
+  const codexVersion = await getCodexVersion(options.codexCommand, options.repoRoot);
+  const git = await getGitProvenance(options.repoRoot);
+  const cliSha256 = sha256(await readFile4(options.cliEntry));
+  const profileSha256 = sha256(EVAL_PROMPT_PROFILES[options.profile]);
+  const judgeContractSha256 = sha256(
+    `${LLM_JUDGE_CONTRACT_VERSION}
+${JSON.stringify(LLM_JUDGE_OUTPUT_SCHEMA)}`
+  );
+  const host = {
+    codexCommand: options.codexCommand,
+    model: options.model,
+    repoRoot: options.repoRoot,
+    cliEntry: options.cliEntry,
+    profile: options.profile
+  };
+  const results = [];
+  for (const [index, testCase] of options.cases.entries()) {
+    process.stdout.write(
+      `[${index + 1}/${options.cases.length}] ${testCase.id}: generating ${testCase.targetFormat.toUpperCase()}
+`
+    );
+    const caseDirectory = path4.join(directory, "cases", testCase.id);
+    await mkdir(caseDirectory, { recursive: true });
+    const result = await runOneCase(host, options, testCase, caseDirectory);
+    results.push(result);
+    await writeFile3(
+      path4.join(caseDirectory, "case-result.json"),
+      `${JSON.stringify(result, null, 2)}
+`
+    );
+    process.stdout.write(
+      `[${index + 1}/${options.cases.length}] ${testCase.id}: ${result.score.toFixed(1)} (${result.passed ? "pass" : "fail"})
+`
+    );
+    if (options.failFast && !result.passed) break;
+  }
+  const completedAt = /* @__PURE__ */ new Date();
+  const provenance = {
+    schemaVersion: 1,
+    runId,
+    label: options.label,
+    startedAt: startedAt.toISOString(),
+    completedAt: completedAt.toISOString(),
+    suite: options.suite,
+    profile: options.profile,
+    judgeMode: options.judgeMode,
+    threshold: options.threshold,
+    model: options.model,
+    codexCommand: options.codexCommand,
+    codexVersion,
+    nodeVersion: process.version,
+    platform: `${process.platform}-${process.arch}`,
+    gitCommit: git.commit,
+    gitDirty: git.dirty,
+    cliEntry: options.cliEntry,
+    cliSha256,
+    promptProfileSha256: profileSha256,
+    judgeContractSha256
+  };
+  const run = {
+    provenance,
+    score: average(results.map((result) => result.score)),
+    passed: results.length === options.cases.length && results.every((result) => result.passed),
+    cases: results
+  };
+  await writeRunReport(directory, run);
+  return { directory, run };
+}
+async function runOneCase(host, options, testCase, caseDirectory) {
+  const expectedArtifactPath = path4.join(caseDirectory, testCase.artifactFilename);
+  try {
+    const generation = await runGeneration(host, testCase, caseDirectory);
+    const traceMarkdown = generation.traceMarkdown;
+    const markdown = traceMarkdown ?? generation.result.markdown;
+    const traceMarkdownMatchesFinal = traceMarkdown ? normalizeMarkdown(traceMarkdown) === normalizeMarkdown(generation.result.markdown) : null;
+    await writeFile3(path4.join(caseDirectory, "authored.md"), markdown);
+    const artifactJudge = await judgeArtifact(expectedArtifactPath, testCase);
+    const markdownJudge = judgeMarkdown(markdown, testCase);
+    await Promise.all([
+      writeFile3(
+        path4.join(caseDirectory, "artifact-static-judge.json"),
+        `${JSON.stringify(artifactJudge, null, 2)}
+`
+      ),
+      writeFile3(
+        path4.join(caseDirectory, "markdown-static-judge.json"),
+        `${JSON.stringify(markdownJudge, null, 2)}
+`
+      )
+    ]);
+    let llmJudge = null;
+    let llmScore = null;
+    let judgeDurationMs = null;
+    let judgeUsage = null;
+    if (options.judgeMode === "both") {
+      process.stdout.write(`[judge] ${testCase.id}: evaluating authored Markdown
+`);
+      const judged = await runLlmJudge(host, testCase, markdown, caseDirectory);
+      llmJudge = judged.result;
+      llmScore = scoreLlmJudge(llmJudge);
+      judgeDurationMs = judged.process.durationMs;
+      judgeUsage = judged.usage;
+    }
+    const score = combinedScore(artifactJudge.score, markdownJudge.score, llmScore);
+    const passed = artifactJudge.passed && markdownJudge.passed && score >= options.threshold && traceMarkdownMatchesFinal !== false;
+    return {
+      caseId: testCase.id,
+      title: testCase.title,
+      targetFormat: testCase.targetFormat,
+      passed,
+      score,
+      artifactScore: artifactJudge.score,
+      markdownScore: markdownJudge.score,
+      llmScore,
+      markdownSource: traceMarkdown ? "trace" : "final-response",
+      traceMarkdownMatchesFinal,
+      artifactPath: expectedArtifactPath,
+      caseDirectory,
+      generationDurationMs: generation.process.durationMs,
+      judgeDurationMs,
+      generationUsage: generation.usage,
+      judgeUsage,
+      mcpTraceMetrics: generation.traceMetrics,
+      artifactJudge,
+      markdownJudge,
+      llmJudge,
+      failure: traceMarkdownMatchesFinal === false ? "Markdown in the final response does not match the Markdown captured from the final conversion tool call" : null
+    };
+  } catch (caught) {
+    const message = caught instanceof Error ? caught.stack ?? caught.message : String(caught);
+    const failedJudge = failureJudge(message);
+    return {
+      caseId: testCase.id,
+      title: testCase.title,
+      targetFormat: testCase.targetFormat,
+      passed: false,
+      score: 0,
+      artifactScore: 0,
+      markdownScore: 0,
+      llmScore: null,
+      markdownSource: "final-response",
+      traceMarkdownMatchesFinal: null,
+      artifactPath: expectedArtifactPath,
+      caseDirectory,
+      generationDurationMs: 0,
+      judgeDurationMs: null,
+      generationUsage: null,
+      judgeUsage: null,
+      mcpTraceMetrics: null,
+      artifactJudge: failedJudge,
+      markdownJudge: failedJudge,
+      llmJudge: null,
+      failure: message
+    };
+  }
+}
+function combinedScore(artifactScore, markdownScore, llmScore) {
+  const components = [
+    { score: artifactScore, weight: 0.35 },
+    { score: markdownScore, weight: 0.25 },
+    ...llmScore === null ? [] : [{ score: llmScore, weight: 0.4 }]
+  ];
+  const totalWeight = components.reduce((sum, component) => sum + component.weight, 0);
+  return round2(
+    components.reduce((sum, component) => sum + component.score * component.weight, 0) / totalWeight
+  );
+}
+function scoreLlmJudge(result) {
+  const scores = result.scores;
+  const weighted = scores.briefAdherence + scores.contentQuality + scores.structure + scores.formatFitness + scores.squisqUsage + scores.overall * 2;
+  const qualitativeScore = weighted / 35 * 100;
+  const groundingPenalty = Math.min(
+    40,
+    result.unsupportedClaims.length * 3 + result.missingRequestedElements.length * 5
+  );
+  return round2(Math.max(0, qualitativeScore - groundingPenalty));
+}
+async function getCodexVersion(command, cwd) {
+  const resolved = await resolveCodexCommand(command);
+  try {
+    const capture = await runCapturedProcess({
+      command: resolved.command,
+      args: [...resolved.prefixArgs, "--version"],
+      cwd,
+      timeoutMs: 1e4
+    });
+    return capture.exitCode === 0 ? capture.stdout.trim() : "unknown";
+  } catch {
+    return "unknown";
+  }
+}
+async function getGitProvenance(cwd) {
+  try {
+    const [commit, status] = await Promise.all([
+      runCapturedProcess({
+        command: "git",
+        args: ["rev-parse", "HEAD"],
+        cwd,
+        timeoutMs: 1e4
+      }),
+      runCapturedProcess({
+        command: "git",
+        args: ["status", "--porcelain"],
+        cwd,
+        timeoutMs: 1e4
+      })
+    ]);
+    return {
+      commit: commit.exitCode === 0 ? commit.stdout.trim() : null,
+      dirty: status.exitCode === 0 ? status.stdout.trim().length > 0 : null
+    };
+  } catch {
+    return { commit: null, dirty: null };
+  }
+}
+function failureJudge(message) {
+  return summarizeChecks(
+    [
+      {
+        id: "harness-execution",
+        passed: false,
+        required: true,
+        message,
+        observed: false,
+        expected: true
+      }
+    ],
+    {}
+  );
+}
+function normalizeMarkdown(markdown) {
+  return markdown.replace(/\r\n/g, "\n").trim();
+}
+function sha256(value) {
+  return createHash("sha256").update(value).digest("hex");
+}
+function timestampId(date) {
+  return date.toISOString().replace(/[-:.]/g, "");
+}
+function sanitizeLabel(label) {
+  const sanitized = label.toLowerCase().replace(/[^a-z0-9_-]+/g, "-").replace(/^-+|-+$/g, "");
+  if (!sanitized) throw new Error("Eval label must contain at least one letter or number");
+  return sanitized.slice(0, 64);
+}
+function average(values) {
+  return values.length === 0 ? 0 : round2(values.reduce((sum, value) => sum + value, 0) / values.length);
+}
+function round2(value) {
+  return Math.round(value * 10) / 10;
+}
+
+// src/eval/cli.ts
+var program = new Command().name("docblocks-mcp-eval").description(
+  "Run reproducible Codex content evaluations against the locally built DocBlocks MCP"
+);
+program.command("list").description("List canonical content evaluation cases").action(() => {
+  for (const testCase of MCP_CONTENT_EVAL_CASES) {
+    process.stdout.write(
+      `${testCase.id}	${testCase.targetFormat}	${testCase.suites.join(",")}	${testCase.title}
+`
+    );
+  }
+});
+program.command("run").description("Build one isolated eval run and preserve traces, artifacts, judges, and provenance").option("--suite <suite-or-ids>", "quick, full, or comma-separated case ids", "quick").option("--label <label>", "run label", "baseline").option("--profile <profile>", "baseline or content-first", "baseline").option("--judge <mode>", "static or both", "both").option("--threshold <score>", "passing score from 0 through 100", "70").option("--model <model>", "pin a Codex model; defaults to the Codex CLI default").option(
+  "--codex-command <command>",
+  "Codex executable",
+  process.platform === "win32" ? "codex.cmd" : "codex"
+).option("--repo-root <directory>", "DocBlocks checkout", process.cwd()).option("--cli-entry <file>", "built DocBlocks CLI entry").option("--reports-dir <directory>", "eval report root").option("--fail-fast", "stop after the first failing case", false).option("--enforce", "exit nonzero when the run misses its threshold", false).action(async (raw) => {
+  const repoRoot = path5.resolve(raw.repoRoot);
+  const profile = requireProfile(raw.profile);
+  const judgeMode = requireJudgeMode(raw.judge);
+  const threshold = requireThreshold(raw.threshold);
+  const cliEntry = path5.resolve(
+    raw.cliEntry ?? path5.join(repoRoot, "packages", "cli", "dist", "bin.js")
+  );
+  const reportsDirectory = path5.resolve(
+    raw.reportsDir ?? path5.join(repoRoot, "reports", "mcp-content-evals")
+  );
+  const result = await runEval({
+    label: raw.label,
+    suite: raw.suite,
+    cases: selectEvalCases(raw.suite),
+    profile,
+    judgeMode,
+    threshold,
+    model: raw.model ?? null,
+    codexCommand: raw.codexCommand,
+    repoRoot,
+    cliEntry,
+    reportsDirectory,
+    failFast: raw.failFast
+  });
+  process.stdout.write(`Run report: ${result.directory}
+`);
+  process.stdout.write(
+    `Aggregate score: ${result.run.score.toFixed(1)} (${result.run.passed ? "pass" : "fail"})
+`
+  );
+  if (raw.enforce && !result.run.passed) process.exitCode = 1;
+});
+program.command("compare").description("Compare paired cases from a baseline and candidate run").argument("<baseline>", "baseline run directory or run.json").argument("<candidate>", "candidate run directory or run.json").option("--output <directory>", "comparison output directory").option(
+  "--enforce-no-regression",
+  "exit nonzero when any paired case regresses or changes from pass to fail",
+  false
+).action(async (baselinePath, candidatePath, raw) => {
+  const [baseline, candidate] = await Promise.all([
+    readEvalRun(path5.resolve(baselinePath)),
+    readEvalRun(path5.resolve(candidatePath))
+  ]);
+  const comparison = compareEvalRuns(baseline, candidate);
+  const output = path5.resolve(
+    raw.output ?? path5.join(
+      path5.extname(candidatePath) === ".json" ? path5.dirname(candidatePath) : candidatePath,
+      `comparison-vs-${baseline.provenance.runId}`
+    )
+  );
+  await import("fs/promises").then(({ mkdir: mkdir2 }) => mkdir2(output, { recursive: true }));
+  await writeComparisonReport(output, comparison);
+  process.stdout.write(`Comparison report: ${output}
+`);
+  process.stdout.write(
+    `Aggregate delta: ${comparison.delta >= 0 ? "+" : ""}${comparison.delta.toFixed(1)}
+`
+  );
+  process.stdout.write(`Case regressions: ${comparison.regressions}
+`);
+  if (raw.enforceNoRegression && comparison.regressions > 0) process.exitCode = 1;
+});
+function requireProfile(value) {
+  if (value in EVAL_PROMPT_PROFILES) return value;
+  throw new Error(`Unknown prompt profile ${JSON.stringify(value)}`);
+}
+function requireJudgeMode(value) {
+  if (value === "static" || value === "both") return value;
+  throw new Error(`Judge mode must be static or both; received ${JSON.stringify(value)}`);
+}
+function requireThreshold(value) {
+  const parsed = Number(value);
+  if (!Number.isFinite(parsed) || parsed < 0 || parsed > 100) {
+    throw new Error(`Threshold must be between 0 and 100; received ${JSON.stringify(value)}`);
+  }
+  return parsed;
+}
+void program.parseAsync().catch((caught) => {
+  const message = caught instanceof Error ? caught.stack ?? caught.message : String(caught);
+  process.stderr.write(`${message}
+`);
+  process.exitCode = 1;
+});