@xera-ai/core 0.1.6 → 0.3.0

package/src/bin-internal/doctor.ts

@@ -0,0 +1,133 @@
+import { existsSync, readFileSync, readdirSync } from 'node:fs';
+import { join } from 'node:path';
+import type { Stage } from '../eval/types';
+import { verifyPrompts } from './verify-prompts';
+
+export interface DoctorOpts {
+  cwd?: string;
+}
+
+interface CheckResult {
+  ok: boolean;
+  message: string;
+}
+
+const REQUIRED_FILES_PER_STAGE: Record<Stage, string[]> = {
+  'feature-from-story': ['golden/test.feature'],
+  'script-from-feature': ['golden/spec-requirements.md'],
+  'diagnose-failure': [],
+};
+
+const REQUIRED_SCRIPTS = [
+  'xera:eval-prepare',
+  'xera:eval-deterministic',
+  'xera:eval-report',
+  'xera:verify-prompts',
+  'xera:doctor',
+];
+
+function frontmatterField(content: string, field: string): string | null {
+  const m = content.match(new RegExp(`^${field}:\\s*(\\S+)\\s*$`, 'm'));
+  return m?.[1] ?? null;
+}
+
+function checkGoldenEvalDir(repoRoot: string): CheckResult[] {
+  const root = join(repoRoot, 'fixtures/golden-eval');
+  if (!existsSync(root)) return [{ ok: false, message: 'fixtures/golden-eval/ does not exist' }];
+  const dirs = readdirSync(root, { withFileTypes: true }).filter(
+    (e) => e.isDirectory() && !e.name.startsWith('.'),
+  );
+  const results: CheckResult[] = [];
+  if (dirs.length < 3) {
+    results.push({
+      ok: false,
+      message: `fixtures/golden-eval/ has ${dirs.length} ticket dir(s); need ≥ 3`,
+    });
+  }
+  for (const entry of dirs) {
+    const dir = join(root, entry.name);
+    const metaPath = join(dir, 'meta.json');
+    if (!existsSync(metaPath)) {
+      results.push({ ok: false, message: `${entry.name}: meta.json missing` });
+      continue;
+    }
+    let meta: { id?: string; stages?: unknown[] };
+    try {
+      meta = JSON.parse(readFileSync(metaPath, 'utf8')) as { id?: string; stages?: unknown[] };
+    } catch (err) {
+      results.push({
+        ok: false,
+        message: `${entry.name}: meta.json parse error: ${(err as Error).message}`,
+      });
+      continue;
+    }
+    const stages = Array.isArray(meta.stages) ? (meta.stages as Stage[]) : [];
+    if (stages.length === 0)
+      results.push({ ok: false, message: `${entry.name}: meta.stages is empty` });
+    if (!existsSync(join(dir, 'story.md')))
+      results.push({ ok: false, message: `${entry.name}: story.md missing` });
+    for (const stage of stages) {
+      const required = REQUIRED_FILES_PER_STAGE[stage] ?? [];
+      for (const rel of required) {
+        if (!existsSync(join(dir, rel))) {
+          results.push({
+            ok: false,
+            message: `${meta.id ?? entry.name}: stage "${stage}" declared but ${rel} missing`,
+          });
+        }
+      }
+    }
+  }
+  return results;
+}
+
+function checkRubricPrompt(repoRoot: string): CheckResult[] {
+  const path = join(repoRoot, 'packages/prompts/eval-rubric.md');
+  if (!existsSync(path)) return [{ ok: false, message: 'packages/prompts/eval-rubric.md missing' }];
+  const text = readFileSync(path, 'utf8');
+  const id = frontmatterField(text, 'id');
+  const version = frontmatterField(text, 'version');
+  if (id !== 'eval-rubric')
+    return [{ ok: false, message: 'eval-rubric.md frontmatter "id" must be "eval-rubric"' }];
+  if (!version) return [{ ok: false, message: 'eval-rubric.md frontmatter "version" missing' }];
+  return [];
+}
+
+function checkEvalSkill(repoRoot: string): CheckResult[] {
+  const path = join(repoRoot, 'packages/skills/xera-eval.md');
+  if (!existsSync(path)) return [{ ok: false, message: 'packages/skills/xera-eval.md missing' }];
+  const text = readFileSync(path, 'utf8');
+  if (!frontmatterField(text, 'name'))
+    return [{ ok: false, message: 'xera-eval.md frontmatter "name" missing' }];
+  return [];
+}
+
+function checkPromptInjectionPreamble(repoRoot: string): CheckResult[] {
+  return verifyPrompts(repoRoot);
+}
+
+function checkRootScripts(repoRoot: string): CheckResult[] {
+  const path = join(repoRoot, 'package.json');
+  if (!existsSync(path)) return [{ ok: false, message: 'root package.json missing' }];
+  const pkg = JSON.parse(readFileSync(path, 'utf8'));
+  const scripts = pkg.scripts ?? {};
+  const missing = REQUIRED_SCRIPTS.filter((s) => typeof scripts[s] !== 'string');
+  return missing.map((s) => ({ ok: false, message: `root package.json missing script: ${s}` }));
+}
+
+export async function doctorCmd(_argv: string[], opts: DoctorOpts = {}): Promise<number> {
+  const repoRoot = opts.cwd ?? process.cwd();
+  const results: CheckResult[] = [
+    ...checkGoldenEvalDir(repoRoot),
+    ...checkRubricPrompt(repoRoot),
+    ...checkEvalSkill(repoRoot),
+    ...checkPromptInjectionPreamble(repoRoot),
+    ...checkRootScripts(repoRoot),
+  ];
+  if (results.length === 0) {
+    console.log('[xera:doctor] ok');
+    return 0;
+  }
+  for (const r of results) console.error(`[xera:doctor] ${r.message}`);
+  return 1;
+}

package/src/bin-internal/eval-deterministic.ts

@@ -0,0 +1,149 @@
+import { existsSync, readFileSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { validateGherkin } from '@xera-ai/web';
+import { resolveEvalPaths } from '../eval/paths';
+import {
+  type DeterministicEntry,
+  type DeterministicScores,
+  DeterministicScoresSchema,
+  ManifestSchema,
+} from '../eval/types';
+
+export interface EvalDeterministicOpts {
+  cwd?: string;
+}
+
+interface ClassifierScenario {
+  name: string;
+  class: string;
+}
+
+function checkFeatureFromStory(actualFeaturePath: string): {
+  passed: boolean;
+  checks: string[];
+  error?: string;
+} {
+  if (!existsSync(actualFeaturePath)) {
+    return { passed: false, checks: ['validate-feature'], error: 'actual missing: test.feature' };
+  }
+  try {
+    const r = validateGherkin(readFileSync(actualFeaturePath, 'utf8'));
+    if (r.ok) return { passed: true, checks: ['validate-feature'] };
+    return {
+      passed: false,
+      checks: ['validate-feature'],
+      error: r.errors.map((e) => `line ${e.line}: ${e.message}`).join('; '),
+    };
+  } catch (err) {
+    return { passed: false, checks: ['validate-feature'], error: (err as Error).message };
+  }
+}
+
+function checkScriptFromFeature(actualTicketDir: string): {
+  passed: boolean;
+  checks: string[];
+  error?: string;
+} {
+  // v0.2 deviation: file-presence only. Full typecheck/lint hookup is deferred to v0.2.1
+  // because v0.1's lintTicket/typecheckTicket resolve paths from .xera/<TICKET>/ (consumer
+  // project layout), not .xera/eval/<run-id>/actual/<ticket>/. The judge dimensions
+  // "Requirements satisfied", "Wait strategy", "No dead code" cover the lint surface.
+  const specPath = join(actualTicketDir, 'spec.ts');
+  if (!existsSync(specPath)) {
+    return { passed: false, checks: ['file-presence'], error: 'actual missing: spec.ts' };
+  }
+  return { passed: true, checks: ['file-presence'] };
+}
+
+function checkDiagnoseFailure(
+  inputsTicketDir: string,
+  actualTicketDir: string,
+): { passed: boolean; checks: string[]; error?: string } {
+  const inputPath = join(inputsTicketDir, 'classifier-input.json');
+  const actualPath = join(actualTicketDir, 'classification.json');
+  if (!existsSync(actualPath)) {
+    return {
+      passed: false,
+      checks: ['bucket-match'],
+      error: 'actual missing: classification.json',
+    };
+  }
+  if (!existsSync(inputPath)) {
+    return {
+      passed: false,
+      checks: ['bucket-match'],
+      error: 'inputs missing: classifier-input.json',
+    };
+  }
+  const golden = JSON.parse(readFileSync(inputPath, 'utf8'));
+  const actual = JSON.parse(readFileSync(actualPath, 'utf8'));
+  const goldScens: ClassifierScenario[] = golden.scenarios ?? [];
+  const actScens: ClassifierScenario[] = actual.scenarios ?? [];
+  const mismatches: string[] = [];
+  for (const g of goldScens) {
+    const a = actScens.find((s) => s.name === g.name);
+    if (!a) {
+      mismatches.push(`missing scenario "${g.name}"`);
+      continue;
+    }
+    if (a.class !== g.class)
+      mismatches.push(`scenario "${g.name}": expected class ${g.class}, got ${a.class}`);
+  }
+  if (mismatches.length > 0) {
+    return {
+      passed: false,
+      checks: ['bucket-match'],
+      error: `bucket mismatch — ${mismatches.join('; ')}`,
+    };
+  }
+  return { passed: true, checks: ['bucket-match'] };
+}
+
+export async function evalDeterministicCmd(
+  argv: string[],
+  opts: EvalDeterministicOpts = {},
+): Promise<number> {
+  const cwd = opts.cwd ?? process.cwd();
+  const runId = argv[0];
+  if (!runId) {
+    console.error('[xera:eval-deterministic] usage: eval-deterministic <run-id>');
+    return 1;
+  }
+  const paths = resolveEvalPaths(cwd, runId);
+  if (!existsSync(paths.manifest)) {
+    console.error(`[xera:eval-deterministic] missing manifest.json at ${paths.manifest}`);
+    return 1;
+  }
+  const manifest = ManifestSchema.parse(JSON.parse(readFileSync(paths.manifest, 'utf8')));
+
+  const entries: DeterministicEntry[] = [];
+  for (const [ticket, ticketStages] of Object.entries(manifest.ticket_stages)) {
+    for (const stage of ticketStages) {
+      const inputsDir = paths.ticketInputsDir(ticket);
+      const actualDir = paths.ticketActualDir(ticket);
+      let result: { passed: boolean; checks: string[]; error?: string };
+      if (stage === 'feature-from-story') {
+        result = checkFeatureFromStory(join(actualDir, 'test.feature'));
+      } else if (stage === 'script-from-feature') {
+        result = checkScriptFromFeature(actualDir);
+      } else {
+        result = checkDiagnoseFailure(inputsDir, actualDir);
+      }
+
+      const entry: DeterministicEntry = {
+        ticket,
+        stage,
+        passed: result.passed,
+        checks: result.checks,
+      };
+      if (result.error !== undefined) entry.error = result.error;
+      entries.push(entry);
+    }
+  }
+
+  const scores: DeterministicScores = { run_id: runId, entries };
+  DeterministicScoresSchema.parse(scores);
+  writeFileSync(paths.deterministicScores, JSON.stringify(scores, null, 2));
+  console.log(`[xera:eval-deterministic] wrote ${entries.length} entries`);
+  return 0;
+}

package/src/bin-internal/eval-prepare.ts

@@ -0,0 +1,214 @@
+import {
+  copyFileSync,
+  existsSync,
+  mkdirSync,
+  readFileSync,
+  readdirSync,
+  writeFileSync,
+} from 'node:fs';
+import { join } from 'node:path';
+import { resolveEvalPaths } from '../eval/paths';
+import { generateRunId } from '../eval/run-id';
+import { type Manifest, ManifestSchema, STAGES, type Stage } from '../eval/types';
+import { acquireLock } from '../lock/file-lock';
+
+export interface EvalPrepareOpts {
+  cwd?: string;
+  now?: () => Date;
+  getGitSha?: () => string | null;
+}
+
+interface ParsedFlags {
+  force: boolean;
+  only_prompt: Stage | null;
+  only_ticket: string | null;
+}
+
+function parseFlags(argv: string[]): ParsedFlags | { error: string } {
+  const flags: ParsedFlags = { force: false, only_prompt: null, only_ticket: null };
+  for (const arg of argv) {
+    if (arg === '--force') flags.force = true;
+    else if (arg.startsWith('--prompt=')) {
+      const v = arg.slice('--prompt='.length);
+      if (!STAGES.includes(v as Stage)) {
+        return { error: `Unknown stage: ${v}. Valid: ${STAGES.join(', ')}.` };
+      }
+      flags.only_prompt = v as Stage;
+    } else if (arg.startsWith('--ticket=')) {
+      flags.only_ticket = arg.slice('--ticket='.length);
+    } else {
+      return { error: `Unknown argument: ${arg}` };
+    }
+  }
+  return flags;
+}
+
+function readPromptVersion(repoRoot: string, name: string): string {
+  const path = join(repoRoot, 'packages/prompts', `${name}.md`);
+  if (!existsSync(path)) return '0.0.0';
+  const text = readFileSync(path, 'utf8');
+  const m = /^version:\s*(\S+)\s*$/m.exec(text);
+  return m?.[1] ?? '0.0.0';
+}
+
+function discoverEvalTickets(repoRoot: string): { id: string; dir: string; stages: Stage[] }[] {
+  const root = join(repoRoot, 'fixtures/golden-eval');
+  if (!existsSync(root)) return [];
+  const out: { id: string; dir: string; stages: Stage[] }[] = [];
+  for (const entry of readdirSync(root, { withFileTypes: true })) {
+    if (!entry.isDirectory()) continue;
+    if (entry.name === 'README.md' || entry.name.startsWith('.')) continue;
+    const dir = join(root, entry.name);
+    const metaPath = join(dir, 'meta.json');
+    if (!existsSync(metaPath)) continue;
+    const meta = JSON.parse(readFileSync(metaPath, 'utf8')) as { id: string; stages: Stage[] };
+    out.push({ id: meta.id, dir, stages: meta.stages });
+  }
+  return out.sort((a, b) => a.id.localeCompare(b.id));
+}
+
+function discoverClassifierTickets(repoRoot: string): { id: string; path: string }[] {
+  const root = join(repoRoot, 'fixtures/golden-tickets');
+  if (!existsSync(root)) return [];
+  const out: { id: string; path: string }[] = [];
+  for (const entry of readdirSync(root, { withFileTypes: true })) {
+    if (!entry.isFile() || !entry.name.endsWith('.json')) continue;
+    const path = join(root, entry.name);
+    const data = JSON.parse(readFileSync(path, 'utf8')) as { ticket?: string };
+    if (typeof data.ticket === 'string') out.push({ id: data.ticket, path });
+  }
+  return out.sort((a, b) => a.id.localeCompare(b.id));
+}
+
+export async function evalPrepareCmd(argv: string[], opts: EvalPrepareOpts = {}): Promise<number> {
+  const repoRoot = opts.cwd ?? process.cwd();
+
+  const flags = parseFlags(argv);
+  if ('error' in flags) {
+    console.error(`[xera:eval-prepare] ${flags.error}`);
+    return 1;
+  }
+
+  const evalTickets = discoverEvalTickets(repoRoot);
+  const classifierTickets = discoverClassifierTickets(repoRoot);
+
+  // Determine which stages to run.
+  const stages: Stage[] = flags.only_prompt ? [flags.only_prompt] : [...STAGES];
+
+  // Determine which tickets are relevant.
+  const wantsEval = stages.some((s) => s !== 'diagnose-failure');
+  const wantsClassifier = stages.includes('diagnose-failure');
+
+  let selectedTickets: string[] = [];
+  if (wantsEval) selectedTickets.push(...evalTickets.map((t) => t.id));
+  if (wantsClassifier) selectedTickets.push(...classifierTickets.map((t) => t.id));
+  selectedTickets = [...new Set(selectedTickets)].sort();
+
+  if (flags.only_ticket) {
+    if (!selectedTickets.includes(flags.only_ticket)) {
+      console.error(`[xera:eval-prepare] No golden fixture for ${flags.only_ticket}`);
+      return 1;
+    }
+    selectedTickets = [flags.only_ticket];
+  }
+
+  if (selectedTickets.length === 0) {
+    console.error('[xera:eval-prepare] No tickets selected (after filters).');
+    return 1;
+  }
+
+  // Compute per-ticket stages by intersecting the ticket's declared stages with the
+  // global stages array, then filter out tickets with an empty intersection.
+  const ticket_stages: Record<string, Stage[]> = {};
+  for (const ticket of selectedTickets) {
+    const evalT = evalTickets.find((t) => t.id === ticket);
+    let ticketDeclared: Stage[];
+    if (evalT) {
+      ticketDeclared = evalT.stages;
+    } else {
+      // Classifier/GOLD ticket — always diagnose-failure.
+      ticketDeclared = ['diagnose-failure'];
+    }
+    const intersection = ticketDeclared.filter((s) => stages.includes(s));
+    if (intersection.length > 0) {
+      ticket_stages[ticket] = intersection;
+    }
+  }
+
+  // Filter selectedTickets to only those with applicable stages.
+  selectedTickets = selectedTickets.filter((t) => ticket_stages[t] !== undefined);
+
+  if (selectedTickets.length === 0) {
+    console.error('[xera:eval-prepare] No tickets applicable to requested stages.');
+    return 1;
+  }
+
+  const runId = generateRunId({
+    ...(opts.now ? { now: opts.now } : {}),
+    ...(opts.getGitSha ? { getGitSha: opts.getGitSha } : {}),
+  });
+  const paths = resolveEvalPaths(repoRoot, runId);
+
+  if (existsSync(paths.root) && !flags.force) {
+    console.error(
+      `[xera:eval-prepare] run dir already exists: ${paths.root}. Pass --force to re-run.`,
+    );
+    return 1;
+  }
+  mkdirSync(paths.inputsDir, { recursive: true });
+  mkdirSync(paths.actualDir, { recursive: true });
+
+  // Copy inputs.
+  for (const ticket of selectedTickets) {
+    const ticketInputs = paths.ticketInputsDir(ticket);
+    mkdirSync(ticketInputs, { recursive: true });
+    const evalT = evalTickets.find((t) => t.id === ticket);
+    const classT = classifierTickets.find((t) => t.id === ticket);
+    if (evalT) {
+      copyFileSync(join(evalT.dir, 'story.md'), join(ticketInputs, 'story.md'));
+      const featurePath = join(evalT.dir, 'golden/test.feature');
+      if (existsSync(featurePath)) copyFileSync(featurePath, join(ticketInputs, 'test.feature'));
+    }
+    if (classT) {
+      copyFileSync(classT.path, join(ticketInputs, 'classifier-input.json'));
+    }
+  }
+
+  // Build manifest.
+  const now = (opts.now ?? (() => new Date()))();
+  const manifest: Manifest = {
+    run_id: runId,
+    started_at: now.toISOString(),
+    git_sha: runId.split('-')[2] ?? 'nogit',
+    tickets: selectedTickets,
+    stages,
+    ticket_stages,
+    prompt_versions: {
+      'feature-from-story': readPromptVersion(repoRoot, 'feature-from-story'),
+      'script-from-feature': readPromptVersion(repoRoot, 'script-from-feature'),
+      'diagnose-failure': readPromptVersion(repoRoot, 'diagnose-failure'),
+      'eval-rubric': readPromptVersion(repoRoot, 'eval-rubric'),
+    },
+    flags: {
+      force: flags.force,
+      only_prompt: flags.only_prompt,
+      only_ticket: flags.only_ticket,
+      judge_only: false,
+    },
+  };
+
+  // Validate before writing.
+  ManifestSchema.parse(manifest);
+  writeFileSync(paths.manifest, JSON.stringify(manifest, null, 2));
+
+  if (!acquireLock(paths.lock, runId)) {
+    console.error(`[xera:eval-prepare] failed to acquire lock at ${paths.lock}`);
+    return 4;
+  }
+
+  console.log(
+    `[xera:eval-prepare] prepared ${selectedTickets.length} ticket(s) for stages: ${stages.join(', ')}`,
+  );
+  console.log(`RUN_ID=${runId}`);
+  return 0;
+}

package/src/bin-internal/eval-report.ts

@@ -0,0 +1,177 @@
+import { existsSync, readFileSync, writeFileSync } from 'node:fs';
+import { resolveEvalPaths } from '../eval/paths';
+import {
+  type DeterministicScores,
+  DeterministicScoresSchema,
+  type JudgeScores,
+  JudgeScoresSchema,
+  type Judgment,
+  ManifestSchema,
+  type Result,
+  type Summary,
+  SummarySchema,
+} from '../eval/types';
+import { releaseLock } from '../lock/file-lock';
+
+export interface EvalReportOpts {
+  cwd?: string;
+}
+
+function scoreJudgment(j: Judgment): { passed: boolean; score: number } {
+  const nonNa = j.dimensions.filter((d) => d.verdict !== 'NA');
+  if (nonNa.length === 0) return { passed: true, score: 1 };
+  const passes = nonNa.filter((d) => d.verdict === 'PASS').length;
+  const score = passes / nonNa.length;
+  const passed = nonNa.every((d) => d.verdict === 'PASS');
+  return { passed, score };
+}
+
+function renderReport(summary: Summary): string {
+  const lines: string[] = [];
+  lines.push(`# xera eval report ${summary.run_id}`);
+  lines.push('');
+  lines.push(`**Git SHA:** \`${summary.git_sha}\``);
+  lines.push('');
+  lines.push('**Prompt versions:**');
+  for (const [k, v] of Object.entries(summary.prompt_versions)) lines.push(`- \`${k}\`: ${v}`);
+  lines.push('');
+  lines.push(
+    `**Overall:** ${summary.overall.passed}/${summary.overall.total} PASS (score ${(summary.overall.score * 100).toFixed(0)}%)`,
+  );
+  lines.push('');
+  lines.push('## Results');
+  lines.push('');
+  lines.push('| Ticket | Stage | Deterministic | Judge | Score |');
+  lines.push('|---|---|---|---|---|');
+  for (const r of summary.results) {
+    const det = r.deterministic.passed ? 'PASS' : `FAIL (${r.deterministic.error ?? ''})`;
+    const judge = r.skipped ? 'SKIPPED' : r.judge ? (r.judge.passed ? 'PASS' : 'FAIL') : 'SKIPPED';
+    const score = r.judge ? `${(r.judge.score * 100).toFixed(0)}%` : '—';
+    lines.push(`| ${r.ticket} | ${r.stage} | ${det} | ${judge} | ${score} |`);
+  }
+  lines.push('');
+  lines.push('## Dimension breakdown');
+  lines.push('');
+  for (const r of summary.results) {
+    if (!r.judge || r.judge.dimensions.length === 0) continue;
+    lines.push(`### ${r.ticket} — ${r.stage}`);
+    lines.push('');
+    for (const d of r.judge.dimensions) lines.push(`- **${d.name}** — ${d.verdict}: ${d.notes}`);
+    lines.push('');
+  }
+  return lines.join('\n');
+}
+
+export async function evalReportCmd(argv: string[], opts: EvalReportOpts = {}): Promise<number> {
+  const cwd = opts.cwd ?? process.cwd();
+  const runId = argv[0];
+  if (!runId) {
+    console.error('[xera:eval-report] usage: eval-report <run-id>');
+    return 1;
+  }
+  const paths = resolveEvalPaths(cwd, runId);
+  if (!existsSync(paths.manifest)) {
+    console.error(`[xera:eval-report] missing manifest.json at ${paths.manifest}`);
+    return 1;
+  }
+  const manifest = ManifestSchema.parse(JSON.parse(readFileSync(paths.manifest, 'utf8')));
+
+  try {
+    let det: DeterministicScores;
+    let judge: JudgeScores;
+    try {
+      det = DeterministicScoresSchema.parse(
+        JSON.parse(readFileSync(paths.deterministicScores, 'utf8')),
+      );
+    } catch (err) {
+      console.error(
+        `[xera:eval-report] invalid deterministic-scores.json: ${(err as Error).message}`,
+      );
+      return 2;
+    }
+    try {
+      judge = JudgeScoresSchema.parse(JSON.parse(readFileSync(paths.judgeScores, 'utf8')));
+    } catch (err) {
+      console.error(`[xera:eval-report] invalid judge-scores.json: ${(err as Error).message}`);
+      return 2;
+    }
+
+    const results: Result[] = [];
+    for (const detEntry of det.entries) {
+      const judgment = judge.judgments.find(
+        (j) => j.ticket === detEntry.ticket && j.stage === detEntry.stage,
+      );
+      if (!judgment && detEntry.error?.startsWith('actual missing')) {
+        const r: Result = {
+          ticket: detEntry.ticket,
+          stage: detEntry.stage,
+          deterministic: {
+            passed: detEntry.passed,
+            checks: detEntry.checks,
+            ...(detEntry.error !== undefined ? { error: detEntry.error } : {}),
+          },
+          judge: null,
+          skipped: true,
+        };
+        results.push(r);
+        continue;
+      }
+      if (!judgment) {
+        // Judge entry expected but missing: count as FAIL not SKIPPED.
+        const r: Result = {
+          ticket: detEntry.ticket,
+          stage: detEntry.stage,
+          deterministic: {
+            passed: detEntry.passed,
+            checks: detEntry.checks,
+            ...(detEntry.error !== undefined ? { error: detEntry.error } : {}),
+          },
+          judge: { passed: false, dimensions: [], score: 0 },
+        };
+        results.push(r);
+        continue;
+      }
+      const { passed, score } = scoreJudgment(judgment);
+      const r: Result = {
+        ticket: detEntry.ticket,
+        stage: detEntry.stage,
+        deterministic: {
+          passed: detEntry.passed,
+          checks: detEntry.checks,
+          ...(detEntry.error !== undefined ? { error: detEntry.error } : {}),
+        },
+        judge: { passed, dimensions: judgment.dimensions, score },
+      };
+      results.push(r);
+    }
+
+    const counted = results.filter((r) => !r.skipped);
+    const passedCount = counted.filter((r) => r.deterministic.passed && r.judge?.passed).length;
+    const failedCount = counted.length - passedCount;
+    const avgScore =
+      counted.length === 0
+        ? 0
+        : counted.reduce(
+            (acc, r) => acc + (r.deterministic.passed && r.judge ? r.judge.score : 0),
+            0,
+          ) / counted.length;
+
+    const summary: Summary = {
+      run_id: runId,
+      git_sha: manifest.git_sha,
+      prompt_versions: manifest.prompt_versions,
+      results,
+      overall: { passed: passedCount, failed: failedCount, total: counted.length, score: avgScore },
+    };
+    SummarySchema.parse(summary);
+    writeFileSync(paths.summary, JSON.stringify(summary, null, 2));
+    writeFileSync(paths.report, renderReport(summary));
+
+    console.log(
+      `[xera:eval-report] ${passedCount}/${counted.length} PASS (avg ${(avgScore * 100).toFixed(0)}%)`,
+    );
+    return 0;
+  } finally {
+    releaseLock(paths.lock);
+  }
+}