@xera-ai/core 0.1.7 → 0.4.0

package/src/bin-internal/eval-report.ts

@@ -0,0 +1,177 @@
+import { existsSync, readFileSync, writeFileSync } from 'node:fs';
+import { resolveEvalPaths } from '../eval/paths';
+import {
+  type DeterministicScores,
+  DeterministicScoresSchema,
+  type JudgeScores,
+  JudgeScoresSchema,
+  type Judgment,
+  ManifestSchema,
+  type Result,
+  type Summary,
+  SummarySchema,
+} from '../eval/types';
+import { releaseLock } from '../lock/file-lock';
+
+export interface EvalReportOpts {
+  cwd?: string;
+}
+
+function scoreJudgment(j: Judgment): { passed: boolean; score: number } {
+  const nonNa = j.dimensions.filter((d) => d.verdict !== 'NA');
+  if (nonNa.length === 0) return { passed: true, score: 1 };
+  const passes = nonNa.filter((d) => d.verdict === 'PASS').length;
+  const score = passes / nonNa.length;
+  const passed = nonNa.every((d) => d.verdict === 'PASS');
+  return { passed, score };
+}
+
+function renderReport(summary: Summary): string {
+  const lines: string[] = [];
+  lines.push(`# xera eval report ${summary.run_id}`);
+  lines.push('');
+  lines.push(`**Git SHA:** \`${summary.git_sha}\``);
+  lines.push('');
+  lines.push('**Prompt versions:**');
+  for (const [k, v] of Object.entries(summary.prompt_versions)) lines.push(`- \`${k}\`: ${v}`);
+  lines.push('');
+  lines.push(
+    `**Overall:** ${summary.overall.passed}/${summary.overall.total} PASS (score ${(summary.overall.score * 100).toFixed(0)}%)`,
+  );
+  lines.push('');
+  lines.push('## Results');
+  lines.push('');
+  lines.push('| Ticket | Stage | Deterministic | Judge | Score |');
+  lines.push('|---|---|---|---|---|');
+  for (const r of summary.results) {
+    const det = r.deterministic.passed ? 'PASS' : `FAIL (${r.deterministic.error ?? ''})`;
+    const judge = r.skipped ? 'SKIPPED' : r.judge ? (r.judge.passed ? 'PASS' : 'FAIL') : 'SKIPPED';
+    const score = r.judge ? `${(r.judge.score * 100).toFixed(0)}%` : '—';
+    lines.push(`| ${r.ticket} | ${r.stage} | ${det} | ${judge} | ${score} |`);
+  }
+  lines.push('');
+  lines.push('## Dimension breakdown');
+  lines.push('');
+  for (const r of summary.results) {
+    if (!r.judge || r.judge.dimensions.length === 0) continue;
+    lines.push(`### ${r.ticket} — ${r.stage}`);
+    lines.push('');
+    for (const d of r.judge.dimensions) lines.push(`- **${d.name}** — ${d.verdict}: ${d.notes}`);
+    lines.push('');
+  }
+  return lines.join('\n');
+}
+
+export async function evalReportCmd(argv: string[], opts: EvalReportOpts = {}): Promise<number> {
+  const cwd = opts.cwd ?? process.cwd();
+  const runId = argv[0];
+  if (!runId) {
+    console.error('[xera:eval-report] usage: eval-report <run-id>');
+    return 1;
+  }
+  const paths = resolveEvalPaths(cwd, runId);
+  if (!existsSync(paths.manifest)) {
+    console.error(`[xera:eval-report] missing manifest.json at ${paths.manifest}`);
+    return 1;
+  }
+  const manifest = ManifestSchema.parse(JSON.parse(readFileSync(paths.manifest, 'utf8')));
+
+  try {
+    let det: DeterministicScores;
+    let judge: JudgeScores;
+    try {
+      det = DeterministicScoresSchema.parse(
+        JSON.parse(readFileSync(paths.deterministicScores, 'utf8')),
+      );
+    } catch (err) {
+      console.error(
+        `[xera:eval-report] invalid deterministic-scores.json: ${(err as Error).message}`,
+      );
+      return 2;
+    }
+    try {
+      judge = JudgeScoresSchema.parse(JSON.parse(readFileSync(paths.judgeScores, 'utf8')));
+    } catch (err) {
+      console.error(`[xera:eval-report] invalid judge-scores.json: ${(err as Error).message}`);
+      return 2;
+    }
+
+    const results: Result[] = [];
+    for (const detEntry of det.entries) {
+      const judgment = judge.judgments.find(
+        (j) => j.ticket === detEntry.ticket && j.stage === detEntry.stage,
+      );
+      if (!judgment && detEntry.error?.startsWith('actual missing')) {
+        const r: Result = {
+          ticket: detEntry.ticket,
+          stage: detEntry.stage,
+          deterministic: {
+            passed: detEntry.passed,
+            checks: detEntry.checks,
+            ...(detEntry.error !== undefined ? { error: detEntry.error } : {}),
+          },
+          judge: null,
+          skipped: true,
+        };
+        results.push(r);
+        continue;
+      }
+      if (!judgment) {
+        // Judge entry expected but missing: count as FAIL not SKIPPED.
+        const r: Result = {
+          ticket: detEntry.ticket,
+          stage: detEntry.stage,
+          deterministic: {
+            passed: detEntry.passed,
+            checks: detEntry.checks,
+            ...(detEntry.error !== undefined ? { error: detEntry.error } : {}),
+          },
+          judge: { passed: false, dimensions: [], score: 0 },
+        };
+        results.push(r);
+        continue;
+      }
+      const { passed, score } = scoreJudgment(judgment);
+      const r: Result = {
+        ticket: detEntry.ticket,
+        stage: detEntry.stage,
+        deterministic: {
+          passed: detEntry.passed,
+          checks: detEntry.checks,
+          ...(detEntry.error !== undefined ? { error: detEntry.error } : {}),
+        },
+        judge: { passed, dimensions: judgment.dimensions, score },
+      };
+      results.push(r);
+    }
+
+    const counted = results.filter((r) => !r.skipped);
+    const passedCount = counted.filter((r) => r.deterministic.passed && r.judge?.passed).length;
+    const failedCount = counted.length - passedCount;
+    const avgScore =
+      counted.length === 0
+        ? 0
+        : counted.reduce(
+            (acc, r) => acc + (r.deterministic.passed && r.judge ? r.judge.score : 0),
+            0,
+          ) / counted.length;
+
+    const summary: Summary = {
+      run_id: runId,
+      git_sha: manifest.git_sha,
+      prompt_versions: manifest.prompt_versions,
+      results,
+      overall: { passed: passedCount, failed: failedCount, total: counted.length, score: avgScore },
+    };
+    SummarySchema.parse(summary);
+    writeFileSync(paths.summary, JSON.stringify(summary, null, 2));
+    writeFileSync(paths.report, renderReport(summary));
+
+    console.log(
+      `[xera:eval-report] ${passedCount}/${counted.length} PASS (avg ${(avgScore * 100).toFixed(0)}%)`,
+    );
+    return 0;
+  } finally {
+    releaseLock(paths.lock);
+  }
+}

package/src/bin-internal/exec.ts

@@ -1,17 +1,20 @@
-import { resolveArtifactPaths, generateRunId } from '../artifact/paths';
-import { acquireLock, releaseLock, isLockStale, readLock, forceUnlock } from '../lock/file-lock';
-import { NdjsonLogger } from '../logging/ndjson-logger';
-import { loadConfig } from '../config/load';
-import { readAuthState } from '../auth/state';
-import { needsRefresh } from '../auth/refresh';
-import { stagePlaywrightState, runAuthSetup, runPlaywright } from '@xera-ai/web';
-import { chromium } from '@playwright/test';
-import { mkdirSync, existsSync } from 'node:fs';
+import { existsSync, mkdirSync } from 'node:fs';
 import { join } from 'node:path';
+import { chromium } from '@playwright/test';
+import { runAuthSetup, runPlaywright, stagePlaywrightState } from '@xera-ai/web';
+import { generateRunId, resolveArtifactPaths } from '../artifact/paths';
+import { needsRefresh } from '../auth/refresh';
+import { readAuthState } from '../auth/state';
+import { loadConfig } from '../config/load';
+import { acquireLock, forceUnlock, isLockStale, readLock, releaseLock } from '../lock/file-lock';
+import { NdjsonLogger } from '../logging/ndjson-logger';
 
 export async function execCmd(argv: string[]): Promise<number> {
   const ticket = argv[0];
-  if (!ticket) { console.error('[xera:exec] usage: exec <TICKET>'); return 1; }
+  if (!ticket) {
+    console.error('[xera:exec] usage: exec <TICKET>');
+    return 1;
+  }
   const cwd = process.cwd();
   const config = await loadConfig(cwd);
   const paths = resolveArtifactPaths(cwd, ticket);
@@ -21,12 +24,16 @@ export async function execCmd(argv: string[]): Promise<number> {
   // Acquire lock
   if (!acquireLock(paths.lockPath, runId)) {
     if (isLockStale(paths.lockPath)) {
-      console.error(`[xera:exec] stale lock detected; force unlocking. Run \`xera-internal unlock ${ticket}\` to clear manually.`);
+      console.error(
+        `[xera:exec] stale lock detected; force unlocking. Run \`xera-internal unlock ${ticket}\` to clear manually.`,
+      );
       forceUnlock(paths.lockPath);
       acquireLock(paths.lockPath, runId);
     } else {
       const existing = readLock(paths.lockPath);
-      console.error(`[xera:exec] another run in progress (PID ${existing?.pid} on ${existing?.hostname}, started ${existing?.started_at}). Wait or run \`xera-internal unlock ${ticket}\`.`);
+      console.error(
+        `[xera:exec] another run in progress (PID ${existing?.pid} on ${existing?.hostname}, started ${existing?.started_at}). Wait or run \`xera-internal unlock ${ticket}\`.`,
+      );
       return 1;
     }
   }
@@ -39,11 +46,18 @@ export async function execCmd(argv: string[]): Promise<number> {
       try {
         for (const [roleName, roleCreds] of Object.entries(config.web.auth.roles)) {
           const entry = readAuthState(paths.authDir, roleName);
-          if (needsRefresh(entry, { ttl: config.web.auth.ttl, refreshBuffer: config.web.auth.refreshBuffer })) {
+          if (
+            needsRefresh(entry, {
+              ttl: config.web.auth.ttl,
+              refreshBuffer: config.web.auth.refreshBuffer,
+            })
+          ) {
             const email = process.env[roleCreds.envEmail];
             const password = process.env[roleCreds.envPassword];
             if (!email || !password) {
-              console.error(`[xera:exec] missing env ${roleCreds.envEmail} or ${roleCreds.envPassword} for role "${roleName}"`);
+              console.error(
+                `[xera:exec] missing env ${roleCreds.envEmail} or ${roleCreds.envPassword} for role "${roleName}"`,
+              );
               return 1;
             }
             await runAuthSetup({
@@ -113,4 +127,3 @@ export async function execCmd(argv: string[]): Promise<number> {
     releaseLock(paths.lockPath);
   }
 }
-

package/src/bin-internal/fetch.ts

@@ -1,13 +1,15 @@
-import { writeFileSync, mkdirSync } from 'node:fs';
+import { mkdirSync, writeFileSync } from 'node:fs';
 import { dirname } from 'node:path';
-import { loadConfig } from '../config/load';
-import { resolveArtifactPaths } from '../artifact/paths';
 import { hashString } from '../artifact/hash';
-import { writeMeta, readMeta } from '../artifact/meta';
+import { readMeta, writeMeta } from '../artifact/meta';
+import { resolveArtifactPaths } from '../artifact/paths';
+import { loadConfig } from '../config/load';
 import { createJiraClient } from '../jira/client';
 import type { JiraTicket } from '../jira/types';
 
-export interface FetchCmdOpts { cwd?: string; }
+export interface FetchCmdOpts {
+  cwd?: string;
+}
 
 export async function fetchCmd(argv: string[], opts: FetchCmdOpts = {}): Promise<number> {
   const cwd = opts.cwd ?? process.cwd();
@@ -31,9 +33,13 @@ export async function fetchCmd(argv: string[], opts: FetchCmdOpts = {}): Promise
         ? { rest: { email: process.env.JIRA_EMAIL, apiToken: process.env.JIRA_API_TOKEN } }
         : {}),
     });
-    const fieldMap = config.jira.fields.acceptanceCriteria !== undefined
-      ? { story: config.jira.fields.story, acceptanceCriteria: config.jira.fields.acceptanceCriteria }
-      : { story: config.jira.fields.story };
+    const fieldMap =
+      config.jira.fields.acceptanceCriteria !== undefined
+        ? {
+            story: config.jira.fields.story,
+            acceptanceCriteria: config.jira.fields.acceptanceCriteria,
+          }
+        : { story: config.jira.fields.story };
     t = await client.fetchTicket(ticket, fieldMap);
   }
 
@@ -69,7 +75,7 @@ function renderStory(t: JiraTicket): string {
     lines.push('## Story', '', story, '');
   }
 
-  if (t.acceptanceCriteria && t.acceptanceCriteria.trim()) {
+  if (t.acceptanceCriteria?.trim()) {
     const ac = t.acceptanceCriteria.trim();
     if (/^##\s+acceptance\s+criteria\b/i.test(ac)) {
       lines.push(ac, '');
@@ -78,7 +84,12 @@ function renderStory(t: JiraTicket): string {
     }
   }
   if (t.attachments.length > 0) {
-    lines.push('## Attachments', '', ...t.attachments.map((a) => `- [${a.filename}](${a.url})`), '');
+    lines.push(
+      '## Attachments',
+      '',
+      ...t.attachments.map((a) => `- [${a.filename}](${a.url})`),
+      '',
+    );
   }
   return lines.join('\n');
 }

package/src/bin-internal/graph-backfill.ts

@@ -0,0 +1,43 @@
+import { existsSync, readdirSync } from 'node:fs';
+import { join } from 'node:path';
+import { recordScriptImpl } from './graph-record-script';
+
+async function backfillTicket(repoRoot: string, ticket: string, dryRun: boolean): Promise<number> {
+  // 1) Synthesize ticket.fetched from story.md (use story.md mtime)
+  const storyPath = join(repoRoot, '.xera', ticket, 'story.md');
+  if (!existsSync(storyPath)) return 0;
+
+  const { recordFetch } = await import('./graph-record');
+  // Re-use the same code path; in dry-run we don't actually call appendEvents.
+  // For simplicity in v0.6.0, dry-run lists ticket count and returns.
+  if (dryRun) {
+    console.log(`[backfill dry-run] would backfill ${ticket}`);
+    return 0;
+  }
+  // recordScriptImpl handles scenario/POM extraction.
+  await recordScriptImpl(repoRoot, ticket);
+  await recordFetch(repoRoot, ticket);
+  return 0;
+}
+
+export async function graphBackfillCmd(argv: string[]): Promise<number> {
+  const dryRun = argv.includes('--dry-run');
+  const repoRoot = process.cwd();
+  const xeraDir = join(repoRoot, '.xera');
+  if (!existsSync(xeraDir)) {
+    console.log('[backfill] no .xera/ directory');
+    return 0;
+  }
+  const tickets: string[] = [];
+  for (const entry of readdirSync(xeraDir, { withFileTypes: true })) {
+    if (!entry.isDirectory()) continue;
+    if (entry.name === 'graph') continue;
+    if (entry.name.startsWith('.')) continue;
+    if (!/^[A-Z]+-\d+$/.test(entry.name)) continue;
+    tickets.push(entry.name);
+  }
+  console.log(`[backfill] found ${tickets.length} tickets`);
+  for (const t of tickets) await backfillTicket(repoRoot, t, dryRun);
+  console.log(`[backfill] done`);
+  return 0;
+}

package/src/bin-internal/graph-query.ts

@@ -0,0 +1,43 @@
+import { deriveSnapshot, loadAllEvents } from '../graph/store';
+import type { Snapshot } from '../graph/types';
+
+function filterByTicket(snap: Snapshot, ticket: string): Snapshot {
+  const out: Snapshot = {
+    ...snap,
+    tickets: snap.tickets[ticket] ? { [ticket]: snap.tickets[ticket]! } : {},
+    scenarios: Object.fromEntries(
+      Object.entries(snap.scenarios).filter(([, s]) => s.ticketId === ticket),
+    ),
+    poms: Object.fromEntries(Object.entries(snap.poms).filter(([, p]) => p.ticketId === ticket)),
+    edges: snap.edges.filter((e) => e.from === ticket || e.to === ticket),
+  };
+  return out;
+}
+
+function renderText(snap: Snapshot): string {
+  const out: string[] = [];
+  out.push(`Graph snapshot — ${snap.event_count} events`);
+  out.push(`Tickets: ${Object.keys(snap.tickets).length}`);
+  out.push(`Scenarios: ${Object.keys(snap.scenarios).length}`);
+  out.push(`POMs: ${Object.keys(snap.poms).length}`);
+  out.push(`Edges: ${snap.edges.length}`);
+  for (const t of Object.values(snap.tickets)) {
+    out.push(`  ${t.id} — ${t.summary}`);
+  }
+  return out.join('\n');
+}
+
+export async function graphQueryCmd(argv: string[]): Promise<number> {
+  let ticket: string | undefined;
+  let format: 'text' | 'json' = 'text';
+  for (let i = 0; i < argv.length; i++) {
+    if (argv[i] === '--ticket') ticket = argv[++i];
+    else if (argv[i] === '--format') format = argv[++i] as 'text' | 'json';
+  }
+  const repoRoot = process.cwd();
+  let snap = deriveSnapshot(loadAllEvents(repoRoot));
+  if (ticket) snap = filterByTicket(snap, ticket);
+  if (format === 'json') process.stdout.write(JSON.stringify(snap, null, 2));
+  else process.stdout.write(renderText(snap));
+  return 0;
+}

package/src/bin-internal/graph-record-script.ts

@@ -0,0 +1,191 @@
+import { createHash } from 'node:crypto';
+import { existsSync, readdirSync, readFileSync } from 'node:fs';
+import { basename, join } from 'node:path';
+import { appendEvents } from '../graph/store';
+import type {
+  EdgeDiscoveredPayload,
+  Event,
+  PomGeneratedPayload,
+  ScenarioGeneratedPayload,
+} from '../graph/types';
+import { SCHEMA_VERSION } from '../graph/types';
+import { ulid } from '../graph/ulid';
+
+const sha1 = (s: string) => createHash('sha1').update(s).digest('hex');
+const sId = (ticket: string, name: string) =>
+  sha1(`${ticket}:${name.trim().toLowerCase().replace(/\s+/g, ' ')}`);
+const pId = (file: string) => sha1(basename(file));
+const nowIso = () => new Date().toISOString();
+const mk = <T extends Event['type']>(
+  actor: string,
+  type: T,
+  payload: Extract<Event, { type: T }>['payload'],
+): Event =>
+  ({
+    event_id: ulid(),
+    schema_version: SCHEMA_VERSION,
+    ts: nowIso(),
+    actor,
+    type,
+    payload,
+  }) as Event;
+
+function parseFeature(
+  text: string,
+): Array<{ name: string; priority: 'p0' | 'p1' | 'p2'; gherkin: string }> {
+  const scenarios: Array<{ name: string; priority: 'p0' | 'p1' | 'p2'; gherkin: string }> = [];
+  const lines = text.split('\n');
+  let currentTagPriority: 'p0' | 'p1' | 'p2' = 'p1';
+  let i = 0;
+  while (i < lines.length) {
+    const line = lines[i]!.trim();
+    if (line.startsWith('@')) {
+      const tag = line.slice(1).split(/\s+/)[0]!.toLowerCase();
+      if (tag === 'p0' || tag === 'p1' || tag === 'p2') currentTagPriority = tag;
+      i++;
+      continue;
+    }
+    if (line.startsWith('Scenario:') || line.startsWith('Scenario Outline:')) {
+      const name = line.replace(/^Scenario( Outline)?:\s*/, '');
+      const start = i;
+      i++;
+      while (
+        i < lines.length &&
+        !lines[i]!.trim().startsWith('Scenario') &&
+        !lines[i]!.trim().startsWith('@')
+      )
+        i++;
+      scenarios.push({
+        name,
+        priority: currentTagPriority,
+        gherkin: lines.slice(start, i).join('\n'),
+      });
+      currentTagPriority = 'p1';
+      continue;
+    }
+    i++;
+  }
+  return scenarios;
+}
+
+function listPomFiles(dir: string): string[] {
+  if (!existsSync(dir)) return [];
+  return readdirSync(dir)
+    .filter((f) => f.endsWith('.ts'))
+    .map((f) => join(dir, f));
+}
+
+function extractRoute(pomContent: string): string {
+  const m = pomContent.match(/goto\s*\(\s*['"]([^'"]+)['"]/);
+  return m ? m[1]! : '';
+}
+
+function extractLocators(pomContent: string): string[] {
+  const out: string[] = [];
+  const re = /\b(getByRole|getByLabel|getByText|getByTestId|locator)\s*\(\s*([^)]+)\)/g;
+  let m = re.exec(pomContent);
+  while (m !== null) {
+    out.push(`${m[1]}(${m[2]})`);
+    m = re.exec(pomContent);
+  }
+  return out;
+}
+
+function extractPomUsage(specContent: string): string[] {
+  const names = new Set<string>();
+  const re = /new\s+([A-Z][A-Za-z0-9]*Page)\s*\(/g;
+  let m = re.exec(specContent);
+  while (m !== null) {
+    names.add(m[1]!);
+    m = re.exec(specContent);
+  }
+  return [...names];
+}
+
+export async function recordScriptImpl(repoRoot: string, ticket: string): Promise<number> {
+  const ticketDir = join(repoRoot, '.xera', ticket);
+  const featurePath = join(ticketDir, 'feature', `${ticket}.feature`);
+  const specPath = join(ticketDir, 'tests', `${ticket}.spec.ts`);
+  const pomDir = join(ticketDir, 'poms');
+
+  if (!existsSync(featurePath)) {
+    console.error(`[graph-record script] feature missing`);
+    return 1;
+  }
+
+  const featureText = readFileSync(featurePath, 'utf8');
+  const featureHash = sha1(featureText);
+  const scenarios = parseFeature(featureText);
+
+  const events: Event[] = [];
+  for (const s of scenarios) {
+    const id = sId(ticket, s.name);
+    const p: ScenarioGeneratedPayload = {
+      scenarioId: id,
+      ticketId: ticket,
+      name: s.name,
+      gherkin: s.gherkin,
+      priority: s.priority,
+      featureHash,
+      generatedAt: nowIso(),
+    };
+    events.push(mk('xera-script', 'scenario.generated', p));
+  }
+
+  const pomFiles = listPomFiles(pomDir);
+  const pomNameToId = new Map<string, string>();
+  for (const pomFile of pomFiles) {
+    const content = readFileSync(pomFile, 'utf8');
+    const id = pId(pomFile);
+    const className = content.match(/export\s+class\s+([A-Z][A-Za-z0-9]*Page)/)?.[1] ?? '';
+    pomNameToId.set(className, id);
+    const pg: PomGeneratedPayload = {
+      pomId: id,
+      ticketId: ticket,
+      filePath: pomFile.replace(`${repoRoot}/`, ''),
+      route: extractRoute(content),
+      locators: extractLocators(content),
+      scope: 'local',
+    };
+    events.push(mk('xera-script', 'pom.generated', pg));
+  }
+
+  if (existsSync(specPath)) {
+    const specContent = readFileSync(specPath, 'utf8');
+    const usedPoms = extractPomUsage(specContent);
+    for (const scenario of scenarios) {
+      const scId = sId(ticket, scenario.name);
+      for (const pomName of usedPoms) {
+        const pid = pomNameToId.get(pomName);
+        if (!pid) continue;
+        const ep: EdgeDiscoveredPayload = {
+          kind: 'uses',
+          from: scId,
+          to: pid,
+          source: 'xera-script',
+        };
+        events.push(mk('xera-script', 'edge.discovered', ep));
+      }
+    }
+  }
+
+  for (const [, id] of pomNameToId) {
+    const pom = events.find(
+      (e) => e.type === 'pom.generated' && (e.payload as PomGeneratedPayload).pomId === id,
+    );
+    if (!pom) continue;
+    const route = (pom.payload as PomGeneratedPayload).route;
+    if (!route) continue;
+    const slug =
+      route
+        .replace(/^\//, '')
+        .split('/')[0]!
+        .replace(/[^a-z0-9-]/gi, '-')
+        .toLowerCase() || 'root';
+    const ep: EdgeDiscoveredPayload = { kind: 'covers', from: id, to: slug, source: 'xera-script' };
+    events.push(mk('xera-script', 'edge.discovered', ep));
+  }
+
+  appendEvents(repoRoot, events, { skill: 'xera-script', ticketId: ticket });
+  return 0;
+}