flake-ledger 0.0.0-stage → 0.1.0

package/examples/runs/run-03/results.xml

@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuites>
+  <testsuite name="com.example.CheckoutTest" timestamp="2026-01-05T10:20:00">
+    <testcase classname="com.example.CheckoutTest" name="appliesCoupon">
+      <flakyFailure message="expected 10% discount, got 0">
+        <stackTrace>org.opentest4j.AssertionFailedError: expected 10% discount, got 0
+  at CheckoutTest.appliesCoupon(CheckoutTest.java:42)</stackTrace>
+      </flakyFailure>
+    </testcase>
+  </testsuite>
+  <testsuite name="com.example.LoginTest" timestamp="2026-01-05T10:20:00">
+    <testcase classname="com.example.LoginTest" name="rejectsExpiredToken"/>
+  </testsuite>
+  <testsuite name="com.example.SearchTest" timestamp="2026-01-05T10:20:00">
+    <testcase classname="com.example.SearchTest" name="filtersByPrice"/>
+  </testsuite>
+  <testsuite name="com.example.CartTest" timestamp="2026-01-05T10:20:00">
+    <testcase classname="com.example.CartTest" name="addsItem"/>
+  </testsuite>
+  <testsuite name="com.example.ReportTest" timestamp="2026-01-05T10:20:00">
+    <testcase classname="com.example.ReportTest" name="exportsCsv">
+      <skipped message="CSV export not configured"/>
+    </testcase>
+  </testsuite>
+</testsuites>

package/examples/runs/run-04/results.xml

@@ -0,0 +1,24 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuites>
+  <testsuite name="com.example.CheckoutTest" timestamp="2026-01-05T10:30:00">
+    <testcase classname="com.example.CheckoutTest" name="appliesCoupon"/>
+  </testsuite>
+  <testsuite name="com.example.LoginTest" timestamp="2026-01-05T10:30:00">
+    <testcase classname="com.example.LoginTest" name="rejectsExpiredToken">
+      <failure message="expected 401, got 200">
+        <stackTrace>org.opentest4j.AssertionFailedError: expected 401, got 200</stackTrace>
+      </failure>
+    </testcase>
+  </testsuite>
+  <testsuite name="com.example.SearchTest" timestamp="2026-01-05T10:30:00">
+    <testcase classname="com.example.SearchTest" name="filtersByPrice"/>
+  </testsuite>
+  <testsuite name="com.example.CartTest" timestamp="2026-01-05T10:30:00">
+    <testcase classname="com.example.CartTest" name="addsItem"/>
+  </testsuite>
+  <testsuite name="com.example.ReportTest" timestamp="2026-01-05T10:30:00">
+    <testcase classname="com.example.ReportTest" name="exportsCsv">
+      <skipped message="CSV export not configured"/>
+    </testcase>
+  </testsuite>
+</testsuites>

package/examples/runs/run-05/results.xml

@@ -0,0 +1,24 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuites>
+  <testsuite name="com.example.CheckoutTest" timestamp="2026-01-05T10:40:00">
+    <testcase classname="com.example.CheckoutTest" name="appliesCoupon"/>
+  </testsuite>
+  <testsuite name="com.example.LoginTest" timestamp="2026-01-05T10:40:00">
+    <testcase classname="com.example.LoginTest" name="rejectsExpiredToken"/>
+  </testsuite>
+  <testsuite name="com.example.SearchTest" timestamp="2026-01-05T10:40:00">
+    <testcase classname="com.example.SearchTest" name="filtersByPrice">
+      <failure message="expected price 10, got 12">
+        <stackTrace>org.opentest4j.AssertionFailedError: expected price 10, got 12</stackTrace>
+      </failure>
+    </testcase>
+  </testsuite>
+  <testsuite name="com.example.CartTest" timestamp="2026-01-05T10:40:00">
+    <testcase classname="com.example.CartTest" name="addsItem"/>
+  </testsuite>
+  <testsuite name="com.example.ReportTest" timestamp="2026-01-05T10:40:00">
+    <testcase classname="com.example.ReportTest" name="exportsCsv">
+      <skipped message="CSV export not configured"/>
+    </testcase>
+  </testsuite>
+</testsuites>

package/lib/index.mjs

@@ -0,0 +1,24 @@
+/**
+ * Programmatic surface for flake-ledger.
+ *
+ * The CLI is the primary interface, but keeping the scanner and parsers
+ * importable makes it easy to embed the ledger in a larger CI report without
+ * shelling out.
+ */
+export {
+  ScanError,
+  discoverJUnitFiles,
+  groupFilesByRun,
+  scanRuns,
+} from './scan.mjs';
+export { parseJUnit, parseJUnitFileContents } from './junit.mjs';
+export { parseXml, XmlParseError, decodeEntities, localName, textContent } from './xml.mjs';
+export {
+  FLAKY_CLASSIFICATIONS,
+  flakyTests,
+  formatExplain,
+  formatPercent,
+  formatTable,
+  isFlaky,
+  toJson,
+} from './report.mjs';

package/lib/junit.mjs

@@ -0,0 +1,332 @@
+import {
+  elementChildren,
+  getAttribute,
+  isElement,
+  localName,
+  parseXml,
+  textContent,
+} from './xml.mjs';
+
+const FAILURE_KINDS = new Set(['failure', 'error']);
+const FLAKY_KINDS = new Set(['flakyfailure', 'flakyerror']);
+const RERUN_KINDS = new Set(['rerunfailure', 'rerunerror']);
+const SKIP_KINDS = new Set(['skipped', 'disabled']);
+
+function lower(value) {
+  return typeof value === 'string' ? value.trim().toLowerCase() : '';
+}
+
+function numericAttribute(node, name) {
+  const raw = getAttribute(node, name);
+  if (raw === undefined || raw === '') return null;
+  const parsed = Number(raw);
+  return Number.isFinite(parsed) ? parsed : null;
+}
+
+const RETRY_ATTRIBUTE_NAMES = [
+  'retry',
+  'retries',
+  'rerun',
+  'reruns',
+  'retryCount',
+  'rerunCount',
+  'retry_count',
+  'rerun_count',
+];
+
+function retryEvidence(node) {
+  // There is no JUnit standard for retries. Some reporters add retry="1",
+  // retries="2", rerun="1" or a *Count attribute to a testcase that only
+  // passed after a rerun. getAttribute is case-insensitive for namespaces,
+  // so both camelCase and snake_case spellings are covered.
+  for (const name of RETRY_ATTRIBUTE_NAMES) {
+    const raw = getAttribute(node, name);
+    if (raw === undefined || raw === '') continue;
+    const trimmed = String(raw).trim();
+    const numeric = Number(trimmed);
+    if (Number.isFinite(numeric)) {
+      if (numeric > 0) return true;
+      continue;
+    }
+    if (/^(true|yes|y)$/i.test(trimmed)) return true;
+  }
+  return false;
+}
+
+function normalizeStatus(value) {
+  const status = lower(value);
+  if (status === 'passed' || status === 'pass' || status === 'success' || status === 'ok' || status === 'resolved') {
+    return 'passed';
+  }
+  if (status === 'failed' || status === 'failure' || status === 'fail' || status === 'broken') {
+    return 'failed';
+  }
+  if (status === 'error' || status === 'errored' || status === 'errored-out') return 'error';
+  if (
+    status === 'skipped' ||
+    status === 'skip' ||
+    status === 'disabled' ||
+    status === 'pending' ||
+    status === 'notrun' ||
+    status === 'not-run' ||
+    status === 'ignored'
+  ) {
+    return 'skipped';
+  }
+  return null;
+}
+
+function statusKindForTag(node) {
+  const tag = lower(localName(node.name));
+  if (tag === 'error' || tag === 'flakyerror' || tag === 'rerunerror') return 'error';
+  return 'failed';
+}
+
+function attemptFromNode(node, status) {
+  const details = textContent(node).replace(/\r\n/g, '\n').trim();
+  const messageAttribute = getAttribute(node, 'message');
+  let message = messageAttribute === undefined ? '' : String(messageAttribute).trim();
+  if (!message && details) {
+    const firstLine = details.split('\n').find((line) => line.trim() !== '');
+    message = firstLine ? firstLine.trim() : '';
+  }
+  return {
+    status,
+    message: message || null,
+    type: getAttribute(node, 'type') || null,
+    details: details || null,
+    time: numericAttribute(node, 'time'),
+    retried: false,
+  };
+}
+
+function skippedAttempt(node) {
+  const message = getAttribute(node, 'message');
+  return {
+    status: 'skipped',
+    message: message === undefined || String(message).trim() === '' ? null : String(message).trim(),
+    type: getAttribute(node, 'type') || null,
+    details: textContent(node).trim() || null,
+    time: numericAttribute(node, 'time'),
+    retried: false,
+  };
+}
+
+function statusAttempt(node, status) {
+  return {
+    status,
+    message: null,
+    type: null,
+    details: null,
+    time: numericAttribute(node, 'time'),
+    retried: status === 'passed' && retryEvidence(node),
+  };
+}
+
+function passedAttempt(node) {
+  const attempt = statusAttempt(node, 'passed');
+  // Reaching here means retry evidence (flakyFailure/rerunFailure/attribute)
+  // describes the earlier attempts even when the recorder did not copy the
+  // retry marker onto the testcase itself.
+  attempt.retried = true;
+  return attempt;
+}
+
+/**
+ * Turn one <testcase> element into one or more attempts.
+ *
+ * Surefire writes a passed-on-retry test as a passing testcase carrying a
+ * <flakyFailure>/<flakyError> child describing the earlier failure. It writes
+ * a retry that ultimately failed as <rerunFailure>/<rerunError>. Tools that
+ * emit one testcase per attempt simply produce one attempt here; the scan
+ * layer merges all of them into a single test record.
+ */
+function attemptsForTestcase(node) {
+  const children = elementChildren(node);
+  const failures = [];
+  const flaky = [];
+  const reruns = [];
+  const skips = [];
+  for (const child of children) {
+    const tag = lower(localName(child.name));
+    if (FAILURE_KINDS.has(tag)) failures.push(child);
+    else if (FLAKY_KINDS.has(tag)) flaky.push(child);
+    else if (RERUN_KINDS.has(tag)) reruns.push(child);
+    else if (SKIP_KINDS.has(tag)) skips.push(child);
+  }
+
+  const declared = normalizeStatus(getAttribute(node, 'status') || getAttribute(node, 'result'));
+  const attempts = [];
+
+  if (failures.length > 0) {
+    for (const child of flaky) attempts.push(attemptFromNode(child, statusKindForTag(child)));
+    // Surefire can emit rerunFailure alongside the final <failure>. Keep the
+    // earlier evidence without letting it replace the final outcome.
+    for (const child of reruns) attempts.push(attemptFromNode(child, statusKindForTag(child)));
+    // The final <failure>/<error> child is the outcome that matters. Adding
+    // every failure child would count an error as an extra retry.
+    const finalNode = failures[0];
+    attempts.push(attemptFromNode(finalNode, statusKindForTag(finalNode)));
+    return attempts;
+  }
+
+  if (flaky.length > 0) {
+    for (const child of flaky) attempts.push(attemptFromNode(child, statusKindForTag(child)));
+    for (const child of reruns) attempts.push(attemptFromNode(child, statusKindForTag(child)));
+    if (declared === 'skipped') {
+      attempts.push(skippedAttempt(skips[0] || node));
+    } else if (declared && declared !== 'passed') {
+      attempts.push(statusAttempt(node, declared));
+    } else if (declared === 'passed' || reruns.length === 0) {
+      // flakyFailure alone means the final attempt passed. An explicit passed
+      // status after rerunFailure means the rerun recovered the test too.
+      attempts.push(passedAttempt(node));
+    }
+    return attempts;
+  }
+
+  if (reruns.length > 0) {
+    for (const child of reruns) attempts.push(attemptFromNode(child, statusKindForTag(child)));
+    if (declared === 'passed') attempts.push(passedAttempt(node));
+    else if (declared === 'skipped') attempts.push(skippedAttempt(skips[0] || node));
+    // Without an explicit pass, rerunFailure means the final retry stayed red.
+    return attempts;
+  }
+
+  if (skips.length > 0) {
+    attempts.push(skippedAttempt(skips[0]));
+    return attempts;
+  }
+
+  if (declared) {
+    attempts.push(statusAttempt(node, declared));
+    return attempts;
+  }
+
+  attempts.push(statusAttempt(node, 'passed'));
+  return attempts;
+}
+
+function suiteMetadata(node, sourcePath) {
+  return {
+    name: getAttribute(node, 'name') || null,
+    timestamp: getAttribute(node, 'timestamp') || null,
+    tests: numericAttribute(node, 'tests'),
+    failures: numericAttribute(node, 'failures'),
+    errors: numericAttribute(node, 'errors'),
+    skipped: numericAttribute(node, 'skipped'),
+    time: numericAttribute(node, 'time'),
+    file: getAttribute(node, 'file') || null,
+    source: sourcePath,
+  };
+}
+
+function testcaseIdentity(node, suiteName) {
+  const classname =
+    getAttribute(node, 'classname') ||
+    getAttribute(node, 'class') ||
+    getAttribute(node, 'class_name') ||
+    suiteName ||
+    '';
+  let name = getAttribute(node, 'name');
+  if (name === undefined || name === null || String(name) === '') {
+    name = getAttribute(node, 'test') || '(unnamed)';
+  }
+  const className = String(classname).trim();
+  const testName = String(name).trim();
+  return {
+    classname: className,
+    name: testName,
+    id: `${className}::${testName}`,
+  };
+}
+
+function addTestcase(node, suiteName, sourcePath, output) {
+  const identity = testcaseIdentity(node, suiteName);
+  const attempts = attemptsForTestcase(node);
+  output.testcaseCount += 1;
+  for (const attempt of attempts) {
+    output.attempts.push({
+      ...identity,
+      ...attempt,
+      source: sourcePath,
+      suite: suiteName,
+    });
+  }
+}
+
+function visitContainer(node, context, output) {
+  for (const child of elementChildren(node)) {
+    if (!isElement(child)) continue;
+    const tag = lower(localName(child.name));
+    if (tag === 'testsuite') {
+      visitSuite(child, context, output);
+    } else if (tag === 'testsuites') {
+      visitContainer(child, context, output);
+    } else if (tag === 'testcase') {
+      addTestcase(child, context.suiteName, context.sourcePath, output);
+    }
+  }
+}
+
+function visitSuite(node, context, output) {
+  const ownName = getAttribute(node, 'name') || context.suiteName || null;
+  output.suites.push(suiteMetadata(node, context.sourcePath));
+  const timestamp = getAttribute(node, 'timestamp');
+  if (timestamp) {
+    const time = Date.parse(timestamp);
+    if (Number.isFinite(time) && (output.timestampMs === null || time < output.timestampMs)) {
+      output.timestampMs = time;
+      output.timestamp = timestamp;
+    }
+  }
+
+  for (const child of elementChildren(node)) {
+    if (!isElement(child)) continue;
+    const tag = lower(localName(child.name));
+    if (tag === 'testsuite') {
+      visitSuite(child, { ...context, suiteName: ownName }, output);
+    } else if (tag === 'testsuites') {
+      visitContainer(child, { ...context, suiteName: ownName }, output);
+    } else if (tag === 'testcase') {
+      addTestcase(child, ownName, context.sourcePath, output);
+    }
+  }
+}
+
+/**
+ * Parse one JUnit XML document.
+ *
+ * Returns the test attempts in document order along with lightweight suite
+ * metadata. Structural XML errors throw XmlParseError; the scan layer catches
+ * them so one broken shard cannot hide the rest of the ledger.
+ */
+export function parseJUnit(source, sourcePath = '<junit.xml>') {
+  const document = parseXml(source, { sourceName: sourcePath });
+  const output = {
+    sourcePath,
+    suites: [],
+    attempts: [],
+    testcaseCount: 0,
+    timestamp: null,
+    timestampMs: null,
+  };
+
+  for (const root of elementChildren(document)) {
+    if (!isElement(root)) continue;
+    const tag = lower(localName(root.name));
+    if (tag === 'testsuite') {
+      visitSuite(root, { suiteName: null, sourcePath }, output);
+    } else if (tag === 'testsuites') {
+      visitContainer(root, { suiteName: null, sourcePath }, output);
+    } else if (tag === 'testcase') {
+      addTestcase(root, null, sourcePath, output);
+    }
+  }
+
+  return output;
+}
+
+export function parseJUnitFileContents(contents, sourcePath) {
+  return parseJUnit(contents, sourcePath);
+}

package/lib/report.mjs

@@ -0,0 +1,133 @@
+const FLAKY_CLASSIFICATIONS = new Set(['cross-run flake', 'passed-on-retry', 'failed-on-retry']);
+
+export function isFlaky(test) {
+  return FLAKY_CLASSIFICATIONS.has(test.classification);
+}
+
+export function flakyTests(tests) {
+  return tests.filter(isFlaky);
+}
+
+export function formatPercent(value, digits = 1) {
+  if (!Number.isFinite(value)) return '0.0%';
+  return `${(value * 100).toFixed(digits)}%`;
+}
+
+function tableRows(tests) {
+  return tests.map((test) => [
+    test.id,
+    String(test.runs),
+    String(test.attempts),
+    String(test.unstableRuns),
+    String(test.passedOnRetryRuns),
+    String(test.failedRuns + test.errorRuns),
+    formatPercent(test.unstableRate),
+    test.score.toFixed(3),
+    test.classification,
+  ]);
+}
+
+export function formatTable(tests, options = {}) {
+  const headers = [
+    'TEST ID',
+    'RUNS',
+    'ATTEMPTS',
+    'UNSTABLE',
+    'PASS-ON-RETRY',
+    'FAILED',
+    'RATE',
+    'SCORE',
+    'CLASS',
+  ];
+  const rows = tableRows(tests);
+  const widths = headers.map((header, index) =>
+    rows.reduce(
+      (width, row) => Math.max(width, String(row[index]).length),
+      header.length,
+    ),
+  );
+  const render = (cells) =>
+    cells
+      .map((cell, index) => (index === cells.length - 1 ? String(cell) : String(cell).padEnd(widths[index])))
+      .join('  ')
+      .replace(/\s+$/, '');
+
+  const lines = [render(headers)];
+  for (const row of rows) lines.push(render(row));
+
+  const totals = options.totals;
+  if (totals) {
+    lines.push('');
+    const shown =
+      options.totalTests !== undefined && options.totalTests > tests.length
+        ? `showing ${tests.length} of ${options.totalTests} tests; `
+        : '';
+    lines.push(
+      `${shown}${totals.runs} run${totals.runs === 1 ? '' : 's'}, ` +
+        `${totals.tests} test${totals.tests === 1 ? '' : 's'}, ` +
+        `${totals.attempts} attempt${totals.attempts === 1 ? '' : 's'}, ` +
+        `${totals.files} XML file${totals.files === 1 ? '' : 's'}.`,
+    );
+  }
+
+  return lines.join('\n');
+}
+
+function indentedMessage(message) {
+  return String(message)
+    .split(/\r?\n/)
+    .map((line) => `         ${line}`)
+    .join('\n');
+}
+
+export function formatExplain(test) {
+  const lines = [
+    test.id,
+    `  classification: ${test.classification}`,
+    `  runs: ${test.runs}  attempts: ${test.attempts}  unstable: ${test.unstableRuns}/${test.eligibleRuns}  ` +
+      `rate: ${formatPercent(test.unstableRate)}  score: ${test.score.toFixed(3)}`,
+  ];
+
+  for (const entry of test.timeline) {
+    lines.push('');
+    lines.push(
+      `  run ${entry.runId}  ${entry.outcome}  ${entry.attemptCount} attempt${
+        entry.attemptCount === 1 ? '' : 's'
+      }`,
+    );
+    lines.push(`    ${entry.runPath}`);
+    entry.attempts.forEach((attempt, index) => {
+      const type = attempt.type ? ` [${attempt.type}]` : '';
+      const retried = attempt.retried ? ' (retried)' : '';
+      lines.push(
+        `    #${index + 1} ${String(attempt.status).padEnd(7)} ${attempt.source}${type}${retried}`,
+      );
+      if (attempt.message) lines.push(indentedMessage(attempt.message));
+    });
+  }
+
+  return lines.join('\n');
+}
+
+export function toJson(result, options = {}) {
+  const tests = options.tests || result.tests;
+  return {
+    tool: 'flake-ledger',
+    version: options.version || null,
+    runsDir: result.runsDir,
+    totals: result.totals,
+    runs: result.runs.map((run) => ({
+      id: run.id,
+      path: run.path,
+      startedAt: run.startedAt,
+      files: run.files,
+      fileCount: run.files.length,
+      testCount: run.testCount,
+      attemptCount: run.attemptCount,
+    })),
+    tests,
+    warnings: result.warnings,
+  };
+}
+
+export { FLAKY_CLASSIFICATIONS };