executable-stories-formatters 1.17.1 → 1.18.0

package/dist/cli.js

@@ -148,8 +148,8 @@ import { formatDuration } from "executable-stories-core/utils/duration";
 
 // src/scenario-failure.ts
 function failingScenarioMessage(tc) {
-  const failingStep = tc.stepResults.find((s) => s.status === "failed" && s.errorMessage);
-  return failingStep?.errorMessage ?? tc.errorMessage;
+  const failingStep2 = tc.stepResults.find((s) => s.status === "failed" && s.errorMessage);
+  return failingStep2?.errorMessage ?? tc.errorMessage;
 }
 
 // src/check.ts
@@ -257,8 +257,8 @@ function renderCheckText(report) {
       lines.push(`${marker}${step.keyword} ${step.text}`);
     }
     if (f.errorMessage) {
-      const firstLine2 = f.errorMessage.split("\n")[0];
-      lines.push(`    \u2192 ${firstLine2}`);
+      const firstLine3 = f.errorMessage.split("\n")[0];
+      lines.push(`    \u2192 ${firstLine3}`);
     }
     if (f.covers.length > 0) {
       lines.push(`  covers: ${f.covers.join(", ")}`);
@@ -2200,8 +2200,8 @@ function escapeCell(value) {
   return value.replace(/\|/g, "\\|").replace(/\n/g, " ");
 }
 function intentSummary(intent) {
-  const firstLine2 = intent.split("\n").find((l) => l.trim().length > 0) ?? "";
-  const trimmed = firstLine2.trim();
+  const firstLine3 = intent.split("\n").find((l) => l.trim().length > 0) ?? "";
+  const trimmed = firstLine3.trim();
   return trimmed.length > 200 ? `${trimmed.slice(0, 197)}\u2026` : trimmed;
 }
 function renderTicket(ticket) {
@@ -2396,6 +2396,138 @@ var ReviewMarkdownFormatter = class {
   }
 };
 
+// src/formatters/review-json.ts
+var SEVERITY_RANK = {
+  blocker: 0,
+  major: 1,
+  minor: 2
+};
+function failingStep(testCase) {
+  const failed = testCase.stepResults.find((s) => s.status === "failed");
+  if (!failed) return void 0;
+  const step = testCase.story.steps?.[failed.index];
+  return step ? `${step.keyword} ${step.text}` : void 0;
+}
+function firstLine(message) {
+  const line = message.split("\n").find((l) => l.trim().length > 0)?.trim() ?? "";
+  return line.length > 300 ? `${line.slice(0, 297)}\u2026` : line;
+}
+function failedFinding(claim) {
+  const testCase = claim.testCase;
+  const evidence = [`the scenario is ${claim.status}`];
+  const step = failingStep(testCase);
+  if (step) evidence.push(`failed at: ${step}`);
+  if (testCase.errorMessage) evidence.push(firstLine(testCase.errorMessage));
+  return {
+    kind: "failed",
+    severity: "blocker",
+    title: `Unproven claim: ${claim.scenario}`,
+    file: claim.sourceFile,
+    line: claim.sourceLine,
+    detail: "This scenario states a claim about the change and does not pass, so the claim is unproven.",
+    evidence,
+    remedy: "Fix the behaviour so the scenario passes, or correct the scenario if it states the wrong claim."
+  };
+}
+function skippedFinding(claim) {
+  return {
+    kind: "skipped",
+    severity: "minor",
+    title: `Claim not exercised: ${claim.scenario}`,
+    file: claim.sourceFile,
+    line: claim.sourceLine,
+    detail: `This scenario is ${claim.status}, so it did not run and its claim about the change is unproven.`,
+    evidence: claim.strengthReasons,
+    remedy: "Run the scenario, or delete it if the claim no longer applies \u2014 a permanently skipped claim is worse than no claim."
+  };
+}
+function unassertedFinding(claim) {
+  return {
+    kind: "unasserted",
+    severity: "major",
+    title: `Green but proves nothing: ${claim.scenario}`,
+    file: claim.sourceFile,
+    line: claim.sourceLine,
+    detail: "This scenario passed without asserting anything, so it cannot fail and proves nothing about the change.",
+    evidence: claim.strengthReasons,
+    remedy: "Assert the outcome the scenario claims, so that breaking the behaviour turns it red."
+  };
+}
+function uncoveredFinding(file) {
+  return {
+    kind: "uncovered",
+    severity: "major",
+    title: "Changed with no evidence",
+    file: file.path,
+    detail: `This file was ${file.changeKind} in the diff and no scenario in the run claims anything about it.`,
+    evidence: ["no claim in this run correlates to this file"],
+    remedy: "Add a scenario covering the behaviour this file changed, or say in the PR why it needs none."
+  };
+}
+function weakFinding(file) {
+  return {
+    kind: "weak",
+    severity: "minor",
+    title: "Weak evidence only",
+    file: file.path,
+    detail: `This file was ${file.changeKind} in the diff and its only claims are weakly evidenced.`,
+    evidence: file.claims.map((c) => `${c.scenario} (${c.strength})`),
+    remedy: "Strengthen the proof: verify the test fails on the base ref, add integration or e2e coverage, or attach a screenshot or trace."
+  };
+}
+function reviewFindings(review) {
+  const findings = [];
+  for (const claim of review.claims) {
+    if (claim.status === "failed") {
+      findings.push(failedFinding(claim));
+    } else if (claim.status === "skipped" || claim.status === "pending") {
+      findings.push(skippedFinding(claim));
+    } else if (claim.strength === "none") {
+      findings.push(unassertedFinding(claim));
+    }
+  }
+  for (const file of review.changedFiles) {
+    if (file.band === "uncovered") findings.push(uncoveredFinding(file));
+    else if (file.band === "weak") findings.push(weakFinding(file));
+  }
+  return findings.sort(
+    (a, b) => SEVERITY_RANK[a.severity] - SEVERITY_RANK[b.severity]
+  );
+}
+function toJsonClaim(claim) {
+  return {
+    id: claim.id,
+    scenario: claim.scenario,
+    sourceFile: claim.sourceFile,
+    sourceLine: claim.sourceLine,
+    status: claim.status,
+    audience: claim.audience,
+    changeType: claim.changeType,
+    strength: claim.strength,
+    strengthReasons: claim.strengthReasons,
+    coversFiles: claim.coversFiles
+  };
+}
+function buildReviewJson(review) {
+  const run = { total: 0, passed: 0, failed: 0, skipped: 0, pending: 0 };
+  for (const testCase of review.run.testCases) {
+    run.total++;
+    if (testCase.status === "passed" || testCase.status === "failed" || testCase.status === "skipped" || testCase.status === "pending") {
+      run[testCase.status]++;
+    }
+  }
+  return {
+    version: 1,
+    baseRef: review.context.baseRef,
+    headRef: review.context.headRef,
+    summary: review.summary,
+    run,
+    findings: reviewFindings(review),
+    changedFiles: review.changedFiles,
+    claims: review.claims.map(toJsonClaim)
+  };
+}
+
 // src/goal.ts
 var ACTIVE = ["passed", "failed"];
 function buildGoal(args, _deps = {}) {
@@ -3045,14 +3177,14 @@ function scenarioLine(diff) {
     case "moved":
       return `- **${title}** moved from \`${diff.baseline?.sourceFile ?? "?"}\` to ${source}`;
     case "regressed":
-      return `- **${title}** (${source})${diff.current?.errorMessage ? ` \u2014 ${firstLine(diff.current.errorMessage)}` : ""}`;
+      return `- **${title}** (${source})${diff.current?.errorMessage ? ` \u2014 ${firstLine2(diff.current.errorMessage)}` : ""}`;
     case "changed":
       return `- **${title}** (${source}) \u2014 ${diff.changedFields.join(", ")}`;
     default:
       return `- **${title}** (${source})`;
   }
 }
-function firstLine(text2) {
+function firstLine2(text2) {
   return text2.split("\n", 1)[0].trim();
 }
 var RunDiffChangelogFormatter = class {
@@ -10966,6 +11098,12 @@ Options:
                      cloud can recommend a test scope for the change.
   --format <fmt>     auto (default), story, junit, playwright or allure.
                      Only needed when detection guesses wrong.
+  --review-json <path>
+                     Write what a CI surface renders \u2014 the cloud run URL, this
+                     run's outcome counts, and, when --gate is used, the org's
+                     verdict and its blocking reasons \u2014 as a StoryReport
+                     ReviewJson. Written on every successful push, not only when
+                     a gate runs, so the PR comment can link the run either way.
   --gate             After pushing, ask the cloud whether this commit is safe
                      to release and exit 5 if it is blocked. The policy lives
                      in your organization's settings, not in a file here.
@@ -10992,6 +11130,7 @@ function defaultDeps() {
       }
     },
     appendFile: (filePath, text2) => fs14.appendFileSync(filePath, text2),
+    writeFile: (filePath, text2) => fs14.writeFileSync(filePath, text2, "utf8"),
     fetchFn: fetch,
     git: (args) => {
       try {
@@ -11105,6 +11244,19 @@ function summaryWriter(deps) {
 `);
   };
 }
+var NO_COUNTS = { total: 0, passed: 0, failed: 0, skipped: 0, pending: 0 };
+function runCounts(report) {
+  const summary = report?.summary;
+  if (!summary) return NO_COUNTS;
+  const read = (key) => typeof summary[key] === "number" ? summary[key] : 0;
+  return {
+    total: read("total"),
+    passed: read("passed"),
+    failed: read("failed"),
+    skipped: read("skipped"),
+    pending: read("pending")
+  };
+}
 function cell(text2) {
   return text2.replaceAll("|", "\\|");
 }
@@ -11124,6 +11276,7 @@ async function runPush(rawArgs, depsOverride = {}) {
         base: { type: "string" },
         format: { type: "string" },
         gate: { type: "boolean" },
+        "review-json": { type: "string" },
         force: { type: "boolean" },
         help: { type: "boolean", short: "h" }
       }
@@ -11205,8 +11358,11 @@ async function runPush(rawArgs, depsOverride = {}) {
     deps.error(`Warning: no changed files found against ${base}; pushing without change metadata.`);
   }
   const forced = parsed.values.force === true;
-  const gateArgs = () => ({
+  const gateArgs = (runUrl, counts2 = NO_COUNTS) => ({
     wanted: parsed.values.gate === true,
+    reviewJson: parsed.values["review-json"],
+    runUrl,
+    counts: counts2,
     forced,
     baseUrl,
     key,
@@ -11279,6 +11435,13 @@ async function runPush(rawArgs, depsOverride = {}) {
     result = JSON.parse(body);
   } catch {
   }
+  const counts = runCounts(report);
+  writeReviewJson(
+    parsed.values["review-json"],
+    void 0,
+    { repo, gitSha, runUrl: result.url, counts },
+    deps
+  );
   const runId = String(result.runId ?? "");
   deps.log(runId ? `Pushed run ${runId} (${repo}${branch ? `@${branch}` : ""})` : "Pushed run.");
   if (result.url) deps.log(result.url);
@@ -11306,7 +11469,7 @@ Recommended scope for this change (${recommendations.length}):`);
       summary(`| ${cell(item.confidence)} | ${cell(item.title)} | ${cell(item.reason)} |`);
     }
   }
-  return await gateIfRequested(gateArgs(), deps);
+  return await gateIfRequested(gateArgs(result.url, counts), deps);
 }
 async function gateIfRequested(args, deps) {
   if (!args.wanted) return EXIT_SUCCESS;
@@ -11317,14 +11480,78 @@ async function gateIfRequested(args, deps) {
   const code = await runGate({ ...args, gitSha: args.gitSha }, deps);
   return args.forced && code !== EXIT_GATE_BLOCKED ? EXIT_SUCCESS : code;
 }
+function gateReviewJson(gate, context) {
+  const where = context.gitSha ? `${context.repo}@${context.gitSha.slice(0, 12)}` : context.repo;
+  const evidence = [`organisation release policy, evaluated for ${where}`];
+  const findings = gate === void 0 ? [] : [
+    ...(gate.blocking ?? []).map((reason) => ({
+      kind: "policy",
+      severity: "blocker",
+      title: "Release policy not satisfied",
+      detail: reason,
+      evidence,
+      remedy: (
+        // Named without naming the product: this text reaches a PR comment on
+        // every gated push, and the cloud is not launched yet.
+        "Satisfy the policy this names, or record a decision against the release."
+      )
+    })),
+    ...(gate.warnings ?? []).map((reason) => ({
+      kind: "policy",
+      severity: "minor",
+      title: "Release policy warning",
+      detail: reason,
+      evidence,
+      remedy: "Not blocking this release. Worth clearing before the next one."
+    }))
+  ];
+  const gateStatus = gate === void 0 ? void 0 : gate.status === "no-release" ? "not-evaluated" : gate.status === "blocked" ? "blocked" : "clear";
+  const review = {
+    version: 1,
+    ...context.gitSha ? { headRef: context.gitSha } : {},
+    summary: {
+      totalClaims: 0,
+      byAudience: { stakeholder: 0, engineer: 0 },
+      byStrength: { none: 0, weak: 0, moderate: 0, strong: 0 },
+      changedSourceFiles: 0,
+      uncovered: 0,
+      weaklyCovered: 0,
+      covered: 0
+    },
+    // Taken from the StoryReport this command just pushed, so a comment written
+    // with no gate still says what the run actually did rather than nothing.
+    run: context.counts,
+    findings,
+    changedFiles: [],
+    claims: [],
+    ...context.runUrl ? { reportUrl: context.runUrl } : {},
+    ...gateStatus ? { gate: gateStatus } : {}
+  };
+  return `${JSON.stringify(review, null, 2)}
+`;
+}
+function writeReviewJson(target, gate, context, deps) {
+  if (!target) return;
+  try {
+    deps.writeFile(target, gateReviewJson(gate, context));
+  } catch (err) {
+    deps.error(
+      `Could not write --review-json to ${target}: ${err instanceof Error ? err.message : String(err)}`
+    );
+  }
+}
 async function runGate({
   baseUrl,
   key,
   repo,
   gitSha,
-  onActions
+  onActions,
+  reviewJson,
+  runUrl,
+  counts
 }, deps) {
   const summary = summaryWriter(deps);
+  const writeGateJson = (gate2) => writeReviewJson(reviewJson, gate2, { repo, gitSha, runUrl, counts }, deps);
   const query = `repo=${encodeURIComponent(repo)}&sha=${encodeURIComponent(gitSha)}`;
   let response;
   try {
@@ -11359,6 +11586,7 @@ async function runGate({
     deps.log(`
 No release recorded for ${commit} \u2014 nothing to gate on.`);
     summary("\n**Release gate: no release recorded for this commit**");
+    writeGateJson(gate);
     return EXIT_SUCCESS;
   }
   if (gate.status === "blocked") {
@@ -11370,11 +11598,13 @@ Release gate: BLOCKED for ${commit}`);
       summary(`- ${reason}`);
       if (onActions) deps.error(`::error::Release gate: ${reason}`);
     }
+    writeGateJson(gate);
     return EXIT_GATE_BLOCKED;
   }
   deps.log(`
 Release gate: clear for ${commit}`);
   summary("\n**Release gate: clear**");
+  writeGateJson(gate);
   return EXIT_SUCCESS;
 }
 
@@ -13601,6 +13831,10 @@ ${presetHelpLines().map((l) => `                                  ${l}`).join("\
   --code-diff <path>            (review) Code Diff annotation sidecar (JSON: {title, annotations: [{file, match, text, label?, scenarioIds?}]})
   --patch <path>                (review) Unified patch for --code-diff; generate with "git diff --histogram"
   --strict-code-diff            (review) Gate: exit non-zero on orphaned/ambiguous anchors or unverified scenario references (default: off)
+
+  review writes three files: <output-name>.md and .html for people, and
+  <output-name>.review.json \u2014 the machine contract CI surfaces render (ranked
+  findings, evidence bands, per-claim strength). See the ReviewJson type.
   --emit-canonical <path>       Write canonical JSON to given path
   --help                        Show this help message
 
@@ -15485,9 +15719,16 @@ function writeReviewReport(review, args) {
   fs19.mkdirSync(outputDir, { recursive: true });
   const mdPath = path25.join(outputDir, `${baseName}${suffix}.md`);
   const htmlPath = path25.join(outputDir, `${baseName}${suffix}.html`);
+  const jsonPath = path25.join(outputDir, `${baseName}${suffix}.review.json`);
   fs19.writeFileSync(mdPath, markdown, "utf8");
   fs19.writeFileSync(htmlPath, html, "utf8");
-  return [mdPath, htmlPath];
+  fs19.writeFileSync(
+    jsonPath,
+    `${JSON.stringify(buildReviewJson(review), null, 2)}
+`,
+    "utf8"
+  );
+  return [mdPath, htmlPath, jsonPath];
 }
 function evaluateReviewGate(review, args) {
   const failures = [];