@flakiness/cucumberjs 1.0.0-alpha.0

package/features/scenario_outlines_multiple.feature

@@ -0,0 +1,44 @@
+Feature: Multiple Scenario Outline Examples
+  Scenario: captures multiple examples blocks
+    Given the project file "features/checkout-totals.feature":
+      """
+      Feature: Checkout totals
+        Scenario Outline: totals
+          Given the subtotal is <subtotal> and tax is <tax> and the total is <total>
+
+          @priority
+          Examples: common orders
+            | subtotal | tax | total |
+            | 1        | 2   | 3     |
+            | 2        | 2   | 4     |
+
+          Examples:
+            | subtotal | tax | total |
+            | 3        | 10  | 13    |
+      """
+    And the project file "features/support/steps.js":
+      """
+      const { Given } = require('@cucumber/cucumber');
+
+      Given('the subtotal is {int} and tax is {int} and the total is {int}', function(subtotal, tax, total) {});
+      """
+    When I generate the Flakiness report for "multiple examples"
+    Then the report contains 3 tests
+
+    When I look at the test named "totals [subtotal=1, tax=2, total=3]"
+    And the test is in file "features/checkout-totals.feature" at line 8
+    And the test has tags "priority"
+    And the test contains 1 attempt
+    When I look at the attempt #1
+    Then the attempt contains 1 step:
+      """
+      Given the subtotal is 1 and tax is 2 and the total is 3
+      """
+
+    When I look at the test named "totals [subtotal=2, tax=2, total=4]"
+    And the test is in file "features/checkout-totals.feature" at line 9
+    And the test has tags "priority"
+
+    When I look at the test named "totals [subtotal=3, tax=10, total=13]"
+    And the test is in file "features/checkout-totals.feature" at line 13
+    And the test has tags ""

package/features/statuses.feature

@@ -0,0 +1,70 @@
+Feature: Statuses
+
+  Scenario: captures non-passing Cucumber statuses
+    Given the project file "features/statuses.feature":
+      """
+      Feature: Statuses
+
+        Scenario: it is pending
+          Given a pending step
+
+        Scenario: it is ambiguous
+          Given an ambiguous step
+
+        Scenario: it is undefined
+          Given an undefined step
+      """
+    And the project file "features/support/steps.js":
+      """
+      const { Given } = require('@cucumber/cucumber');
+
+      Given('a pending step', function() {
+        return 'pending';
+      });
+
+      Given('an ambiguous step', function() {});
+      Given('an ambiguous step', function() {});
+      """
+    When I generate the Flakiness report for "cucumber statuses"
+    Then the report contains 3 tests
+
+    When I look at the test named "it is pending"
+    Then the test contains 1 attempt
+    And attempt #1 is "failed"
+    When I look at the "failed" attempt
+    Then the attempt contains 1 error
+    And the attempt error #1 has message "Step is pending"
+    And the attempt contains 1 step:
+      """
+      Given a pending step
+      """
+    When I look at the step #1
+    Then the step has an error with message "Step is pending"
+
+    When I look at the test named "it is ambiguous"
+    Then the test contains 1 attempt
+    And attempt #1 is "failed"
+    When I look at the "failed" attempt
+    Then the attempt contains 1 error
+    And the attempt error #1 has message containing "Multiple step definitions match:"
+    And the attempt contains 1 step:
+      """
+      Given an ambiguous step
+      """
+    When I look at the step #1
+    Then the step has an error with message containing "Multiple step definitions match:"
+
+    When I look at the test named "it is undefined"
+    Then the test contains 1 attempt
+    And attempt #1 is "failed"
+    When I look at the "failed" attempt
+    Then the attempt contains 1 error
+    And the attempt error #1 has message "Undefined step"
+    And the attempt error #1 has a snippet containing "an undefined step"
+    And the attempt contains 1 step:
+      """
+      Given an undefined step
+      """
+    When I look at the step #1
+    Then the step has an error with message "Undefined step"
+    And the step error has a snippet containing "an undefined step"

package/features/stdio.feature

@@ -0,0 +1,29 @@
+Feature: STDIO
+  Scenario: captures Cucumber log output as attempt stdio
+    Given the project file "features/logging.feature":
+      """
+      Feature: Logging
+        Scenario: it logs
+          Given a logging step
+      """
+    And the project file "features/support/steps.js":
+      """
+      const { Given } = require('@cucumber/cucumber');
+      Given('a logging step', async function() {
+        this.log('hello from log');
+        await new Promise(resolve => setTimeout(resolve, 30));
+        this.log('second line');
+      });
+      """
+    When I generate the Flakiness report for "scenario with cucumber log output"
+    When I look at the test named "it logs"
+    Then the test contains 1 attempt
+    When I look at the attempt #1
+    Then the attempt contains 2 stdio entries
+
+    When I look at the stdio entry #1
+    Then the stdio entry has text "hello from log"
+
+    When I look at the stdio entry #2
+    Then the stdio entry has text "second line"
+    And the stdio entry happened after the previous stdio entry

package/features/steps.feature

@@ -0,0 +1,24 @@
+Feature: Steps
+  Scenario: captures test steps
+    Given the project file "features/passing.feature":
+      """
+      Feature: Passing Test Suite
+        Scenario: it passes
+          Given a passing step
+      """
+    And the project file "features/support/steps.js":
+      """
+      const { Given } = require('@cucumber/cucumber');
+      Given('a passing step', function() {});
+      """
+    When I generate the Flakiness report for "passing steps"
+    When I look at the test named "it passes"
+    And the test contains 1 attempt
+
+    When I look at the attempt #1
+    And the attempt contains 1 step:
+      """
+      Given a passing step
+      """
+    When I look at the step #1
+    And the step is in file "features/passing.feature" at line 3

package/features/support/attachments_steps.ts

@@ -0,0 +1,32 @@
+import { Then, When } from '@cucumber/cucumber';
+import assert from 'node:assert/strict';
+import fs from 'node:fs';
+import type { TestWorld } from './harness.ts';
+import { assertCount } from './harness.ts';
+
+When<TestWorld>('I look at the attachment #{int}', function(attachmentIdx: number) {
+  this.attachment = this.attempt?.attachments?.[attachmentIdx - 1];
+});
+
+Then<TestWorld>('the attempt contains {int} attachments', function(expectedAttachments: number) {
+  assertCount(this.attempt?.attachments, expectedAttachments);
+});
+
+Then<TestWorld>('the report contains {int} missing attachments', function(expectedMissingAttachments: number) {
+  assertCount(this.reportResult?.missingAttachments, expectedMissingAttachments);
+});
+
+Then<TestWorld>('the attachment is called {string}', function(expectedName: string) {
+  assert.equal(this.attachment?.name, expectedName);
+});
+
+Then<TestWorld>('the attachment has content type {string}', function(expectedContentType: string) {
+  assert.equal(this.attachment?.contentType, expectedContentType);
+});
+
+Then<TestWorld>('the stored attachment has text {string}', function(expectedText: string) {
+  assert.ok(this.attachment, 'Expected attachment to be selected');
+  const storedAttachment = this.reportResult?.attachments.find(attachment => attachment.id === this.attachment?.id);
+  assert.ok(storedAttachment, `Expected stored attachment for ${this.attachment.id}`);
+  assert.equal(fs.readFileSync(storedAttachment.path, 'utf8'), expectedText);
+});

package/features/support/basic_steps.ts

@@ -0,0 +1,235 @@
+import { BeforeAll, Then, When } from '@cucumber/cucumber';
+import assert from 'node:assert/strict';
+import fs from 'node:fs';
+import type { TestWorld } from './harness.ts';
+import { ARTIFACTS_DIR, assertCount } from './harness.ts';
+import type { FlakinessReport as FK } from '@flakiness/flakiness-report';
+
+BeforeAll(function() {
+  fs.rmSync(ARTIFACTS_DIR, { recursive: true, force: true });
+});
+
+Then<TestWorld>('the report should contain the basic metadata', function() {
+  assert.ok(this.reportResult, 'Expected report result to be defined');
+  const { report, log } = this.reportResult;
+
+  assert.equal(report.category, 'cucumberjs');
+  assert.equal(report.url, 'https://ci.example.test/build/123');
+  assert.equal(report.environments.length, 1);
+  assert.equal(report.environments[0]?.name, 'cucumberjs');
+  assert.ok(report.commitId, 'Expected commitId to be present');
+  assert.ok(report.startTimestamp > 0, 'Expected startTimestamp to be present');
+  assert.ok(report.duration > 0, 'Expected duration to be positive');
+
+  assert.ok((report.cpuCount ?? 0) > 0, 'Expected cpuCount to be populated');
+  assert.ok((report.cpuAvg?.length ?? 0) > 0, 'Expected cpuAvg telemetry to be populated');
+  assert.ok((report.cpuMax?.length ?? 0) > 0, 'Expected cpuMax telemetry to be populated');
+  assert.ok((report.ramBytes ?? 0) > 0, 'Expected ramBytes to be populated');
+  assert.ok((report.ram?.length ?? 0) > 0, 'Expected ram telemetry to be populated');
+
+  assert.equal(log.stderr, '', `Expected stderr to be empty.\n\nSTDERR:\n${log.stderr}`);
+  assert.ok(log.stdout.includes('npx flakiness show'), `Expected report hint in stdout.\n\nSTDOUT:\n${log.stdout}`);
+});
+
+When<TestWorld>('I look at the suite named {string}', function(title: string) {
+  this.suite = findUnique(
+    collectSuites(this.reportResult?.report?.suites ?? []),
+    suite => suite.title === title,
+    `suite named ${JSON.stringify(title)}`,
+  );
+  this.test = undefined;
+  this.attempt = undefined;
+  this.step = undefined;
+  this.attachment = undefined;
+  this.stdio = undefined;
+});
+
+When<TestWorld>('I look at the test named {string}', function(title: string) {
+  this.test = findUnique(
+    collectTests(this.reportResult?.report?.suites ?? []),
+    test => test.title === title,
+    `test named ${JSON.stringify(title)}`,
+  );
+  this.attempt = undefined;
+  this.step = undefined;
+  this.attachment = undefined;
+  this.stdio = undefined;
+});
+
+When<TestWorld>('I look at the attempt #{int}', function(attemptIdx) {
+  this.attempt = this.test?.attempts[attemptIdx - 1];
+  this.step = undefined;
+  this.attachment = undefined;
+  this.stdio = undefined;
+});
+
+When<TestWorld>('I look at the {string} attempt', function(status: string) {
+  this.attempt = findUnique(
+    this.test?.attempts ?? [],
+    attempt => (attempt.status ?? 'passed') === status,
+    `${JSON.stringify(status)} attempt`,
+  );
+  this.step = undefined;
+  this.attachment = undefined;
+  this.stdio = undefined;
+});
+
+When<TestWorld>('I look at the step #{int}', function(stepIdx) {
+  this.step = this.attempt?.steps?.[stepIdx - 1];
+  this.attachment = undefined;
+  this.stdio = undefined;
+});
+
+When<TestWorld>('I look at the step named {string}', function(title: string) {
+  this.step = findUnique(
+    this.attempt?.steps ?? [],
+    step => step.title === title,
+    `step named ${JSON.stringify(title)}`,
+  );
+  this.attachment = undefined;
+  this.stdio = undefined;
+});
+
+Then<TestWorld>('the report contains {int} test(s)', function(expectedTests: number) {
+  assertCount(collectTests(this.reportResult?.report?.suites ?? []), expectedTests);
+});
+
+Then<TestWorld>('the report hierarchy is:', function(expectedHierarchy: string) {
+  const hierarchy = renderReportHierarchy(this.reportResult?.report);
+  assert.equal(normalizeMultiline(hierarchy), normalizeMultiline(expectedHierarchy));
+});
+
+Then<TestWorld>('the suite contains {int} test(s)', function(expectedTests: number) {
+  assertCount(this.suite?.tests, expectedTests);
+});
+
+Then<TestWorld>('the test contains {int} attempt(s)', function(expectedAttempts: number) {
+  assertCount(this.test?.attempts, expectedAttempts);
+});
+
+Then<TestWorld>('the attempt contains {int} step(s)', function(expectedSteps: number) {
+  assertCount(this.attempt?.steps, expectedSteps);
+});
+
+Then<TestWorld>('the attempt contains {int} step(s):', function(expectedSteps: number, expectedStepTitles: string) {
+  const steps = assertCount(this.attempt?.steps, expectedSteps);
+  assert.deepEqual(
+    steps.map(step => step.title),
+    parseMultilineList(expectedStepTitles),
+  );
+});
+
+Then<TestWorld>('the step contains {int} steps(s)', function(expectedSteps: number) {
+  assertCount(this.step?.steps, expectedSteps);
+});
+
+Then<TestWorld>('attempt #{int} is {string}', function(attemptIdx, status) {
+  assert.equal(this.test?.attempts[attemptIdx - 1]?.status ?? 'passed', status);
+});
+
+Then<TestWorld>('the suite is called {string}', function(title) {
+  assert.equal(this.suite?.title, title);
+});
+
+Then<TestWorld>('the suite is in file {string} at line {int}', function(file, line) {
+  assert.equal(this.suite?.location?.file, file);
+  assert.equal(this.suite?.location?.line, line);
+});
+
+Then<TestWorld>('the test is called {string}', function(title) {
+  assert.equal(this.test?.title, title);
+});
+
+Then<TestWorld>('the test is in file {string} at line {int}', function(file, line) {
+  assert.equal(this.test?.location?.file, file);
+  assert.equal(this.test?.location?.line, line);
+});
+
+Then<TestWorld>('the step is called {string}', function(title) {
+  assert.equal(this.step?.title, title);
+});
+
+Then<TestWorld>('the step #{int} is called {string}', function(stepIdx: number, title: string) {
+  assert.equal(this.attempt?.steps?.[stepIdx - 1]?.title, title);
+});
+
+Then<TestWorld>('the step is in file {string} at line {int}', function(file, line) {
+  assert.equal(this.step?.location?.file, file);
+  assert.equal(this.step?.location?.line, line);
+});
+
+function collectSuites(suites: FK.Suite[]): FK.Suite[] {
+  return suites.flatMap(suite => [suite, ...collectSuites(suite.suites ?? [])]);
+}
+
+function collectTests(suites: FK.Suite[]): FK.Test[] {
+  return suites.flatMap(suite => [...(suite.tests ?? []), ...collectTests(suite.suites ?? [])]);
+}
+
+function findUnique<T>(elements: T[], predicate: (element: T) => boolean, description: string): T {
+  const matches = elements.filter(predicate);
+  assert.equal(matches.length, 1, `Expected exactly 1 ${description}, got ${matches.length}`);
+  return matches[0]!;
+}
+
+function renderReportHierarchy(report: FK.Report | undefined): string {
+  return renderTree(
+    (report?.suites ?? []).map(suite => renderSuiteNode(suite)),
+  );
+}
+
+type TreeNode = {
+  label: string,
+  children?: TreeNode[],
+};
+
+function renderSuiteNode(suite: FK.Suite): TreeNode {
+  const label = suite.type === 'file' ? `file ${suite.title}` : `suite ${suite.title}`;
+  return {
+    label,
+    children: [
+      ...(suite.suites ?? []).map(renderSuiteNode),
+      ...(suite.tests ?? []).map(renderTestNode),
+    ],
+  };
+}
+
+function renderTestNode(test: FK.Test): TreeNode {
+  return {
+    label: `test ${test.title}`,
+    children: test.attempts.map((attempt, index) => ({
+      label: `attempt #${index + 1} ${attempt.status ?? 'passed'}`,
+      children: (attempt.steps ?? []).map(step => ({
+        label: `step ${step.title}`,
+      })),
+    })),
+  };
+}
+
+function renderTree(nodes: TreeNode[]): string {
+  return nodes.flatMap((node, index) => renderTreeNode(node, '', index === nodes.length - 1)).join('\n');
+}
+
+function renderTreeNode(node: TreeNode, prefix: string, isLast: boolean): string[] {
+  const branch = isLast ? '`- ' : '|- ';
+  const childPrefix = prefix + (isLast ? '   ' : '|  ');
+  return [
+    `${prefix}${branch}${node.label}`,
+    ...(node.children ?? []).flatMap((child, index, children) => renderTreeNode(child, childPrefix, index === children.length - 1)),
+  ];
+}
+
+function normalizeMultiline(text: string): string {
+  return text
+    .trim()
+    .split('\n')
+    .map(line => line.trimEnd())
+    .join('\n');
+}
+
+function parseMultilineList(text: string): string[] {
+  return normalizeMultiline(text)
+    .split('\n')
+    .map(line => line.trim())
+    .filter(Boolean);
+}

package/features/support/description_steps.ts

@@ -0,0 +1,37 @@
+import { Then } from '@cucumber/cucumber';
+import type { FlakinessReport as FK } from '@flakiness/flakiness-report';
+import assert from 'node:assert/strict';
+import type { TestWorld } from './harness.ts';
+import { assertCount } from './harness.ts';
+
+Then<TestWorld>('the attempt contains {int} annotation(s)', function(expectedAnnotations: number) {
+  assertCount(this.attempt?.annotations, expectedAnnotations);
+});
+
+Then<TestWorld>('the attempt has an annotation {string} with description:', function(type: string, description: string) {
+  const annotation = findAnnotation(this.attempt?.annotations, type);
+  assert.equal(normalizeMultiline(annotation.description ?? ''), normalizeMultiline(description));
+});
+
+Then<TestWorld>('the attempt has no annotation {string}', function(type: string) {
+  const annotations = this.attempt?.annotations ?? [];
+  assert.equal(
+    annotations.filter(annotation => annotation.type === type).length,
+    0,
+    `Expected no annotation of type ${JSON.stringify(type)}, got ${annotations.filter(annotation => annotation.type === type).length}`,
+  );
+});
+
+function findAnnotation(annotations: FK.Annotation[] | undefined, type: string): FK.Annotation {
+  const matches = (annotations ?? []).filter(annotation => annotation.type === type);
+  assert.equal(matches.length, 1, `Expected exactly 1 annotation of type ${JSON.stringify(type)}, got ${matches.length}`);
+  return matches[0]!;
+}
+
+function normalizeMultiline(text: string): string {
+  return text
+    .trim()
+    .split('\n')
+    .map(line => line.trimEnd())
+    .join('\n');
+}

package/features/support/errors_steps.ts

@@ -0,0 +1,48 @@
+import { Then } from '@cucumber/cucumber';
+import assert from 'node:assert/strict';
+import type { TestWorld } from './harness.ts';
+import { assertCount } from './harness.ts';
+
+Then<TestWorld>('the attempt contains {int} error', function(expectedErrors: number) {
+  assertCount(this.attempt?.errors, expectedErrors);
+});
+
+Then<TestWorld>('the attempt error #{int} has message {string}', function(errorIdx: number, message: string) {
+  assert.equal(this.attempt?.errors?.[errorIdx - 1]?.message, message);
+});
+
+Then<TestWorld>('the attempt error #{int} has message containing {string}', function(errorIdx: number, text: string) {
+  assert.ok(
+    this.attempt?.errors?.[errorIdx - 1]?.message?.includes(text),
+    `Expected attempt error message to contain ${JSON.stringify(text)}, got ${JSON.stringify(this.attempt?.errors?.[errorIdx - 1]?.message)}`,
+  );
+});
+
+Then<TestWorld>('the attempt error #{int} has a stack trace', function(errorIdx: number) {
+  assert.ok(this.attempt?.errors?.[errorIdx - 1]?.stack, 'Expected error stack to be present');
+});
+
+Then<TestWorld>('the attempt error #{int} has a snippet containing {string}', function(errorIdx: number, text: string) {
+  assert.ok(
+    this.attempt?.errors?.[errorIdx - 1]?.snippet?.includes(text),
+    `Expected attempt error snippet to contain ${JSON.stringify(text)}, got ${JSON.stringify(this.attempt?.errors?.[errorIdx - 1]?.snippet)}`,
+  );
+});
+
+Then<TestWorld>('the step has an error with message {string}', function(message: string) {
+  assert.equal(this.step?.error?.message, message);
+});
+
+Then<TestWorld>('the step has an error with message containing {string}', function(text: string) {
+  assert.ok(
+    this.step?.error?.message?.includes(text),
+    `Expected step error message to contain ${JSON.stringify(text)}, got ${JSON.stringify(this.step?.error?.message)}`,
+  );
+});
+
+Then<TestWorld>('the step error has a snippet containing {string}', function(text: string) {
+  assert.ok(
+    this.step?.error?.snippet?.includes(text),
+    `Expected step error snippet to contain ${JSON.stringify(text)}, got ${JSON.stringify(this.step?.error?.snippet)}`,
+  );
+});