@flakiness/cucumberjs 1.0.0-alpha.0

package/.github/workflows/flakiness-upload-fork-prs.yml

@@ -0,0 +1,30 @@
+name: Upload Flakiness.io report (fork PRs)
+on:
+  workflow_run:
+    # Must match the name(s) of workflows that produce flakiness-report artifacts
+    workflows: ["Tests"]
+    types: [completed]
+
+jobs:
+  upload-flakiness-report:
+    runs-on: ubuntu-latest
+    if: >-
+      (github.event.workflow_run.conclusion == 'success' || github.event.workflow_run.conclusion == 'failure')
+      && github.event.workflow_run.event == 'pull_request'
+      && github.event.workflow_run.head_repository.full_name != github.event.workflow_run.repository.full_name
+    permissions:
+      actions: read
+      contents: read
+      id-token: write
+    steps:
+      - name: Install Flakiness CLI
+        run: curl -LsSf https://cli.flakiness.io/install.sh | sh
+
+      - name: Download flakiness-report artifacts
+        env:
+          GH_TOKEN: ${{ github.token }}
+          RUN_ID: ${{ github.event.workflow_run.id }}
+        run: gh run download "$RUN_ID" --repo "$GITHUB_REPOSITORY" --pattern 'flakiness-report-*' --dir .
+
+      - name: Upload to Flakiness.io
+        run: find . -path '*/flakiness-report-*/report.json' -exec flakiness upload {} \;

package/.github/workflows/publish-npm.yml

@@ -0,0 +1,41 @@
+name: Publish to NPM
+
+on:
+  release:
+    types: [published]
+
+permissions:
+  id-token: write  # Required for OIDC
+  contents: read
+
+jobs:
+  publish-to-npm:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: 24
+
+      - name: Setup PNPM
+        uses: pnpm/action-setup@v4
+        with:
+          version: 10
+
+      - name: Build & Publish
+        run: |
+          pnpm i --frozen-lockfile
+          pnpm build
+
+          VERSION=${GITHUB_REF_NAME#v}
+
+          if [[ "$VERSION" == *"-"* ]]; then
+            echo "Publishing prerelease to @next"
+            pnpm publish --access=public --no-git-checks --tag next
+          else
+            echo "Publishing stable to @latest"
+            pnpm publish --access=public --no-git-checks
+          fi

package/.github/workflows/tests.yml

@@ -0,0 +1,45 @@
+name: Tests
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+permissions:
+  contents: read
+  id-token: write
+
+jobs:
+  test:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          version: 10
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: lts/*
+          cache: pnpm
+
+      # Since tests initialize git repositories, we have
+      # to configure git to avoid warnings.
+      - run: git config --global init.defaultBranch main
+      - run: pnpm install --frozen-lockfile
+      - run: pnpm build
+      - run: pnpm test
+      - name: Upload Flakiness report artifact (for Pull Requests from forks)
+        if: always() && github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork
+        uses: actions/upload-artifact@v4
+        with:
+          name: flakiness-report-${{ github.job }}-${{ strategy.job-index }}
+          path: flakiness-report/
+          retention-days: 1

package/CONTRIBUTING.md

@@ -0,0 +1,58 @@
+# Contributing
+
+## Prerequisites
+
+- Node.js 22+
+- [pnpm](https://pnpm.io/)
+
+## Getting Started
+
+Clone the repo and install dependencies:
+
+```bash
+git clone https://github.com/flakiness/cucumberjs.git fk-cucumber
+cd fk-cucumber
+pnpm install
+```
+
+## Building
+
+This project uses [Kubik](https://github.com/flakiness/kubik) as its build system. The build script is defined in `build.mts`.
+
+To build:
+
+```bash
+pnpm build
+```
+
+To watch:
+
+```bash
+pnpm build -w
+```
+
+This will bundle the source with esbuild and generate TypeScript declarations.
+
+## Releasing
+
+To release a new version:
+
+1. Bump the version:
+
+   ```bash
+   # For a stable minor release
+   pnpm version minor
+
+   # For an alpha pre-release
+   pnpm version preminor --preid=alpha
+   ```
+
+2. Push the commit and tag:
+
+   ```bash
+   git push --follow-tags
+   ```
+
+3. [Create a GitHub Release](https://github.com/flakiness/cucumberjs/releases/new) for the new tag and publish it.
+
+   CI will handle publishing to npm. Pre-releases are published under @next tag.

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023-2026 Degu Labs, Inc
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/agenda.md

@@ -0,0 +1,2 @@
+- descriptions as annotations
+

package/build.mts

@@ -0,0 +1,34 @@
+#!/usr/bin/env pnpm kubik
+
+import esbuild from 'esbuild';
+import fs from 'fs';
+import { Task } from 'kubik';
+import path from 'path';
+
+const { __dirname, $ } = Task.init(import.meta, {
+  name: 'cucubmer',
+  watch: [ './src' ],
+});
+
+const outDir = path.join(__dirname, 'lib');
+const typesDir = path.join(__dirname, 'types');
+const srcDir = path.join(__dirname, 'src');
+await fs.promises.rm(outDir, { recursive: true, force: true });
+await fs.promises.rm(typesDir, { recursive: true, force: true });
+
+const { errors } = await esbuild.build({
+  color: true,
+  entryPoints: [
+    path.join(srcDir, 'formatter.ts'),
+  ],
+  outdir: outDir,
+  format: 'esm',
+  platform: 'node',
+  target: ['node22'],
+  sourcemap: true,
+  bundle: false,
+  minify: false,
+});
+
+if (!errors.length)
+  await $`tsc --pretty -p .`;

package/cucumber.mjs

@@ -0,0 +1,6 @@
+export default {
+  paths: ['features/**/*.feature'],
+  import: ['tsx/esm', 'features/support/**/*.ts'],
+  format: ['@cucumber/pretty-formatter'],
+  parallel: 0,
+};

package/features/attachments.feature

@@ -0,0 +1,32 @@
+Feature: Attachments
+  Scenario: captures Cucumber attachments
+    Given the project file "features/attachments.feature":
+      """
+      Feature: Attachments
+        Scenario: it attaches data
+          Given a step with attachments
+      """
+    And the project file "features/support/steps.js":
+      """
+      const { Given } = require('@cucumber/cucumber');
+      Given('a step with attachments', function() {
+        this.attach('hello attachment', { mediaType: 'text/plain', fileName: 'note.txt' });
+        this.attach(Buffer.from('{"ok":true}', 'utf8'), { mediaType: 'application/json', fileName: 'data.json' });
+      });
+      """
+    When I generate the Flakiness report for "scenario with attachments"
+    When I look at the test named "it attaches data"
+    Then the test contains 1 attempt
+    When I look at the attempt #1
+    Then the attempt contains 2 attachments
+    And the report contains 0 missing attachments
+
+    When I look at the attachment #1
+    Then the attachment is called "note.txt"
+    And the attachment has content type "text/plain"
+    And the stored attachment has text "hello attachment"
+
+    When I look at the attachment #2
+    Then the attachment is called "data.json"
+    And the attachment has content type "application/json"
+    And the stored attachment has text "{\"ok\":true}"

package/features/basic.feature

@@ -0,0 +1,27 @@
+Feature: Basic Functionality
+  Scenario: generates a basic report
+    Given the project file "features/passing.feature":
+      """
+      Feature: Passing Test Suite
+        Scenario: it passes
+          Given a passing step
+      """
+    And the project file "features/support/steps.js":
+      """
+      const { Given } = require('@cucumber/cucumber');
+      Given('a passing step', function() {});
+      """
+    And the environment variable "BUILD_URL" is "https://ci.example.test/build/123"
+    When I generate the Flakiness report for "passing scenario"
+    Then the report should contain the basic metadata
+    And the report hierarchy is:
+      """
+      `- file passing.feature
+         `- suite Passing Test Suite
+            `- test it passes
+               `- attempt #1 passed
+                  `- step Given a passing step
+      """
+
+    When I look at the test named "it passes"
+    Then the test is in file "features/passing.feature" at line 2

package/features/data_tables.feature

@@ -0,0 +1,45 @@
+Feature: Data Tables
+  Scenario: keeps step titles single-line for data tables
+    Given the project file "features/data-table.feature":
+      """
+      Feature: Data table
+
+        Scenario: addition
+          Given a table
+            | a | b | result |
+            | 2 | 2 | 4 |
+            | 1 | 7 | 8 |
+          When I add the values
+          Then I get the expected results
+      """
+    And the project file "features/support/steps.js":
+      """
+      const assert = require('node:assert/strict');
+      const { Given, When, Then } = require('@cucumber/cucumber');
+
+      Given('a table', function(dataTable) {
+        this.rows = dataTable.hashes().map(row => ({
+          a: Number(row.a),
+          b: Number(row.b),
+          result: Number(row.result),
+        }));
+      });
+
+      When('I add the values', function() {
+        this.results = this.rows.map(row => row.a + row.b);
+      });
+
+      Then('I get the expected results', function() {
+        assert.deepEqual(this.results, this.rows.map(row => row.result));
+      });
+      """
+    When I generate the Flakiness report for "scenario with a data table"
+    When I look at the test named "addition"
+    Then the test contains 1 attempt
+    When I look at the attempt #1
+    Then the attempt contains 3 steps:
+      """
+      Given a table
+      When I add the values
+      Then I get the expected results
+      """

package/features/description.feature

@@ -0,0 +1,49 @@
+Feature: Descriptions
+
+  Scenario: captures native Gherkin descriptions as annotations
+    Given the project file "features/help-center.feature":
+      """
+      Feature: Help center
+        These scenarios describe what support agents can read.
+        They should stay visible in the report.
+
+        Scenario: opens a pinned article
+          Support agents can inspect a pinned article before replying.
+          Given a pinned article exists
+
+        Scenario: lists pinned articles
+          Given several pinned articles exist
+      """
+    And the project file "features/support/steps.js":
+      """
+      const { Given } = require('@cucumber/cucumber');
+
+      Given('a pinned article exists', function() {});
+      Given('several pinned articles exist', function() {});
+      """
+    When I generate the Flakiness report for "descriptions"
+
+    When I look at the test named "opens a pinned article"
+    Then the test contains 1 attempt
+    When I look at the attempt #1
+    Then the attempt contains 2 annotations
+    And the attempt has an annotation "feature" with description:
+      """
+      These scenarios describe what support agents can read.
+      They should stay visible in the report.
+      """
+    And the attempt has an annotation "scenario" with description:
+      """
+      Support agents can inspect a pinned article before replying.
+      """
+
+    When I look at the test named "lists pinned articles"
+    Then the test contains 1 attempt
+    When I look at the attempt #1
+    Then the attempt contains 1 annotation
+    And the attempt has an annotation "feature" with description:
+      """
+      These scenarios describe what support agents can read.
+      They should stay visible in the report.
+      """
+    And the attempt has no annotation "scenario"

package/features/errors.feature

@@ -0,0 +1,28 @@
+Feature: Errors
+  Scenario: captures thrown errors
+    Given the project file "features/failing.feature":
+      """
+      Feature: Failing
+        Scenario: it fails
+          Given a step that throws an error
+      """
+    And the project file "features/support/steps.js":
+      """
+      const { Given } = require('@cucumber/cucumber');
+      Given('a step that throws an error', function() {
+        throw new Error('intentional failure');
+      });
+      """
+    When I generate the Flakiness report for "failing scenario"
+    When I look at the test named "it fails"
+    Then the test contains 1 attempt
+    When I look at the "failed" attempt
+    # Make sure the attempt has error
+    Then the attempt contains 1 error
+    And the attempt error #1 has message "intentional failure"
+    And the attempt error #1 has a stack trace
+
+    # Make sure the same error exists in step too
+    Then the attempt contains 1 step
+    When I look at the step named "Given a step that throws an error"
+    And the step has an error with message "intentional failure"

package/features/hooks_named.feature

@@ -0,0 +1,32 @@
+Feature: Named Hooks
+  Scenario: captures named hooks
+    Given the project file "features/passing.feature":
+      """
+      Feature: Hooks
+        Scenario: it runs named hooks
+          Given a passing step
+      """
+    And the project file "features/support/steps.js":
+      """
+      const { Before, After, Given } = require('@cucumber/cucumber');
+      Before({ name: 'database setup' }, function() {});
+      After({ name: 'database cleanup' }, function() {});
+      Given('a passing step', function() {});
+      """
+    When I generate the Flakiness report for "scenario with named hooks"
+    When I look at the test named "it runs named hooks"
+    And the test contains 1 attempt
+    When I look at the attempt #1
+    And the attempt contains 3 steps
+
+    When I look at the step #1
+    Then the step is called "Before (database setup)"
+    And the step is in file "features/support/steps.js" at line 2
+
+    When I look at the step #2
+    Then the step is called "Given a passing step"
+    And the step is in file "features/passing.feature" at line 3
+
+    When I look at the step #3
+    Then the step is called "After (database cleanup)"
+    And the step is in file "features/support/steps.js" at line 3

package/features/hooks_unnamed.feature

@@ -0,0 +1,33 @@
+Feature: Unnamed Hooks
+  Scenario: captures before and after hooks
+    Given the project file "features/passing.feature":
+      """
+      Feature: Hooks
+        Scenario: it runs hooks
+          Given a passing step
+      """
+    And the project file "features/support/steps.js":
+      """
+      const { Before, After, Given } = require('@cucumber/cucumber');
+      Before(function() {});
+      After(function() {});
+      Given('a passing step', function() {});
+      """
+    And the environment variable "BUILD_URL" is "https://ci.example.test/build/123"
+    When I generate the Flakiness report for "scenario with hooks"
+    When I look at the test named "it runs hooks"
+    And the test contains 1 attempt
+    When I look at the attempt #1
+    And the attempt contains 3 steps
+
+    When I look at the step #1
+    Then the step is called "Before"
+    And the step is in file "features/support/steps.js" at line 2
+
+    When I look at the step #2
+    Then the step is called "Given a passing step"
+    And the step is in file "features/passing.feature" at line 3
+
+    When I look at the step #3
+    Then the step is called "After"
+    And the step is in file "features/support/steps.js" at line 3

package/features/locations.feature

@@ -0,0 +1,37 @@
+Feature: Locations
+  Scenario: computes locations relative to the git root
+    Given the project file "features/orders/nested/checkout.feature":
+      """
+      Feature: Nested checkout
+
+        Scenario: computes locations from git root
+          Given a passing step
+      """
+    And the project file "features/support/nested/steps.js":
+      """
+      const { Before, Given } = require('@cucumber/cucumber');
+
+      Before(function() {});
+      Given('a passing step', function() {});
+      """
+    When I generate the Flakiness report for "nested locations"
+    When I look at the suite named "checkout.feature"
+    Then the suite is in file "features/orders/nested/checkout.feature" at line 0
+    When I look at the suite named "Nested checkout"
+    Then the suite is in file "features/orders/nested/checkout.feature" at line 1
+    When I look at the test named "computes locations from git root"
+    Then the test is in file "features/orders/nested/checkout.feature" at line 3
+    And the test contains 1 attempt
+
+    When I look at the attempt #1
+    Then the attempt contains 2 steps:
+      """
+      Before
+      Given a passing step
+      """
+
+    When I look at the step #1
+    Then the step is in file "features/support/nested/steps.js" at line 3
+
+    When I look at the step #2
+    Then the step is in file "features/orders/nested/checkout.feature" at line 4

package/features/retries.feature

@@ -0,0 +1,30 @@
+Feature: Retries
+  Scenario: captures a scenario that succeeds on retry
+    Given the project file "features/eventually-passing.feature":
+      """
+      Feature: Eventually passing
+        Scenario: it succeeds on retry
+          Given a step that succeeds on retry
+      """
+    And the project file "features/support/steps.js":
+      """
+      const { Given } = require('@cucumber/cucumber');
+      let hasFailedOnce = false;
+
+      Given('a step that succeeds on retry', function() {
+        if (hasFailedOnce)
+          return;
+        hasFailedOnce = true;
+        throw new Error('intentional first-attempt failure');
+      });
+      """
+    And the Cucumber arguments are:
+      | --retry |
+      | 1 |
+    When I generate the Flakiness report for "flaky scenario"
+    When I look at the test named "it succeeds on retry"
+    Then the test contains 2 attempts
+    When I look at the "failed" attempt
+    Then the attempt contains 1 step
+    When I look at the "passed" attempt
+    Then the attempt contains 1 step

package/features/rules.feature

@@ -0,0 +1,25 @@
+Feature: Rules
+  Scenario: captures rules as suites
+    Given the project file "features/rules.feature":
+      """
+      Feature: Wallet
+
+        Rule: Balance cannot go negative
+          Scenario: rejects overspending
+            Given an account with a balance
+      """
+    And the project file "features/support/steps.js":
+      """
+      const { Given } = require('@cucumber/cucumber');
+      Given('an account with a balance', function() {});
+      """
+    When I generate the Flakiness report for "rules"
+    Then the report hierarchy is:
+      """
+      `- file rules.feature
+         `- suite Wallet
+            `- suite Balance cannot go negative
+               `- test rejects overspending
+                  `- attempt #1 passed
+                     `- step Given an account with a balance
+      """

package/features/scenario_outlines.feature

@@ -0,0 +1,57 @@
+Feature: Scenario Outlines
+  Scenario: captures one reported test per example row
+    Given the project file "features/addition.feature":
+      """
+      Feature: Addition
+
+        Scenario Outline: addition
+          Given a calculator
+          When I add <a> to <b>
+          Then I get <result>
+
+          Examples:
+            | a | b | result |
+            | 2 | 2 | 4 |
+            | 1 | 7 | 8 |
+      """
+    And the project file "features/support/steps.js":
+      """
+      const assert = require('node:assert/strict');
+      const { Given, When, Then } = require('@cucumber/cucumber');
+
+      Given('a calculator', function() {});
+
+      When('I add {int} to {int}', function(a, b) {
+        this.result = a + b;
+      });
+
+      Then('I get {int}', function(expected) {
+        assert.equal(this.result, expected);
+      });
+      """
+    When I generate the Flakiness report for "scenario outline"
+    Then the report contains 2 tests
+
+    When I look at the test named "addition [a=2, b=2, result=4]"
+    And the test is in file "features/addition.feature" at line 10
+    And the test contains 1 attempt
+
+    When I look at the attempt #1
+    Then the attempt contains 3 steps:
+      """
+      Given a calculator
+      When I add 2 to 2
+      Then I get 4
+      """
+
+    When I look at the test named "addition [a=1, b=7, result=8]"
+    And the test is in file "features/addition.feature" at line 11
+    And the test contains 1 attempt
+
+    When I look at the attempt #1
+    Then the attempt contains 3 steps:
+      """
+      Given a calculator
+      When I add 1 to 7
+      Then I get 8
+      """