@flakiness/cucumberjs 1.0.0-alpha.0

package/features/support/harness.ts

@@ -0,0 +1,196 @@
+import type { IWorld } from '@cucumber/cucumber';
+import type { FlakinessReport as FK } from '@flakiness/flakiness-report';
+import { readReport } from '@flakiness/sdk';
+import assert from 'node:assert/strict';
+import { execFileSync, spawnSync } from 'node:child_process';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { setTimeout as delay } from 'node:timers/promises';
+import { pathToFileURL } from 'node:url';
+
+export type TestWorld = IWorld & {
+  reportResult?: GenerateFlakinessReportResult,
+  projectArgs?: string[],
+  projectEnv?: Record<string, string | undefined>,
+  projectFiles?: ProjectFiles,
+  suite?: FK.Suite,
+  test?: FK.Test,
+  attempt?: FK.RunAttempt,
+  attachment?: FK.Attachment,
+  step?: FK.TestStep,
+  stdio?: FK.TimedSTDIOEntry,
+};
+
+export type ProjectFiles = Record<string, string>;
+
+export type SampleProjectRun = {
+  status: number,
+  stdout: string,
+  stderr: string,
+  targetDir: string,
+  reportDir: string,
+};
+
+export type ProjectRunOptions = {
+  args?: string[],
+  env?: Record<string, string | undefined>,
+  formatOptions?: Record<string, unknown>,
+};
+
+export type GenerateFlakinessReportResult = Awaited<ReturnType<typeof readReport>> & {
+  log: {
+    stdout: string,
+    stderr: string,
+  },
+};
+
+export const ARTIFACTS_DIR = process.platform === 'darwin'
+  ? '/private/tmp/flakiness-cucumber'
+  : path.join(os.tmpdir(), 'flakiness-cucumber');
+
+const PROJECT_ROOT = path.resolve(import.meta.dirname, '..', '..');
+const CUCUMBER_BIN = path.join(PROJECT_ROOT, 'node_modules', '@cucumber', 'cucumber', 'bin', 'cucumber.js');
+const FORMATTER_PATH = path.join(PROJECT_ROOT, 'lib', 'formatter.js');
+const FORMATTER_DESCRIPTOR = JSON.stringify(pathToFileURL(FORMATTER_PATH).href);
+const NODE_MODULES_PATH = path.join(PROJECT_ROOT, 'node_modules');
+
+const CLEARED_CI_ENV: Record<string, undefined> = {
+  BUILD_BUILDID: undefined,
+  BUILD_URL: undefined,
+  CI_JOB_URL: undefined,
+  GITHUB_REPOSITORY: undefined,
+  GITHUB_RUN_ATTEMPT: undefined,
+  GITHUB_RUN_ID: undefined,
+  GITHUB_SERVER_URL: undefined,
+  SYSTEM_TEAMFOUNDATIONCOLLECTIONURI: undefined,
+  SYSTEM_TEAMPROJECT: undefined,
+};
+
+const DEFAULT_FILES: ProjectFiles = {
+  'package.json': JSON.stringify({
+    name: 'sample-cucumber-project',
+    version: '1.0.0',
+  }, null, 2),
+};
+
+function runSampleProject(name: string, files: ProjectFiles, options: ProjectRunOptions = {}): SampleProjectRun {
+  const targetDir = path.join(ARTIFACTS_DIR, slugify(name));
+  const reportDir = path.join(targetDir, 'flakiness-report');
+  fs.rmSync(targetDir, { recursive: true, force: true });
+  fs.mkdirSync(targetDir, { recursive: true });
+
+  for (const [filePath, content] of Object.entries({ ...DEFAULT_FILES, ...files })) {
+    const fullPath = path.join(targetDir, ...filePath.split('/'));
+    fs.mkdirSync(path.dirname(fullPath), { recursive: true });
+    fs.writeFileSync(fullPath, content);
+  }
+
+  initGitRepo(targetDir);
+
+  const result = spawnSync(
+    process.execPath,
+    [
+      CUCUMBER_BIN,
+      'features/**/*.feature',
+      '--require',
+      'features/support/**/*.js',
+      ...(options.args ?? []),
+      '--format',
+      FORMATTER_DESCRIPTOR,
+      ...formatOptionsArgs(options.formatOptions),
+    ],
+    {
+      cwd: targetDir,
+      encoding: 'utf8',
+      env: {
+        ...process.env,
+        ...CLEARED_CI_ENV,
+        NODE_PATH: [NODE_MODULES_PATH, process.env.NODE_PATH].filter(Boolean).join(path.delimiter),
+        ...options.env,
+      },
+    },
+  );
+
+  return {
+    status: result.status ?? 0,
+    stdout: result.stdout ?? '',
+    stderr: result.stderr ?? '',
+    targetDir,
+    reportDir,
+  };
+}
+
+export async function generateFlakinessReport(
+  name: string,
+  files: ProjectFiles,
+  options: ProjectRunOptions = {},
+): Promise<GenerateFlakinessReportResult> {
+  const run = runSampleProject(name, files, {
+    args: options.args,
+    env: options.env,
+    formatOptions: {
+      ...options.formatOptions,
+      outputFolder: path.join(ARTIFACTS_DIR, slugify(name), 'flakiness-report'),
+      disableUpload: true,
+      open: 'never',
+    },
+  });
+  await waitForReport(run);
+
+  return {
+    ...(await readReport(run.reportDir)),
+    log: {
+      stdout: run.stdout,
+      stderr: run.stderr,
+    },
+  };
+}
+
+function initGitRepo(targetDir: string): void {
+  execFileSync('git', ['init'], { cwd: targetDir, stdio: 'pipe' });
+  execFileSync('git', ['add', '.'], { cwd: targetDir, stdio: 'pipe' });
+  execFileSync(
+    'git',
+    ['-c', 'user.email=john@example.com', '-c', 'user.name=john', '-c', 'commit.gpgsign=false', 'commit', '-m', 'staging'],
+    {
+      cwd: targetDir,
+      stdio: 'pipe',
+    },
+  );
+}
+
+function slugify(value: string): string {
+  return value
+    .replace(/[^.a-zA-Z0-9-]+/g, '-')
+    .replace(/-+/g, '-')
+    .replace(/^-|-$/g, '')
+    .toLowerCase();
+}
+
+function formatOptionsArgs(formatOptions: Record<string, unknown> | undefined): string[] {
+  if (!formatOptions || !Object.keys(formatOptions).length)
+    return [];
+  return ['--format-options', JSON.stringify(formatOptions)];
+}
+
+export function assertCount<T>(elements: T[] | undefined, count: number): T[] {
+  assert.equal(elements?.length ?? 0, count);
+  return elements!;
+}
+
+async function waitForReport(run: SampleProjectRun): Promise<void> {
+  const reportPath = path.join(run.reportDir, 'report.json');
+  for (let attempt = 0; attempt < 50; ++attempt) {
+    if (fs.existsSync(reportPath))
+      return;
+    await delay(20);
+  }
+
+  throw new Error([
+    `Report not found at ${reportPath}`,
+    `Exit status: ${run.status}`,
+    `STDOUT:\n${run.stdout}`,
+    `STDERR:\n${run.stderr}`,
+  ].join('\n\n'));
+}

package/features/support/project_steps.ts

@@ -0,0 +1,24 @@
+import { DataTable, Given, When } from '@cucumber/cucumber';
+import type { TestWorld } from './harness.ts';
+import { generateFlakinessReport } from './harness.ts';
+
+Given<TestWorld>('the project file {string}:', function(filePath: string, content: string) {
+  this.projectFiles ??= {};
+  this.projectFiles[filePath] = content;
+});
+
+Given<TestWorld>('the environment variable {string} is {string}', function(name: string, value: string) {
+  this.projectEnv ??= {};
+  this.projectEnv[name] = value;
+});
+
+Given<TestWorld>('the Cucumber arguments are:', function(args: DataTable) {
+  this.projectArgs = args.raw().flatMap(row => row);
+});
+
+When<TestWorld>('I generate the Flakiness report for {string}', async function(name: string) {
+  this.reportResult = await generateFlakinessReport(name, this.projectFiles ?? {}, {
+    args: this.projectArgs,
+    env: this.projectEnv,
+  });
+});

package/features/support/stdio_steps.ts

@@ -0,0 +1,21 @@
+import { Then, When } from '@cucumber/cucumber';
+import assert from 'node:assert/strict';
+import type { TestWorld } from './harness.ts';
+import { assertCount } from './harness.ts';
+
+When<TestWorld>('I look at the stdio entry #{int}', function(entryIdx: number) {
+  this.stdio = this.attempt?.stdio?.[entryIdx - 1];
+});
+
+Then<TestWorld>('the attempt contains {int} stdio entries', function(expectedEntries: number) {
+  assertCount(this.attempt?.stdio, expectedEntries);
+});
+
+Then<TestWorld>('the stdio entry has text {string}', function(expectedText: string) {
+  assert.ok(this.stdio && 'text' in this.stdio, 'Expected a text stdio entry');
+  assert.equal(this.stdio.text, expectedText);
+});
+
+Then<TestWorld>('the stdio entry happened after the previous stdio entry', function() {
+  assert.ok((this.stdio?.dts ?? 0) > 0, 'Expected stdio entry delta to be positive');
+});

package/features/support/tags_steps.ts

@@ -0,0 +1,10 @@
+import { Then } from '@cucumber/cucumber';
+import assert from 'node:assert/strict';
+import type { TestWorld } from './harness.ts';
+
+Then<TestWorld>('the test has tags {string}', function(tags: string) {
+  assert.deepEqual(
+    [...(this.test?.tags ?? [])].sort(),
+    tags.split(',').map(tag => tag.trim()).filter(Boolean).sort(),
+  );
+});

package/features/tags.feature

@@ -0,0 +1,19 @@
+Feature: Tags
+  Scenario: captures test tags
+    Given the project file "features/tagged.feature":
+      """
+      @feature-tag
+      Feature: Tagged
+        @smoke @fast
+        Scenario: it has tags
+          Given a passing step
+      """
+    And the project file "features/support/steps.js":
+      """
+      const { Given } = require('@cucumber/cucumber');
+      Given('a passing step', function() {});
+      """
+    And the environment variable "BUILD_URL" is "https://ci.example.test/build/123"
+    When I generate the Flakiness report for "tagged scenario"
+    When I look at the test named "it has tags"
+    Then the test has tags "feature-tag, smoke, fast"

package/features/tags_hierarchy.feature

@@ -0,0 +1,37 @@
+Feature: Labels
+  Scenario: captures inherited tags across a feature tree
+    Given the project file "features/hierarchy-tags.feature":
+      """
+      @checkout
+      Feature: Checkout flow
+
+        Scenario: guest checkout
+          Given a guest begins checkout
+
+        @card
+        Rule: Card payments
+          @visa
+          Scenario: paying with a saved card
+            Given a saved card is available
+
+          Scenario: paying with a new card
+            Given a new card is entered
+      """
+    And the project file "features/support/steps.js":
+      """
+      const { Given } = require('@cucumber/cucumber');
+
+      Given('a guest begins checkout', function() {});
+      Given('a saved card is available', function() {});
+      Given('a new card is entered', function() {});
+      """
+    When I generate the Flakiness report for "hierarchy tags"
+
+    When I look at the test named "guest checkout"
+    Then the test has tags "checkout"
+
+    When I look at the test named "paying with a saved card"
+    Then the test has tags "checkout, card, visa"
+
+    When I look at the test named "paying with a new card"
+    Then the test has tags "checkout, card"

package/package.json

@@ -0,0 +1,37 @@
+{
+  "name": "@flakiness/cucumberjs",
+  "version": "1.0.0-alpha.0",
+  "description": "",
+  "exports": {
+    ".": {
+      "types": "./types/src/formatter.d.ts",
+      "import": "./lib/formatter.js",
+      "require": "./lib/formatter.js"
+    }
+  },
+  "main": "./lib/formatter.js",
+  "types": "./types/src/formatter.d.ts",
+  "scripts": {
+    "build": "kubik build.mts",
+    "test": "cucumber-js",
+    "coverage": "c8 --reporter=text --reporter=html --include=lib/formatter.js cucumber-js"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "MIT",
+  "type": "module",
+  "dependencies": {
+    "@flakiness/flakiness-report": "^0.28.0",
+    "@flakiness/sdk": "^2.2.2"
+  },
+  "devDependencies": {
+    "@cucumber/cucumber": "^12.7.0",
+    "@cucumber/messages": "^32.2.0",
+    "@types/node": "^25.5.0",
+    "c8": "^11.0.0",
+    "esbuild": "^0.27.4",
+    "kubik": "^0.24.0",
+    "tsx": "^4.21.0",
+    "typescript": "^5.9.3"
+  }
+}

package/plan.md

@@ -0,0 +1,59 @@
+# Gherkin Compatibility Review
+
+Scope:
+- Compared the sample `.feature` files from `/Users/aslushnikov/prog/allure-js/packages/allure-cucumberjs/test/samples/features`
+- Ignored Allure-only runtime/config behavior
+- Focused only on Gherkin/Cucumber constructs that our reporter should handle
+
+## Supported, but with caveats
+
+- `examples-name-change.feature`
+  Outline title with `<a>`, `<b>`, `<c>` should work.
+  Caveat: our final test title still appends `[a=..., b=..., c=...]`.
+
+- `tags_hierarchy.feature`
+  As Gherkin, this is feature tags plus `Rule` plus scenario tags under the rule.
+  Caveat: tags are stored without `@`.
+
+- `links.feature`
+  As Gherkin, this is also just tags plus `Rule`.
+  Caveat: tags are stored without `@`.
+
+- `stepArguments.feature`
+  Substituted step arguments should report correctly.
+  Caveat: step titles omit `Given/When/Then`.
+
+## Needs work
+
+- `dataTable.feature`
+  The outline itself is fine.
+  Missing: the data table content is not preserved structurally in the report.
+
+- `dataTableAndExamples.feature`
+  The outline itself is fine.
+  Missing: neither the data table nor the examples table is preserved structurally.
+
+- `examples-multi.feature`
+  Multiple `Examples` blocks should execute fine.
+  Missing: example-block identity/name is lost, and we do not preserve examples tables structurally.
+
+- `nested/simple.feature`
+  Nested path should execute fine.
+  Missing: file suite title is only `simple.feature`, so same-basename files would collide visually.
+
+- `undefinedStepDefs.feature`
+  This probably runs through okay.
+  Missing: we do not have direct coverage, and `UNDEFINED` is currently flattened to `failed`.
+
+## Clearly missing
+
+- `description.feature`
+  Feature descriptions and scenario descriptions are not reported at all.
+
+## Suggested implementation order
+
+1. Add coverage for `undefinedStepDefs.feature`
+2. Add coverage for multiple `Examples` blocks
+3. Preserve feature/scenario descriptions
+4. Preserve data tables and examples tables structurally
+5. Improve nested feature-file identity beyond basename-only file suites

package/pnpm-workspace.yaml

@@ -0,0 +1,2 @@
+allowBuilds:
+  esbuild: true