buildkite-test_collector 2.8.0 → 2.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a06be6db9195629f49724e87ac755466db10f8e8ae5289ca7f3a1803a335ff66
-  data.tar.gz: 8b0110c802047ac6f169156f8f07e0808a2d23e8cc3f47156a5208a92167e0a4
+  metadata.gz: 497ced90aac96227b8e4d2fc9ad1685eb4f554742b597568efa7e0f8bb1a0879
+  data.tar.gz: 709f96eaa2af7732b73c2838862bb98f350e9f8cda5aa7e209b00e96e00ab127
 SHA512:
-  metadata.gz: caad67348cde90de584225d0c64e1c1cc3ab50aa222879679ab1399063ea2e9ff97ce3c19f3c85725560d288a034d0ee25e1d0e5e6eb79bcf89d9ee0e51e9f38
-  data.tar.gz: 550c9d2ba8120b5fc2464f47094223f28c5103a836123179fb43f8b6399ce45490a20c67454921a7f192cda575e6b19c3296654af28a9a3b7fc2a90cbcf03fe0
+  metadata.gz: d559f0b297b7402a3f57203ea6226ee861b81b72682896c8a49d13ecaa036ca6ba6b3cf6b22e5693d0de8133500b4dd0b84a10db171d17c93237a0d2873e2704
+  data.tar.gz: e6d23fe2be97f26f4c1f753585340220d46ea926ed490878a4b02627b3488da57a859a1e3948796e84055dce99d01380e60ca77e96c33970c03ba38683259b7c

data/.rspec

@@ -1,3 +1,3 @@
 --format documentation
 --color
---require spec_helper
+--require ./spec/spec_helper.rb

data/CHANGELOG.md

@@ -1,5 +1,15 @@
 # CHANGELOG
 
+## v2.10.0
+
+* Add support for Cucumber
+
+## v2.9.0
+
+* `Buildkite::TestCollector.tag_execution(key, value)` by @pda in https://github.com/buildkite/test-collector-ruby/pull/240
+
+**Full Changelog**: https://github.com/buildkite/test-collector-ruby/compare/v2.8.0...v2.9.0
+
 ## v2.8.0
 
 * Buildkite::TestCollector.tags: specify tags for all executions by @pda in https://github.com/buildkite/test-collector-ruby/pull/235

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    buildkite-test_collector (2.8.0)
+    buildkite-test_collector (2.10.0)
 
 GEM
   remote: https://rubygems.org/
@@ -22,14 +22,43 @@ GEM
     base64 (0.2.0)
     benchmark (0.4.0)
     bigdecimal (3.1.9)
+    builder (3.3.0)
     concurrent-ruby (1.3.5)
     connection_pool (2.5.0)
-    diff-lcs (1.5.1)
+    cucumber (9.2.1)
+      builder (~> 3.2)
+      cucumber-ci-environment (> 9, < 11)
+      cucumber-core (> 13, < 14)
+      cucumber-cucumber-expressions (~> 17.0)
+      cucumber-gherkin (> 24, < 28)
+      cucumber-html-formatter (> 20.3, < 22)
+      cucumber-messages (> 19, < 25)
+      diff-lcs (~> 1.5)
+      mini_mime (~> 1.1)
+      multi_test (~> 1.1)
+      sys-uname (~> 1.2)
+    cucumber-ci-environment (10.0.1)
+    cucumber-core (13.0.3)
+      cucumber-gherkin (>= 27, < 28)
+      cucumber-messages (>= 20, < 23)
+      cucumber-tag-expressions (> 5, < 7)
+    cucumber-cucumber-expressions (17.1.0)
+      bigdecimal
+    cucumber-gherkin (27.0.0)
+      cucumber-messages (>= 19.1.4, < 23)
+    cucumber-html-formatter (21.9.0)
+      cucumber-messages (> 19, < 28)
+    cucumber-messages (22.0.0)
+    cucumber-tag-expressions (6.1.2)
+    diff-lcs (1.6.2)
     drb (2.2.1)
+    ffi (1.17.2)
     i18n (1.14.7)
       concurrent-ruby (~> 1.0)
-    logger (1.6.5)
+    logger (1.7.0)
+    mini_mime (1.1.5)
     minitest (5.25.4)
+    multi_test (1.1.0)
     rake (13.2.1)
     rspec (3.13.0)
       rspec-core (~> 3.13.0)
@@ -45,6 +74,8 @@ GEM
       rspec-support (~> 3.13.0)
     rspec-support (3.13.2)
     securerandom (0.4.1)
+    sys-uname (1.3.1)
+      ffi (~> 1.1)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
     uri (1.0.2)
@@ -55,6 +86,7 @@ PLATFORMS
 DEPENDENCIES
   activesupport (>= 4.2)
   buildkite-test_collector!
+  cucumber (~> 9.0)
   rake (~> 13.0)
   rspec (~> 3.0)
   rspec-core (~> 3.10)

data/README.md

@@ -5,7 +5,7 @@ Versions prior to 2.1.x are unsupported and will not work after mid-2023. Please
 
 Official [Buildkite Test Engine](https://buildkite.com/platform/test-engine) collectors for Ruby test frameworks ✨
 
-⚒ **Supported test frameworks:** RSpec, Minitest, and [more coming soon](https://github.com/buildkite/test-collector-ruby/issues?q=is%3Aissue+is%3Aopen+label%3A%22test+frameworks%22).
+⚒ **Supported test frameworks:** RSpec, Minitest, Cucumber, and [more coming soon](https://github.com/buildkite/test-collector-ruby/issues?q=is%3Aissue+is%3Aopen+label%3A%22test+frameworks%22).
 
 📦 **Supported CI systems:** Buildkite, GitHub Actions, CircleCI, Codeship, and others via the `BUILDKITE_ANALYTICS_*` environment variables.
 
@@ -63,47 +63,30 @@ Run your tests locally:
 BUILDKITE_ANALYTICS_TOKEN=xyz rake
 ```
 
-### Step 3
-
-Add the `BUILDKITE_ANALYTICS_TOKEN` secret to your CI, push your changes to a branch, and open a pull request 🎉
-
-```bash
-git checkout -b add-buildkite-test-engine
-git commit -am "Add Buildkite Test Engine"
-git push origin add-buildkite-test-engine
-```
+#### Cucumber
 
-### VCR
-If your test suites use [VCR](https://github.com/vcr/vcr) to stub network requests, you'll need to modify the config to allow actual network requests to Test Engine.
+Add the following code to your Cucumber setup file:
 
-```
-VCR.configure do |c|
-  c.ignore_hosts "analytics-api.buildkite.com"
-end
+```ruby
+# features/support/env.rb
+require 'buildkite/test_collector'
+Buildkite::TestCollector.configure(hook: :cucumber)
 ```
 
-## 🗨️ Annotations
-
-This gem allows adding custom annotations to the span data sent to Buildkite using the [.annotate](https://github.com/buildkite/test-collector-ruby/blob/d9fe11341e4aa470e766febee38124b644572360/lib/buildkite/test_collector.rb#L64) method. For example:
+Run your tests locally:
 
-```ruby
-Buildkite::TestCollector.annotate("User logged in successfully")
+```shell
+BUILDKITE_ANALYTICS_TOKEN=xyz cucumber
 ```
 
-This is particularly useful for tests that generate a lot of span data such as system/feature tests.
-
-## 🏷️ Tagging duplicate test executions with a prefix/suffix
+### Step 3
 
-For builds that execute the same test multiple times - such as when running the same test suite against multiple versions of ruby/rails - it's possible to tag each test execution with a prefix/suffix. This prefix/suffix is displayed for each execution on the test show page to differentiate the build environment. The prefix/suffix is specified using these environment variables:
+Add the `BUILDKITE_ANALYTICS_TOKEN` secret to your CI, push your changes to a branch, and open a pull request 🎉
 
-```
-BUILDKITE_ANALYTICS_EXECUTION_NAME_PREFIX
-BUILDKITE_ANALYTICS_EXECUTION_NAME_SUFFIX
-```
 
-## 🔜 Roadmap
+## More information
 
-See the [GitHub 'enhancement' issues](https://github.com/buildkite/test-collector-ruby/issues?q=is%3Aissue+is%3Aopen+label%3Aenhancement) for planned features. Pull requests are always welcome, and we’ll give you feedback and guidance if you choose to contribute 💚
+For more use cases such as custom tags, annotations, and span tracking, please visit our [official Ruby collector documentation](https://buildkite.com/docs/test-engine/ruby-collectors) for details.
 
 ## ⚒ Developing
 

data/buildkite-test_collector.gemspec

@@ -27,4 +27,9 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "activesupport", ">= 4.2"
   spec.add_development_dependency "rspec-core", '~> 3.10'
   spec.add_development_dependency "rspec-expectations", '~> 3.10'
+
+  # When running the legacy CI builds against versions of Ruby pre 2.7 we cannot include cucumber 9 as it's not supported.
+  if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new('2.7')
+    spec.add_development_dependency "cucumber", '~> 9.0'
+  end
 end

data/buildkite.yaml

@@ -7,7 +7,20 @@ steps:
       - "gem install bundler:2.3.25"
       - "bundle"
       - "bundle exec rake"
+      - "bundle exec cucumber"
     plugins:
+      - aws-assume-role-with-web-identity#v1.2.0:
+          role-arn: "${OIDC_ASSUME_ROLE_ARN:-arn:aws:iam::445615400570:role/pipeline-buildkite-test-collector-ruby-build}"
+          session-tags:
+            - organization_slug
+            - organization_id
+            - pipeline_slug
+            - build_branch
+
+      - ecr#v2.9.0:
+          login: true
+          account_ids: "public.ecr.aws"
+          region: "us-east-1"
       - docker#v3.7.0:
           image: "public.ecr.aws/docker/library/ruby:{{matrix}}"
     matrix:
@@ -27,6 +40,17 @@ steps:
           - "bundle"
           - "bundle exec rake"
         plugins:
+          - aws-assume-role-with-web-identity#v1.2.0:
+              role-arn: "${OIDC_ASSUME_ROLE_ARN:-arn:aws:iam::445615400570:role/pipeline-buildkite-test-collector-ruby-build}"
+              session-tags:
+                - organization_slug
+                - organization_id
+                - pipeline_slug
+                - build_branch
+          - ecr#v2.9.0:
+              login: true
+              account_ids: "public.ecr.aws"
+              region: "us-east-1"
           - docker#v3.7.0:
               # Images for older Ruby versions aren't available on AWS ECR
               # so fall back to Docker Hub

data/cucumber.yml

@@ -0,0 +1 @@
+default: --publish-quiet

data/lib/buildkite/test_collector/cucumber_plugin/trace.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module Buildkite::TestCollector::CucumberPlugin
+  class Trace
+    attr_accessor :scenario, :failure_reason, :failure_expanded
+    attr_reader :history, :tags
+
+    FILE_PATH_REGEX = /^(.*?\.(rb|feature))/
+
+    def initialize(scenario, history:, failure_reason: nil, failure_expanded: [], tags: nil)
+      @scenario         = scenario
+      @history          = history
+      @failure_reason   = failure_reason
+      @failure_expanded = failure_expanded
+      @tags             = tags
+    end
+
+    def result
+      if scenario.passed?
+        'passed'
+      elsif scenario.failed?
+        'failed'
+      else
+        'skipped'
+      end
+    end
+
+    def as_hash
+      parser = Gherkin::Parser.new
+      document = parser.parse(File.read(file_name))
+      feature_name = document.feature.name
+
+      strip_invalid_utf8_chars(
+        scope:          feature_name,
+        name:           scenario.name,
+        location:       scenario.location&.to_s,
+        file_name:      file_name,
+        result:         result,
+        failure_reason: failure_reason,
+        failure_expanded: failure_expanded,
+        history:        history,
+        tags:           tags,
+      ).select { |_, v| !v.nil? }
+    end
+
+    private
+
+    def file_name
+      @file_name ||= scenario.location&.to_s[FILE_PATH_REGEX]
+    end
+
+    def strip_invalid_utf8_chars(object)
+      case object
+      when Hash
+        object.transform_values { |v| strip_invalid_utf8_chars(v) }
+      when Array
+        object.map { |v| strip_invalid_utf8_chars(v) }
+      when String
+        object.encode('UTF-8', invalid: :replace, undef: :replace)
+      else
+        object
+      end
+    end
+  end
+end

data/lib/buildkite/test_collector/error.rb

@@ -1,4 +1,5 @@
 module Buildkite::TestCollector
   class Error < StandardError; end
   class TimeoutError < ::Timeout::Error; end
+  class UnsupportedFrameworkError < Error; end
 end

data/lib/buildkite/test_collector/library_hooks/cucumber.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+require 'cucumber'
+
+require_relative '../cucumber_plugin/trace'
+
+# Use Buildkite's uploader for Cucumber as well
+Buildkite::TestCollector.uploader = Buildkite::TestCollector::Uploader
+
+# -------------------------------------------------
+# Hooks
+# -------------------------------------------------
+
+Before do |scenario|
+  tracer = Buildkite::TestCollector::Tracer.new(
+    min_duration: Buildkite::TestCollector.trace_min_duration,
+  )
+
+  tags = {}
+
+  # _buildkite prefix reduces chance of collisions in this almost-global (per-fiber) namespace.
+  Thread.current[:_buildkite_tracer] = tracer
+  Thread.current[:_buildkite_tags] = tags
+end
+
+After do |scenario|
+  tracer = Thread.current[:_buildkite_tracer]
+  tags   = Thread.current[:_buildkite_tags]
+
+  Thread.current[:_buildkite_tracer] = nil
+  Thread.current[:_buildkite_tags]   = nil
+
+  if tracer
+    tracer.finalize
+
+    failure_reason   = nil
+    failure_expanded = []
+
+    if scenario.failed?
+      exception = scenario.exception
+      if exception
+        failure_reason = exception.message
+        failure_expanded << {
+          backtrace: exception.backtrace,
+        }
+      end
+    end
+
+    trace = Buildkite::TestCollector::CucumberPlugin::Trace.new(
+      scenario,
+      history:          tracer.history,
+      failure_reason:   failure_reason,
+      failure_expanded: failure_expanded,
+      tags:             tags,
+    )
+
+    Buildkite::TestCollector.uploader.traces[scenario.location.to_s] = trace
+
+    if Buildkite::TestCollector.session
+      Buildkite::TestCollector.session.add_example_to_send_queue(scenario.location.to_s)
+    end
+  end
+end
+
+at_exit do
+  if Buildkite::TestCollector.session
+    Buildkite::TestCollector.session.send_remaining_data
+    Buildkite::TestCollector.session.close
+  end
+end
+
+# Initialise shared tracing & session behaviour once hooks file is loaded.
+Buildkite::TestCollector.enable_tracing!
+Buildkite::TestCollector.session = Buildkite::TestCollector::Session.new

data/lib/buildkite/test_collector/library_hooks/rspec.rb

@@ -18,21 +18,27 @@ RSpec.configure do |config|
       min_duration: Buildkite::TestCollector.trace_min_duration,
     )
 
-    # The _buildkite prefix here is added as a safeguard against name collisions
-    # as we are in the main thread
+    tags = {}
+
+    # _buildkite prefix reduces chance of collisions in this almost-global (per-fiber) namespace.
     Thread.current[:_buildkite_tracer] = tracer
-    # It's important to use begin/ensure here, because otherwise if other hooks fail,
-    # the cleanup code won't run, meaning we will miss some data.
-    #
-    # Having said that, this behavior isn't documented by RSpec.
+    Thread.current[:_buildkite_tags] = tags
+
+    # example.run can raise errors (including from other middleware/hooks) so clean up in `ensure`.
     begin
       example.run
     ensure
       Thread.current[:_buildkite_tracer] = nil
+      Thread.current[:_buildkite_tags] = nil
 
       tracer.finalize
 
-      trace = Buildkite::TestCollector::RSpecPlugin::Trace.new(example, history: tracer.history)
+      trace = Buildkite::TestCollector::RSpecPlugin::Trace.new(
+        example,
+        history: tracer.history,
+        tags: tags,
+      )
+
       Buildkite::TestCollector.uploader.traces[example.id] = trace
     end
   end

data/lib/buildkite/test_collector/minitest_plugin/trace.rb

@@ -5,6 +5,7 @@ module Buildkite::TestCollector::MinitestPlugin
     attr_accessor :example
     attr_writer :failure_reason, :failure_expanded
     attr_reader :history
+    attr_reader :tags
 
     RESULT_CODES = {
       '.' => 'passed',
@@ -15,9 +16,10 @@ module Buildkite::TestCollector::MinitestPlugin
 
     FILE_PATH_REGEX = /^(.*?\.(rb|feature))/
 
-    def initialize(example, history:)
+    def initialize(example, history:, tags: nil)
       @example = example
       @history = history
+      @tags = tags
     end
 
     def result
@@ -38,6 +40,7 @@ module Buildkite::TestCollector::MinitestPlugin
         failure_reason: failure_reason,
         failure_expanded: failure_expanded,
         history: history,
+        tags: tags,
       ).select { |_, value| !value.nil? }
     end
 

data/lib/buildkite/test_collector/minitest_plugin.rb

@@ -14,18 +14,29 @@ module Buildkite::TestCollector::MinitestPlugin
       min_duration: Buildkite::TestCollector.trace_min_duration,
     )
 
-    # The _buildkite prefix here is added as a safeguard against name collisions
-    # as we are in the main thread
+    tags = {}
+
+    # _buildkite prefix reduces chance of collisions in this almost-global (per-fiber) namespace.
     Thread.current[:_buildkite_tracer] = tracer
+    Thread.current[:_buildkite_tags] = tags
   end
 
   def after_teardown
     tracer = Thread.current[:_buildkite_tracer]
+    tags = Thread.current[:_buildkite_tags]
+
+    Thread.current[:_buildkite_tracer] = nil
+    Thread.current[:_buildkite_tags] = nil
+
     if !tracer.nil?
-      Thread.current[:_buildkite_tracer] = nil
       tracer.finalize
 
-      trace = Buildkite::TestCollector::MinitestPlugin::Trace.new(self, history: tracer.history)
+      trace = Buildkite::TestCollector::MinitestPlugin::Trace.new(
+        self,
+        history: tracer.history,
+        tags: tags,
+      )
+
       Buildkite::TestCollector.uploader.traces[trace.source_location] = trace
     end
 

data/lib/buildkite/test_collector/rspec_plugin/trace.rb

@@ -4,14 +4,16 @@ module Buildkite::TestCollector::RSpecPlugin
   class Trace
     attr_accessor :example, :failure_reason, :failure_expanded
     attr_reader :history
+    attr_reader :tags
 
     FILE_PATH_REGEX = /^(.*?\.(rb|feature))/
 
-    def initialize(example, history:, failure_reason: nil, failure_expanded: [])
+    def initialize(example, history:, failure_reason: nil, failure_expanded: [], tags: nil)
       @example = example
       @history = history
       @failure_reason = failure_reason
       @failure_expanded = failure_expanded
+      @tags = tags
     end
 
     def result
@@ -32,6 +34,7 @@ module Buildkite::TestCollector::RSpecPlugin
         failure_reason: failure_reason,
         failure_expanded: failure_expanded,
         history: history,
+        tags: tags,
       ).select { |_, value| !value.nil? }
     end
 

data/lib/buildkite/test_collector/version.rb

@@ -2,7 +2,7 @@
 
 module Buildkite
   module TestCollector
-    VERSION = "2.8.0"
+    VERSION = "2.10.0"
     NAME = "buildkite-test_collector"
   end
 end

data/lib/buildkite/test_collector.rb

@@ -43,6 +43,10 @@ module Buildkite
     end
 
     def self.configure(hook:, token: nil, url: nil, tracing_enabled: true, artifact_path: nil, env: {}, tags: {})
+      if hook.to_sym == :cucumber && Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.7')
+        raise UnsupportedFrameworkError.new("Cucumber is only supported in versions of Ruby >= 2.7")
+      end
+
       self.api_token = (token || ENV["BUILDKITE_ANALYTICS_TOKEN"])&.strip
       self.url = url || DEFAULT_URL
       self.tracing_enabled = tracing_enabled
@@ -77,6 +81,18 @@ module Buildkite
       tracer&.leave
     end
 
+    # Set a key=value tag on the current test execution.
+    def self.tag_execution(key, value)
+      tags = Thread.current[:_buildkite_tags]
+      raise "_buildkite_tags not available" unless tags
+
+      unless key.is_a?(String) && value.is_a?(String)
+        raise ArgumentError, "tag key and value expected string"
+      end
+
+      tags[key] = value
+    end
+
     def self.enable_tracing!
       return unless self.tracing_enabled
 

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: buildkite-test_collector
 version: !ruby/object:Gem::Version
-  version: 2.8.0
+  version: 2.10.0
 platform: ruby
 authors:
 - Buildkite
 bindir: bin
 cert_chain: []
-date: 2025-02-04 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -51,6 +51,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.10'
+- !ruby/object:Gem::Dependency
+  name: cucumber
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '9.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '9.0'
 email:
 - support+analytics@buildkite.com
 executables: []
@@ -74,10 +88,13 @@ files:
 - bin/setup
 - buildkite-test_collector.gemspec
 - buildkite.yaml
+- cucumber.yml
 - lib/buildkite/test_collector.rb
 - lib/buildkite/test_collector/ci.rb
+- lib/buildkite/test_collector/cucumber_plugin/trace.rb
 - lib/buildkite/test_collector/error.rb
 - lib/buildkite/test_collector/http_client.rb
+- lib/buildkite/test_collector/library_hooks/cucumber.rb
 - lib/buildkite/test_collector/library_hooks/minitest.rb
 - lib/buildkite/test_collector/library_hooks/rspec.rb
 - lib/buildkite/test_collector/minitest_plugin.rb
@@ -113,7 +130,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.2
+rubygems_version: 3.6.7
 specification_version: 4
 summary: Track test executions and report to Buildkite Test Engine
 test_files: []