buildkite-test_collector 2.14.0 → 2.16.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 10535c306b5b29856f36335ba71e8f703b994605136545e9b1e3ed206ca4fe9d
-  data.tar.gz: 3c8115481fb1cfd98a7cee11e7f05347cc3398bcc81f2f7494044a48cb811d27
+  metadata.gz: 83d45811e314ed3a6defa2daebb928af60cf113a555e0409c5a842ad44cf0385
+  data.tar.gz: bb1687220b76611bb509f12d1804bf9ec530423e55b66a89cfd4ac0bb66acfd4
 SHA512:
-  metadata.gz: d4188a10501a2bb5b32bde498ed360ac8bbbb733e199899c7bb51c40411f6c3ab96b6f3bef90fb79e4901168fad21f88f7e911765988df63ae47d03a120dbaef
-  data.tar.gz: 7903643c37b1b65b9f0a38612253d6c8da084b56cab623f2e3782deacdb0ba3d631b6d17e2e40c796542226b9b00b996606ef38bd67181df6bdc9f99c7594d38
+  metadata.gz: 42527c632311eafe2cc64e48b92b3a4da9e148b0a32fb72a687393f7ccf227cbee57be3fc8e50ba555844ef3d27e9dfae8b923285588ccf8cd0c635f9e92baab
+  data.tar.gz: 554ca53a2271210bdc6c796187f60bc5069038c98137514c156c5d9f0ef877767608a4d44d6f9357687c4892e7743e55550ca49e097bb08099cefdcc5872dfc4

data/.buildkite/pipeline.release.yml

@@ -1,12 +1,54 @@
+# Monorepo release pipeline for test-collector-ruby, uploaded by the
+# release dispatcher (.buildkite/pipeline.release.yml) running on the
+# separate bktest-release Buildkite pipeline — never by CI: as a rehearsal
+# when this project changes (no tag), and for real on a
+# test-collector-ruby/vX.Y.Z tag. Injected by `ctg gather` from the ctg
+# repo's overlay/ directory during the dry-run era; post-cutover this file
+# is maintained here, in the monorepo. It replaces the source repo's
+# .buildkite/pipeline.release.yml, which assumed the repo root was the
+# project root.
+#
+# Safety model (see docs/publishing.md in the ctg repo):
+#   - The build step is side-effect-free: it builds the gem, checks
+#     tag/version consistency, and queries RubyGems read-only. It runs on
+#     every rehearsal, so rehearsals exercise the release code. It never sees
+#     RELEASE_PUBLISH (set only on the publish step below), so it stops
+#     before `gem push` even on tag builds.
+#   - The publish steps mint a push token via the rubygems.org OIDC API key
+#     role, whose access policy trusts this pipeline (see the credential
+#     cut-over section of ctg docs/publishing.md).
+#   - The publish step exists only on an exact test-collector-ruby/vX.Y.Z
+#     tag, behind a block step; it sets RELEASE_PUBLISH=true in its own
+#     environment, and the release script independently re-checks the tag
+#     and RELEASE_PUBLISH before `gem push`.
+
 agents:
   queue: hosted
 
 steps:
-  - block: "OK to release?"
+  - label: ":rubygems: test-collector-ruby: build + rehearse release"
+    key: ruby-build
+    command: test-collector-ruby/.buildkite/steps/release
+    plugins:
+      - docker#v5.12.0:
+          image: "ruby:3.4"
+          environment:
+            - BUILDKITE_TAG
 
-  - command: ".buildkite/steps/release-gem"
-    label: ":rubygems:"
-    if: build.tag != null
+  - block: ":rocket: Publish test-collector-ruby to RubyGems?"
+    key: ruby-confirm
+    if: build.tag =~ /^test-collector-ruby\/v[0-9]+\.[0-9]+\.[0-9]+$/
+
+  - label: ":rubygems: test-collector-ruby: publish to RubyGems"
+    if: build.tag =~ /^test-collector-ruby\/v[0-9]+\.[0-9]+\.[0-9]+$/
+    depends_on:
+      - ruby-build
+      - ruby-confirm
+    env:
+      # Step-local publish signal: distinguishes this credentialed step from
+      # the rehearsal build step, which runs the same script.
+      RELEASE_PUBLISH: "true"
+    command: test-collector-ruby/.buildkite/steps/release
     plugins:
       - rubygems-oidc#v0.2.0:
           role: "rg_oidc_akr_fy1x4px4yjwd1rdhkkda"
@@ -15,4 +57,4 @@ steps:
           environment:
             - GEM_HOST_API_KEY
             - BUILDKITE_TAG
-
+            - RELEASE_PUBLISH

data/.buildkite/pipeline.yml

@@ -0,0 +1,30 @@
+# Monorepo pipeline for test-collector-ruby, uploaded by the top-level
+# dispatcher when this project changes. Injected by `ctg gather` from the
+# ctg repo's overlay/ directory during the dry-run era; post-cutover this
+# file is maintained here, in the monorepo.
+#
+# Dry-run-safe: pure test steps only, derived from the source repo's
+# buildkite.yaml with credentialed plugins (OIDC role assumption, ECR)
+# removed, Docker Hub images, and a trimmed matrix.
+#
+# Jobs check out the monorepo root, so commands cd into this project.
+
+agents:
+  queue: hosted
+
+steps:
+  - group: ":ruby: test-collector-ruby"
+    steps:
+      - label: ":rspec: Ruby {{matrix}}"
+        command:
+          - "cd test-collector-ruby"
+          - "gem install bundler:2.3.25"
+          - "bundle"
+          - "bundle exec rake"
+          - "bundle exec cucumber"
+        plugins:
+          - docker#v3.7.0:
+              image: "ruby:{{matrix}}"
+        matrix:
+          - "latest"
+          - "3.3"

data/.buildkite/steps/release

@@ -0,0 +1,74 @@
+#!/usr/bin/env bash
+# Monorepo release script for test-collector-ruby: build the gem, check
+# tag/version consistency and RubyGems state, then publish — but only in
+# the publish step, which sets RELEASE_PUBLISH=true in its own container
+# (see pipeline.release.yml); the pre-confirmation build step runs this
+# same script as a side-effect-free rehearsal, even on tag builds.
+# Everything before the push runs in rehearsals too, so dry-runs exercise
+# this code.
+#
+# Injected by `ctg gather` from the ctg repo's overlay/ directory during the
+# dry-run era; post-cutover this file is maintained here, in the monorepo.
+
+set -euo pipefail
+
+cd "$(dirname "$0")/../.." # this project's root, wherever the checkout cwd is
+
+PROJECT="test-collector-ruby"
+GEM_NAME="buildkite-test_collector"
+TAG_PATTERN="^${PROJECT}/v([0-9]+\.[0-9]+\.[0-9]+)$"
+
+echo "--- Determining version"
+if [ -n "${BUILDKITE_TAG:-}" ]; then
+  if [[ ! "${BUILDKITE_TAG}" =~ ${TAG_PATTERN} ]]; then
+    echo "BUILDKITE_TAG '${BUILDKITE_TAG}' does not match ${PROJECT}/vX.Y.Z; refusing" >&2
+    exit 1
+  fi
+  VERSION="${BASH_REMATCH[1]}"
+  echo "version ${VERSION} (from tag ${BUILDKITE_TAG})"
+else
+  VERSION="$(ruby -Ilib -rbuildkite/test_collector/version -e 'puts Buildkite::TestCollector::VERSION')"
+  echo "version ${VERSION} (from lib/buildkite/test_collector/version.rb; no tag)"
+fi
+
+GEM_FILENAME="${GEM_NAME}-${VERSION}.gem"
+
+echo "--- Building gem"
+gem build "${GEM_NAME}.gemspec"
+if [ ! -f "${GEM_FILENAME}" ]; then
+  echo "ERROR: expected ${GEM_FILENAME}; does the gemspec/version.rb say ${VERSION}?" >&2
+  echo "gem files found:" >&2
+  ls -- *.gem >&2 || true
+  exit 1
+fi
+
+echo "--- Checking rubygems.org for existing version ${VERSION} (read-only)"
+STATUS="$(curl -s -o /dev/null -w "%{http_code}" "https://rubygems.org/api/v2/rubygems/${GEM_NAME}/versions/${VERSION}.json")"
+if [ "${STATUS}" = "200" ]; then
+  ALREADY_PUBLISHED=true
+  echo "${GEM_NAME} ${VERSION} is already on rubygems.org"
+else
+  ALREADY_PUBLISHED=false
+  echo "${GEM_NAME} ${VERSION} is not on rubygems.org (HTTP ${STATUS})"
+fi
+
+if [ "${RELEASE_PUBLISH:-}" != "true" ]; then
+  echo "--- Rehearsal only"
+  echo "RELEASE_PUBLISH is not 'true': stopping before 'gem push ${GEM_FILENAME}'."
+  exit 0
+fi
+
+echo "--- Publishing ${GEM_FILENAME} to RubyGems"
+if [ -z "${BUILDKITE_TAG:-}" ]; then
+  echo "RELEASE_PUBLISH=true without a ${PROJECT}/vX.Y.Z tag; refusing" >&2
+  exit 1
+fi
+if [ "${ALREADY_PUBLISHED}" = "true" ]; then
+  echo "version ${VERSION} already published; refusing" >&2
+  exit 1
+fi
+if [ -z "${GEM_HOST_API_KEY:-}" ]; then
+  echo "GEM_HOST_API_KEY not set (rubygems-oidc plugin missing or failed)" >&2
+  exit 1
+fi
+gem push "${GEM_FILENAME}"

data/CHANGELOG.md

@@ -1,5 +1,102 @@
 # CHANGELOG
 
+## v2.16.0
+
+* Report `code.line.number` for shared example inclusions (`it_behaves_like`,
+  `include_examples`, `include_context`) in OpenTelemetry test spans; the line
+  was previously omitted because RSpec's inclusion location includes the frame
+  label.
+* Prevent OpenTelemetry export failures outside `StandardError`, including
+  WebMock network-blocking errors, from failing the test suite.
+* Add the OTLP endpoint host to WebMock's allow list when WebMock is loaded.
+* Validate OpenTelemetry run keys before export and fall back to the JSON upload
+  with a warning when they are invalid (or report that results cannot be
+  uploaded when no JSON token is available).
+* Ignore empty `BUILDKITE_ANALYTICS_*` metadata overrides so they do not replace
+  values detected from the CI environment.
+* Read bktec's relay credential from `BUILDKITE_TESTS_OTLP_TOKEN` and stop
+  reading `OTEL_EXPORTER_OTLP_*_HEADERS`, so credentials the process configures
+  for other OpenTelemetry destinations are never sent to Buildkite.
+* Pin OpenTelemetry export to gzip, peer-verified HTTPS, and the system
+  certificate store, ignoring process-wide `OTEL_EXPORTER_OTLP_*` transport
+  overrides. Requires `opentelemetry-exporter-otlp` 0.29+; older versions fall
+  back to the JSON upload with a warning.
+* Reduce the experimental OpenTelemetry test span batch size from 512 to 120 so
+  a batch of failures with maximum-length output fits the server's 900 KiB
+  request limit. Override the batch and queue sizes with
+  `BUILDKITE_TESTS_OTEL_TEST_SPAN_BATCH_SIZE` and
+  `BUILDKITE_TESTS_OTEL_TEST_SPAN_QUEUE_SIZE`; invalid values warn and use
+  the defaults.
+* Override the OpenTelemetry child span batch and queue sizes with
+  `BUILDKITE_TESTS_OTEL_CHILD_SPAN_BATCH_SIZE` and
+  `BUILDKITE_TESTS_OTEL_CHILD_SPAN_QUEUE_SIZE`, and warn when child spans are
+  dropped instead of discarding them silently. `test.execution` results are
+  unaffected by child span drops.
+* Set every OpenTelemetry batch processor option explicitly, including the export
+  timeout and starting the export thread on boot, so `OTEL_BSP_*` and
+  `OTEL_RUBY_BSP_START_THREAD_ON_BOOT` no longer affect Buildkite export.
+* Truncate OpenTelemetry test span attributes to 10,243 characters (test names
+  keep the same identity as the JSON upload), exception messages to 10,243
+  characters, other event attributes to 16,384 characters, events to the newest
+  100 per span, and the failure summary in the span status to 1,024 characters.
+* **Breaking change to the experimental OpenTelemetry support:** opt-in
+  submission is now OTLP-only for RSpec. `otel_enabled: true` submits executions
+  as spans without also uploading legacy JSON, and the separate `otel_only`
+  option has been removed (passing it now raises `ArgumentError`). OpenTelemetry
+  remains off by default, so suites that never opted in are unaffected. When
+  OpenTelemetry cannot be used, the collector warns and uploads JSON instead:
+  at configure time for a non-RSpec hook, and at suite start when the
+  OpenTelemetry gems are missing or Ruby is older than 3.3. Without a token,
+  `otel_enabled` stays off and nothing is exported, the same as the JSON path,
+  which skips uploading without a token. When only `BUILDKITE_TESTS_OTLP_TOKEN`
+  held the credential, the fallback warning says results will not be uploaded,
+  since the JSON path has no token to use. Execution name
+  affixes (`BUILDKITE_ANALYTICS_EXECUTION_NAME_PREFIX`/`SUFFIX`) and custom
+  `env:` values have no OTLP equivalent and are not sent when `otel_enabled` is
+  on; the JSON upload that carried them in the old dual mode no longer happens.
+* **Breaking change to the experimental OpenTelemetry support:** remove the
+  `otel_instrumentations` option (passing it now raises `ArgumentError`). It
+  only accepted `nil` or `[]`, and `[]` skipped installing the instrumentation
+  gems the suite had already required. Choose instrumentation by requiring the
+  gems you want; the collector installs every registered instrumentation when it
+  configures the SDK, and installs none when the suite configures the SDK
+  itself. There is no replacement switch that disables all instrumentation at
+  once: to skip one gem your bundle requires anyway, set the SDK's
+  `OTEL_RUBY_INSTRUMENTATION_<NAME>_ENABLED=false`; to skip them all, configure
+  the SDK yourself without `use_all` (see [Choosing
+  instrumentation](docs/opentelemetry.md#choosing-instrumentation)).
+* Warn prominently, usually naming the HTTP status or connection error, the first
+  time Buildkite rejects or the collector drops `test.execution` spans, since
+  OTLP is now the only upload path. A test span that could not be started counts
+  as dropped too. When more are dropped after that, report the total at the
+  suite-end flush and again at process exit for anything still draining.
+* Keep stable suite, CI worker/run, and VCS identity on OpenTelemetry resources,
+  while moving Test Engine run metadata, framework details, and configured tags
+  to each `test.execution` span. Child spans no longer carry execution-only
+  metadata as duplicated resource attributes.
+* Send the Ruby version and configured location prefix with OpenTelemetry test
+  executions so OTLP-only uploads retain their classic upload tags and selector
+  paths.
+* Add `otel_span_filter` to drop selected OpenTelemetry child spans before they
+  are queued for export. `test.execution` spans are never filtered, and a
+  filter that fails retains the span and warns once. Passing a value that does
+  not respond to `call` raises `ArgumentError` at configure time.
+
+## v2.15.0
+
+* Add [experimental, opt-in OpenTelemetry trace export](docs/opentelemetry.md)
+  for RSpec suites. Both modes share execution spans and run resources while
+  preserving execution roots; `otel_only` skips legacy trace capture and JSON
+  uploads. Standard OTLP header environment variables are supported for export
+  authentication. Export requires Ruby 3.3 or newer and the OpenTelemetry SDK
+  and OTLP exporter gems.
+
+* Automatically tag executions with `ci.worker.id` from `BUILDKITE_AGENT_ID`,
+  so failures can be grouped by the agent that ran them. An explicit
+  caller-supplied `ci.worker.id` tag still takes precedence.
+
+* Drop support for Ruby 2.x. Ruby 3.0 or newer is now required.
+
 ## v2.14.0
 
 * Send `location_prefix` in `run_env`

data/Gemfile

@@ -8,3 +8,8 @@ gemspec
 gem "rake", "~> 13.0"
 
 gem "rspec", "~> 3.0"
+
+if Gem.ruby_version >= Gem::Version.new("3.3")
+  gem "opentelemetry-exporter-otlp", "~> 0.34"
+  gem "opentelemetry-sdk", "~> 1.13"
+end

data/README.md

@@ -5,7 +5,7 @@ Versions prior to 2.1.x are unsupported and will not work after mid-2023. Please
 
 Official [Buildkite Test Engine](https://buildkite.com/platform/test-engine) collectors for Ruby test frameworks ✨
 
-⚒ **Supported test frameworks:** RSpec, Minitest, Cucumber, and [more coming soon](https://github.com/buildkite/test-collector-ruby/issues?q=is%3Aissue+is%3Aopen+label%3A%22test+frameworks%22).
+⚒ **Supported test frameworks:** RSpec, Minitest, Cucumber, and [more coming soon](https://github.com/buildkite/bktest/issues).
 
 📦 **Supported CI systems:** Buildkite, GitHub Actions, CircleCI, Codeship, and others via the `BUILDKITE_ANALYTICS_*` environment variables.
 
@@ -83,6 +83,35 @@ BUILDKITE_ANALYTICS_TOKEN=xyz cucumber
 
 Add the `BUILDKITE_ANALYTICS_TOKEN` secret to your CI, push your changes to a branch, and open a pull request 🎉
 
+### OpenTelemetry submission (experimental)
+
+RSpec suites can submit each test execution as an OpenTelemetry trace. Each
+trace is rooted in a `test.execution` span carrying the test's name, location,
+result, and failure detail. Instrumented child spans show what the test did and
+where it spent its time.
+
+This feature is still under development and may change. OpenTelemetry is off by
+default. Enable it when you configure the collector:
+
+```ruby
+Buildkite::TestCollector.configure(hook: :rspec, otel_enabled: true)
+```
+
+When enabled, OpenTelemetry is the only submission path. Every `test.execution` span includes
+`buildkite.execution.via=otlp`, which tells Buildkite to synthesize the test
+execution from the span. Nothing is uploaded to `/v1/uploads`.
+
+This requires Ruby 3.3 or newer and the optional `opentelemetry-sdk` and
+`opentelemetry-exporter-otlp` gems, which the collector does not install. If
+OpenTelemetry cannot be configured, the collector warns and uploads results as
+JSON instead.
+
+If Buildkite rejects the exported `test.execution` spans, the collector prints a
+prominent warning. Tests continue to run, but the affected results are not
+uploaded. See the
+[OpenTelemetry guide](docs/opentelemetry.md) for dependency setup,
+instrumentation, filtering noisy child spans with `otel_span_filter`,
+authentication, attributes, and current limitations.
 
 ## More information
 
@@ -108,20 +137,11 @@ See [DESIGN.md](DESIGN.md) for an overview of the design of this gem.
 
 ## 👩‍💻 Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/buildkite/test-collector-ruby
+Bug reports and pull requests are welcome on GitHub at https://github.com/buildkite/bktest
 
 ## 🚀 Releasing
 
-1. Bump the version in `version.rb` and run `bundle` to update the `Gemfile.lock`.
-2. Update the CHANGELOG.md with your new version and a description of your changes.
-3. Once your PR is merged to `main` git tag the merge commit and push:
-
-```
-git tag vX.X.X
-git push origin vX.X.X
-```
-4. Visit the [release pipeline](https://buildkite.com/buildkite/test-collector-ruby-release) to unblock it and confirm the new version is pushed to rubygems.org
-5. Create a [new release in github](https://github.com/buildkite/test-collector-ruby/releases).
+See the monorepo's [collector release guide](../RELEASING.md#ruby-rubygems).
 
 ## 📜 MIT License
 

data/buildkite-test_collector.gemspec

@@ -9,11 +9,13 @@ Gem::Specification.new do |spec|
   spec.email         = ["support+analytics@buildkite.com"]
 
   spec.summary       = "Track test executions and report to Buildkite Test Engine"
-  spec.homepage      = "https://github.com/buildkite/test-collector-ruby"
+  spec.homepage      = "https://github.com/buildkite/bktest/tree/main/test-collector-ruby"
   spec.license       = "MIT"
 
   spec.metadata["homepage_uri"] = spec.homepage
-  spec.metadata["source_code_uri"] = "https://github.com/buildkite/test-collector-ruby"
+  spec.metadata["source_code_uri"] = "https://github.com/buildkite/bktest"
+  spec.metadata["changelog_uri"] = "https://github.com/buildkite/bktest/blob/main/test-collector-ruby/CHANGELOG.md"
+  spec.metadata["bug_tracker_uri"] = "https://github.com/buildkite/bktest/issues"
 
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
@@ -22,17 +24,14 @@ Gem::Specification.new do |spec|
   end
   spec.require_paths = ["lib"]
 
-  spec.required_ruby_version = Gem::Requirement.new(">= 2.3.0")
+  spec.required_ruby_version = Gem::Requirement.new(">= 3.0.0")
 
   spec.add_dependency "concurrent-ruby"
 
   spec.add_development_dependency "activesupport", ">= 4.2"
+  spec.add_development_dependency "cucumber", '~> 9.0'
   spec.add_development_dependency "ostruct"
   spec.add_development_dependency "rspec-core", '~> 3.10'
   spec.add_development_dependency "rspec-expectations", '~> 3.10'
-
-  # When running the legacy CI builds against versions of Ruby pre 2.7 we cannot include cucumber 9 as it's not supported.
-  if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new('2.7')
-    spec.add_development_dependency "cucumber", '~> 9.0'
-  end
+  spec.add_development_dependency "webmock"
 end

data/buildkite.yaml

@@ -27,36 +27,29 @@ steps:
       - "latest"
       - "3.4"
       - "3.3"
+
+  - label: ":rspec: Tests (EOL Ruby) :ruby: {{matrix}}"
+    command:
+      - "gem install bundler:2.3.25"
+      - "bundle"
+      - "bundle exec rake"
+      - "bundle exec cucumber"
+    plugins:
+      - aws-assume-role-with-web-identity#v1.2.0:
+          role-arn: "${OIDC_ASSUME_ROLE_ARN:-arn:aws:iam::445615400570:role/pipeline-buildkite-test-collector-ruby-build}"
+          session-tags:
+            - organization_slug
+            - organization_id
+            - pipeline_slug
+            - build_branch
+
+      - ecr#v2.9.0:
+          login: true
+          account_ids: "public.ecr.aws"
+          region: "us-east-1"
+      - docker#v3.7.0:
+          image: "public.ecr.aws/docker/library/ruby:{{matrix}}"
+    matrix:
       - "3.2"
       - "3.1"
       - "3.0"
-      - "2.7"
-
-  - group: ":rspec: Legacy Ruby :ruby:"
-    steps:
-      - label: ":rspec: Tests :ruby: {{matrix}}"
-        command:
-          - "gem install bundler:2.3.25"
-          - "bundle"
-          - "bundle exec rake"
-        plugins:
-          - aws-assume-role-with-web-identity#v1.2.0:
-              role-arn: "${OIDC_ASSUME_ROLE_ARN:-arn:aws:iam::445615400570:role/pipeline-buildkite-test-collector-ruby-build}"
-              session-tags:
-                - organization_slug
-                - organization_id
-                - pipeline_slug
-                - build_branch
-          - ecr#v2.9.0:
-              login: true
-              account_ids: "public.ecr.aws"
-              region: "us-east-1"
-          - docker#v3.7.0:
-              # Images for older Ruby versions aren't available on AWS ECR
-              # so fall back to Docker Hub
-              image: "ruby:{{matrix}}"
-        matrix:
-          - "2.6"
-          - "2.5"
-          - "2.4"
-          - "2.3"