buildkite-test_collector 2.15.0 → 2.16.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: de04f3cd718a70ec62965dd2e491c5eca981e9781118b3ab7c6a7c05e6ad7655
-  data.tar.gz: c4a0fdbf14634c404ff6ade3db7d639b3f22fa4c5c6e21ba083ef92313ca6743
+  metadata.gz: 83d45811e314ed3a6defa2daebb928af60cf113a555e0409c5a842ad44cf0385
+  data.tar.gz: bb1687220b76611bb509f12d1804bf9ec530423e55b66a89cfd4ac0bb66acfd4
 SHA512:
-  metadata.gz: 8682918b8784f2544a3da68e0a6dfd00ce8b2ab8decd9eb24bb0d458381c402697f3d20799bff9d38bd9ebec4a4f48e5ffe5b89d80ce88c89ffc7f54130e573a
-  data.tar.gz: 2c1316f5b43bc3ea14df5aeae2e48496755edfe1892594403df6af499efe5b2aecbef4b1c3f8c5289100b4cacaeffc63da917109611cc5b84d0122a39eb8052c
+  metadata.gz: 42527c632311eafe2cc64e48b92b3a4da9e148b0a32fb72a687393f7ccf227cbee57be3fc8e50ba555844ef3d27e9dfae8b923285588ccf8cd0c635f9e92baab
+  data.tar.gz: 554ca53a2271210bdc6c796187f60bc5069038c98137514c156c5d9f0ef877767608a4d44d6f9357687c4892e7743e55550ca49e097bb08099cefdcc5872dfc4

data/.buildkite/pipeline.release.yml

@@ -12,13 +12,15 @@
 #   - The build step is side-effect-free: it builds the gem, checks
 #     tag/version consistency, and queries RubyGems read-only. It runs on
 #     every rehearsal, so rehearsals exercise the release code. It never sees
-#     RELEASE_EXECUTE (deliberately not propagated into its container).
+#     RELEASE_PUBLISH (set only on the publish step below), so it stops
+#     before `gem push` even on tag builds.
 #   - The publish steps mint a push token via the rubygems.org OIDC API key
 #     role, whose access policy trusts this pipeline (see the credential
 #     cut-over section of ctg docs/publishing.md).
-#   - The publish steps exist only on an exact test-collector-ruby/vX.Y.Z
-#     tag AND RELEASE_EXECUTE=true set on the build, behind a block step;
-#     the release script independently re-checks both before `gem push`.
+#   - The publish step exists only on an exact test-collector-ruby/vX.Y.Z
+#     tag, behind a block step; it sets RELEASE_PUBLISH=true in its own
+#     environment, and the release script independently re-checks the tag
+#     and RELEASE_PUBLISH before `gem push`.
 
 agents:
   queue: hosted
@@ -35,13 +37,17 @@ steps:
 
   - block: ":rocket: Publish test-collector-ruby to RubyGems?"
     key: ruby-confirm
-    if: build.tag =~ /^test-collector-ruby\/v[0-9]+\.[0-9]+\.[0-9]+$/ && build.env("RELEASE_EXECUTE") == "true"
+    if: build.tag =~ /^test-collector-ruby\/v[0-9]+\.[0-9]+\.[0-9]+$/
 
   - label: ":rubygems: test-collector-ruby: publish to RubyGems"
-    if: build.tag =~ /^test-collector-ruby\/v[0-9]+\.[0-9]+\.[0-9]+$/ && build.env("RELEASE_EXECUTE") == "true"
+    if: build.tag =~ /^test-collector-ruby\/v[0-9]+\.[0-9]+\.[0-9]+$/
     depends_on:
       - ruby-build
       - ruby-confirm
+    env:
+      # Step-local publish signal: distinguishes this credentialed step from
+      # the rehearsal build step, which runs the same script.
+      RELEASE_PUBLISH: "true"
     command: test-collector-ruby/.buildkite/steps/release
     plugins:
       - rubygems-oidc#v0.2.0:
@@ -51,4 +57,4 @@ steps:
           environment:
             - GEM_HOST_API_KEY
             - BUILDKITE_TAG
-            - RELEASE_EXECUTE
+            - RELEASE_PUBLISH

data/.buildkite/steps/release

@@ -1,9 +1,11 @@
 #!/usr/bin/env bash
 # Monorepo release script for test-collector-ruby: build the gem, check
-# tag/version consistency and RubyGems state, then publish — but the publish
-# side effect only happens when RELEASE_EXECUTE=true (set explicitly on the
-# release build; see pipeline.release.yml). Everything before the push runs
-# in rehearsals too, so dry-runs exercise this code.
+# tag/version consistency and RubyGems state, then publish — but only in
+# the publish step, which sets RELEASE_PUBLISH=true in its own container
+# (see pipeline.release.yml); the pre-confirmation build step runs this
+# same script as a side-effect-free rehearsal, even on tag builds.
+# Everything before the push runs in rehearsals too, so dry-runs exercise
+# this code.
 #
 # Injected by `ctg gather` from the ctg repo's overlay/ directory during the
 # dry-run era; post-cutover this file is maintained here, in the monorepo.
@@ -50,15 +52,15 @@ else
   echo "${GEM_NAME} ${VERSION} is not on rubygems.org (HTTP ${STATUS})"
 fi
 
-if [ "${RELEASE_EXECUTE:-}" != "true" ]; then
+if [ "${RELEASE_PUBLISH:-}" != "true" ]; then
   echo "--- Rehearsal only"
-  echo "RELEASE_EXECUTE is not 'true': stopping before 'gem push ${GEM_FILENAME}'."
+  echo "RELEASE_PUBLISH is not 'true': stopping before 'gem push ${GEM_FILENAME}'."
   exit 0
 fi
 
 echo "--- Publishing ${GEM_FILENAME} to RubyGems"
 if [ -z "${BUILDKITE_TAG:-}" ]; then
-  echo "RELEASE_EXECUTE=true without a ${PROJECT}/vX.Y.Z tag; refusing" >&2
+  echo "RELEASE_PUBLISH=true without a ${PROJECT}/vX.Y.Z tag; refusing" >&2
   exit 1
 fi
 if [ "${ALREADY_PUBLISHED}" = "true" ]; then

data/CHANGELOG.md

@@ -1,5 +1,87 @@
 # CHANGELOG
 
+## v2.16.0
+
+* Report `code.line.number` for shared example inclusions (`it_behaves_like`,
+  `include_examples`, `include_context`) in OpenTelemetry test spans; the line
+  was previously omitted because RSpec's inclusion location includes the frame
+  label.
+* Prevent OpenTelemetry export failures outside `StandardError`, including
+  WebMock network-blocking errors, from failing the test suite.
+* Add the OTLP endpoint host to WebMock's allow list when WebMock is loaded.
+* Validate OpenTelemetry run keys before export and fall back to the JSON upload
+  with a warning when they are invalid (or report that results cannot be
+  uploaded when no JSON token is available).
+* Ignore empty `BUILDKITE_ANALYTICS_*` metadata overrides so they do not replace
+  values detected from the CI environment.
+* Read bktec's relay credential from `BUILDKITE_TESTS_OTLP_TOKEN` and stop
+  reading `OTEL_EXPORTER_OTLP_*_HEADERS`, so credentials the process configures
+  for other OpenTelemetry destinations are never sent to Buildkite.
+* Pin OpenTelemetry export to gzip, peer-verified HTTPS, and the system
+  certificate store, ignoring process-wide `OTEL_EXPORTER_OTLP_*` transport
+  overrides. Requires `opentelemetry-exporter-otlp` 0.29+; older versions fall
+  back to the JSON upload with a warning.
+* Reduce the experimental OpenTelemetry test span batch size from 512 to 120 so
+  a batch of failures with maximum-length output fits the server's 900 KiB
+  request limit. Override the batch and queue sizes with
+  `BUILDKITE_TESTS_OTEL_TEST_SPAN_BATCH_SIZE` and
+  `BUILDKITE_TESTS_OTEL_TEST_SPAN_QUEUE_SIZE`; invalid values warn and use
+  the defaults.
+* Override the OpenTelemetry child span batch and queue sizes with
+  `BUILDKITE_TESTS_OTEL_CHILD_SPAN_BATCH_SIZE` and
+  `BUILDKITE_TESTS_OTEL_CHILD_SPAN_QUEUE_SIZE`, and warn when child spans are
+  dropped instead of discarding them silently. `test.execution` results are
+  unaffected by child span drops.
+* Set every OpenTelemetry batch processor option explicitly, including the export
+  timeout and starting the export thread on boot, so `OTEL_BSP_*` and
+  `OTEL_RUBY_BSP_START_THREAD_ON_BOOT` no longer affect Buildkite export.
+* Truncate OpenTelemetry test span attributes to 10,243 characters (test names
+  keep the same identity as the JSON upload), exception messages to 10,243
+  characters, other event attributes to 16,384 characters, events to the newest
+  100 per span, and the failure summary in the span status to 1,024 characters.
+* **Breaking change to the experimental OpenTelemetry support:** opt-in
+  submission is now OTLP-only for RSpec. `otel_enabled: true` submits executions
+  as spans without also uploading legacy JSON, and the separate `otel_only`
+  option has been removed (passing it now raises `ArgumentError`). OpenTelemetry
+  remains off by default, so suites that never opted in are unaffected. When
+  OpenTelemetry cannot be used, the collector warns and uploads JSON instead:
+  at configure time for a non-RSpec hook, and at suite start when the
+  OpenTelemetry gems are missing or Ruby is older than 3.3. Without a token,
+  `otel_enabled` stays off and nothing is exported, the same as the JSON path,
+  which skips uploading without a token. When only `BUILDKITE_TESTS_OTLP_TOKEN`
+  held the credential, the fallback warning says results will not be uploaded,
+  since the JSON path has no token to use. Execution name
+  affixes (`BUILDKITE_ANALYTICS_EXECUTION_NAME_PREFIX`/`SUFFIX`) and custom
+  `env:` values have no OTLP equivalent and are not sent when `otel_enabled` is
+  on; the JSON upload that carried them in the old dual mode no longer happens.
+* **Breaking change to the experimental OpenTelemetry support:** remove the
+  `otel_instrumentations` option (passing it now raises `ArgumentError`). It
+  only accepted `nil` or `[]`, and `[]` skipped installing the instrumentation
+  gems the suite had already required. Choose instrumentation by requiring the
+  gems you want; the collector installs every registered instrumentation when it
+  configures the SDK, and installs none when the suite configures the SDK
+  itself. There is no replacement switch that disables all instrumentation at
+  once: to skip one gem your bundle requires anyway, set the SDK's
+  `OTEL_RUBY_INSTRUMENTATION_<NAME>_ENABLED=false`; to skip them all, configure
+  the SDK yourself without `use_all` (see [Choosing
+  instrumentation](docs/opentelemetry.md#choosing-instrumentation)).
+* Warn prominently, usually naming the HTTP status or connection error, the first
+  time Buildkite rejects or the collector drops `test.execution` spans, since
+  OTLP is now the only upload path. A test span that could not be started counts
+  as dropped too. When more are dropped after that, report the total at the
+  suite-end flush and again at process exit for anything still draining.
+* Keep stable suite, CI worker/run, and VCS identity on OpenTelemetry resources,
+  while moving Test Engine run metadata, framework details, and configured tags
+  to each `test.execution` span. Child spans no longer carry execution-only
+  metadata as duplicated resource attributes.
+* Send the Ruby version and configured location prefix with OpenTelemetry test
+  executions so OTLP-only uploads retain their classic upload tags and selector
+  paths.
+* Add `otel_span_filter` to drop selected OpenTelemetry child spans before they
+  are queued for export. `test.execution` spans are never filtered, and a
+  filter that fails retains the span and warns once. Passing a value that does
+  not respond to `call` raises `ArgumentError` at configure time.
+
 ## v2.15.0
 
 * Add [experimental, opt-in OpenTelemetry trace export](docs/opentelemetry.md)

data/README.md

@@ -5,7 +5,7 @@ Versions prior to 2.1.x are unsupported and will not work after mid-2023. Please
 
 Official [Buildkite Test Engine](https://buildkite.com/platform/test-engine) collectors for Ruby test frameworks ✨
 
-⚒ **Supported test frameworks:** RSpec, Minitest, Cucumber, and [more coming soon](https://github.com/buildkite/test-collector-ruby/issues?q=is%3Aissue+is%3Aopen+label%3A%22test+frameworks%22).
+⚒ **Supported test frameworks:** RSpec, Minitest, Cucumber, and [more coming soon](https://github.com/buildkite/bktest/issues).
 
 📦 **Supported CI systems:** Buildkite, GitHub Actions, CircleCI, Codeship, and others via the `BUILDKITE_ANALYTICS_*` environment variables.
 
@@ -83,102 +83,35 @@ BUILDKITE_ANALYTICS_TOKEN=xyz cucumber
 
 Add the `BUILDKITE_ANALYTICS_TOKEN` secret to your CI, push your changes to a branch, and open a pull request 🎉
 
-### OpenTelemetry export (experimental)
+### OpenTelemetry submission (experimental)
 
-RSpec suites can also send an OpenTelemetry trace per test execution to Buildkite,
-showing what each test did and where it spent its time. Each trace is rooted in a
-`test.execution` span carrying its name, location, result, and any failure detail.
-Tags passed to `configure` appear as resource attributes, while `tag_execution`
-adds attributes to the current test's root span.
+RSpec suites can submit each test execution as an OpenTelemetry trace. Each
+trace is rooted in a `test.execution` span carrying the test's name, location,
+result, and failure detail. Instrumented child spans show what the test did and
+where it spent its time.
 
-This feature is still under development and may change. This first release is
-intended for suites that do not already configure OpenTelemetry. It may work
-with an existing OpenTelemetry setup, but that configuration is not yet
-supported or guaranteed to work.
-
-OpenTelemetry export is off by default. Opt in when you configure the collector:
+This feature is still under development and may change. OpenTelemetry is off by
+default. Enable it when you configure the collector:
 
 ```ruby
 Buildkite::TestCollector.configure(hook: :rspec, otel_enabled: true)
 ```
 
-Execution roots use a private AlwaysOn provider so a suite's sampling policy
-cannot remove them.
-
-The collector configures a global provider for child spans and installs all
-applicable instrumentation registered when the suite starts. Because export is
-optional and its dependencies require Ruby 3.3+, the collector does not install
-them automatically. Add the OpenTelemetry SDK and OTLP exporter, plus any
-instrumentation you want to use:
-
-```ruby
-# Gemfile
-gem "opentelemetry-exporter-otlp", "~> 0.34", require: false
-gem "opentelemetry-sdk", "~> 1.13", require: false
-gem "opentelemetry-instrumentation-pg", require: false
-
-# spec/spec_helper.rb
-require "opentelemetry-instrumentation-pg"
-require "buildkite/test_collector"
-
-Buildkite::TestCollector.configure(hook: :rspec, otel_enabled: true)
-```
+When enabled, OpenTelemetry is the only submission path. Every `test.execution` span includes
+`buildkite.execution.via=otlp`, which tells Buildkite to synthesize the test
+execution from the span. Nothing is uploaded to `/v1/uploads`.
 
-Adding a gem to the Gemfile may auto-require it in applications that call
-`Bundler.require`, but that is not guaranteed. An explicit `require` is the
-recommended setup. To disable instrumentations and export only root
-`test.execution` spans, set `otel_instrumentations: []`. Any other value is
-reserved for a future release and disables span export with a warning. See the
-[OpenTelemetry guide](docs/opentelemetry.md#choosing-instrumentation) for more.
-
-Export needs Ruby 3.3 or newer, which is what the OpenTelemetry gems require. If
-those gems are unavailable, the option is accepted and export remains disabled.
-
-The collector honors standard `OTEL_EXPORTER_OTLP_TRACES_HEADERS` (or the
-generic `OTEL_EXPORTER_OTLP_HEADERS`) and gives them precedence over its own
-headers, including `Authorization`. bktec's OTLP relay uses this to provide its
-local credential without changing `BUILDKITE_ANALYTICS_TOKEN`, which remains
-available for normal JSON uploads in `otel_enabled` mode. Without an OTLP
-Authorization header, spans use `BUILDKITE_ANALYTICS_TOKEN`, which must be an
-agent OIDC token with the `write_uploads` scope; a suite API token still uploads
-executions, but its spans are rejected.
-
-Export failures never fail a test or block the normal Test Engine upload. See the
-[OpenTelemetry guide](docs/opentelemetry.md) for setup details and current
-limitations.
-
-### OTLP-only submission (experimental)
-
-RSpec suites can go one step further and submit results *only* over OTLP, with
-no JSON upload at all. It exports the same spans as `otel_enabled` and adds
-`buildkite.execution.via=otlp`, which tells Buildkite to synthesize each test
-execution from its span server-side:
-
-```ruby
-Buildkite::TestCollector.configure(hook: :rspec, otel_only: true)
-```
+This requires Ruby 3.3 or newer and the optional `opentelemetry-sdk` and
+`opentelemetry-exporter-otlp` gems, which the collector does not install. If
+OpenTelemetry cannot be configured, the collector warns and uploads results as
+JSON instead.
 
-In this mode the collector's legacy machinery is switched off: nothing is
-uploaded to `/v1/uploads`, and `Net::HTTP` and `Object` are left unpatched. The
-gem's whole job is to configure OpenTelemetry so each test gets a suitable span:
-
-- `Buildkite::TestCollector.annotate` adds a `test.annotation` event to the
-  current span.
-- `Buildkite::TestCollector.tag_execution` sets attributes on the test span.
-- `tags:` given to `configure` become resource attributes on every span.
-- Instrumentation works exactly as it does with `otel_enabled`: everything you
-  require and register installs, and `otel_instrumentations: []` exports only
-  the `test.execution` spans. See
-  [choosing instrumentation](docs/opentelemetry.md#choosing-instrumentation).
-- Your code can also talk to OpenTelemetry directly — the collector configures
-  the global tracer provider, so
-  `OpenTelemetry::Trace.current_span.set_attribute(...)` works during a test,
-  and any instrumentation joins the test's trace.
-
-`otel_only` is currently RSpec-only and has the same Ruby 3.3+ and OpenTelemetry
-gem requirements as `otel_enabled`. It's an alternative to `otel_enabled`; the
-two are mutually exclusive, and passing both (either value) raises
-`ArgumentError`.
+If Buildkite rejects the exported `test.execution` spans, the collector prints a
+prominent warning. Tests continue to run, but the affected results are not
+uploaded. See the
+[OpenTelemetry guide](docs/opentelemetry.md) for dependency setup,
+instrumentation, filtering noisy child spans with `otel_span_filter`,
+authentication, attributes, and current limitations.
 
 ## More information
 
@@ -204,7 +137,7 @@ See [DESIGN.md](DESIGN.md) for an overview of the design of this gem.
 
 ## 👩‍💻 Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/buildkite/test-collector-ruby
+Bug reports and pull requests are welcome on GitHub at https://github.com/buildkite/bktest
 
 ## 🚀 Releasing
 

data/buildkite-test_collector.gemspec

@@ -9,11 +9,13 @@ Gem::Specification.new do |spec|
   spec.email         = ["support+analytics@buildkite.com"]
 
   spec.summary       = "Track test executions and report to Buildkite Test Engine"
-  spec.homepage      = "https://github.com/buildkite/test-collector-ruby"
+  spec.homepage      = "https://github.com/buildkite/bktest/tree/main/test-collector-ruby"
   spec.license       = "MIT"
 
   spec.metadata["homepage_uri"] = spec.homepage
-  spec.metadata["source_code_uri"] = "https://github.com/buildkite/test-collector-ruby"
+  spec.metadata["source_code_uri"] = "https://github.com/buildkite/bktest"
+  spec.metadata["changelog_uri"] = "https://github.com/buildkite/bktest/blob/main/test-collector-ruby/CHANGELOG.md"
+  spec.metadata["bug_tracker_uri"] = "https://github.com/buildkite/bktest/issues"
 
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
@@ -31,4 +33,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "ostruct"
   spec.add_development_dependency "rspec-core", '~> 3.10'
   spec.add_development_dependency "rspec-expectations", '~> 3.10'
+  spec.add_development_dependency "webmock"
 end