buildkite-test_collector 2.11.0 → 2.13.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 423b9692a49e1cea2d180d684f9f54c82a6fd84289087efe256cf4a2d1dd5f4b
-  data.tar.gz: 7ec78a72fb4506ee301b44f2830adfc22c83eb0508cab3ef7d455ec4a1f3049a
+  metadata.gz: 3b5079c7b62d8a883afd7c607f452aa0f91b5b73d4b431f1e32137343bfbb5ba
+  data.tar.gz: f1fddd40906655afb5d0fc5fc83a5d81e6f44c6548ae85bb32be0ec03b806f38
 SHA512:
-  metadata.gz: dcf9a81dd2c1b74f8241c22ea94aff3dc75a9c1190afda0b12d4c2ac06bd79b89694fcee5d43a07a051864d404e140cf665064778dbb5d73b389abbba28af1ec
-  data.tar.gz: eb43eb66820451723296546c70b3b2fc7b63572aa393ae4ea2380bb8e195e7faa0c6e8b7eff56c04a70223585c55162872a82d647a7a4d74e8cfe7ffa7cad3c6
+  metadata.gz: d75687ce79c27f11f56244b1707deb6a54d822c5b2244cb3bea6b25a9454140f4997e9654137f1cffaefa9eadec7e77d9f10cb93bac5b36ac73fd4b760d45701
+  data.tar.gz: 55e339451c85f5c95c0a664cd9f34f9f554f06c864e1eab0730d217fb3ae9298de7cd61fd46546156c1081de4a7895c25039f8a9eb776a72c313103596699744

data/.gitignore

@@ -7,6 +7,7 @@
 /spec/reports/
 /tmp/
 /vendor/bundle
+Gemfile.lock
 
 # rspec failure tracking
 .rspec_status

data/CHANGELOG.md

@@ -1,5 +1,17 @@
 # CHANGELOG
 
+## v2.13.0
+
+* Remove `./` prefix from minitest collector file paths
+
+**Full Changelog**: https://github.com/buildkite/test-collector-ruby/compare/v2.12.0...v2.13.0
+
+## v2.12.0
+
+* Prevent OOM with bounded thread pool and memory cleanup
+
+**Full Changelog**: https://github.com/buildkite/test-collector-ruby/compare/v2.11.0...v2.12.0
+
 ## v2.11.0
 
 * Add `location_prefix` option. Allows a file name prefix to be set which will be prepended to `file_name` and `location` in the test results.

data/buildkite-test_collector.gemspec

@@ -24,7 +24,10 @@ Gem::Specification.new do |spec|
 
   spec.required_ruby_version = Gem::Requirement.new(">= 2.3.0")
 
+  spec.add_dependency "concurrent-ruby"
+
   spec.add_development_dependency "activesupport", ">= 4.2"
+  spec.add_development_dependency "ostruct"
   spec.add_development_dependency "rspec-core", '~> 3.10'
   spec.add_development_dependency "rspec-expectations", '~> 3.10'
 

data/lib/buildkite/test_collector/minitest_plugin/trace.rb

@@ -49,7 +49,7 @@ module Buildkite::TestCollector::MinitestPlugin
     end
 
     def file_name
-      @file_name ||= File.join('./', source_location[0].sub(/\A#{project_dir}/, ""))
+      @file_name ||= source_location[0].sub(/\A#{project_dir}\//, "")
     end
 
     def line_number

data/lib/buildkite/test_collector/session.rb

@@ -1,14 +1,15 @@
 # frozen_string_literal: true
 
+require "concurrent-ruby"
+
 module Buildkite::TestCollector
   class Session
-    UPLOAD_THREAD_TIMEOUT = 60
     UPLOAD_SESSION_TIMEOUT = 60
     UPLOAD_API_MAX_RESULTS = 5000
 
     def initialize
       @send_queue_ids = []
-      @upload_threads = []
+      @upload_futures = []
     end
 
     def add_example_to_send_queue(id)
@@ -24,20 +25,20 @@ module Buildkite::TestCollector
       return if @send_queue_ids.empty?
 
       upload_data(@send_queue_ids)
+      @send_queue_ids.clear
     end
 
     def close
-      # There are two thread joins here, because the inner join will wait up to
-      # UPLOAD_THREAD_TIMEOUT seconds PER thread that is uploading data, i.e.
-      # n_threads x UPLOAD_THREAD_TIMEOUT latency if Buildkite happens to be
-      # down. By wrapping that in an outer thread join with the
-      # UPLOAD_SESSION_TIMEOUT, we ensure that we only wait a max of
-      # UPLOAD_SESSION_TIMEOUT seconds before the session exits.
-      Thread.new do
-        @upload_threads.each { |t| t.join(UPLOAD_THREAD_TIMEOUT) }
-      end.join(UPLOAD_SESSION_TIMEOUT)
-
-      @upload_threads.each { |t| t&.kill }
+      return if @upload_futures.empty?
+
+      begin
+        Concurrent::Promises.zip(*@upload_futures).value!(UPLOAD_SESSION_TIMEOUT)
+      rescue StandardError
+        # Timeout or other error - futures will continue running in the thread pool
+        # with their own per-upload timeouts, we just stop waiting for them
+      ensure
+        @upload_futures.clear
+      end
     end
 
     private
@@ -45,12 +46,17 @@ module Buildkite::TestCollector
     def upload_data(ids)
       data = Buildkite::TestCollector.uploader.traces.values_at(*ids).compact
 
-      # we do this in batches of UPLOAD_API_MAX_RESULTS in case the number of
-      # results exceeds this due to a bug, or user error in configuring the
-      # batch size
-      data.each_slice(UPLOAD_API_MAX_RESULTS) do |batch|
-        new_thread = Buildkite::TestCollector::Uploader.upload(batch)
-        @upload_threads << new_thread if new_thread
+      begin
+        # We do this in batches of UPLOAD_API_MAX_RESULTS in case the number of
+        # results exceeds this due to a bug, or user error in configuring the
+        # batch size
+        data.each_slice(UPLOAD_API_MAX_RESULTS) do |batch|
+          new_future = Buildkite::TestCollector::Uploader.upload(batch)
+          @upload_futures << new_future if new_future
+        end
+      ensure
+        # Free memory by removing uploaded traces from the in-memory cache
+        ids.each { |id| Buildkite::TestCollector.uploader.traces.delete(id) }
       end
     end
   end

data/lib/buildkite/test_collector/uploader.rb

@@ -1,12 +1,14 @@
 # frozen_string_literal: true
 
+require "concurrent-ruby"
+
 module Buildkite::TestCollector
   class Uploader
     MAX_UPLOAD_ATTEMPTS = 3
+    UPLOAD_TIMEOUT = 60
 
-    def self.traces
-      @traces ||= {}
-    end
+    THREAD_POOL_SIZE = 10
+    THREAD_POOL = Concurrent::FixedThreadPool.new(THREAD_POOL_SIZE)
 
     REQUEST_EXCEPTIONS = [
       URI::InvalidURIError,
@@ -29,19 +31,23 @@ module Buildkite::TestCollector
       # TODO: some retries for server-side error would be great.
     ]
 
+    def self.traces
+      @traces ||= {}
+    end
+
     def self.tracer
       Thread.current[:_buildkite_tracer]
     end
 
     def self.upload(data)
-      return false unless Buildkite::TestCollector.api_token
+      return unless Buildkite::TestCollector.api_token
 
       http = Buildkite::TestCollector::HTTPClient.new(
         url: Buildkite::TestCollector.url,
         api_token: Buildkite::TestCollector.api_token,
       )
 
-      Thread.new do
+      upload_future = Concurrent::Promises.future_on(THREAD_POOL) do
         begin
           upload_attempts ||= 0
           http.post_upload(
@@ -58,6 +64,10 @@ module Buildkite::TestCollector
           $stderr.puts "#{Buildkite::TestCollector::NAME} #{Buildkite::TestCollector::VERSION} experienced an error when sending your data, you may be missing some executions for this run."
         end
       end
+
+      timeout_future = Concurrent::Promises.schedule(UPLOAD_TIMEOUT) { :timeout }
+
+      Concurrent::Promises.any(upload_future, timeout_future)
     end
   end
 end

data/lib/buildkite/test_collector/version.rb

@@ -2,7 +2,7 @@
 
 module Buildkite
   module TestCollector
-    VERSION = "2.11.0"
+    VERSION = "2.13.0"
     NAME = "buildkite-test_collector"
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: buildkite-test_collector
 version: !ruby/object:Gem::Version
-  version: 2.11.0
+  version: 2.13.0
 platform: ruby
 authors:
 - Buildkite
@@ -9,6 +9,20 @@ bindir: bin
 cert_chain: []
 date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
@@ -23,6 +37,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '4.2'
+- !ruby/object:Gem::Dependency
+  name: ostruct
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec-core
   requirement: !ruby/object:Gem::Requirement
@@ -80,7 +108,6 @@ files:
 - CODE_OF_CONDUCT.md
 - DESIGN.md
 - Gemfile
-- Gemfile.lock
 - LICENSE.txt
 - README.md
 - Rakefile

data/Gemfile.lock

@@ -1,96 +0,0 @@
-PATH
-  remote: .
-  specs:
-    buildkite-test_collector (2.11.0)
-
-GEM
-  remote: https://rubygems.org/
-  specs:
-    activesupport (8.0.1)
-      base64
-      benchmark (>= 0.3)
-      bigdecimal
-      concurrent-ruby (~> 1.0, >= 1.3.1)
-      connection_pool (>= 2.2.5)
-      drb
-      i18n (>= 1.6, < 2)
-      logger (>= 1.4.2)
-      minitest (>= 5.1)
-      securerandom (>= 0.3)
-      tzinfo (~> 2.0, >= 2.0.5)
-      uri (>= 0.13.1)
-    base64 (0.2.0)
-    benchmark (0.4.0)
-    bigdecimal (3.1.9)
-    builder (3.3.0)
-    concurrent-ruby (1.3.5)
-    connection_pool (2.5.0)
-    cucumber (9.2.1)
-      builder (~> 3.2)
-      cucumber-ci-environment (> 9, < 11)
-      cucumber-core (> 13, < 14)
-      cucumber-cucumber-expressions (~> 17.0)
-      cucumber-gherkin (> 24, < 28)
-      cucumber-html-formatter (> 20.3, < 22)
-      cucumber-messages (> 19, < 25)
-      diff-lcs (~> 1.5)
-      mini_mime (~> 1.1)
-      multi_test (~> 1.1)
-      sys-uname (~> 1.2)
-    cucumber-ci-environment (10.0.1)
-    cucumber-core (13.0.3)
-      cucumber-gherkin (>= 27, < 28)
-      cucumber-messages (>= 20, < 23)
-      cucumber-tag-expressions (> 5, < 7)
-    cucumber-cucumber-expressions (17.1.0)
-      bigdecimal
-    cucumber-gherkin (27.0.0)
-      cucumber-messages (>= 19.1.4, < 23)
-    cucumber-html-formatter (21.9.0)
-      cucumber-messages (> 19, < 28)
-    cucumber-messages (22.0.0)
-    cucumber-tag-expressions (6.1.2)
-    diff-lcs (1.6.2)
-    drb (2.2.1)
-    ffi (1.17.2)
-    i18n (1.14.7)
-      concurrent-ruby (~> 1.0)
-    logger (1.7.0)
-    mini_mime (1.1.5)
-    minitest (5.25.4)
-    multi_test (1.1.0)
-    rake (13.2.1)
-    rspec (3.13.0)
-      rspec-core (~> 3.13.0)
-      rspec-expectations (~> 3.13.0)
-      rspec-mocks (~> 3.13.0)
-    rspec-core (3.13.2)
-      rspec-support (~> 3.13.0)
-    rspec-expectations (3.13.3)
-      diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.13.0)
-    rspec-mocks (3.13.2)
-      diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.13.0)
-    rspec-support (3.13.2)
-    securerandom (0.4.1)
-    sys-uname (1.3.1)
-      ffi (~> 1.1)
-    tzinfo (2.0.6)
-      concurrent-ruby (~> 1.0)
-    uri (1.0.2)
-
-PLATFORMS
-  ruby
-
-DEPENDENCIES
-  activesupport (>= 4.2)
-  buildkite-test_collector!
-  cucumber (~> 9.0)
-  rake (~> 13.0)
-  rspec (~> 3.0)
-  rspec-core (~> 3.10)
-  rspec-expectations (~> 3.10)
-
-BUNDLED WITH
-   2.3.25