buildkite-test_collector 2.0.0.pre → 2.1.0.pre

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: b7fc00254b8a2002e8680ca265b00f5b2700dace5d6948aa56f779102f8ce0c6
4
- data.tar.gz: 6c05b872c740262ca0c66f4d6c09b5b253a366466241513b74706c8d0a56328a
3
+ metadata.gz: e85ba163eccc317f8e03168153e2018d6b3033b760c3479d7e569848dee8be64
4
+ data.tar.gz: 2ee8f7c0088bd8cf0eff5b94bff12eaf93a5eec5b34e55e25f4bc31d130dbb53
5
5
  SHA512:
6
- metadata.gz: 2f646c1a4564960ba78494efef9ec64fa43eb9a4107d7c004884d977649d5cdf07aa19abec49b887d046d1c7a15ec7b61afbde60f4b12769c77f7578ba55df8e
7
- data.tar.gz: b80f9fd21f045f396bceea9af860e471191669dc80b1d3e57ae38c5d1b7dcebe3fdfc1646774af043d6b8f243cd7335c9b8f044aff9232146e3c0a4dad8a58d8
6
+ metadata.gz: 302e0420cf87e321faaaa4cf7843d322d1b9589a31411e4e7239c781f6176ab7d2a8ff2c0d3650912bccd0f863946d8815d7f0667e1cf132497b35410dcfa247
7
+ data.tar.gz: ff0a31165893a7368465be84a8ba8fbc6f863198a3b8d8965cee9941a29b785a6c547eb70cfa6c6e1283d4378970c0e217dfff87b657cc91837736ccf6c7a978
data/CHANGELOG.md CHANGED
@@ -1,9 +1,13 @@
1
1
  # CHANGELOG
2
2
 
3
+ ## v2.1.0.pre
4
+ - Minitest plugin to use HTTP Upload API instead of websocket connection to send test data #178 #179 - @niceking
5
+
3
6
  ## v2.0.0.pre
4
7
 
5
8
  - Major change: RSpec plugin to use HTTP Upload API instead of websocket connection to send test data #174 #175 - @niceking
6
9
  - `identifier` field removed from trace #176 - @amybiyuliu
10
+ - Only warn on EOF errors and also catch SSLErrors #160 - @gchan
7
11
 
8
12
  ## v1.5.0
9
13
 
data/Gemfile.lock CHANGED
@@ -1,14 +1,14 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- buildkite-test_collector (2.0.0.pre)
4
+ buildkite-test_collector (2.1.0.pre)
5
5
  activesupport (>= 4.2)
6
6
  websocket (~> 1.2)
7
7
 
8
8
  GEM
9
9
  remote: https://rubygems.org/
10
10
  specs:
11
- activesupport (7.0.4.2)
11
+ activesupport (7.0.4.3)
12
12
  concurrent-ruby (~> 1.0, >= 1.0.2)
13
13
  i18n (>= 1.6, < 2)
14
14
  minitest (>= 5.1)
data/README.md CHANGED
@@ -89,10 +89,6 @@ BUILDKITE_ANALYTICS_EXECUTION_NAME_PREFIX
89
89
  BUILDKITE_ANALYTICS_EXECUTION_NAME_SUFFIX
90
90
  ```
91
91
 
92
- ## 🔍 Debugging
93
-
94
- To enable debugging output, set the `BUILDKITE_ANALYTICS_DEBUG_ENABLED` environment variable to `true`.
95
-
96
92
  ## 🔜 Roadmap
97
93
 
98
94
  See the [GitHub 'enhancement' issues](https://github.com/buildkite/test-collector-ruby/issues?q=is%3Aissue+is%3Aopen+label%3Aenhancement) for planned features. Pull requests are always welcome, and we’ll give you feedback and guidance if you choose to contribute 💚
@@ -34,7 +34,6 @@ class Buildkite::TestCollector::CI
34
34
  "number" => ENV["BUILDKITE_ANALYTICS_NUMBER"],
35
35
  "job_id" => ENV["BUILDKITE_ANALYTICS_JOB_ID"],
36
36
  "message" => ENV["BUILDKITE_ANALYTICS_MESSAGE"],
37
- "debug" => ENV["BUILDKITE_ANALYTICS_DEBUG_ENABLED"],
38
37
  "execution_name_prefix" => ENV["BUILDKITE_ANALYTICS_EXECUTION_NAME_PREFIX"],
39
38
  "execution_name_suffix" => ENV["BUILDKITE_ANALYTICS_EXECUTION_NAME_SUFFIX"],
40
39
  "version" => Buildkite::TestCollector::VERSION,
@@ -12,4 +12,4 @@ end
12
12
 
13
13
  Buildkite::TestCollector.enable_tracing!
14
14
 
15
- Buildkite::TestCollector.safe { Buildkite::TestCollector::Uploader.configure }
15
+ Buildkite::TestCollector.session = Buildkite::TestCollector::Session.new
@@ -13,7 +13,7 @@ module Buildkite::TestCollector::MinitestPlugin
13
13
 
14
14
  if Buildkite::TestCollector.uploader
15
15
  if trace = Buildkite::TestCollector.uploader.traces[result.source_location]
16
- Buildkite::TestCollector.session&.write_result(trace)
16
+ Buildkite::TestCollector.session.add_example_to_send_queue(result.source_location)
17
17
  end
18
18
  end
19
19
  end
@@ -21,16 +21,8 @@ module Buildkite::TestCollector::MinitestPlugin
21
21
  def report
22
22
  super
23
23
 
24
- if Buildkite::TestCollector.session.present?
25
- examples_count = {
26
- examples: count,
27
- failed: failures,
28
- pending: skips,
29
- errors_outside_examples: 0, # Minitest does not report this
30
- }
31
-
32
- Buildkite::TestCollector.session.close(examples_count)
33
- end
24
+ Buildkite::TestCollector.session.send_remaining_data
25
+ Buildkite::TestCollector.session.close
34
26
  end
35
27
  end
36
28
  end
@@ -27,40 +27,6 @@ module Buildkite::TestCollector
27
27
  EOFError
28
28
  ]
29
29
 
30
- def self.configure
31
- Buildkite::TestCollector.logger.debug("hello from main thread")
32
-
33
- if Buildkite::TestCollector.api_token
34
- http = Buildkite::TestCollector::HTTPClient.new(Buildkite::TestCollector.url)
35
-
36
- response = begin
37
- http.post
38
- rescue *Buildkite::TestCollector::Uploader::REQUEST_EXCEPTIONS => e
39
- Buildkite::TestCollector.logger.error "Buildkite Test Analytics: Error communicating with the server: #{e.message}"
40
- end
41
-
42
- return unless response
43
-
44
- case response.code
45
- when "401"
46
- Buildkite::TestCollector.logger.info "Buildkite Test Analytics: Invalid Suite API key. Please double check your Suite API key."
47
- when "200"
48
- json = JSON.parse(response.body)
49
-
50
- if (socket_url = json["cable"]) && (channel = json["channel"])
51
- Buildkite::TestCollector.session = Buildkite::TestCollector::SocketSession.new(socket_url, http.authorization_header, channel)
52
- end
53
- else
54
- request_id = response.to_hash["x-request-id"]
55
- Buildkite::TestCollector.logger.info "buildkite-test_collector could not establish an initial connection with Buildkite. You may be missing some data for this test suite, please contact support with request ID #{request_id}."
56
- end
57
- else
58
- if !!ENV["BUILDKITE_BUILD_ID"]
59
- Buildkite::TestCollector.logger.info "Buildkite Test Analytics: No Suite API key provided. You can get the API key from your Suite settings page."
60
- end
61
- end
62
- end
63
-
64
30
  def self.tracer
65
31
  Thread.current[:_buildkite_tracer]
66
32
  end
@@ -2,7 +2,7 @@
2
2
 
3
3
  module Buildkite
4
4
  module TestCollector
5
- VERSION = "2.0.0.pre"
5
+ VERSION = "2.1.0.pre"
6
6
  NAME = "buildkite-test_collector"
7
7
  end
8
8
  end
@@ -22,15 +22,12 @@ require "active_support/notifications"
22
22
 
23
23
  require_relative "test_collector/version"
24
24
  require_relative "test_collector/error"
25
- require_relative "test_collector/logger"
26
25
  require_relative "test_collector/ci"
27
26
  require_relative "test_collector/http_client"
28
27
  require_relative "test_collector/uploader"
29
28
  require_relative "test_collector/network"
30
29
  require_relative "test_collector/object"
31
30
  require_relative "test_collector/tracer"
32
- require_relative "test_collector/socket_connection"
33
- require_relative "test_collector/socket_session"
34
31
  require_relative "test_collector/session"
35
32
 
36
33
  module Buildkite
@@ -43,17 +40,15 @@ module Buildkite
43
40
  attr_accessor :url
44
41
  attr_accessor :uploader
45
42
  attr_accessor :session
46
- attr_accessor :debug_enabled
47
43
  attr_accessor :tracing_enabled
48
44
  attr_accessor :artifact_path
49
45
  attr_accessor :env
50
46
  attr_accessor :batch_size
51
47
  end
52
48
 
53
- def self.configure(hook:, token: nil, url: nil, debug_enabled: false, tracing_enabled: true, artifact_path: nil, env: {})
49
+ def self.configure(hook:, token: nil, url: nil, tracing_enabled: true, artifact_path: nil, env: {})
54
50
  self.api_token = (token || ENV["BUILDKITE_ANALYTICS_TOKEN"])&.strip
55
51
  self.url = url || DEFAULT_URL
56
- self.debug_enabled = debug_enabled || !!(ENV["BUILDKITE_ANALYTICS_DEBUG_ENABLED"])
57
52
  self.tracing_enabled = tracing_enabled
58
53
  self.artifact_path = artifact_path
59
54
  self.env = env
@@ -74,30 +69,6 @@ module Buildkite
74
69
  tracer&.leave
75
70
  end
76
71
 
77
- def self.log_formatter
78
- @log_formatter ||= Buildkite::TestCollector::Logger::Formatter.new
79
- end
80
-
81
- def self.log_formatter=(log_formatter)
82
- @log_formatter = log_formatter
83
- logger.formatter = log_formatter
84
- end
85
-
86
- def self.logger=(logger)
87
- @logger = logger
88
- end
89
-
90
- def self.logger
91
- return @logger if defined?(@logger)
92
-
93
- debug_mode = ENV.fetch("BUILDKITE_ANALYTICS_DEBUG_ENABLED") do
94
- $DEBUG
95
- end
96
-
97
- level = !!debug_mode ? ::Logger::DEBUG : ::Logger::WARN
98
- @logger ||= Buildkite::TestCollector::Logger.new($stderr, level: level)
99
- end
100
-
101
72
  def self.enable_tracing!
102
73
  return unless self.tracing_enabled
103
74
 
@@ -108,12 +79,5 @@ module Buildkite
108
79
  Buildkite::TestCollector::Uploader.tracer&.backfill(:sql, finish - start, **{ query: payload[:sql] })
109
80
  end
110
81
  end
111
-
112
- def self.safe(&block)
113
- block.call
114
- rescue StandardError => e
115
- logger.error("Buildkite::TestCollector received exception: #{e}")
116
- logger.error("Backtrace:\n#{e.backtrace.join("\n")}")
117
- end
118
82
  end
119
83
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: buildkite-test_collector
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.0.0.pre
4
+ version: 2.1.0.pre
5
5
  platform: ruby
6
6
  authors:
7
7
  - Buildkite
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2023-03-07 00:00:00.000000000 Z
11
+ date: 2023-03-30 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: activesupport
@@ -94,7 +94,6 @@ files:
94
94
  - lib/buildkite/test_collector/http_client.rb
95
95
  - lib/buildkite/test_collector/library_hooks/minitest.rb
96
96
  - lib/buildkite/test_collector/library_hooks/rspec.rb
97
- - lib/buildkite/test_collector/logger.rb
98
97
  - lib/buildkite/test_collector/minitest_plugin.rb
99
98
  - lib/buildkite/test_collector/minitest_plugin/reporter.rb
100
99
  - lib/buildkite/test_collector/minitest_plugin/trace.rb
@@ -103,8 +102,6 @@ files:
103
102
  - lib/buildkite/test_collector/rspec_plugin/reporter.rb
104
103
  - lib/buildkite/test_collector/rspec_plugin/trace.rb
105
104
  - lib/buildkite/test_collector/session.rb
106
- - lib/buildkite/test_collector/socket_connection.rb
107
- - lib/buildkite/test_collector/socket_session.rb
108
105
  - lib/buildkite/test_collector/tracer.rb
109
106
  - lib/buildkite/test_collector/uploader.rb
110
107
  - lib/buildkite/test_collector/version.rb
@@ -1,16 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Buildkite::TestCollector
4
- class Logger < ::Logger
5
- class Formatter < ::Logger::Formatter
6
- def call(severity, time, _program, message)
7
- "#{time.utc.iso8601(9)} pid=#{::Process.pid} tid=#{::Thread.current.object_id} #{severity}: #{message}\n"
8
- end
9
- end
10
-
11
- def initialize(*args, **kwargs)
12
- super
13
- self.formatter = Buildkite::TestCollector.log_formatter
14
- end
15
- end
16
- end
@@ -1,156 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Buildkite::TestCollector
4
- class SocketConnection
5
- class HandshakeError < StandardError; end
6
- class SocketError < StandardError; end
7
-
8
- def initialize(session, url, headers)
9
- uri = URI.parse(url)
10
- @session = session
11
- protocol = "http"
12
-
13
- begin
14
- socket = TCPSocket.new(uri.host, uri.port || (uri.scheme == "wss" ? 443 : 80))
15
-
16
- if uri.scheme == "wss"
17
- ctx = OpenSSL::SSL::SSLContext.new
18
- protocol = "https"
19
-
20
- ctx.min_version = :TLS1_2
21
- ctx.verify_mode = OpenSSL::SSL::VERIFY_PEER
22
- ctx.cert_store = OpenSSL::X509::Store.new.tap(&:set_default_paths)
23
-
24
- socket = OpenSSL::SSL::SSLSocket.new(socket, ctx)
25
- socket.connect
26
- end
27
- rescue
28
- # We are rescuing all here, as there are a range of Errno errors that could be
29
- # raised when we fail to establish a TCP connection
30
- raise SocketError
31
- end
32
-
33
- @socket = socket
34
-
35
- headers = { "Origin" => "#{protocol}://#{uri.host}" }.merge(headers)
36
- handshake = WebSocket::Handshake::Client.new(url: url, headers: headers)
37
-
38
- @socket.write handshake.to_s
39
-
40
- until handshake.finished?
41
- if byte = @socket.getc
42
- handshake << byte
43
- end
44
- end
45
-
46
- # The errors below are raised when we establish the TCP connection, but get back
47
- # an error, i.e. in dev we can still connect to puma-dev while nginx isn't
48
- # running, or in prod we can hit a load balancer while app is down
49
- unless handshake.valid?
50
- case handshake.error
51
- when Exception, String
52
- raise HandshakeError.new(handshake.error)
53
- when nil
54
- raise HandshakeError.new("Invalid handshake")
55
- else
56
- raise HandshakeError.new(handshake.error.inspect)
57
- end
58
- end
59
-
60
- @version = handshake.version
61
-
62
- # Setting up a new thread that listens on the socket, and processes incoming
63
- # comms from the server
64
- @read_thread = Thread.new do
65
- Buildkite::TestCollector.logger.debug("listening in on socket")
66
- frame = WebSocket::Frame::Incoming::Client.new
67
-
68
- while @socket
69
- frame << @socket.readpartial(4096)
70
-
71
- while data = frame.next
72
- @session.handle(self, data.data)
73
- end
74
- end
75
- # These get re-raise from session, we should fail gracefully
76
- rescue *Buildkite::TestCollector::SocketSession::DISCONNECTED_EXCEPTIONS => e
77
- Buildkite::TestCollector.logger.error("We could not establish a connection with Buildkite Test Analytics. The error was: #{e.message}. If this is a problem, please contact support.")
78
- rescue EOFError, OpenSSL::SSL::SSLError => e
79
- # https://github.com/buildkite/test-collector-ruby/pull/147#issuecomment-1250485611
80
- raise if e.class == OpenSSL::SSL::SSLError && e.message != "SSL_read: unexpected eof while reading"
81
-
82
- Buildkite::TestCollector.logger.warn("#{e}")
83
- if @socket
84
- Buildkite::TestCollector.logger.warn("attempting disconnected flow")
85
- @session.disconnected(self)
86
- disconnect
87
- end
88
- rescue Errno::ECONNRESET, Errno::ETIMEDOUT => e
89
- Buildkite::TestCollector.logger.error("#{e}")
90
- if @socket
91
- Buildkite::TestCollector.logger.error("attempting disconnected flow")
92
- @session.disconnected(self)
93
- disconnect
94
- end
95
- rescue IOError
96
- # This is fine to ignore
97
- Buildkite::TestCollector.logger.error("IOError")
98
- rescue IndexError
99
- # I don't like that we're doing this but I think it's the best of the options
100
- #
101
- # This relates to this issue https://github.com/ruby/openssl/issues/452
102
- # A fix for it has been released but the repercussions of overriding
103
- # the OpenSSL version in the stdlib seem worse than catching this error here.
104
- Buildkite::TestCollector.logger.error("IndexError")
105
- if @socket
106
- Buildkite::TestCollector.logger.error("attempting disconnected flow")
107
- @session.disconnected(self)
108
- disconnect
109
- end
110
- end
111
- end
112
-
113
- def transmit(data, type: :text)
114
- # this line prevents us from calling disconnect twice
115
- return if @socket.nil?
116
-
117
- raw_data = data.to_json
118
- frame = WebSocket::Frame::Outgoing::Client.new(data: raw_data, type: :text, version: @version)
119
- @socket.write(frame.to_s)
120
- rescue Errno::EPIPE, Errno::ECONNRESET, OpenSSL::SSL::SSLError => e
121
- return unless @socket
122
- return if type == :close
123
- Buildkite::TestCollector.logger.error("got #{e}, attempting disconnected flow")
124
- @session.disconnected(self)
125
- disconnect
126
- rescue IndexError
127
- # I don't like that we're doing this but I think it's the best of the options
128
- #
129
- # This relates to this issue https://github.com/ruby/openssl/issues/452
130
- # A fix for it has been released but the repercussions of overriding
131
- # the OpenSSL version in the stdlib seem worse than catching this error here.
132
- Buildkite::TestCollector.logger.error("IndexError")
133
- if @socket
134
- Buildkite::TestCollector.logger.error("attempting disconnected flow")
135
- @session.disconnected(self)
136
- disconnect
137
- end
138
- end
139
-
140
- def close
141
- Buildkite::TestCollector.logger.debug("socket close")
142
- transmit(nil, type: :close)
143
- disconnect
144
- end
145
-
146
- private
147
-
148
- def disconnect
149
- Buildkite::TestCollector.logger.debug("socket disconnect")
150
- socket = @socket
151
- @socket = nil
152
- socket&.close
153
- @read_thread&.join unless @read_thread == Thread.current
154
- end
155
- end
156
- end
@@ -1,333 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Buildkite::TestCollector
4
- class SocketSession
5
- # Picked 75 as the magic timeout number as it's longer than the TCP timeout of 60s 🤷‍♀️
6
- CONFIRMATION_TIMEOUT = ENV.fetch("BUILDKITE_ANALYTICS_CONFIRMATION_TIMEOUT") { 75 }.to_i
7
- MAX_RECONNECTION_ATTEMPTS = ENV.fetch("BUILDKITE_ANALYTICS_RECONNECTION_ATTEMPTS") { 3 }.to_i
8
- WAIT_BETWEEN_RECONNECTIONS = ENV.fetch("BUILDKITE_ANALYTICS_RECONNECTION_WAIT") { 5 }.to_i
9
-
10
- # We keep a private reference so that mocking libraries won't break JSON
11
- JSON_PARSE = JSON.method(:parse)
12
- private_constant :JSON_PARSE
13
-
14
- class RejectedSubscription < StandardError; end
15
- class InitialConnectionFailure < StandardError; end
16
-
17
- DISCONNECTED_EXCEPTIONS = [
18
- Buildkite::TestCollector::SocketConnection::HandshakeError,
19
- Buildkite::TestCollector::TimeoutError,
20
- Buildkite::TestCollector::SocketConnection::SocketError,
21
- RejectedSubscription,
22
- InitialConnectionFailure,
23
- ]
24
-
25
- def initialize(url, authorization_header, channel)
26
- @establish_subscription_queue = Queue.new
27
- @channel = channel
28
-
29
- @unconfirmed_idents = {}
30
- @idents_mutex = Mutex.new
31
- @send_queue = Queue.new
32
- @empty = ConditionVariable.new
33
- @closing = false
34
- @eot_queued = false
35
- @eot_queued_mutex = Mutex.new
36
- @reconnection_mutex = Mutex.new
37
-
38
- @url = url
39
- @authorization_header = authorization_header
40
-
41
- reconnection_count = 0
42
-
43
- begin
44
- reconnection_count += 1
45
- connect
46
- rescue Buildkite::TestCollector::TimeoutError, InitialConnectionFailure => e
47
- Buildkite::TestCollector.logger.warn("buildkite-test_collector could not establish an initial connection with Buildkite due to #{e}. Attempting retry #{reconnection_count} of #{MAX_RECONNECTION_ATTEMPTS}...")
48
- if reconnection_count > MAX_RECONNECTION_ATTEMPTS
49
- Buildkite::TestCollector.logger.error "buildkite-test_collector could not establish an initial connection with Buildkite due to #{e.message} after #{MAX_RECONNECTION_ATTEMPTS} attempts. You may be missing some data for this test suite, please contact support if this issue persists."
50
- else
51
- sleep(WAIT_BETWEEN_RECONNECTIONS)
52
- Buildkite::TestCollector.logger.warn("retrying reconnection")
53
- retry
54
- end
55
- end
56
- init_write_thread
57
- end
58
-
59
- def disconnected(connection)
60
- @reconnection_mutex.synchronize do
61
- # When the first thread detects a disconnection, it calls the disconnect method
62
- # with the current connection. This thread grabs the reconnection mutex and does the
63
- # reconnection, which then updates the value of @connection.
64
- #
65
- # At some point in that process, the second thread would have detected the
66
- # disconnection too, and it also calls it with the current connection. However, the
67
- # second thread can't run the reconnection code because of the mutex. By the
68
- # time the mutex is released, the value of @connection has been refreshed, and so
69
- # the second thread returns early and does not reattempt the reconnection.
70
- return unless connection == @connection
71
- Buildkite::TestCollector.logger.debug("starting reconnection")
72
-
73
- reconnection_count = 0
74
-
75
- begin
76
- reconnection_count += 1
77
- connect
78
- init_write_thread
79
- rescue *DISCONNECTED_EXCEPTIONS => e
80
- Buildkite::TestCollector.logger.warn("failed reconnection attempt #{reconnection_count} due to #{e}")
81
- if reconnection_count > MAX_RECONNECTION_ATTEMPTS
82
- Buildkite::TestCollector.logger.error "buildkite-test_collector experienced a disconnection and could not reconnect to Buildkite due to #{e.message}. Please contact support."
83
- raise e
84
- else
85
- sleep(WAIT_BETWEEN_RECONNECTIONS)
86
- Buildkite::TestCollector.logger.warn("retrying reconnection")
87
- retry
88
- end
89
- end
90
- end
91
- retransmit
92
- end
93
-
94
- def close(examples_count)
95
- @closing = true
96
- @examples_count = examples_count
97
- Buildkite::TestCollector.logger.debug("closing socket connection")
98
-
99
- # Because the server only sends us confirmations after every 10mb of
100
- # data it uploads to S3, we'll never get confirmation of the
101
- # identifiers of the last upload part unless we send an explicit finish,
102
- # to which the server will respond with the last bits of data
103
- send_eot
104
-
105
- # After EOT, we wait for 75 seconds for the send queue to be drained and for the
106
- # server to confirm the last idents. If everything has already been confirmed we can
107
- # proceed without waiting.
108
- @idents_mutex.synchronize do
109
- if @unconfirmed_idents.any?
110
- Buildkite::TestCollector.logger.debug "Waiting for Buildkite Test Analytics to send results..."
111
- Buildkite::TestCollector.logger.debug("waiting for last confirm")
112
-
113
- @empty.wait(@idents_mutex, CONFIRMATION_TIMEOUT)
114
- end
115
- end
116
-
117
- # Then we always disconnect cos we can't wait forever? 🤷‍♀️
118
- @connection.close
119
- # We kill the write thread cos it's got a while loop in it, so it won't finish otherwise
120
- @write_thread&.kill
121
-
122
- Buildkite::TestCollector.logger.info "Buildkite Test Analytics completed"
123
- Buildkite::TestCollector.logger.debug("socket connection closed")
124
- end
125
-
126
- def handle(_connection, data)
127
- data = JSON_PARSE.call(data)
128
- case data["type"]
129
- when "ping"
130
- # In absence of other message, the server sends us a ping every 3 seconds
131
- # We are currently not doing anything with these
132
- Buildkite::TestCollector.logger.debug("received ping")
133
- when "welcome", "confirm_subscription"
134
- # Push these two messages onto the queue, so that we block on waiting for the
135
- # initializing phase to complete
136
- @establish_subscription_queue.push(data)
137
- Buildkite::TestCollector.logger.debug("received #{data['type']}")
138
- when "reject_subscription"
139
- Buildkite::TestCollector.logger.debug("received rejected_subscription")
140
- raise RejectedSubscription
141
- else
142
- process_message(data)
143
- end
144
- end
145
-
146
- def write_result(result)
147
- queue_and_track_result(result.id, result.as_hash)
148
-
149
- Buildkite::TestCollector.logger.debug("added #{result.id} to send queue")
150
- end
151
-
152
- def unconfirmed_idents_count
153
- @idents_mutex.synchronize do
154
- @unconfirmed_idents.count
155
- end
156
- end
157
-
158
- private
159
-
160
- def connect
161
- Buildkite::TestCollector.logger.debug("starting socket connection process")
162
-
163
- @connection = SocketConnection.new(self, @url, {
164
- "Authorization" => @authorization_header,
165
- })
166
-
167
- wait_for_welcome
168
-
169
- @connection.transmit({
170
- "command" => "subscribe",
171
- "identifier" => @channel
172
- })
173
-
174
- wait_for_confirm
175
-
176
- Buildkite::TestCollector.logger.info "Connected to Buildkite Test Analytics!"
177
- Buildkite::TestCollector.logger.debug("connected")
178
- end
179
-
180
- def init_write_thread
181
- # As this method can be called multiple times in the
182
- # reconnection process, kill prev write threads (if any) before
183
- # setting up the new one
184
- @write_thread&.kill
185
-
186
- @write_thread = Thread.new do
187
- Buildkite::TestCollector.logger.debug("hello from write thread")
188
- # Pretty sure this eternal loop is fine cos the call to queue.pop is blocking
189
- loop do
190
- data = @send_queue.pop
191
- message_type = data["action"]
192
-
193
- if message_type == "end_of_transmission"
194
- # Because of the unpredictable sequencing between the test suite finishing
195
- # (EOT gets queued) and disconnections happening (retransmit results gets
196
- # queued), we don't want to send an EOT before any retransmits are sent.
197
- if @send_queue.length > 0
198
- @send_queue << data
199
- Buildkite::TestCollector.logger.debug("putting eot at back of queue")
200
- next
201
- end
202
- @eot_queued_mutex.synchronize do
203
- @eot_queued = false
204
- end
205
- end
206
-
207
- @connection.transmit({
208
- "identifier" => @channel,
209
- "command" => "message",
210
- "data" => data.to_json
211
- })
212
-
213
- if Buildkite::TestCollector.debug_enabled
214
- ids = if message_type == "record_results"
215
- data["results"].map { |result| result["id"] }
216
- end
217
- Buildkite::TestCollector.logger.debug("transmitted #{message_type} #{ids}")
218
- end
219
- end
220
- end
221
- end
222
-
223
- def pop_with_timeout(message_type)
224
- Timeout.timeout(30, Buildkite::TestCollector::TimeoutError, "Timeout: Waited 30 seconds for #{message_type}") do
225
- @establish_subscription_queue.pop
226
- end
227
- end
228
-
229
- def wait_for_welcome
230
- welcome = pop_with_timeout("welcome")
231
-
232
- if welcome && welcome != { "type" => "welcome" }
233
- raise InitialConnectionFailure.new("Wrong message received, expected a welcome, but received: #{welcome.inspect}")
234
- end
235
- end
236
-
237
- def wait_for_confirm
238
- confirm = pop_with_timeout("confirm")
239
-
240
- if confirm && confirm != { "type" => "confirm_subscription", "identifier" => @channel }
241
- raise InitialConnectionFailure.new("Wrong message received, expected a confirm, but received: #{confirm.inspect}")
242
- end
243
- end
244
-
245
- def queue_and_track_result(ident, result_as_hash)
246
- @idents_mutex.synchronize do
247
- @unconfirmed_idents[ident] = result_as_hash
248
-
249
- @send_queue << {
250
- "action" => "record_results",
251
- "results" => [result_as_hash]
252
- }
253
- end
254
- end
255
-
256
- def confirm_idents(idents)
257
- retransmit_required = @closing
258
-
259
- @idents_mutex.synchronize do
260
- # Remove received idents from unconfirmed_idents
261
- idents.each { |key| @unconfirmed_idents.delete(key) }
262
-
263
- Buildkite::TestCollector.logger.debug("received confirm for indentifiers: #{idents}")
264
-
265
- # This @empty ConditionVariable broadcasts every time that @unconfirmed_idents is
266
- # empty, which will happen about every 10mb of data as that's when the server
267
- # sends back confirmations.
268
- #
269
- # However, there aren't any threads waiting on this signal until after we
270
- # send the EOT message, so the prior broadcasts shouldn't do anything.
271
- if @unconfirmed_idents.empty?
272
- @empty.broadcast
273
-
274
- retransmit_required = false
275
-
276
- Buildkite::TestCollector.logger.debug("all identifiers have been confirmed")
277
- else
278
- Buildkite::TestCollector.logger.debug("still waiting on confirm for identifiers: #{@unconfirmed_idents.keys}")
279
- end
280
- end
281
-
282
- # If we're closing, any unconfirmed results need to be retransmitted.
283
- retransmit if retransmit_required
284
- end
285
-
286
- def send_eot
287
- @eot_queued_mutex.synchronize do
288
- return if @eot_queued
289
-
290
- @send_queue << {
291
- "action" => "end_of_transmission",
292
- "examples_count" => @examples_count.to_json
293
- }
294
- @eot_queued = true
295
-
296
- Buildkite::TestCollector.logger.debug("added EOT to send queue")
297
- end
298
- end
299
-
300
- def process_message(data)
301
- # Check we're getting the data we expect
302
- return unless data["identifier"] == @channel
303
-
304
- case
305
- when data["message"].key?("confirm")
306
- confirm_idents(data["message"]["confirm"])
307
- else
308
- # unhandled message
309
- Buildkite::TestCollector.logger.debug("received unhandled message #{data["message"]}")
310
- end
311
- end
312
-
313
- def retransmit
314
- @idents_mutex.synchronize do
315
- results = @unconfirmed_idents.values
316
-
317
- # queue the contents of the buffer, unless it's empty
318
- if results.any?
319
- @send_queue << {
320
- "action" => "record_results",
321
- "results" => results
322
- }
323
-
324
- Buildkite::TestCollector.logger.debug("queueing up retransmitted results #{@unconfirmed_idents.keys}")
325
- end
326
- end
327
-
328
- # if we were disconnected in the closing phase, then resend the EOT
329
- # message so the server can persist the last upload part
330
- send_eot if @closing
331
- end
332
- end
333
- end