gitlab-test-balancing 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: d0847c642c34fc38dbbb8fe6f468c7c906d04a5d756b08d92b0c1fd8f9b715f0
4
- data.tar.gz: af8f00f37096a128eb0d0cd0ecc7b190ef15915d660ef64eb7c1c4a0825b9ba5
3
+ metadata.gz: 284c90dd11463a6371ec4f4b1d5991c8739b925cd7b7883c69532f06245a9050
4
+ data.tar.gz: 0dc4221f45ec3ebcbdf9e482acf9e9253e8621aabf4106fd69fd22e1aecd3b6a
5
5
  SHA512:
6
- metadata.gz: 4ff4da41f151730ca3d72413bcbce9872c4f3947e0682196b1acdbd1f565d525626287bdd8e055020c6bd9e194b7e5facb0394c918eb9b38faf65437fdfff358
7
- data.tar.gz: 2fa0983c324d7d90edcbfbf1d809f7de1a1adfbe3ceb92bf6635ea4ffce6196b0c296d1f5fceb5fdfdd82ecfa3ec741f5f0a864adcb98da6a67361306517562f
6
+ metadata.gz: 51b7d39b9f6cb86617ab42a991ca4f5775a8ee49593ad65bd4bb40399f5e26d1da97b801e1acb62da1706ed720232e5e28d26e60d01bdc6d38576809d756dd15
7
+ data.tar.gz: a0d78179e9af4c43823ab1826466f3f1273988b8e9a0a946df671424140f8c0f8ed4d61854948c6b494c2a43271b7a0ef117065d8be8f8a4c56c20cfb724c6b8
@@ -0,0 +1,6 @@
1
+ #!/usr/bin/env ruby
2
+ # frozen_string_literal: true
3
+
4
+ require 'gitlab/test_balancing/cli/merge_durations'
5
+
6
+ exit Gitlab::TestBalancing::CLI::MergeDurations.new(ARGV).run
@@ -0,0 +1,9 @@
1
+ #!/usr/bin/env ruby
2
+ # frozen_string_literal: true
3
+
4
+ require 'gitlab/test_balancing/cli/rspec'
5
+
6
+ # `exec` would discard anything still buffered.
7
+ $stdout.sync = true
8
+
9
+ exit Gitlab::TestBalancing::CLI::Rspec.new(ARGV).run
@@ -0,0 +1,132 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'fileutils'
4
+ require 'json'
5
+ require 'logger'
6
+ require 'gitlab/test_balancing/durations'
7
+
8
+ module Gitlab
9
+ module TestBalancing
10
+ module CLI
11
+ # Averages the nodes' duration reports into the previous merged report:
12
+ #
13
+ # gitlab-test-balancing-merge-durations OUTPUT [NODE_REPORT...]
14
+ #
15
+ # With TEST_BALANCING_NODE_REPORT_JOBS, a comma-separated list of job names, it
16
+ # reads each node's TEST_BALANCING_NODE_REPORT from those jobs in this pipeline
17
+ # through the API, instead of from NODE_REPORT files. The previous report
18
+ # is read like the nodes read it, from TEST_BALANCING_DURATIONS_FILE or the
19
+ # TEST_BALANCING_DURATIONS_JOB artifact.
20
+ #
21
+ # Files that no node reported are dropped, as long as every node's job
22
+ # succeeded and its report was read. Without NODE_REPORT files, the previous
23
+ # report is kept as it is. With TEST_BALANCING_NODE_REPORT_JOBS, it fails when
24
+ # it finds no reports, or can't fetch the previous report.
25
+ class MergeDurations
26
+ CONCURRENCY = 8
27
+
28
+ def initialize(argv, env: ENV, logger: nil)
29
+ @output, *@report_paths = argv
30
+ @env = env
31
+ @logger = logger
32
+ end
33
+
34
+ def run
35
+ abort 'usage: gitlab-test-balancing-merge-durations OUTPUT [NODE_REPORT...]' unless @output
36
+
37
+ job_mode = env['TEST_BALANCING_NODE_REPORT_JOBS']
38
+ reports, complete = job_mode ? job_reports : file_reports
39
+
40
+ # Otherwise a typo in a job name or the report path would stop updates unnoticed.
41
+ if job_mode && reports.empty?
42
+ logger.error 'No node reports found. Check TEST_BALANCING_NODE_REPORT_JOBS and TEST_BALANCING_NODE_REPORT.'
43
+ return 1
44
+ end
45
+
46
+ merged = Durations.merge(previous, reports, prune: complete && reports.any?)
47
+
48
+ FileUtils.mkdir_p(File.dirname(@output))
49
+ File.write(@output, JSON.pretty_generate(merged))
50
+ logger.info "Merged #{reports.size} node reports into #{@output}: #{merged.size} files, " \
51
+ "#{(merged.values.sum / 60).round(1)} minutes in total."
52
+
53
+ 0
54
+ rescue Durations::FetchError => e
55
+ # Publishing only this run's durations would replace the averaged history.
56
+ logger.error "#{e.message} Not publishing a new report."
57
+ 1
58
+ end
59
+
60
+ private
61
+
62
+ attr_reader :env
63
+
64
+ # Returns the reports, and whether every report could be read.
65
+ def file_reports
66
+ existing = @report_paths.select { |path| File.exist?(path) }
67
+
68
+ [existing.map { |path| Durations.load(path) }, existing.size == @report_paths.size]
69
+ end
70
+
71
+ # Returns the reports, and whether every job succeeded and its report could be
72
+ # read. A successful job without a report ran no spec files.
73
+ def job_reports
74
+ names = env['TEST_BALANCING_NODE_REPORT_JOBS'].split(',').map(&:strip)
75
+ jobs = pipeline_jobs.select { |job| names.include?(job['name'].sub(%r{ \d+/\d+\z}, '')) }
76
+ logger.info "Reading reports from #{jobs.size} jobs."
77
+
78
+ results = jobs.each_slice(CONCURRENCY).flat_map do |slice|
79
+ slice.map { |job| Thread.new { job_report(job) } }.map(&:value)
80
+ end
81
+
82
+ complete = results.none?(&:nil?) && jobs.all? { |job| job['status'] == 'success' }
83
+
84
+ [results.select { |result| result.is_a?(Hash) }, complete]
85
+ end
86
+
87
+ def job_report(job)
88
+ api_get("/projects/#{env['CI_PROJECT_ID']}/jobs/#{job['id']}/artifacts/#{env['TEST_BALANCING_NODE_REPORT']}")
89
+ end
90
+
91
+ # The jobs API leaves out retried jobs, so each node's latest attempt is listed once.
92
+ def pipeline_jobs
93
+ (1..).each_with_object([]) do |page, jobs|
94
+ batch = api_get("/projects/#{env['CI_PROJECT_ID']}/pipelines/#{env['CI_PIPELINE_ID']}/jobs" \
95
+ "?per_page=100&page=#{page}")
96
+ abort 'Could not list the pipeline jobs.' unless batch.is_a?(Array)
97
+
98
+ jobs.concat(batch)
99
+ break jobs if batch.size < 100
100
+ end
101
+ end
102
+
103
+ def previous
104
+ return Durations.load(env['TEST_BALANCING_DURATIONS_FILE']) if env['TEST_BALANCING_DURATIONS_FILE']
105
+ return {} unless env['TEST_BALANCING_DURATIONS_JOB'] && env['CI_JOB_TOKEN']
106
+
107
+ Durations.fetch(
108
+ job: env['TEST_BALANCING_DURATIONS_JOB'],
109
+ path: env.fetch('TEST_BALANCING_DURATIONS_ARTIFACT', Durations::DEFAULT_PATH),
110
+ ref: env.fetch('TEST_BALANCING_DURATIONS_REF', env['CI_DEFAULT_BRANCH']),
111
+ api_url: env['CI_API_V4_URL'],
112
+ project_id: env['CI_PROJECT_ID'],
113
+ job_token: env['CI_JOB_TOKEN'],
114
+ logger: logger
115
+ ) || {}
116
+ end
117
+
118
+ def api_get(path)
119
+ url = "#{env['CI_API_V4_URL'].chomp('/')}#{path}"
120
+
121
+ Durations.get_json(url, job_token: env['CI_JOB_TOKEN'], logger: logger)
122
+ end
123
+
124
+ def logger
125
+ @logger ||= Logger.new($stdout, formatter: ->(severity, _time, _progname, msg) {
126
+ "[test balancing] #{severity}: #{msg}\n"
127
+ })
128
+ end
129
+ end
130
+ end
131
+ end
132
+ end
@@ -0,0 +1,163 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'logger'
4
+ require 'gitlab/test_balancing/durations'
5
+ require 'gitlab/test_balancing/runner/rspec'
6
+
7
+ module Gitlab
8
+ module TestBalancing
9
+ module CLI
10
+ # Runs this node's spec files through GitLab test balancing:
11
+ #
12
+ # bundle exec gitlab-test-balancing-rspec [RSPEC_ARGS] [-- SPEC_FILES]
13
+ #
14
+ # Configured with environment variables:
15
+ #
16
+ # - TEST_BALANCING_PATTERN: spec files to run, when SPEC_FILES isn't given.
17
+ # Defaults to spec/**/*_spec.rb.
18
+ # - TEST_BALANCING_EXCLUDE_PATTERN: spec files to leave out.
19
+ # - TEST_BALANCING_DURATIONS_FILE: a local durations report to seed from.
20
+ # - TEST_BALANCING_DURATIONS_JOB: otherwise, the job whose artifact holds the
21
+ # durations report, read from the latest successful pipeline on
22
+ # TEST_BALANCING_DURATIONS_REF (default: the default branch), at
23
+ # TEST_BALANCING_DURATIONS_ARTIFACT (default: test-balancing/durations.json).
24
+ # - TEST_BALANCING_NODE_REPORT: where to write this node's measured durations.
25
+ #
26
+ # Each node seeds its share of the files into the job group's shared queue,
27
+ # then runs the files it claims until the queue is empty. Without test
28
+ # balancing, it runs its share with plain `rspec`.
29
+ class Rspec
30
+ DEFAULT_PATTERN = 'spec/**/*_spec.rb'
31
+ LIB_DIR = File.expand_path('../../..', __dir__)
32
+
33
+ def initialize(argv, env: ENV, logger: nil)
34
+ separator = argv.index('--')
35
+ @rspec_args = separator ? argv[0...separator] : argv
36
+ @given_files = separator ? argv[(separator + 1)..] : nil
37
+ @env = env
38
+ @logger = logger
39
+ end
40
+
41
+ # Returns the exit status. The static path replaces the process instead.
42
+ def run
43
+ # An empty list after -- is deliberate, for example a predictive run with nothing to test.
44
+ if files.empty? && given_files
45
+ logger.info 'No spec files to run.'
46
+ return 0
47
+ end
48
+
49
+ # A pattern that matches nothing is a misconfiguration, which shouldn't pass the job.
50
+ if files.empty?
51
+ logger.error 'No spec files match TEST_BALANCING_PATTERN and TEST_BALANCING_EXCLUDE_PATTERN.'
52
+ return 1
53
+ end
54
+
55
+ if node_files.empty?
56
+ logger.info 'No spec files allocated to this node.'
57
+ return 0
58
+ end
59
+
60
+ return run_static unless parallel?
61
+
62
+ result = Runner::Rspec.new(test_splits: test_splits, rspec_args: rspec_args_with_report, logger: logger).run
63
+ return run_static if result.unavailable?
64
+
65
+ logger.info "Test balancing queue drained. Exiting with status #{result.status}."
66
+ result.status
67
+ end
68
+
69
+ private
70
+
71
+ attr_reader :rspec_args, :given_files, :env
72
+
73
+ def parallel?
74
+ env['CI_JOB_TOKEN'] && env['CI_API_V4_URL'] && node_total > 1
75
+ end
76
+
77
+ def test_splits
78
+ node_files.map { |path| { path: path, expected_duration: expected_durations[path] }.compact }
79
+ end
80
+
81
+ def run_static
82
+ command = ['bundle', 'exec', 'rspec', *rspec_args_with_report, '--', *node_files]
83
+ logger.info "Running #{node_files.size} spec files without test balancing."
84
+
85
+ exec(*command)
86
+ end
87
+
88
+ def rspec_args_with_report
89
+ return rspec_args unless env['TEST_BALANCING_NODE_REPORT']
90
+
91
+ [*rspec_args, '-I', LIB_DIR, '--require', 'gitlab/test_balancing/rspec/durations_report']
92
+ end
93
+
94
+ # This node's share, which it seeds, or runs when test balancing is unavailable.
95
+ # It depends only on the file list, not on the durations report: nodes can
96
+ # fetch different reports, and shares that don't add up would leave files unseeded.
97
+ def node_files
98
+ @node_files ||= files.select.with_index { |_path, i| i % node_total == node_index - 1 }
99
+ end
100
+
101
+ # Sorted, so that every node computes the same split. Without a leading ./,
102
+ # like the durations report's paths.
103
+ def files
104
+ @files ||= (given_files || Dir.glob(env.fetch('TEST_BALANCING_PATTERN', DEFAULT_PATTERN)).reject do |path|
105
+ exclude = env['TEST_BALANCING_EXCLUDE_PATTERN']
106
+ exclude && File.fnmatch?(exclude, path, File::FNM_PATHNAME | File::FNM_EXTGLOB)
107
+ end).map { |path| path.delete_prefix('./') }.sort
108
+ end
109
+
110
+ # Files missing from the report get the median, so they aren't treated as the
111
+ # slowest ones, as they would be with the server's default duration.
112
+ def expected_durations
113
+ @expected_durations ||= begin
114
+ known = files.filter_map { |path| durations[path] }
115
+ default = Durations.median(known)
116
+
117
+ files.to_h { |path| [path, durations.fetch(path, default)] }.compact
118
+ end
119
+ end
120
+
121
+ def durations
122
+ @durations ||= if env['TEST_BALANCING_DURATIONS_FILE']
123
+ Durations.load(env['TEST_BALANCING_DURATIONS_FILE'])
124
+ elsif env['TEST_BALANCING_DURATIONS_JOB'] && env['CI_JOB_TOKEN']
125
+ fetch_durations || {}
126
+ else
127
+ {}
128
+ end
129
+ end
130
+
131
+ def fetch_durations
132
+ Durations.fetch(
133
+ job: env['TEST_BALANCING_DURATIONS_JOB'],
134
+ path: env.fetch('TEST_BALANCING_DURATIONS_ARTIFACT', Durations::DEFAULT_PATH),
135
+ ref: env.fetch('TEST_BALANCING_DURATIONS_REF', env['CI_DEFAULT_BRANCH']),
136
+ api_url: env['CI_API_V4_URL'],
137
+ project_id: env['CI_PROJECT_ID'],
138
+ job_token: env['CI_JOB_TOKEN'],
139
+ logger: logger
140
+ ).tap { |report| logger.info "Loaded durations for #{report.size} files." if report }
141
+ rescue Durations::FetchError => e
142
+ # Without durations, nodes still balance, just less well.
143
+ logger.warn "#{e.message} Seeding without durations."
144
+ nil
145
+ end
146
+
147
+ def node_index
148
+ env.fetch('CI_NODE_INDEX', 1).to_i
149
+ end
150
+
151
+ def node_total
152
+ env.fetch('CI_NODE_TOTAL', 1).to_i
153
+ end
154
+
155
+ def logger
156
+ @logger ||= Logger.new($stdout, formatter: ->(severity, _time, _progname, msg) {
157
+ "[test balancing] #{severity}: #{msg}\n"
158
+ })
159
+ end
160
+ end
161
+ end
162
+ end
163
+ end
@@ -0,0 +1,108 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'cgi'
4
+ require 'json'
5
+ require 'net/http'
6
+ require 'uri'
7
+
8
+ module Gitlab
9
+ module TestBalancing
10
+ # Per-file test durations, as a JSON object mapping file paths to seconds.
11
+ #
12
+ # The merged report is published as a job artifact and read back from the
13
+ # latest successful pipeline that has it.
14
+ module Durations
15
+ FetchError = Class.new(StandardError)
16
+
17
+ DEFAULT_PATH = 'test-balancing/durations.json'
18
+ ATTEMPTS = 3
19
+ TIMEOUT = 30
20
+
21
+ # Weight of the newest measurement when averaging into the previous report.
22
+ NEW_WEIGHT = 0.5
23
+
24
+ module_function
25
+
26
+ def load(path)
27
+ return {} unless path && File.exist?(path)
28
+
29
+ normalize(JSON.parse(File.read(path)))
30
+ end
31
+
32
+ # Downloads a report from the job artifacts of the latest successful pipeline
33
+ # on `ref` that has `job`. Returns nil when there is none, and raises FetchError
34
+ # when it can't be fetched.
35
+ def fetch(job:, path:, ref:, api_url:, project_id:, job_token:, logger:)
36
+ url = "#{api_url.chomp('/')}/projects/#{project_id}/jobs/artifacts/#{CGI.escape(ref)}/raw/#{path}" \
37
+ "?job=#{CGI.escape(job)}&search_recent_successful_pipelines=true"
38
+
39
+ report = get_json(url, job_token: job_token, logger: logger)
40
+ return normalize(report) if report.is_a?(Hash)
41
+ raise FetchError, "Could not fetch the durations report from job '#{job}' on '#{ref}'." if report.nil?
42
+
43
+ logger.info "No durations report found in job '#{job}' on '#{ref}'."
44
+ nil
45
+ end
46
+
47
+ # Returns the parsed response, :not_found on 404, or nil when the request failed.
48
+ def get_json(url, job_token:, logger:)
49
+ uri = URI.parse(url)
50
+ error = nil
51
+
52
+ ATTEMPTS.times do |attempt|
53
+ sleep(2 * attempt)
54
+ response = get(uri, job_token)
55
+
56
+ return JSON.parse(response.body) if response.is_a?(Net::HTTPSuccess)
57
+ return :not_found if response.is_a?(Net::HTTPNotFound)
58
+
59
+ error = "#{response.code} #{response.body.to_s[0, 200]}"
60
+ rescue StandardError => e
61
+ error = "#{e.class}: #{e.message}"
62
+ end
63
+
64
+ logger.warn "Request to #{uri.path} failed: #{error}"
65
+ nil
66
+ end
67
+
68
+ # Averages new measurements into the previous report. Files that no report
69
+ # measured keep their previous value, or are dropped with `prune`, for
70
+ # example because they were deleted.
71
+ def merge(previous, reports, prune: false)
72
+ measured = Hash.new { |hash, path| hash[path] = [] }
73
+ reports.each { |report| report.each { |path, seconds| measured[path] << seconds.to_f } }
74
+
75
+ merged = previous.merge(measured.transform_values { |values| values.sum / values.size }) do |_path, old, new|
76
+ (old.to_f * (1 - NEW_WEIGHT)) + (new * NEW_WEIGHT)
77
+ end
78
+ merged = merged.slice(*measured.keys) if prune
79
+
80
+ merged.transform_values { |seconds| seconds.round(3) }.sort.to_h
81
+ end
82
+
83
+ def median(values)
84
+ return if values.empty?
85
+
86
+ sorted = values.sort
87
+ mid = sorted.size / 2
88
+ sorted.size.odd? ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2.0
89
+ end
90
+
91
+ # Some Knapsack setups write paths with a leading ./, which the spec file list doesn't have.
92
+ def normalize(report)
93
+ report.transform_keys { |path| path.delete_prefix('./') }
94
+ end
95
+
96
+ def get(uri, job_token)
97
+ request = Net::HTTP::Get.new(uri)
98
+ request['JOB-TOKEN'] = job_token
99
+
100
+ Net::HTTP.start(uri.hostname, uri.port, use_ssl: uri.scheme == 'https', open_timeout: TIMEOUT,
101
+ read_timeout: TIMEOUT) do |http|
102
+ http.request(request)
103
+ end
104
+ end
105
+ private_class_method :normalize, :get
106
+ end
107
+ end
108
+ end
@@ -0,0 +1,62 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Writes the wall-clock time of each spec file to TEST_BALANCING_NODE_REPORT, as a
4
+ # JSON object mapping file paths to seconds. Load it with `--require`.
5
+
6
+ require 'fileutils'
7
+ require 'json'
8
+ require 'rspec/core'
9
+
10
+ module Gitlab
11
+ module TestBalancing
12
+ module Rspec
13
+ # Times each top-level example group, so a file's time includes its
14
+ # before(:all) and let_it_be setup, not only its examples.
15
+ class DurationsFormatter
16
+ ::RSpec::Core::Formatters.register self, :example_group_started, :example_group_finished, :close
17
+
18
+ def initialize(output)
19
+ @output = output
20
+ @durations = Hash.new(0.0)
21
+ @started = {}
22
+ end
23
+
24
+ def example_group_started(notification)
25
+ group = notification.group
26
+ @started[group] = now if group.top_level?
27
+ end
28
+
29
+ def example_group_finished(notification)
30
+ started = @started.delete(notification.group)
31
+ return unless started
32
+
33
+ @durations[notification.group.metadata[:file_path].delete_prefix('./')] += now - started
34
+ end
35
+
36
+ def close(_notification)
37
+ @output.write(JSON.pretty_generate(@durations.transform_values { |seconds| seconds.round(3) }.sort.to_h))
38
+ @output.flush
39
+ end
40
+
41
+ private
42
+
43
+ def now
44
+ Process.clock_gettime(Process::CLOCK_MONOTONIC)
45
+ end
46
+ end
47
+ end
48
+ end
49
+ end
50
+
51
+ report_path = ENV['TEST_BALANCING_NODE_REPORT']
52
+
53
+ if report_path
54
+ RSpec.configure do |config|
55
+ # Added once the suite starts, so it doesn't stop RSpec from setting up its
56
+ # default formatter or replace the formatters the project configured.
57
+ config.before(:suite) do
58
+ FileUtils.mkdir_p(File.dirname(report_path))
59
+ config.add_formatter(Gitlab::TestBalancing::Rspec::DurationsFormatter, report_path)
60
+ end
61
+ end
62
+ end
@@ -26,6 +26,11 @@ module Gitlab
26
26
  # The only deviation from `run_specs` is keeping ONE outer `reporter.report`
27
27
  # across all batches, and `with_suite_hooks` wrapping the whole lane so
28
28
  # before(:suite)/after(:suite) run once.
29
+ #
30
+ # RSpec loads every spec file before running before(:suite) hooks. We load
31
+ # the first batch before them, so hooks registered by helpers that spec files
32
+ # require (e.g. `spec_helper`) still run. Hooks first registered by a later
33
+ # batch never run, and we log a warning for them.
29
34
  class Rspec
30
35
  # status: process exit status (0 on success).
31
36
  # unavailable: true when the feature was unavailable (404)
@@ -104,8 +109,13 @@ module Gitlab
104
109
  # progress/summary formatters treat this as "unknown total".
105
110
  @config.reporter.report(0) do |reporter|
106
111
  @reporter = reporter
112
+ first_example_groups = load_batch(first_batch)
113
+
114
+ # Like Runner#run, skip the suite when loading files failed.
115
+ next false if @world.wants_to_quit
116
+
107
117
  @config.with_suite_hooks do # before(:suite)/after(:suite) run once for the lane
108
- drain_queue(first_batch)
118
+ drain_queue(first_example_groups)
109
119
  end
110
120
  end
111
121
  end
@@ -115,25 +125,25 @@ module Gitlab
115
125
  #
116
126
  # Stops pulling once RSpec sets world.wants_to_quit (fail-fast limit met or
117
127
  # an interrupt), mirroring how Runner#run guards run_specs.
118
- def drain_queue(first_batch)
119
- all_passed = true
120
- batch = first_batch
121
-
122
- loop do
123
- all_passed = false unless run_batch(batch)
124
-
125
- break if @world.wants_to_quit
128
+ def drain_queue(first_example_groups)
129
+ @before_suite_hook_count = before_suite_hook_count
130
+ all_passed = run_example_groups(first_example_groups)
126
131
 
132
+ until @world.wants_to_quit
127
133
  batch = client.request
128
134
  break if batch.empty?
135
+
136
+ example_groups = load_batch(batch)
137
+ warn_about_skipped_suite_hooks(batch)
138
+ all_passed = false unless run_example_groups(example_groups)
129
139
  end
130
140
 
131
141
  all_passed
132
142
  end
133
143
 
134
- # Run a whole pulled batch through the full `run_specs` lifecycle against
135
- # the shared reporter. Returns true if every example in the batch passed.
136
- def run_batch(batch)
144
+ # Load a pulled batch's files into a fresh world, as `Runner#setup` does,
145
+ # and return its example groups in run order.
146
+ def load_batch(batch)
137
147
  paths = batch.map { |split| split[:path] }
138
148
  expected_duration = batch.sum { |split| split[:expected_duration].to_f }
139
149
  logger.info "Pulled #{paths.size} test splits (expected duration: #{expected_duration.round(2)}s) " \
@@ -152,10 +162,14 @@ module Gitlab
152
162
  @config.load_spec_files
153
163
  @world.announce_filters
154
164
 
155
- groups = @world.ordered_example_groups
156
- @world.example_count(groups) # force filtered_examples computation, as run_specs does
165
+ example_groups = @world.ordered_example_groups
166
+ @world.example_count(example_groups) # force filtered_examples computation, as run_specs does
167
+ example_groups
168
+ end
157
169
 
158
- passed = groups.map { |group| group.run(@reporter) }.all?
170
+ # Returns true if every example in the example groups passed.
171
+ def run_example_groups(example_groups)
172
+ passed = example_groups.map { |example_group| example_group.run(@reporter) }.all?
159
173
 
160
174
  # Capture this batch's examples before the next iteration's world.reset
161
175
  @executed_examples.concat(@world.all_examples)
@@ -163,6 +177,21 @@ module Gitlab
163
177
  passed
164
178
  end
165
179
 
180
+ def warn_about_skipped_suite_hooks(batch)
181
+ count = before_suite_hook_count
182
+ return if count <= @before_suite_hook_count
183
+
184
+ @before_suite_hook_count = count
185
+ logger.warn "before(:suite) hooks registered while loading " \
186
+ "#{batch.map { |split| split[:path] }.join(', ')} did not run, because the suite had already started. " \
187
+ "Register suite hooks in a file loaded with `--require`, for example in `.rspec`."
188
+ end
189
+
190
+ # RSpec keeps no public reader for these hooks.
191
+ def before_suite_hook_count
192
+ @config.instance_variable_get(:@before_suite_hooks)&.size.to_i
193
+ end
194
+
166
195
  # Mirrors RSpec::Core::Runner#exit_code: errors raised outside of examples
167
196
  # (e.g. a before(:suite) hook, or "errors occurred outside of examples")
168
197
  # fail the run even when every example passed.
@@ -2,6 +2,6 @@
2
2
 
3
3
  module Gitlab
4
4
  module TestBalancing
5
- VERSION = '0.1.0'
5
+ VERSION = '0.2.0'
6
6
  end
7
7
  end
metadata CHANGED
@@ -1,12 +1,12 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: gitlab-test-balancing
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.0
4
+ version: 0.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - GitLab Engineers
8
8
  autorequire:
9
- bindir: bin
9
+ bindir: exe
10
10
  cert_chain: []
11
11
  date: 2026-11-06 00:00:00.000000000 Z
12
12
  dependencies:
@@ -70,12 +70,20 @@ description: Distributes tests across the nodes of a parallel CI/CD job using th
70
70
  GitLab test balancing API
71
71
  email:
72
72
  - engineering@gitlab.com
73
- executables: []
73
+ executables:
74
+ - gitlab-test-balancing-rspec
75
+ - gitlab-test-balancing-merge-durations
74
76
  extensions: []
75
77
  extra_rdoc_files: []
76
78
  files:
79
+ - exe/gitlab-test-balancing-merge-durations
80
+ - exe/gitlab-test-balancing-rspec
77
81
  - lib/gitlab-test-balancing.rb
82
+ - lib/gitlab/test_balancing/cli/merge_durations.rb
83
+ - lib/gitlab/test_balancing/cli/rspec.rb
78
84
  - lib/gitlab/test_balancing/client.rb
85
+ - lib/gitlab/test_balancing/durations.rb
86
+ - lib/gitlab/test_balancing/rspec/durations_report.rb
79
87
  - lib/gitlab/test_balancing/runner/rspec.rb
80
88
  - lib/gitlab/test_balancing/version.rb
81
89
  homepage: https://gitlab.com/gitlab-org/ruby/gems/gitlab-test-balancing