gitlab-test-balancing 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/exe/gitlab-test-balancing-merge-durations +6 -0
- data/exe/gitlab-test-balancing-rspec +9 -0
- data/lib/gitlab/test_balancing/cli/merge_durations.rb +132 -0
- data/lib/gitlab/test_balancing/cli/rspec.rb +163 -0
- data/lib/gitlab/test_balancing/durations.rb +108 -0
- data/lib/gitlab/test_balancing/rspec/durations_report.rb +62 -0
- data/lib/gitlab/test_balancing/runner/rspec.rb +44 -15
- data/lib/gitlab/test_balancing/version.rb +1 -1
- metadata +11 -3
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 284c90dd11463a6371ec4f4b1d5991c8739b925cd7b7883c69532f06245a9050
|
|
4
|
+
data.tar.gz: 0dc4221f45ec3ebcbdf9e482acf9e9253e8621aabf4106fd69fd22e1aecd3b6a
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 51b7d39b9f6cb86617ab42a991ca4f5775a8ee49593ad65bd4bb40399f5e26d1da97b801e1acb62da1706ed720232e5e28d26e60d01bdc6d38576809d756dd15
|
|
7
|
+
data.tar.gz: a0d78179e9af4c43823ab1826466f3f1273988b8e9a0a946df671424140f8c0f8ed4d61854948c6b494c2a43271b7a0ef117065d8be8f8a4c56c20cfb724c6b8
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'fileutils'
|
|
4
|
+
require 'json'
|
|
5
|
+
require 'logger'
|
|
6
|
+
require 'gitlab/test_balancing/durations'
|
|
7
|
+
|
|
8
|
+
module Gitlab
|
|
9
|
+
module TestBalancing
|
|
10
|
+
module CLI
|
|
11
|
+
# Averages the nodes' duration reports into the previous merged report:
|
|
12
|
+
#
|
|
13
|
+
# gitlab-test-balancing-merge-durations OUTPUT [NODE_REPORT...]
|
|
14
|
+
#
|
|
15
|
+
# With TEST_BALANCING_NODE_REPORT_JOBS, a comma-separated list of job names, it
|
|
16
|
+
# reads each node's TEST_BALANCING_NODE_REPORT from those jobs in this pipeline
|
|
17
|
+
# through the API, instead of from NODE_REPORT files. The previous report
|
|
18
|
+
# is read like the nodes read it, from TEST_BALANCING_DURATIONS_FILE or the
|
|
19
|
+
# TEST_BALANCING_DURATIONS_JOB artifact.
|
|
20
|
+
#
|
|
21
|
+
# Files that no node reported are dropped, as long as every node's job
|
|
22
|
+
# succeeded and its report was read. Without NODE_REPORT files, the previous
|
|
23
|
+
# report is kept as it is. With TEST_BALANCING_NODE_REPORT_JOBS, it fails when
|
|
24
|
+
# it finds no reports, or can't fetch the previous report.
|
|
25
|
+
class MergeDurations
|
|
26
|
+
CONCURRENCY = 8
|
|
27
|
+
|
|
28
|
+
def initialize(argv, env: ENV, logger: nil)
|
|
29
|
+
@output, *@report_paths = argv
|
|
30
|
+
@env = env
|
|
31
|
+
@logger = logger
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def run
|
|
35
|
+
abort 'usage: gitlab-test-balancing-merge-durations OUTPUT [NODE_REPORT...]' unless @output
|
|
36
|
+
|
|
37
|
+
job_mode = env['TEST_BALANCING_NODE_REPORT_JOBS']
|
|
38
|
+
reports, complete = job_mode ? job_reports : file_reports
|
|
39
|
+
|
|
40
|
+
# Otherwise a typo in a job name or the report path would stop updates unnoticed.
|
|
41
|
+
if job_mode && reports.empty?
|
|
42
|
+
logger.error 'No node reports found. Check TEST_BALANCING_NODE_REPORT_JOBS and TEST_BALANCING_NODE_REPORT.'
|
|
43
|
+
return 1
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
merged = Durations.merge(previous, reports, prune: complete && reports.any?)
|
|
47
|
+
|
|
48
|
+
FileUtils.mkdir_p(File.dirname(@output))
|
|
49
|
+
File.write(@output, JSON.pretty_generate(merged))
|
|
50
|
+
logger.info "Merged #{reports.size} node reports into #{@output}: #{merged.size} files, " \
|
|
51
|
+
"#{(merged.values.sum / 60).round(1)} minutes in total."
|
|
52
|
+
|
|
53
|
+
0
|
|
54
|
+
rescue Durations::FetchError => e
|
|
55
|
+
# Publishing only this run's durations would replace the averaged history.
|
|
56
|
+
logger.error "#{e.message} Not publishing a new report."
|
|
57
|
+
1
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
private
|
|
61
|
+
|
|
62
|
+
attr_reader :env
|
|
63
|
+
|
|
64
|
+
# Returns the reports, and whether every report could be read.
|
|
65
|
+
def file_reports
|
|
66
|
+
existing = @report_paths.select { |path| File.exist?(path) }
|
|
67
|
+
|
|
68
|
+
[existing.map { |path| Durations.load(path) }, existing.size == @report_paths.size]
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
# Returns the reports, and whether every job succeeded and its report could be
|
|
72
|
+
# read. A successful job without a report ran no spec files.
|
|
73
|
+
def job_reports
|
|
74
|
+
names = env['TEST_BALANCING_NODE_REPORT_JOBS'].split(',').map(&:strip)
|
|
75
|
+
jobs = pipeline_jobs.select { |job| names.include?(job['name'].sub(%r{ \d+/\d+\z}, '')) }
|
|
76
|
+
logger.info "Reading reports from #{jobs.size} jobs."
|
|
77
|
+
|
|
78
|
+
results = jobs.each_slice(CONCURRENCY).flat_map do |slice|
|
|
79
|
+
slice.map { |job| Thread.new { job_report(job) } }.map(&:value)
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
complete = results.none?(&:nil?) && jobs.all? { |job| job['status'] == 'success' }
|
|
83
|
+
|
|
84
|
+
[results.select { |result| result.is_a?(Hash) }, complete]
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
def job_report(job)
|
|
88
|
+
api_get("/projects/#{env['CI_PROJECT_ID']}/jobs/#{job['id']}/artifacts/#{env['TEST_BALANCING_NODE_REPORT']}")
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
# The jobs API leaves out retried jobs, so each node's latest attempt is listed once.
|
|
92
|
+
def pipeline_jobs
|
|
93
|
+
(1..).each_with_object([]) do |page, jobs|
|
|
94
|
+
batch = api_get("/projects/#{env['CI_PROJECT_ID']}/pipelines/#{env['CI_PIPELINE_ID']}/jobs" \
|
|
95
|
+
"?per_page=100&page=#{page}")
|
|
96
|
+
abort 'Could not list the pipeline jobs.' unless batch.is_a?(Array)
|
|
97
|
+
|
|
98
|
+
jobs.concat(batch)
|
|
99
|
+
break jobs if batch.size < 100
|
|
100
|
+
end
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
def previous
|
|
104
|
+
return Durations.load(env['TEST_BALANCING_DURATIONS_FILE']) if env['TEST_BALANCING_DURATIONS_FILE']
|
|
105
|
+
return {} unless env['TEST_BALANCING_DURATIONS_JOB'] && env['CI_JOB_TOKEN']
|
|
106
|
+
|
|
107
|
+
Durations.fetch(
|
|
108
|
+
job: env['TEST_BALANCING_DURATIONS_JOB'],
|
|
109
|
+
path: env.fetch('TEST_BALANCING_DURATIONS_ARTIFACT', Durations::DEFAULT_PATH),
|
|
110
|
+
ref: env.fetch('TEST_BALANCING_DURATIONS_REF', env['CI_DEFAULT_BRANCH']),
|
|
111
|
+
api_url: env['CI_API_V4_URL'],
|
|
112
|
+
project_id: env['CI_PROJECT_ID'],
|
|
113
|
+
job_token: env['CI_JOB_TOKEN'],
|
|
114
|
+
logger: logger
|
|
115
|
+
) || {}
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
def api_get(path)
|
|
119
|
+
url = "#{env['CI_API_V4_URL'].chomp('/')}#{path}"
|
|
120
|
+
|
|
121
|
+
Durations.get_json(url, job_token: env['CI_JOB_TOKEN'], logger: logger)
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
def logger
|
|
125
|
+
@logger ||= Logger.new($stdout, formatter: ->(severity, _time, _progname, msg) {
|
|
126
|
+
"[test balancing] #{severity}: #{msg}\n"
|
|
127
|
+
})
|
|
128
|
+
end
|
|
129
|
+
end
|
|
130
|
+
end
|
|
131
|
+
end
|
|
132
|
+
end
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'logger'
|
|
4
|
+
require 'gitlab/test_balancing/durations'
|
|
5
|
+
require 'gitlab/test_balancing/runner/rspec'
|
|
6
|
+
|
|
7
|
+
module Gitlab
|
|
8
|
+
module TestBalancing
|
|
9
|
+
module CLI
|
|
10
|
+
# Runs this node's spec files through GitLab test balancing:
|
|
11
|
+
#
|
|
12
|
+
# bundle exec gitlab-test-balancing-rspec [RSPEC_ARGS] [-- SPEC_FILES]
|
|
13
|
+
#
|
|
14
|
+
# Configured with environment variables:
|
|
15
|
+
#
|
|
16
|
+
# - TEST_BALANCING_PATTERN: spec files to run, when SPEC_FILES isn't given.
|
|
17
|
+
# Defaults to spec/**/*_spec.rb.
|
|
18
|
+
# - TEST_BALANCING_EXCLUDE_PATTERN: spec files to leave out.
|
|
19
|
+
# - TEST_BALANCING_DURATIONS_FILE: a local durations report to seed from.
|
|
20
|
+
# - TEST_BALANCING_DURATIONS_JOB: otherwise, the job whose artifact holds the
|
|
21
|
+
# durations report, read from the latest successful pipeline on
|
|
22
|
+
# TEST_BALANCING_DURATIONS_REF (default: the default branch), at
|
|
23
|
+
# TEST_BALANCING_DURATIONS_ARTIFACT (default: test-balancing/durations.json).
|
|
24
|
+
# - TEST_BALANCING_NODE_REPORT: where to write this node's measured durations.
|
|
25
|
+
#
|
|
26
|
+
# Each node seeds its share of the files into the job group's shared queue,
|
|
27
|
+
# then runs the files it claims until the queue is empty. Without test
|
|
28
|
+
# balancing, it runs its share with plain `rspec`.
|
|
29
|
+
class Rspec
|
|
30
|
+
DEFAULT_PATTERN = 'spec/**/*_spec.rb'
|
|
31
|
+
LIB_DIR = File.expand_path('../../..', __dir__)
|
|
32
|
+
|
|
33
|
+
def initialize(argv, env: ENV, logger: nil)
|
|
34
|
+
separator = argv.index('--')
|
|
35
|
+
@rspec_args = separator ? argv[0...separator] : argv
|
|
36
|
+
@given_files = separator ? argv[(separator + 1)..] : nil
|
|
37
|
+
@env = env
|
|
38
|
+
@logger = logger
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
# Returns the exit status. The static path replaces the process instead.
|
|
42
|
+
def run
|
|
43
|
+
# An empty list after -- is deliberate, for example a predictive run with nothing to test.
|
|
44
|
+
if files.empty? && given_files
|
|
45
|
+
logger.info 'No spec files to run.'
|
|
46
|
+
return 0
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
# A pattern that matches nothing is a misconfiguration, which shouldn't pass the job.
|
|
50
|
+
if files.empty?
|
|
51
|
+
logger.error 'No spec files match TEST_BALANCING_PATTERN and TEST_BALANCING_EXCLUDE_PATTERN.'
|
|
52
|
+
return 1
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
if node_files.empty?
|
|
56
|
+
logger.info 'No spec files allocated to this node.'
|
|
57
|
+
return 0
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
return run_static unless parallel?
|
|
61
|
+
|
|
62
|
+
result = Runner::Rspec.new(test_splits: test_splits, rspec_args: rspec_args_with_report, logger: logger).run
|
|
63
|
+
return run_static if result.unavailable?
|
|
64
|
+
|
|
65
|
+
logger.info "Test balancing queue drained. Exiting with status #{result.status}."
|
|
66
|
+
result.status
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
private
|
|
70
|
+
|
|
71
|
+
attr_reader :rspec_args, :given_files, :env
|
|
72
|
+
|
|
73
|
+
def parallel?
|
|
74
|
+
env['CI_JOB_TOKEN'] && env['CI_API_V4_URL'] && node_total > 1
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
def test_splits
|
|
78
|
+
node_files.map { |path| { path: path, expected_duration: expected_durations[path] }.compact }
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
def run_static
|
|
82
|
+
command = ['bundle', 'exec', 'rspec', *rspec_args_with_report, '--', *node_files]
|
|
83
|
+
logger.info "Running #{node_files.size} spec files without test balancing."
|
|
84
|
+
|
|
85
|
+
exec(*command)
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
def rspec_args_with_report
|
|
89
|
+
return rspec_args unless env['TEST_BALANCING_NODE_REPORT']
|
|
90
|
+
|
|
91
|
+
[*rspec_args, '-I', LIB_DIR, '--require', 'gitlab/test_balancing/rspec/durations_report']
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
# This node's share, which it seeds, or runs when test balancing is unavailable.
|
|
95
|
+
# It depends only on the file list, not on the durations report: nodes can
|
|
96
|
+
# fetch different reports, and shares that don't add up would leave files unseeded.
|
|
97
|
+
def node_files
|
|
98
|
+
@node_files ||= files.select.with_index { |_path, i| i % node_total == node_index - 1 }
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
# Sorted, so that every node computes the same split. Without a leading ./,
|
|
102
|
+
# like the durations report's paths.
|
|
103
|
+
def files
|
|
104
|
+
@files ||= (given_files || Dir.glob(env.fetch('TEST_BALANCING_PATTERN', DEFAULT_PATTERN)).reject do |path|
|
|
105
|
+
exclude = env['TEST_BALANCING_EXCLUDE_PATTERN']
|
|
106
|
+
exclude && File.fnmatch?(exclude, path, File::FNM_PATHNAME | File::FNM_EXTGLOB)
|
|
107
|
+
end).map { |path| path.delete_prefix('./') }.sort
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
# Files missing from the report get the median, so they aren't treated as the
|
|
111
|
+
# slowest ones, as they would be with the server's default duration.
|
|
112
|
+
def expected_durations
|
|
113
|
+
@expected_durations ||= begin
|
|
114
|
+
known = files.filter_map { |path| durations[path] }
|
|
115
|
+
default = Durations.median(known)
|
|
116
|
+
|
|
117
|
+
files.to_h { |path| [path, durations.fetch(path, default)] }.compact
|
|
118
|
+
end
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
def durations
|
|
122
|
+
@durations ||= if env['TEST_BALANCING_DURATIONS_FILE']
|
|
123
|
+
Durations.load(env['TEST_BALANCING_DURATIONS_FILE'])
|
|
124
|
+
elsif env['TEST_BALANCING_DURATIONS_JOB'] && env['CI_JOB_TOKEN']
|
|
125
|
+
fetch_durations || {}
|
|
126
|
+
else
|
|
127
|
+
{}
|
|
128
|
+
end
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
def fetch_durations
|
|
132
|
+
Durations.fetch(
|
|
133
|
+
job: env['TEST_BALANCING_DURATIONS_JOB'],
|
|
134
|
+
path: env.fetch('TEST_BALANCING_DURATIONS_ARTIFACT', Durations::DEFAULT_PATH),
|
|
135
|
+
ref: env.fetch('TEST_BALANCING_DURATIONS_REF', env['CI_DEFAULT_BRANCH']),
|
|
136
|
+
api_url: env['CI_API_V4_URL'],
|
|
137
|
+
project_id: env['CI_PROJECT_ID'],
|
|
138
|
+
job_token: env['CI_JOB_TOKEN'],
|
|
139
|
+
logger: logger
|
|
140
|
+
).tap { |report| logger.info "Loaded durations for #{report.size} files." if report }
|
|
141
|
+
rescue Durations::FetchError => e
|
|
142
|
+
# Without durations, nodes still balance, just less well.
|
|
143
|
+
logger.warn "#{e.message} Seeding without durations."
|
|
144
|
+
nil
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
def node_index
|
|
148
|
+
env.fetch('CI_NODE_INDEX', 1).to_i
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
def node_total
|
|
152
|
+
env.fetch('CI_NODE_TOTAL', 1).to_i
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
def logger
|
|
156
|
+
@logger ||= Logger.new($stdout, formatter: ->(severity, _time, _progname, msg) {
|
|
157
|
+
"[test balancing] #{severity}: #{msg}\n"
|
|
158
|
+
})
|
|
159
|
+
end
|
|
160
|
+
end
|
|
161
|
+
end
|
|
162
|
+
end
|
|
163
|
+
end
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'cgi'
|
|
4
|
+
require 'json'
|
|
5
|
+
require 'net/http'
|
|
6
|
+
require 'uri'
|
|
7
|
+
|
|
8
|
+
module Gitlab
|
|
9
|
+
module TestBalancing
|
|
10
|
+
# Per-file test durations, as a JSON object mapping file paths to seconds.
|
|
11
|
+
#
|
|
12
|
+
# The merged report is published as a job artifact and read back from the
|
|
13
|
+
# latest successful pipeline that has it.
|
|
14
|
+
module Durations
|
|
15
|
+
FetchError = Class.new(StandardError)
|
|
16
|
+
|
|
17
|
+
DEFAULT_PATH = 'test-balancing/durations.json'
|
|
18
|
+
ATTEMPTS = 3
|
|
19
|
+
TIMEOUT = 30
|
|
20
|
+
|
|
21
|
+
# Weight of the newest measurement when averaging into the previous report.
|
|
22
|
+
NEW_WEIGHT = 0.5
|
|
23
|
+
|
|
24
|
+
module_function
|
|
25
|
+
|
|
26
|
+
def load(path)
|
|
27
|
+
return {} unless path && File.exist?(path)
|
|
28
|
+
|
|
29
|
+
normalize(JSON.parse(File.read(path)))
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
# Downloads a report from the job artifacts of the latest successful pipeline
|
|
33
|
+
# on `ref` that has `job`. Returns nil when there is none, and raises FetchError
|
|
34
|
+
# when it can't be fetched.
|
|
35
|
+
def fetch(job:, path:, ref:, api_url:, project_id:, job_token:, logger:)
|
|
36
|
+
url = "#{api_url.chomp('/')}/projects/#{project_id}/jobs/artifacts/#{CGI.escape(ref)}/raw/#{path}" \
|
|
37
|
+
"?job=#{CGI.escape(job)}&search_recent_successful_pipelines=true"
|
|
38
|
+
|
|
39
|
+
report = get_json(url, job_token: job_token, logger: logger)
|
|
40
|
+
return normalize(report) if report.is_a?(Hash)
|
|
41
|
+
raise FetchError, "Could not fetch the durations report from job '#{job}' on '#{ref}'." if report.nil?
|
|
42
|
+
|
|
43
|
+
logger.info "No durations report found in job '#{job}' on '#{ref}'."
|
|
44
|
+
nil
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
# Returns the parsed response, :not_found on 404, or nil when the request failed.
|
|
48
|
+
def get_json(url, job_token:, logger:)
|
|
49
|
+
uri = URI.parse(url)
|
|
50
|
+
error = nil
|
|
51
|
+
|
|
52
|
+
ATTEMPTS.times do |attempt|
|
|
53
|
+
sleep(2 * attempt)
|
|
54
|
+
response = get(uri, job_token)
|
|
55
|
+
|
|
56
|
+
return JSON.parse(response.body) if response.is_a?(Net::HTTPSuccess)
|
|
57
|
+
return :not_found if response.is_a?(Net::HTTPNotFound)
|
|
58
|
+
|
|
59
|
+
error = "#{response.code} #{response.body.to_s[0, 200]}"
|
|
60
|
+
rescue StandardError => e
|
|
61
|
+
error = "#{e.class}: #{e.message}"
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
logger.warn "Request to #{uri.path} failed: #{error}"
|
|
65
|
+
nil
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
# Averages new measurements into the previous report. Files that no report
|
|
69
|
+
# measured keep their previous value, or are dropped with `prune`, for
|
|
70
|
+
# example because they were deleted.
|
|
71
|
+
def merge(previous, reports, prune: false)
|
|
72
|
+
measured = Hash.new { |hash, path| hash[path] = [] }
|
|
73
|
+
reports.each { |report| report.each { |path, seconds| measured[path] << seconds.to_f } }
|
|
74
|
+
|
|
75
|
+
merged = previous.merge(measured.transform_values { |values| values.sum / values.size }) do |_path, old, new|
|
|
76
|
+
(old.to_f * (1 - NEW_WEIGHT)) + (new * NEW_WEIGHT)
|
|
77
|
+
end
|
|
78
|
+
merged = merged.slice(*measured.keys) if prune
|
|
79
|
+
|
|
80
|
+
merged.transform_values { |seconds| seconds.round(3) }.sort.to_h
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
def median(values)
|
|
84
|
+
return if values.empty?
|
|
85
|
+
|
|
86
|
+
sorted = values.sort
|
|
87
|
+
mid = sorted.size / 2
|
|
88
|
+
sorted.size.odd? ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2.0
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
# Some Knapsack setups write paths with a leading ./, which the spec file list doesn't have.
|
|
92
|
+
def normalize(report)
|
|
93
|
+
report.transform_keys { |path| path.delete_prefix('./') }
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
def get(uri, job_token)
|
|
97
|
+
request = Net::HTTP::Get.new(uri)
|
|
98
|
+
request['JOB-TOKEN'] = job_token
|
|
99
|
+
|
|
100
|
+
Net::HTTP.start(uri.hostname, uri.port, use_ssl: uri.scheme == 'https', open_timeout: TIMEOUT,
|
|
101
|
+
read_timeout: TIMEOUT) do |http|
|
|
102
|
+
http.request(request)
|
|
103
|
+
end
|
|
104
|
+
end
|
|
105
|
+
private_class_method :normalize, :get
|
|
106
|
+
end
|
|
107
|
+
end
|
|
108
|
+
end
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# Writes the wall-clock time of each spec file to TEST_BALANCING_NODE_REPORT, as a
|
|
4
|
+
# JSON object mapping file paths to seconds. Load it with `--require`.
|
|
5
|
+
|
|
6
|
+
require 'fileutils'
|
|
7
|
+
require 'json'
|
|
8
|
+
require 'rspec/core'
|
|
9
|
+
|
|
10
|
+
module Gitlab
|
|
11
|
+
module TestBalancing
|
|
12
|
+
module Rspec
|
|
13
|
+
# Times each top-level example group, so a file's time includes its
|
|
14
|
+
# before(:all) and let_it_be setup, not only its examples.
|
|
15
|
+
class DurationsFormatter
|
|
16
|
+
::RSpec::Core::Formatters.register self, :example_group_started, :example_group_finished, :close
|
|
17
|
+
|
|
18
|
+
def initialize(output)
|
|
19
|
+
@output = output
|
|
20
|
+
@durations = Hash.new(0.0)
|
|
21
|
+
@started = {}
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def example_group_started(notification)
|
|
25
|
+
group = notification.group
|
|
26
|
+
@started[group] = now if group.top_level?
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def example_group_finished(notification)
|
|
30
|
+
started = @started.delete(notification.group)
|
|
31
|
+
return unless started
|
|
32
|
+
|
|
33
|
+
@durations[notification.group.metadata[:file_path].delete_prefix('./')] += now - started
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def close(_notification)
|
|
37
|
+
@output.write(JSON.pretty_generate(@durations.transform_values { |seconds| seconds.round(3) }.sort.to_h))
|
|
38
|
+
@output.flush
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
private
|
|
42
|
+
|
|
43
|
+
def now
|
|
44
|
+
Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
report_path = ENV['TEST_BALANCING_NODE_REPORT']
|
|
52
|
+
|
|
53
|
+
if report_path
|
|
54
|
+
RSpec.configure do |config|
|
|
55
|
+
# Added once the suite starts, so it doesn't stop RSpec from setting up its
|
|
56
|
+
# default formatter or replace the formatters the project configured.
|
|
57
|
+
config.before(:suite) do
|
|
58
|
+
FileUtils.mkdir_p(File.dirname(report_path))
|
|
59
|
+
config.add_formatter(Gitlab::TestBalancing::Rspec::DurationsFormatter, report_path)
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
end
|
|
@@ -26,6 +26,11 @@ module Gitlab
|
|
|
26
26
|
# The only deviation from `run_specs` is keeping ONE outer `reporter.report`
|
|
27
27
|
# across all batches, and `with_suite_hooks` wrapping the whole lane so
|
|
28
28
|
# before(:suite)/after(:suite) run once.
|
|
29
|
+
#
|
|
30
|
+
# RSpec loads every spec file before running before(:suite) hooks. We load
|
|
31
|
+
# the first batch before them, so hooks registered by helpers that spec files
|
|
32
|
+
# require (e.g. `spec_helper`) still run. Hooks first registered by a later
|
|
33
|
+
# batch never run, and we log a warning for them.
|
|
29
34
|
class Rspec
|
|
30
35
|
# status: process exit status (0 on success).
|
|
31
36
|
# unavailable: true when the feature was unavailable (404)
|
|
@@ -104,8 +109,13 @@ module Gitlab
|
|
|
104
109
|
# progress/summary formatters treat this as "unknown total".
|
|
105
110
|
@config.reporter.report(0) do |reporter|
|
|
106
111
|
@reporter = reporter
|
|
112
|
+
first_example_groups = load_batch(first_batch)
|
|
113
|
+
|
|
114
|
+
# Like Runner#run, skip the suite when loading files failed.
|
|
115
|
+
next false if @world.wants_to_quit
|
|
116
|
+
|
|
107
117
|
@config.with_suite_hooks do # before(:suite)/after(:suite) run once for the lane
|
|
108
|
-
drain_queue(
|
|
118
|
+
drain_queue(first_example_groups)
|
|
109
119
|
end
|
|
110
120
|
end
|
|
111
121
|
end
|
|
@@ -115,25 +125,25 @@ module Gitlab
|
|
|
115
125
|
#
|
|
116
126
|
# Stops pulling once RSpec sets world.wants_to_quit (fail-fast limit met or
|
|
117
127
|
# an interrupt), mirroring how Runner#run guards run_specs.
|
|
118
|
-
def drain_queue(
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
loop do
|
|
123
|
-
all_passed = false unless run_batch(batch)
|
|
124
|
-
|
|
125
|
-
break if @world.wants_to_quit
|
|
128
|
+
def drain_queue(first_example_groups)
|
|
129
|
+
@before_suite_hook_count = before_suite_hook_count
|
|
130
|
+
all_passed = run_example_groups(first_example_groups)
|
|
126
131
|
|
|
132
|
+
until @world.wants_to_quit
|
|
127
133
|
batch = client.request
|
|
128
134
|
break if batch.empty?
|
|
135
|
+
|
|
136
|
+
example_groups = load_batch(batch)
|
|
137
|
+
warn_about_skipped_suite_hooks(batch)
|
|
138
|
+
all_passed = false unless run_example_groups(example_groups)
|
|
129
139
|
end
|
|
130
140
|
|
|
131
141
|
all_passed
|
|
132
142
|
end
|
|
133
143
|
|
|
134
|
-
#
|
|
135
|
-
#
|
|
136
|
-
def
|
|
144
|
+
# Load a pulled batch's files into a fresh world, as `Runner#setup` does,
|
|
145
|
+
# and return its example groups in run order.
|
|
146
|
+
def load_batch(batch)
|
|
137
147
|
paths = batch.map { |split| split[:path] }
|
|
138
148
|
expected_duration = batch.sum { |split| split[:expected_duration].to_f }
|
|
139
149
|
logger.info "Pulled #{paths.size} test splits (expected duration: #{expected_duration.round(2)}s) " \
|
|
@@ -152,10 +162,14 @@ module Gitlab
|
|
|
152
162
|
@config.load_spec_files
|
|
153
163
|
@world.announce_filters
|
|
154
164
|
|
|
155
|
-
|
|
156
|
-
@world.example_count(
|
|
165
|
+
example_groups = @world.ordered_example_groups
|
|
166
|
+
@world.example_count(example_groups) # force filtered_examples computation, as run_specs does
|
|
167
|
+
example_groups
|
|
168
|
+
end
|
|
157
169
|
|
|
158
|
-
|
|
170
|
+
# Returns true if every example in the example groups passed.
|
|
171
|
+
def run_example_groups(example_groups)
|
|
172
|
+
passed = example_groups.map { |example_group| example_group.run(@reporter) }.all?
|
|
159
173
|
|
|
160
174
|
# Capture this batch's examples before the next iteration's world.reset
|
|
161
175
|
@executed_examples.concat(@world.all_examples)
|
|
@@ -163,6 +177,21 @@ module Gitlab
|
|
|
163
177
|
passed
|
|
164
178
|
end
|
|
165
179
|
|
|
180
|
+
def warn_about_skipped_suite_hooks(batch)
|
|
181
|
+
count = before_suite_hook_count
|
|
182
|
+
return if count <= @before_suite_hook_count
|
|
183
|
+
|
|
184
|
+
@before_suite_hook_count = count
|
|
185
|
+
logger.warn "before(:suite) hooks registered while loading " \
|
|
186
|
+
"#{batch.map { |split| split[:path] }.join(', ')} did not run, because the suite had already started. " \
|
|
187
|
+
"Register suite hooks in a file loaded with `--require`, for example in `.rspec`."
|
|
188
|
+
end
|
|
189
|
+
|
|
190
|
+
# RSpec keeps no public reader for these hooks.
|
|
191
|
+
def before_suite_hook_count
|
|
192
|
+
@config.instance_variable_get(:@before_suite_hooks)&.size.to_i
|
|
193
|
+
end
|
|
194
|
+
|
|
166
195
|
# Mirrors RSpec::Core::Runner#exit_code: errors raised outside of examples
|
|
167
196
|
# (e.g. a before(:suite) hook, or "errors occurred outside of examples")
|
|
168
197
|
# fail the run even when every example passed.
|
metadata
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: gitlab-test-balancing
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.2.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- GitLab Engineers
|
|
8
8
|
autorequire:
|
|
9
|
-
bindir:
|
|
9
|
+
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
11
|
date: 2026-11-06 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
@@ -70,12 +70,20 @@ description: Distributes tests across the nodes of a parallel CI/CD job using th
|
|
|
70
70
|
GitLab test balancing API
|
|
71
71
|
email:
|
|
72
72
|
- engineering@gitlab.com
|
|
73
|
-
executables:
|
|
73
|
+
executables:
|
|
74
|
+
- gitlab-test-balancing-rspec
|
|
75
|
+
- gitlab-test-balancing-merge-durations
|
|
74
76
|
extensions: []
|
|
75
77
|
extra_rdoc_files: []
|
|
76
78
|
files:
|
|
79
|
+
- exe/gitlab-test-balancing-merge-durations
|
|
80
|
+
- exe/gitlab-test-balancing-rspec
|
|
77
81
|
- lib/gitlab-test-balancing.rb
|
|
82
|
+
- lib/gitlab/test_balancing/cli/merge_durations.rb
|
|
83
|
+
- lib/gitlab/test_balancing/cli/rspec.rb
|
|
78
84
|
- lib/gitlab/test_balancing/client.rb
|
|
85
|
+
- lib/gitlab/test_balancing/durations.rb
|
|
86
|
+
- lib/gitlab/test_balancing/rspec/durations_report.rb
|
|
79
87
|
- lib/gitlab/test_balancing/runner/rspec.rb
|
|
80
88
|
- lib/gitlab/test_balancing/version.rb
|
|
81
89
|
homepage: https://gitlab.com/gitlab-org/ruby/gems/gitlab-test-balancing
|