duel 0.1.1 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +134 -1
- data/README.md +42 -7
- data/docs/benchmarks.md +2 -1
- data/docs/ci.md +630 -61
- data/docs/configuration.md +91 -4
- data/docs/how-it-works.md +34 -5
- data/docs/json.md +2 -0
- data/docs/plugins.md +2 -2
- data/docs/troubleshooting.md +14 -0
- data/lib/duel/baseline.rb +34 -8
- data/lib/duel/cli.rb +174 -29
- data/lib/duel/config.rb +84 -6
- data/lib/duel/context.rb +6 -5
- data/lib/duel/init.rb +45 -7
- data/lib/duel/json_report.rb +4 -3
- data/lib/duel/lock.rb +52 -8
- data/lib/duel/native.rb +20 -0
- data/lib/duel/resource.rb +37 -2
- data/lib/duel/runner.rb +98 -9
- data/lib/duel/summary.rb +8 -3
- data/lib/duel/templates/duel.gitlab-ci.yml.erb +167 -0
- data/lib/duel/templates/duel.rb.erb +13 -0
- data/lib/duel/templates/perf-comment.yml.erb +59 -0
- data/lib/duel/templates/perf.yml.erb +196 -46
- data/lib/duel/testing.rb +2 -2
- data/lib/duel/version.rb +1 -1
- data/lib/duel/worker.rb +1 -1
- data/sig/duel/baseline.rbs +14 -1
- data/sig/duel/cli.rbs +64 -2
- data/sig/duel/config.rbs +39 -0
- data/sig/duel/init.rbs +16 -0
- data/sig/duel/lock.rbs +11 -0
- data/sig/duel/native.rbs +2 -0
- data/sig/duel/resource.rbs +10 -1
- data/sig/duel/runner.rbs +50 -3
- data/sig/duel/summary.rbs +6 -1
- metadata +3 -1
data/lib/duel/lock.rb
CHANGED
|
@@ -8,7 +8,9 @@ module Duel
|
|
|
8
8
|
# baseline's checkout, each checking out and building over the other's.
|
|
9
9
|
# A run holds an flock on tmp/duel/lock, which ends with its process
|
|
10
10
|
# however that ends, so a run that crashed leaves nothing to clear. The
|
|
11
|
-
# file says whose it is, for a run that finds it taken.
|
|
11
|
+
# file says whose it is, for a run that finds it taken. A run that ends
|
|
12
|
+
# removes it, and the directories that leaves empty: tmp/duel, and tmp
|
|
13
|
+
# if Duel made it.
|
|
12
14
|
# @api private
|
|
13
15
|
module Lock
|
|
14
16
|
# An exclusive lock, or none if another holds it.
|
|
@@ -17,21 +19,63 @@ module Duel
|
|
|
17
19
|
# Runs the block holding +dir+'s lock. Fails, naming the run that
|
|
18
20
|
# holds it, if another does.
|
|
19
21
|
def self.hold(dir)
|
|
22
|
+
made = missing(dir)
|
|
20
23
|
FileUtils.mkdir_p(dir)
|
|
21
|
-
|
|
22
|
-
file =
|
|
24
|
+
path = File.join(dir, "lock")
|
|
25
|
+
file = acquire(path, dir)
|
|
23
26
|
begin
|
|
24
|
-
unless file.flock(EXCLUSIVE)
|
|
25
|
-
# (Empty for a moment, while its run writes its pid.)
|
|
26
|
-
pid = file.read
|
|
27
|
-
raise Error, "another duel run#{" (pid #{pid})" unless pid.empty?} is using #{dir}: wait for it to end, or stop it"
|
|
28
|
-
end
|
|
29
27
|
file.truncate(0)
|
|
30
28
|
file.write(Process.pid)
|
|
31
29
|
file.flush
|
|
32
30
|
yield
|
|
33
31
|
ensure
|
|
32
|
+
# (Removed while it's held, so a run that opens it after makes a new
|
|
33
|
+
# one. `duel --clean` may have removed it already.)
|
|
34
|
+
FileUtils.rm_f(path)
|
|
34
35
|
file.close
|
|
36
|
+
tidy(dir, made)
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# The lock file at +path+, locked: the one there now (a run that
|
|
41
|
+
# ended may have removed the one this opened, after this opened it).
|
|
42
|
+
def self.acquire(path, dir)
|
|
43
|
+
loop do
|
|
44
|
+
# (Read from the start, and written at the end: there's no more.)
|
|
45
|
+
file = File.new(path, "a+")
|
|
46
|
+
unless file.flock(EXCLUSIVE)
|
|
47
|
+
# (Empty for a moment, while its run writes its pid.)
|
|
48
|
+
pid = file.read
|
|
49
|
+
file.close
|
|
50
|
+
raise Error, "another duel run#{" (pid #{pid})" unless pid.empty?} is using #{dir}: wait for it to end, or stop it"
|
|
51
|
+
end
|
|
52
|
+
return file if File.exist?(path) && File.identical?(path, file)
|
|
53
|
+
|
|
54
|
+
file.close
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
# The directories, +dir+ and those it's in, that aren't there yet,
|
|
59
|
+
# deepest first.
|
|
60
|
+
def self.missing(dir)
|
|
61
|
+
made = [] #: Array[String]
|
|
62
|
+
path = dir
|
|
63
|
+
until File.exist?(path)
|
|
64
|
+
made << path
|
|
65
|
+
path = File.dirname(path)
|
|
66
|
+
end
|
|
67
|
+
made
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
# Removes the empty directories in +dir+ (cache/, say: not what's in
|
|
71
|
+
# them, nor in a baseline's checkout there), then +dir+, then those of
|
|
72
|
+
# +made+ (Duel made them), where they're empty. (One that isn't empty
|
|
73
|
+
# holds those it's in.)
|
|
74
|
+
def self.tidy(dir, made)
|
|
75
|
+
[*Dir.glob(File.join(dir, "*/")), dir, *made].each do |path|
|
|
76
|
+
Dir.rmdir(path)
|
|
77
|
+
rescue SystemCallError # (not empty, or gone already)
|
|
78
|
+
nil
|
|
35
79
|
end
|
|
36
80
|
end
|
|
37
81
|
end
|
data/lib/duel/native.rb
CHANGED
|
@@ -157,6 +157,26 @@ module Duel
|
|
|
157
157
|
page_size = Etc.sysconf(Etc::SC_PAGESIZE) #: Integer
|
|
158
158
|
Integer(File.read(statm).split.fetch(1)) * page_size
|
|
159
159
|
end
|
|
160
|
+
|
|
161
|
+
# The memory a benchmark's peak is measured in (see MemorySample),
|
|
162
|
+
# in bytes, or nil where it can't tell: resident memory, but on
|
|
163
|
+
# JRuby, the JVM's heap in use.
|
|
164
|
+
def self.used = resident
|
|
165
|
+
end
|
|
166
|
+
|
|
167
|
+
# simplecov:disable JRuby's heap is the JVM's (JRuby's test run covers
|
|
168
|
+
# this).
|
|
169
|
+
if RUBY_ENGINE == "jruby"
|
|
170
|
+
module Memory
|
|
171
|
+
# The JVM takes memory for its heap ahead of time, and keeps what it
|
|
172
|
+
# used while it started, so resident memory barely moves as a
|
|
173
|
+
# benchmark's objects fill it: 24 MB more of them can show as none.
|
|
174
|
+
# The heap in use shows them.
|
|
175
|
+
HEAP = java.lang.management.ManagementFactory.getMemoryMXBean
|
|
176
|
+
|
|
177
|
+
def self.used = HEAP.getHeapMemoryUsage.getUsed
|
|
178
|
+
end
|
|
160
179
|
end
|
|
180
|
+
# simplecov:enable
|
|
161
181
|
end
|
|
162
182
|
end
|
data/lib/duel/resource.rb
CHANGED
|
@@ -42,7 +42,36 @@ module Duel
|
|
|
42
42
|
# @api private
|
|
43
43
|
CACHE = Mutex.new
|
|
44
44
|
|
|
45
|
-
#
|
|
45
|
+
# What a cache key names, without the version and the hash it ends
|
|
46
|
+
# with: "mysql", of mysql-0.1.2-3f9c2a1b7e5d4c6a. Builds of one name
|
|
47
|
+
# are of one thing, and the one a run used replaces the rest (see
|
|
48
|
+
# .evict).
|
|
49
|
+
# @api private
|
|
50
|
+
NAME = /\A(.+?)(?:-\d[\w.]*)?-\h{8,}\z/
|
|
51
|
+
|
|
52
|
+
# The builds this process's runs used (#cached), by path.
|
|
53
|
+
# @api private
|
|
54
|
+
USED = Set.new
|
|
55
|
+
|
|
56
|
+
# Removes from +cache+ the builds that ones this process used replace:
|
|
57
|
+
# of the same name (see NAME), with another version or hash. A server
|
|
58
|
+
# or a plugin upgraded, or a lockfile changed, left its old build
|
|
59
|
+
# behind otherwise: tens of megabytes of a database's files, each
|
|
60
|
+
# time. A key that doesn't end with a hash is never removed, nor is
|
|
61
|
+
# any that this process used (each version's bundle, say). (Once a
|
|
62
|
+
# run's workers have started: none adds to USED then.)
|
|
63
|
+
# @api private
|
|
64
|
+
def self.evict(cache)
|
|
65
|
+
used = USED.select { |dir| File.dirname(dir) == cache }.map { |dir| File.basename(dir) }
|
|
66
|
+
names = used.filter_map { |key| key[NAME, 1] }
|
|
67
|
+
Dir.children(cache).each do |build|
|
|
68
|
+
FileUtils.rm_rf(File.join(cache, build)) if !used.include?(build) && names.include?(build[NAME, 1])
|
|
69
|
+
end
|
|
70
|
+
rescue Errno::ENOENT # (no cache)
|
|
71
|
+
nil
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
# The worker it's for ("base-1", say), or the version ("base",
|
|
46
75
|
# for Config#prepare(:version)), the directory it has of its own, the
|
|
47
76
|
# tree of the version that worker runs, and the Ruby it runs on (see
|
|
48
77
|
# Config#ruby).
|
|
@@ -88,11 +117,17 @@ module Duel
|
|
|
88
117
|
# block is given, and the rest get it built. A key names what's built
|
|
89
118
|
# (put a version in it), as a file name: tmp/duel/cache/+key+, which
|
|
90
119
|
# `duel --clean` removes. A build that fails leaves nothing behind.
|
|
120
|
+
# A key that ends with a hash (and a version before it, if you like:
|
|
121
|
+
# mysql-0.1.2-3f9c2a1b) replaces the builds of its name when the run
|
|
122
|
+
# ends (see .evict): the one a run used is the one that's kept.
|
|
91
123
|
def cached(key)
|
|
92
124
|
raise ArgumentError, "a cache key is a file name (letters, digits, - _ .), not #{key.inspect}" unless key.match?(KEY)
|
|
93
125
|
|
|
94
126
|
dir = File.join(@cache, key)
|
|
95
|
-
CACHE.synchronize
|
|
127
|
+
CACHE.synchronize do
|
|
128
|
+
USED << dir
|
|
129
|
+
build(dir) { |building| yield building } unless File.directory?(dir)
|
|
130
|
+
end
|
|
96
131
|
dir
|
|
97
132
|
end
|
|
98
133
|
|
data/lib/duel/runner.rb
CHANGED
|
@@ -27,13 +27,17 @@ module Duel
|
|
|
27
27
|
# @api private
|
|
28
28
|
YJIT_WARM = 2.0
|
|
29
29
|
|
|
30
|
+
# The seconds a run measures for, at most, by default (see
|
|
31
|
+
# Settings#time_limit and Runner#out_of_time?).
|
|
32
|
+
TIME_LIMIT = 1800.0
|
|
33
|
+
|
|
30
34
|
# How a run is set up (see Duel::CLI for where each comes from).
|
|
31
35
|
# @api private
|
|
32
36
|
Settings = Struct.new(:base, :base_dir, :candidate_dir, :filter, :jobs, :processes, :limit, :confidence, :warm, :force,
|
|
33
|
-
:config_path, :timeout, :ruby, :comment) do
|
|
37
|
+
:config_path, :timeout, :ruby, :comment, :fail_on, :pushed, :target, :time_limit) do
|
|
34
38
|
# @type self: singleton(Settings)
|
|
35
39
|
def self.defaults = new(filter: //, jobs: 1, processes: 3, limit: 1.02, confidence: 0.99, warm: nil, force: false,
|
|
36
|
-
timeout: 60.0)
|
|
40
|
+
timeout: 60.0, time_limit: TIME_LIMIT)
|
|
37
41
|
end
|
|
38
42
|
|
|
39
43
|
# One version as a run has it (see Runner#prepare), +name+d "baseline"
|
|
@@ -51,6 +55,21 @@ module Duel
|
|
|
51
55
|
end
|
|
52
56
|
end
|
|
53
57
|
|
|
58
|
+
class Side
|
|
59
|
+
# What each version's workers, their directories and their sockets
|
|
60
|
+
# are named for (see #label): "base" and "head", as long as each
|
|
61
|
+
# other, so that the two versions' workers get arguments and
|
|
62
|
+
# environments of the same length. Where a process's stack starts
|
|
63
|
+
# depends on how long they are, and at some lengths, the same code
|
|
64
|
+
# runs the same instructions in more cycles: a version whose workers'
|
|
65
|
+
# were a byte longer could measure slower in every one.
|
|
66
|
+
LABELS = {"baseline" => "base", "candidate" => "head"}.freeze
|
|
67
|
+
|
|
68
|
+
# What its workers, and its version's resources, are named for: "base"
|
|
69
|
+
# or "head".
|
|
70
|
+
def label = LABELS.fetch(name)
|
|
71
|
+
end
|
|
72
|
+
|
|
54
73
|
# A worker process (Duel::Worker) running one version of the code.
|
|
55
74
|
# @api private
|
|
56
75
|
class WorkerProcess
|
|
@@ -324,12 +343,12 @@ module Duel
|
|
|
324
343
|
# A worker of +side+, with its own resources, and its version's Ruby
|
|
325
344
|
# and environment.
|
|
326
345
|
def spawn(side, index)
|
|
327
|
-
label = "#{side}-#{index + 1}"
|
|
328
346
|
version = @sides.fetch(side)
|
|
347
|
+
label = "#{version.label}-#{index + 1}"
|
|
329
348
|
env = version.env
|
|
330
349
|
@config.prepares.fetch(:worker).each_with_index do |block, i|
|
|
331
350
|
resource = Resource.new(File.join(@dir, "#{label}-#{i + 1}"), label, version.root, timeout: @settings.timeout,
|
|
332
|
-
cache: File.join(@config.
|
|
351
|
+
cache: File.join(@config.state, "cache"), ruby: version.ruby, env: version.env)
|
|
333
352
|
keep(@resources, resource)
|
|
334
353
|
env = Resource.environment(label, env, block.call(resource))
|
|
335
354
|
end
|
|
@@ -851,23 +870,41 @@ module Duel
|
|
|
851
870
|
# seconds the run took. Set by #run.
|
|
852
871
|
attr_reader :rubies, :seconds
|
|
853
872
|
|
|
873
|
+
# What the run left undone when it reached its time limit
|
|
874
|
+
# (Settings#time_limit): {"unmeasured" => the benchmarks it didn't
|
|
875
|
+
# measure, "unconfirmed" => those it judged without every
|
|
876
|
+
# measurement again (see #confirm), "across" => whether it judged
|
|
877
|
+
# across the board without measuring every benchmark again (see
|
|
878
|
+
# #judge_across)}, or nil where it reached none. Set by #run.
|
|
879
|
+
def cut_short
|
|
880
|
+
return if @unmeasured.empty? && @unconfirmed.empty? && !@across_cut
|
|
881
|
+
|
|
882
|
+
{"unmeasured" => @unmeasured, "unconfirmed" => @unconfirmed, "across" => @across_cut}
|
|
883
|
+
end
|
|
884
|
+
|
|
854
885
|
def initialize(config, settings, base_root, out: $stdout)
|
|
855
886
|
@config = config
|
|
856
887
|
@settings = settings
|
|
857
888
|
@base_root = base_root
|
|
858
889
|
@out = out
|
|
859
890
|
@prepared = []
|
|
891
|
+
@started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
892
|
+
@lock = Mutex.new
|
|
893
|
+
@unmeasured = [] #: Array[String]
|
|
894
|
+
@unconfirmed = [] #: Array[String]
|
|
895
|
+
@across_cut = false
|
|
860
896
|
end
|
|
861
897
|
|
|
862
898
|
# Returns the Results.
|
|
863
899
|
def run
|
|
864
|
-
started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
900
|
+
started = @started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
865
901
|
@dir = Lane.tmpdir
|
|
866
902
|
begin
|
|
867
903
|
compare_all(started)
|
|
868
904
|
ensure
|
|
869
905
|
@prepared.reverse_each(&:stop)
|
|
870
906
|
FileUtils.rm_rf(@dir)
|
|
907
|
+
Resource.evict(File.join(@config.state, "cache"))
|
|
871
908
|
end
|
|
872
909
|
end
|
|
873
910
|
|
|
@@ -877,6 +914,27 @@ module Duel
|
|
|
877
914
|
|
|
878
915
|
def regressed_across = @across.select { |_, v| v["confirmed"] }.keys
|
|
879
916
|
|
|
917
|
+
# What a run cut short at its time limit left undone (see #cut_short),
|
|
918
|
+
# or nil where it wasn't.
|
|
919
|
+
def cut_short_note
|
|
920
|
+
return unless cut_short
|
|
921
|
+
|
|
922
|
+
undone = [("#{Runner.count(@unmeasured.size, "benchmark")} not measured" unless @unmeasured.empty?),
|
|
923
|
+
("#{Runner.count(@unconfirmed.size, "benchmark")} judged without measuring every one again" unless @unconfirmed.empty?),
|
|
924
|
+
("across the board judged without measuring again" if @across_cut)].compact
|
|
925
|
+
"the run measured for longer than #{limit_text}: #{undone.join(", ")}. Raise --time-limit, or make the slowest " \
|
|
926
|
+
"benchmarks quicker to settle"
|
|
927
|
+
end
|
|
928
|
+
|
|
929
|
+
# Whether the run has measured for longer than its time limit
|
|
930
|
+
# (Settings#time_limit). Then it starts nothing more: a benchmark it
|
|
931
|
+
# hasn't measured is an error, unmeasured, and what looked worse is
|
|
932
|
+
# judged on the measurements it has, without measuring it again. It
|
|
933
|
+
# fails what still looks worse in all of them, so a run cut short
|
|
934
|
+
# never passes a regression it would have failed. Each measurement it
|
|
935
|
+
# started runs to its end.
|
|
936
|
+
def out_of_time? = Process.clock_gettime(Process::CLOCK_MONOTONIC) - @started > @settings.time_limit
|
|
937
|
+
|
|
880
938
|
# The plugins the configuration used (Config#plugins), by name.
|
|
881
939
|
def plugins = @config.plugins
|
|
882
940
|
|
|
@@ -918,10 +976,10 @@ module Duel
|
|
|
918
976
|
side.ruby = Rubies.find(choice, side.root, side.name)
|
|
919
977
|
side.env = Rubies.environment(side.ruby)
|
|
920
978
|
@config.prepares.fetch(:version).each_with_index do |block, i|
|
|
921
|
-
resource = Resource.new(File.join(@dir, "#{side.
|
|
922
|
-
cache: File.join(@config.
|
|
979
|
+
resource = Resource.new(File.join(@dir, "#{side.label}-#{i + 1}"), side.label, side.root, timeout: @settings.timeout,
|
|
980
|
+
cache: File.join(@config.state, "cache"), ruby: side.ruby, env: side.env)
|
|
923
981
|
@prepared << resource
|
|
924
|
-
side.env = Resource.environment(side.
|
|
982
|
+
side.env = Resource.environment(side.label, side.env, block.call(resource))
|
|
925
983
|
end
|
|
926
984
|
side
|
|
927
985
|
end
|
|
@@ -944,6 +1002,14 @@ module Duel
|
|
|
944
1002
|
slower = verdicts.select { |_, v| v.fetch("verdict") == "slower" }.keys
|
|
945
1003
|
return verdicts if slower.empty?
|
|
946
1004
|
|
|
1005
|
+
if out_of_time?
|
|
1006
|
+
# (Judged on the one measurement: slower in it, so slower.)
|
|
1007
|
+
@across_cut = true
|
|
1008
|
+
@out.puts "\nSlower across the board in #{slower.join(", ")}, not measured again: the run reached its time limit (#{limit_text})"
|
|
1009
|
+
slower.each { |metric| verdicts.fetch(metric)["confirmed"] = true }
|
|
1010
|
+
return verdicts
|
|
1011
|
+
end
|
|
1012
|
+
|
|
947
1013
|
@out.puts "\nMeasuring every benchmark again, in fresh processes: slower across the board in #{slower.join(", ")}?"
|
|
948
1014
|
again = Across.new(with_lanes { |lanes| measure_all(lanes, names) }, confidence: @settings.confidence).verdicts
|
|
949
1015
|
slower.each { |metric| verdicts.fetch(metric)["confirmed"] = again.dig(metric, "verdict") == "slower" }
|
|
@@ -960,6 +1026,11 @@ module Duel
|
|
|
960
1026
|
lanes.each_index.map do |i|
|
|
961
1027
|
Thread.new do
|
|
962
1028
|
while (name, index = queue.deq)
|
|
1029
|
+
if out_of_time?
|
|
1030
|
+
results[index] = unmeasured(name)
|
|
1031
|
+
next
|
|
1032
|
+
end
|
|
1033
|
+
|
|
963
1034
|
result = results[index] = attempt(lanes.fetch(i), name)
|
|
964
1035
|
@out.print "#{line(result)}\n" # (in one write: other threads write too)
|
|
965
1036
|
next unless result.error
|
|
@@ -979,6 +1050,16 @@ module Duel
|
|
|
979
1050
|
Result.new(name).tap { |r| r.error = "#{e.class}: #{e}" }
|
|
980
1051
|
end
|
|
981
1052
|
|
|
1053
|
+
# A benchmark the run reached its time limit before it measured: an
|
|
1054
|
+
# error, so the run fails. (Each lane's thread adds to the list.)
|
|
1055
|
+
# mutant:disable -- CRuby's Array#<< is atomic anyway, and JRuby's isn't
|
|
1056
|
+
def unmeasured(name)
|
|
1057
|
+
@lock.synchronize { @unmeasured << name }
|
|
1058
|
+
Result.new(name).tap { |r| r.error = "not measured: the run reached its time limit (#{limit_text})" }
|
|
1059
|
+
end
|
|
1060
|
+
|
|
1061
|
+
def limit_text = "#{@settings.time_limit.round}s, --time-limit"
|
|
1062
|
+
|
|
982
1063
|
# Measures each suspect again, from scratch, in fresh processes, until
|
|
983
1064
|
# one measurement clears it or CONFIRM agree (and once more for each
|
|
984
1065
|
# too close to call, up to CONFIRM more: see Result#confirm).
|
|
@@ -989,6 +1070,12 @@ module Duel
|
|
|
989
1070
|
"#{CONFIRM} times each (#{CONFIRM + 1} if what's slower differs, and once more for each too close to call):"
|
|
990
1071
|
suspects.each do |result|
|
|
991
1072
|
while result.confirming?
|
|
1073
|
+
if out_of_time?
|
|
1074
|
+
@unconfirmed << result.name
|
|
1075
|
+
@out.puts format(" %-48s judged on %s: the run reached its time limit (%s)", result.name,
|
|
1076
|
+
Runner.count(result.confirmations.size + 1, "measurement"), limit_text)
|
|
1077
|
+
break
|
|
1078
|
+
end
|
|
992
1079
|
again = Lane.open(@config, @settings, @sides) { |lane| attempt(lane, result.name) }
|
|
993
1080
|
@out.puts line(again) + (result.confirm(again) ? " (too close to call: once more)" : "")
|
|
994
1081
|
end
|
|
@@ -1019,8 +1106,10 @@ module Duel
|
|
|
1019
1106
|
@out.puts format("across the board: %s %.3f [%.3f, %.3f] %s (limit %.4f, %d benchmarks)", metric, v.fetch("median"),
|
|
1020
1107
|
low, high, v.fetch("verdict"), v.fetch("limit"), v.fetch("benchmarks"))
|
|
1021
1108
|
end
|
|
1022
|
-
|
|
1109
|
+
both = @across_cut ? "slower, not measured again (the run reached its time limit)" : "slower in both measurements"
|
|
1110
|
+
regressed_across.each { |metric| @out.puts "REGRESSION across the board: #{metric} #{both}" }
|
|
1023
1111
|
errors.each { |r| @out.puts "ERROR #{r.name}: #{r.error}" }
|
|
1112
|
+
cut_short_note&.then { |note| @out.puts "TIME LIMIT: #{note}" }
|
|
1024
1113
|
regressed.each do |r|
|
|
1025
1114
|
@out.puts "REGRESSION #{r.name}: " +
|
|
1026
1115
|
(r.mixed? ? "slower in every measurement, by one of #{r.regressed.join(", ")}" : "worse #{r.regressed.join(", ")} in every measurement")
|
data/lib/duel/summary.rb
CHANGED
|
@@ -90,8 +90,10 @@ module Duel
|
|
|
90
90
|
|
|
91
91
|
# +runner+ ran, and made +results+, comparing +baseline+ with
|
|
92
92
|
# +candidate+ (each a description, in Markdown, whose commit stands
|
|
93
|
-
# bare, so GitHub links it) as +settings+ say
|
|
94
|
-
|
|
93
|
+
# bare, so GitHub links it) as +settings+ say, with a +note+ (in
|
|
94
|
+
# Markdown) under the heading, if there's one.
|
|
95
|
+
def initialize(runner, results, settings:, baseline:, candidate:, note: nil)
|
|
96
|
+
@note = note
|
|
95
97
|
@runner = runner
|
|
96
98
|
@results = results
|
|
97
99
|
@settings = settings
|
|
@@ -114,7 +116,7 @@ module Duel
|
|
|
114
116
|
|
|
115
117
|
# The Markdown, with the first +explained+ regressions explained.
|
|
116
118
|
def render(explained)
|
|
117
|
-
[heading, compared, failures, across(confirmed: true), details(explained)].compact.join("\n\n") + "\n"
|
|
119
|
+
[heading, @note, compared, cut_short, failures, across(confirmed: true), details(explained)].compact.join("\n\n") + "\n"
|
|
118
120
|
end
|
|
119
121
|
|
|
120
122
|
# The verdict: what failed (benchmarks, or a metric across the board),
|
|
@@ -141,6 +143,9 @@ module Duel
|
|
|
141
143
|
"Compared #{@candidate} with #{@baseline}.#{why}"
|
|
142
144
|
end
|
|
143
145
|
|
|
146
|
+
# What the run left undone at its time limit, if it reached it.
|
|
147
|
+
def cut_short = @runner.cut_short_note&.then { |note| "Time limit: #{note}." }
|
|
148
|
+
|
|
144
149
|
# The benchmarks that failed: what got worse and by how much, or the
|
|
145
150
|
# error, and where the most extra CPU went, if a profile says.
|
|
146
151
|
def failures
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
# Duel on GitLab CI: https://github.com/sferik/duel-ruby/blob/main/docs/ci.md#run-duel-on-gitlab-ci
|
|
2
|
+
#
|
|
3
|
+
# Include it from .gitlab-ci.yml:
|
|
4
|
+
#
|
|
5
|
+
# include:
|
|
6
|
+
# - local: <%= gitlab_file %>
|
|
7
|
+
#
|
|
8
|
+
# It compares a merge request with the branch it targets, a push to the
|
|
9
|
+
# default branch with the commit before it (or as c.push_base in .duel.rb
|
|
10
|
+
# says, the last commit Duel passed), and on a schedule, the last release.
|
|
11
|
+
# The Duel job's log, and its duel.md artifact, have the report. Comments
|
|
12
|
+
# need project access tokens, in masked CI/CD variables:
|
|
13
|
+
#
|
|
14
|
+
# - DUEL_COMMENT_TOKEN (the Reporter role, with the api scope, and not
|
|
15
|
+
# protected), to comment on a merge request. Every job of a merge
|
|
16
|
+
# request's pipeline has it, the Duel job too, which runs the merge
|
|
17
|
+
# request's code, so the token can do no more than comment.
|
|
18
|
+
# - DUEL_TOKEN (the Developer role, with the api scope, and protected, so
|
|
19
|
+
# only the default branch's pipelines have it), to comment on a pushed
|
|
20
|
+
# commit (see c.fail_on), and to mark a push passed (see c.push_base).
|
|
21
|
+
|
|
22
|
+
<%= gitlab_job %>:
|
|
23
|
+
image: ruby:<%= gitlab_ruby %>
|
|
24
|
+
stage: test
|
|
25
|
+
variables:
|
|
26
|
+
GIT_DEPTH: 0 # the baseline comes from history, tags included
|
|
27
|
+
DUEL_OUTPUT: duel.env # whether to comment, and mark a push passed
|
|
28
|
+
rules:
|
|
29
|
+
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
|
|
30
|
+
- if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
|
|
31
|
+
variables:
|
|
32
|
+
DUEL_BASE: $CI_COMMIT_BEFORE_SHA
|
|
33
|
+
- if: $CI_PIPELINE_SOURCE == "schedule"
|
|
34
|
+
variables:
|
|
35
|
+
DUEL_BASE: last-release
|
|
36
|
+
# (Run by hand, it compares anyway: with nothing changed, the code
|
|
37
|
+
# with itself, which checks the setup.)
|
|
38
|
+
- if: $CI_PIPELINE_SOURCE == "web"
|
|
39
|
+
variables:
|
|
40
|
+
DUEL_FORCE: "1"
|
|
41
|
+
# A regression outside a merge request, where only a merge request
|
|
42
|
+
# fails (c.fail_on in .duel.rb), exits 3: a warning.
|
|
43
|
+
allow_failure:
|
|
44
|
+
exit_codes: [3]
|
|
45
|
+
before_script:
|
|
46
|
+
<% if working_directory -%>
|
|
47
|
+
- cd <%= working_directory %>
|
|
48
|
+
<% end -%>
|
|
49
|
+
# The branch a merge request targets, the last commits Duel passed,
|
|
50
|
+
# and the releases.
|
|
51
|
+
- git fetch --quiet --tags origin "+refs/heads/duel/*:refs/remotes/origin/duel/*"
|
|
52
|
+
- if [ -n "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" ]; then git fetch --quiet origin "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME"; fi
|
|
53
|
+
- bundle install
|
|
54
|
+
script:
|
|
55
|
+
- bundle exec duel --markdown duel.md
|
|
56
|
+
artifacts:
|
|
57
|
+
when: always
|
|
58
|
+
paths:
|
|
59
|
+
- <%= working_directory ? "#{working_directory}/" : "" %>duel.md
|
|
60
|
+
reports:
|
|
61
|
+
dotenv: <%= working_directory ? "#{working_directory}/" : "" %>duel.env
|
|
62
|
+
expire_in: 1 day
|
|
63
|
+
|
|
64
|
+
# The report, as a new comment on the merge request, when Duel says to:
|
|
65
|
+
# always (post), or only when its last comment's status differs from this
|
|
66
|
+
# run's (change), so the latest comment is never stale. It never edits or
|
|
67
|
+
# deletes a comment.
|
|
68
|
+
<%= gitlab_job %>:comment:
|
|
69
|
+
image: alpine:3
|
|
70
|
+
stage: .post
|
|
71
|
+
needs: [<%= gitlab_job %>]
|
|
72
|
+
rules:
|
|
73
|
+
- if: $DUEL_COMMENT_TOKEN && $CI_PIPELINE_SOURCE == "merge_request_event"
|
|
74
|
+
when: always
|
|
75
|
+
variables:
|
|
76
|
+
GIT_STRATEGY: none # it runs none of the code
|
|
77
|
+
script:
|
|
78
|
+
- apk add --no-cache --quiet curl jq
|
|
79
|
+
- |
|
|
80
|
+
case "$DUEL_COMMENT" in post|change) ;; *) exit 0 ;; esac
|
|
81
|
+
report=<%= working_directory ? "#{working_directory}/" : "" %>duel.md
|
|
82
|
+
[ -f "$report" ] || exit 0
|
|
83
|
+
api="$CI_API_V4_URL/projects/$CI_PROJECT_ID"
|
|
84
|
+
token="PRIVATE-TOKEN: $DUEL_COMMENT_TOKEN"
|
|
85
|
+
marker="<!-- <%= marker %>"
|
|
86
|
+
me=$(curl -sf -H "$token" "$CI_API_V4_URL/user" | jq .id)
|
|
87
|
+
# The first line of the last comment Duel posted here: its marker,
|
|
88
|
+
# with its status.
|
|
89
|
+
last=$(curl -sf -H "$token" "$api/merge_requests/$CI_MERGE_REQUEST_IID/notes?sort=desc&order_by=created_at&per_page=100" |
|
|
90
|
+
jq -r --argjson me "$me" --arg marker "$marker status=" '.[] | select(.author.id == $me) | .body | split("\n")[0] | rtrimstr("\r") | select(startswith($marker))' | head -n 1)
|
|
91
|
+
if [ "$DUEL_COMMENT" = post ] || { [ -n "$last" ] && [ "$last" != "$marker status=$DUEL_STATUS -->" ]; }; then
|
|
92
|
+
{ echo "$marker status=$DUEL_STATUS -->"; cat "$report"; } | jq -Rs '{body: .}' |
|
|
93
|
+
curl -sf -H "$token" -H "Content-Type: application/json" --data @- "$api/merge_requests/$CI_MERGE_REQUEST_IID/notes" > /dev/null
|
|
94
|
+
fi
|
|
95
|
+
|
|
96
|
+
# The report, as a new comment on a pushed commit, when Duel says to, as
|
|
97
|
+
# for a merge request: only where a regression in a push passes (see
|
|
98
|
+
# c.fail_on), so the comment says what the check doesn't. Its last comment
|
|
99
|
+
# is the commit before's.
|
|
100
|
+
<%= gitlab_job %>:commit-comment:
|
|
101
|
+
image: alpine:3
|
|
102
|
+
stage: .post
|
|
103
|
+
needs: [<%= gitlab_job %>]
|
|
104
|
+
rules:
|
|
105
|
+
- if: $DUEL_TOKEN && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
|
|
106
|
+
when: always
|
|
107
|
+
variables:
|
|
108
|
+
GIT_STRATEGY: none # it runs none of the code
|
|
109
|
+
script:
|
|
110
|
+
- apk add --no-cache --quiet curl jq
|
|
111
|
+
- |
|
|
112
|
+
case "$DUEL_COMMENT" in post|change) ;; *) exit 0 ;; esac
|
|
113
|
+
report=<%= working_directory ? "#{working_directory}/" : "" %>duel.md
|
|
114
|
+
[ -f "$report" ] || exit 0
|
|
115
|
+
api="$CI_API_V4_URL/projects/$CI_PROJECT_ID"
|
|
116
|
+
token="PRIVATE-TOKEN: $DUEL_TOKEN"
|
|
117
|
+
marker="<!-- <%= marker %>"
|
|
118
|
+
me=$(curl -sf -H "$token" "$CI_API_V4_URL/user" | jq .id)
|
|
119
|
+
last=$(curl -sf -H "$token" "$api/repository/commits/$CI_COMMIT_BEFORE_SHA/comments?per_page=100" |
|
|
120
|
+
jq -r --argjson me "$me" --arg marker "$marker status=" '.[] | select(.author.id == $me) | .note | split("\n")[0] | rtrimstr("\r") | select(startswith($marker))' | tail -n 1)
|
|
121
|
+
if [ "$DUEL_COMMENT" = post ] || { [ -n "$last" ] && [ "$last" != "$marker status=$DUEL_STATUS -->" ]; }; then
|
|
122
|
+
{ echo "$marker status=$DUEL_STATUS -->"; cat "$report"; } | jq -Rs '{note: .}' |
|
|
123
|
+
curl -sf -H "$token" -H "Content-Type: application/json" --data @- "$api/repository/commits/$CI_COMMIT_SHA/comments" > /dev/null
|
|
124
|
+
fi
|
|
125
|
+
|
|
126
|
+
# With c.push_base = :last_passed in .duel.rb, a push is compared with the
|
|
127
|
+
# last commit Duel passed, which a branch marks (<%= passed_branch %>/main for
|
|
128
|
+
# main), so slowdowns too small for any one push to fail add up until they
|
|
129
|
+
# fail one. This job moves the branch to a push that passed (or makes it),
|
|
130
|
+
# but never back: where a newer push's job moved it first, it stays. To
|
|
131
|
+
# accept a regression, move the branch yourself (for main, git push
|
|
132
|
+
# origin main:<%= passed_branch %>/main).
|
|
133
|
+
<%= gitlab_job %>:passed:
|
|
134
|
+
image: alpine:3
|
|
135
|
+
stage: .post
|
|
136
|
+
needs: [<%= gitlab_job %>]
|
|
137
|
+
rules:
|
|
138
|
+
- if: $DUEL_TOKEN && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
|
|
139
|
+
variables:
|
|
140
|
+
GIT_STRATEGY: none # it runs none of the code
|
|
141
|
+
script:
|
|
142
|
+
- apk add --no-cache --quiet curl jq
|
|
143
|
+
- |
|
|
144
|
+
[ "$DUEL_PASSED" = true ] || exit 0
|
|
145
|
+
api="$CI_API_V4_URL/projects/$CI_PROJECT_ID"
|
|
146
|
+
token="PRIVATE-TOKEN: $DUEL_TOKEN"
|
|
147
|
+
branch="<%= passed_branch %>/$CI_COMMIT_BRANCH"
|
|
148
|
+
at=$(curl -sf -H "$token" "$api/repository/branches/$(echo "$branch" | sed 's|/|%2F|g')" | jq -r '.commit.id // empty') || at=
|
|
149
|
+
if [ -n "$at" ]; then
|
|
150
|
+
base=$(curl -sf -H "$token" "$api/repository/merge_base?refs%5B%5D=$CI_COMMIT_SHA&refs%5B%5D=$at" | jq -r '.id // empty') || base=
|
|
151
|
+
[ -n "$base" ] || { echo "Couldn't compare $branch with this push" && exit 1; }
|
|
152
|
+
# (A newer push's job moved it, or this one's did, run again.)
|
|
153
|
+
[ "$base" = "$CI_COMMIT_SHA" ] && exit 0
|
|
154
|
+
fi
|
|
155
|
+
curl -s -o /dev/null -H "$token" -X DELETE "$api/repository/branches/$(echo "$branch" | sed 's|/|%2F|g')"
|
|
156
|
+
curl -sf -o /dev/null -H "$token" --data-urlencode "branch=$branch" --data-urlencode "ref=$CI_COMMIT_SHA" "$api/repository/branches" && exit 0
|
|
157
|
+
# (Git can't make a branch where another's name is a folder of its
|
|
158
|
+
# name: a branch named duel blocks duel/passed/main. If one does,
|
|
159
|
+
# this names it.)
|
|
160
|
+
prefix=$branch
|
|
161
|
+
while [ "${prefix%/*}" != "$prefix" ]; do
|
|
162
|
+
prefix=${prefix%/*}
|
|
163
|
+
if curl -sf -o /dev/null -H "$token" "$api/repository/branches/$(echo "$prefix" | sed 's|/|%2F|g')"; then
|
|
164
|
+
echo "The branch $prefix keeps Duel from making $branch, as Git can't have both. Delete it (git push origin --delete $prefix), or rename it."
|
|
165
|
+
fi
|
|
166
|
+
done
|
|
167
|
+
exit 1
|
|
@@ -37,4 +37,17 @@ Duel.configure do |c|
|
|
|
37
37
|
# :results (passed or failed), :failures (failed, or couldn't compare),
|
|
38
38
|
# or :never.
|
|
39
39
|
# c.comment = :failures
|
|
40
|
+
|
|
41
|
+
# Which runs a regression fails: :every_run, or :pull_requests, for a
|
|
42
|
+
# project where people push to main directly (on GitHub Actions, another
|
|
43
|
+
# run warns and passes, and a pushed commit gets the comment instead).
|
|
44
|
+
# c.fail_on = :every_run
|
|
45
|
+
|
|
46
|
+
# A label that accepts a regression in a pull request (nil for none).
|
|
47
|
+
# c.accept_label = "duel: accept"
|
|
48
|
+
|
|
49
|
+
# What the workflow compares a push with: :before (the commit before it),
|
|
50
|
+
# or :last_passed (the last commit Duel passed, which a branch,
|
|
51
|
+
# duel/passed/main, marks), so small slowdowns add up until they fail.
|
|
52
|
+
# c.push_base = :before
|
|
40
53
|
end
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
name: Perf comment<%= " (#{working_directory})" if working_directory %>
|
|
2
|
+
|
|
3
|
+
# Duel's report, as a new comment on the pull request, when Duel says to:
|
|
4
|
+
# always (post), or only when its last comment's status differs from the
|
|
5
|
+
# run's (change), so the latest comment is never stale. It never edits or
|
|
6
|
+
# deletes a comment. It runs once the Perf workflow has, with this
|
|
7
|
+
# repository's own workflow and a token that can write, so a pull request
|
|
8
|
+
# from a fork gets a comment too. It checks nothing out and runs none of
|
|
9
|
+
# the pull request's code, and of what the Duel job kept (where that code
|
|
10
|
+
# ran), it trusts only what that job could have said: the pull request's
|
|
11
|
+
# number, if it names the pull request the run compared, and the comment
|
|
12
|
+
# and the status, if they're ones Duel says. (The workflow must be on the
|
|
13
|
+
# default branch before it runs at all.)
|
|
14
|
+
on:
|
|
15
|
+
workflow_run:
|
|
16
|
+
workflows: ["Perf<%= " (#{working_directory})" if working_directory %>"]
|
|
17
|
+
types: [completed]
|
|
18
|
+
|
|
19
|
+
permissions: {}
|
|
20
|
+
|
|
21
|
+
jobs:
|
|
22
|
+
comment:
|
|
23
|
+
name: Comment
|
|
24
|
+
if: ${{ github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.conclusion != 'cancelled' }}
|
|
25
|
+
runs-on: ubuntu-latest
|
|
26
|
+
permissions:
|
|
27
|
+
actions: read # to get the report from the Perf run
|
|
28
|
+
pull-requests: write # to comment on the pull request with the report
|
|
29
|
+
steps:
|
|
30
|
+
- name: Get the report
|
|
31
|
+
id: report
|
|
32
|
+
continue-on-error: true # there's none when the Duel job failed before comparing
|
|
33
|
+
uses: actions/download-artifact@v8
|
|
34
|
+
with:
|
|
35
|
+
name: duel-report
|
|
36
|
+
path: ${{ runner.temp }}/report
|
|
37
|
+
run-id: ${{ github.event.workflow_run.id }}
|
|
38
|
+
github-token: ${{ github.token }}
|
|
39
|
+
- name: Comment on the pull request
|
|
40
|
+
if: steps.report.outcome == 'success'
|
|
41
|
+
env:
|
|
42
|
+
GH_TOKEN: ${{ github.token }}
|
|
43
|
+
HEAD_SHA: ${{ github.event.workflow_run.head_sha }}
|
|
44
|
+
run: |
|
|
45
|
+
said() { grep -xE "$1=($2)" "$RUNNER_TEMP/report/comment.env" | tail -n 1 | cut -d= -f2; }
|
|
46
|
+
PR=$(said pr '[0-9]+')
|
|
47
|
+
COMMENT=$(said comment 'post|change')
|
|
48
|
+
STATUS=$(said status 'passed|failed|neutral')
|
|
49
|
+
[ -n "$PR" ] && [ -n "$COMMENT" ] && [ -n "$STATUS" ] || exit 0
|
|
50
|
+
[ "$(gh api "repos/$GITHUB_REPOSITORY/pulls/$PR" --jq .head.sha)" = "$HEAD_SHA" ] || exit 0
|
|
51
|
+
marker="<!-- <%= marker %>"
|
|
52
|
+
# The first line of the last comment Duel posted here: its marker,
|
|
53
|
+
# with its status (none in Duel 0.1.0's, which counts as another).
|
|
54
|
+
last=$(gh api --paginate "repos/$GITHUB_REPOSITORY/issues/$PR/comments" \
|
|
55
|
+
--jq ".[] | select(.user.login == \"github-actions[bot]\") | .body | split(\"\n\")[0] | rtrimstr(\"\r\") | select(startswith(\"$marker status=\") or . == \"$marker -->\")" | tail -n 1)
|
|
56
|
+
if [ "$COMMENT" = post ] || { [ -n "$last" ] && [ "$last" != "$marker status=$STATUS -->" ]; }; then
|
|
57
|
+
{ echo "$marker status=$STATUS -->"; cat "$RUNNER_TEMP/report/duel.md"; } > "$RUNNER_TEMP/comment.md"
|
|
58
|
+
gh api --method POST "repos/$GITHUB_REPOSITORY/issues/$PR/comments" -F body=@"$RUNNER_TEMP/comment.md" > /dev/null
|
|
59
|
+
fi
|