duel 0.1.2 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +68 -1
- data/README.md +22 -5
- data/docs/benchmarks.md +2 -1
- data/docs/ci.md +147 -12
- data/docs/configuration.md +11 -4
- data/docs/how-it-works.md +30 -4
- data/docs/json.md +2 -0
- data/docs/plugins.md +2 -2
- data/docs/troubleshooting.md +14 -0
- data/lib/duel/baseline.rb +4 -4
- data/lib/duel/cli.rb +4 -1
- data/lib/duel/config.rb +13 -6
- data/lib/duel/context.rb +6 -5
- data/lib/duel/init.rb +4 -0
- data/lib/duel/json_report.rb +4 -3
- data/lib/duel/lock.rb +52 -8
- data/lib/duel/native.rb +20 -0
- data/lib/duel/resource.rb +37 -2
- data/lib/duel/runner.rb +98 -9
- data/lib/duel/summary.rb +4 -1
- data/lib/duel/templates/duel.gitlab-ci.yml.erb +23 -4
- data/lib/duel/templates/perf.yml.erb +83 -3
- data/lib/duel/testing.rb +2 -2
- data/lib/duel/version.rb +1 -1
- data/lib/duel/worker.rb +1 -1
- data/sig/duel/config.rbs +3 -0
- data/sig/duel/init.rbs +3 -0
- data/sig/duel/lock.rbs +11 -0
- data/sig/duel/native.rbs +2 -0
- data/sig/duel/resource.rbs +10 -1
- data/sig/duel/runner.rbs +37 -2
- data/sig/duel/summary.rbs +3 -0
- metadata +1 -1
data/lib/duel/json_report.rb
CHANGED
|
@@ -22,7 +22,7 @@ module Duel
|
|
|
22
22
|
none = [] #: Array[value]
|
|
23
23
|
nothing = {} #: report
|
|
24
24
|
head(status, exit, message, settings, comment).merge("seconds" => nil, "baseline" => nil, "candidate" => nil, "plugins" => none,
|
|
25
|
-
"benchmarks" => none, "across" => nothing, "errors" => none)
|
|
25
|
+
"benchmarks" => none, "across" => nothing, "errors" => none, "cut_short" => nil)
|
|
26
26
|
end
|
|
27
27
|
|
|
28
28
|
# What every report starts with.
|
|
@@ -30,7 +30,7 @@ module Duel
|
|
|
30
30
|
{"schema" => SCHEMA, "duel_version" => VERSION, "status" => status, "exit" => exit, "message" => message, "comment" => comment,
|
|
31
31
|
"settings" => {"confidence" => settings.confidence, "limit" => settings.limit, "filter" => settings.filter.source,
|
|
32
32
|
"jobs" => settings.jobs, "processes" => settings.processes, "timeout" => settings.timeout,
|
|
33
|
-
"warm" => settings.warm, "force" => settings.force}}
|
|
33
|
+
"time_limit" => settings.time_limit, "warm" => settings.warm, "force" => settings.force}}
|
|
34
34
|
end
|
|
35
35
|
|
|
36
36
|
# +runner+ ran, and made +results+, comparing +baseline+ with
|
|
@@ -54,7 +54,8 @@ module Duel
|
|
|
54
54
|
"plugins" => @runner.plugins.map(&:to_s),
|
|
55
55
|
"benchmarks" => @results.map { |result| benchmark(result) },
|
|
56
56
|
"across" => across,
|
|
57
|
-
"errors" => @results.select(&:error).map { |result| {"benchmark" => result.name, "message" => result.error} }
|
|
57
|
+
"errors" => @results.select(&:error).map { |result| {"benchmark" => result.name, "message" => result.error} },
|
|
58
|
+
"cut_short" => @runner.cut_short
|
|
58
59
|
)
|
|
59
60
|
end
|
|
60
61
|
|
data/lib/duel/lock.rb
CHANGED
|
@@ -8,7 +8,9 @@ module Duel
|
|
|
8
8
|
# baseline's checkout, each checking out and building over the other's.
|
|
9
9
|
# A run holds an flock on tmp/duel/lock, which ends with its process
|
|
10
10
|
# however that ends, so a run that crashed leaves nothing to clear. The
|
|
11
|
-
# file says whose it is, for a run that finds it taken.
|
|
11
|
+
# file says whose it is, for a run that finds it taken. A run that ends
|
|
12
|
+
# removes it, and the directories that leaves empty: tmp/duel, and tmp
|
|
13
|
+
# if Duel made it.
|
|
12
14
|
# @api private
|
|
13
15
|
module Lock
|
|
14
16
|
# An exclusive lock, or none if another holds it.
|
|
@@ -17,21 +19,63 @@ module Duel
|
|
|
17
19
|
# Runs the block holding +dir+'s lock. Fails, naming the run that
|
|
18
20
|
# holds it, if another does.
|
|
19
21
|
def self.hold(dir)
|
|
22
|
+
made = missing(dir)
|
|
20
23
|
FileUtils.mkdir_p(dir)
|
|
21
|
-
|
|
22
|
-
file =
|
|
24
|
+
path = File.join(dir, "lock")
|
|
25
|
+
file = acquire(path, dir)
|
|
23
26
|
begin
|
|
24
|
-
unless file.flock(EXCLUSIVE)
|
|
25
|
-
# (Empty for a moment, while its run writes its pid.)
|
|
26
|
-
pid = file.read
|
|
27
|
-
raise Error, "another duel run#{" (pid #{pid})" unless pid.empty?} is using #{dir}: wait for it to end, or stop it"
|
|
28
|
-
end
|
|
29
27
|
file.truncate(0)
|
|
30
28
|
file.write(Process.pid)
|
|
31
29
|
file.flush
|
|
32
30
|
yield
|
|
33
31
|
ensure
|
|
32
|
+
# (Removed while it's held, so a run that opens it after makes a new
|
|
33
|
+
# one. `duel --clean` may have removed it already.)
|
|
34
|
+
FileUtils.rm_f(path)
|
|
34
35
|
file.close
|
|
36
|
+
tidy(dir, made)
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# The lock file at +path+, locked: the one there now (a run that
|
|
41
|
+
# ended may have removed the one this opened, after this opened it).
|
|
42
|
+
def self.acquire(path, dir)
|
|
43
|
+
loop do
|
|
44
|
+
# (Read from the start, and written at the end: there's no more.)
|
|
45
|
+
file = File.new(path, "a+")
|
|
46
|
+
unless file.flock(EXCLUSIVE)
|
|
47
|
+
# (Empty for a moment, while its run writes its pid.)
|
|
48
|
+
pid = file.read
|
|
49
|
+
file.close
|
|
50
|
+
raise Error, "another duel run#{" (pid #{pid})" unless pid.empty?} is using #{dir}: wait for it to end, or stop it"
|
|
51
|
+
end
|
|
52
|
+
return file if File.exist?(path) && File.identical?(path, file)
|
|
53
|
+
|
|
54
|
+
file.close
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
# The directories, +dir+ and those it's in, that aren't there yet,
|
|
59
|
+
# deepest first.
|
|
60
|
+
def self.missing(dir)
|
|
61
|
+
made = [] #: Array[String]
|
|
62
|
+
path = dir
|
|
63
|
+
until File.exist?(path)
|
|
64
|
+
made << path
|
|
65
|
+
path = File.dirname(path)
|
|
66
|
+
end
|
|
67
|
+
made
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
# Removes the empty directories in +dir+ (cache/, say: not what's in
|
|
71
|
+
# them, nor in a baseline's checkout there), then +dir+, then those of
|
|
72
|
+
# +made+ (Duel made them), where they're empty. (One that isn't empty
|
|
73
|
+
# holds those it's in.)
|
|
74
|
+
def self.tidy(dir, made)
|
|
75
|
+
[*Dir.glob(File.join(dir, "*/")), dir, *made].each do |path|
|
|
76
|
+
Dir.rmdir(path)
|
|
77
|
+
rescue SystemCallError # (not empty, or gone already)
|
|
78
|
+
nil
|
|
35
79
|
end
|
|
36
80
|
end
|
|
37
81
|
end
|
data/lib/duel/native.rb
CHANGED
|
@@ -157,6 +157,26 @@ module Duel
|
|
|
157
157
|
page_size = Etc.sysconf(Etc::SC_PAGESIZE) #: Integer
|
|
158
158
|
Integer(File.read(statm).split.fetch(1)) * page_size
|
|
159
159
|
end
|
|
160
|
+
|
|
161
|
+
# The memory a benchmark's peak is measured in (see MemorySample),
|
|
162
|
+
# in bytes, or nil where it can't tell: resident memory, but on
|
|
163
|
+
# JRuby, the JVM's heap in use.
|
|
164
|
+
def self.used = resident
|
|
165
|
+
end
|
|
166
|
+
|
|
167
|
+
# simplecov:disable JRuby's heap is the JVM's (JRuby's test run covers
|
|
168
|
+
# this).
|
|
169
|
+
if RUBY_ENGINE == "jruby"
|
|
170
|
+
module Memory
|
|
171
|
+
# The JVM takes memory for its heap ahead of time, and keeps what it
|
|
172
|
+
# used while it started, so resident memory barely moves as a
|
|
173
|
+
# benchmark's objects fill it: 24 MB more of them can show as none.
|
|
174
|
+
# The heap in use shows them.
|
|
175
|
+
HEAP = java.lang.management.ManagementFactory.getMemoryMXBean
|
|
176
|
+
|
|
177
|
+
def self.used = HEAP.getHeapMemoryUsage.getUsed
|
|
178
|
+
end
|
|
160
179
|
end
|
|
180
|
+
# simplecov:enable
|
|
161
181
|
end
|
|
162
182
|
end
|
data/lib/duel/resource.rb
CHANGED
|
@@ -42,7 +42,36 @@ module Duel
|
|
|
42
42
|
# @api private
|
|
43
43
|
CACHE = Mutex.new
|
|
44
44
|
|
|
45
|
-
#
|
|
45
|
+
# What a cache key names, without the version and the hash it ends
|
|
46
|
+
# with: "mysql", of mysql-0.1.2-3f9c2a1b7e5d4c6a. Builds of one name
|
|
47
|
+
# are of one thing, and the one a run used replaces the rest (see
|
|
48
|
+
# .evict).
|
|
49
|
+
# @api private
|
|
50
|
+
NAME = /\A(.+?)(?:-\d[\w.]*)?-\h{8,}\z/
|
|
51
|
+
|
|
52
|
+
# The builds this process's runs used (#cached), by path.
|
|
53
|
+
# @api private
|
|
54
|
+
USED = Set.new
|
|
55
|
+
|
|
56
|
+
# Removes from +cache+ the builds that ones this process used replace:
|
|
57
|
+
# of the same name (see NAME), with another version or hash. A server
|
|
58
|
+
# or a plugin upgraded, or a lockfile changed, left its old build
|
|
59
|
+
# behind otherwise: tens of megabytes of a database's files, each
|
|
60
|
+
# time. A key that doesn't end with a hash is never removed, nor is
|
|
61
|
+
# any that this process used (each version's bundle, say). (Once a
|
|
62
|
+
# run's workers have started: none adds to USED then.)
|
|
63
|
+
# @api private
|
|
64
|
+
def self.evict(cache)
|
|
65
|
+
used = USED.select { |dir| File.dirname(dir) == cache }.map { |dir| File.basename(dir) }
|
|
66
|
+
names = used.filter_map { |key| key[NAME, 1] }
|
|
67
|
+
Dir.children(cache).each do |build|
|
|
68
|
+
FileUtils.rm_rf(File.join(cache, build)) if !used.include?(build) && names.include?(build[NAME, 1])
|
|
69
|
+
end
|
|
70
|
+
rescue Errno::ENOENT # (no cache)
|
|
71
|
+
nil
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
# The worker it's for ("base-1", say), or the version ("base",
|
|
46
75
|
# for Config#prepare(:version)), the directory it has of its own, the
|
|
47
76
|
# tree of the version that worker runs, and the Ruby it runs on (see
|
|
48
77
|
# Config#ruby).
|
|
@@ -88,11 +117,17 @@ module Duel
|
|
|
88
117
|
# block is given, and the rest get it built. A key names what's built
|
|
89
118
|
# (put a version in it), as a file name: tmp/duel/cache/+key+, which
|
|
90
119
|
# `duel --clean` removes. A build that fails leaves nothing behind.
|
|
120
|
+
# A key that ends with a hash (and a version before it, if you like:
|
|
121
|
+
# mysql-0.1.2-3f9c2a1b) replaces the builds of its name when the run
|
|
122
|
+
# ends (see .evict): the one a run used is the one that's kept.
|
|
91
123
|
def cached(key)
|
|
92
124
|
raise ArgumentError, "a cache key is a file name (letters, digits, - _ .), not #{key.inspect}" unless key.match?(KEY)
|
|
93
125
|
|
|
94
126
|
dir = File.join(@cache, key)
|
|
95
|
-
CACHE.synchronize
|
|
127
|
+
CACHE.synchronize do
|
|
128
|
+
USED << dir
|
|
129
|
+
build(dir) { |building| yield building } unless File.directory?(dir)
|
|
130
|
+
end
|
|
96
131
|
dir
|
|
97
132
|
end
|
|
98
133
|
|
data/lib/duel/runner.rb
CHANGED
|
@@ -27,13 +27,17 @@ module Duel
|
|
|
27
27
|
# @api private
|
|
28
28
|
YJIT_WARM = 2.0
|
|
29
29
|
|
|
30
|
+
# The seconds a run measures for, at most, by default (see
|
|
31
|
+
# Settings#time_limit and Runner#out_of_time?).
|
|
32
|
+
TIME_LIMIT = 1800.0
|
|
33
|
+
|
|
30
34
|
# How a run is set up (see Duel::CLI for where each comes from).
|
|
31
35
|
# @api private
|
|
32
36
|
Settings = Struct.new(:base, :base_dir, :candidate_dir, :filter, :jobs, :processes, :limit, :confidence, :warm, :force,
|
|
33
|
-
:config_path, :timeout, :ruby, :comment, :fail_on, :pushed, :target) do
|
|
37
|
+
:config_path, :timeout, :ruby, :comment, :fail_on, :pushed, :target, :time_limit) do
|
|
34
38
|
# @type self: singleton(Settings)
|
|
35
39
|
def self.defaults = new(filter: //, jobs: 1, processes: 3, limit: 1.02, confidence: 0.99, warm: nil, force: false,
|
|
36
|
-
timeout: 60.0)
|
|
40
|
+
timeout: 60.0, time_limit: TIME_LIMIT)
|
|
37
41
|
end
|
|
38
42
|
|
|
39
43
|
# One version as a run has it (see Runner#prepare), +name+d "baseline"
|
|
@@ -51,6 +55,21 @@ module Duel
|
|
|
51
55
|
end
|
|
52
56
|
end
|
|
53
57
|
|
|
58
|
+
class Side
|
|
59
|
+
# What each version's workers, their directories and their sockets
|
|
60
|
+
# are named for (see #label): "base" and "head", as long as each
|
|
61
|
+
# other, so that the two versions' workers get arguments and
|
|
62
|
+
# environments of the same length. Where a process's stack starts
|
|
63
|
+
# depends on how long they are, and at some lengths, the same code
|
|
64
|
+
# runs the same instructions in more cycles: a version whose workers'
|
|
65
|
+
# were a byte longer could measure slower in every one.
|
|
66
|
+
LABELS = {"baseline" => "base", "candidate" => "head"}.freeze
|
|
67
|
+
|
|
68
|
+
# What its workers, and its version's resources, are named for: "base"
|
|
69
|
+
# or "head".
|
|
70
|
+
def label = LABELS.fetch(name)
|
|
71
|
+
end
|
|
72
|
+
|
|
54
73
|
# A worker process (Duel::Worker) running one version of the code.
|
|
55
74
|
# @api private
|
|
56
75
|
class WorkerProcess
|
|
@@ -324,12 +343,12 @@ module Duel
|
|
|
324
343
|
# A worker of +side+, with its own resources, and its version's Ruby
|
|
325
344
|
# and environment.
|
|
326
345
|
def spawn(side, index)
|
|
327
|
-
label = "#{side}-#{index + 1}"
|
|
328
346
|
version = @sides.fetch(side)
|
|
347
|
+
label = "#{version.label}-#{index + 1}"
|
|
329
348
|
env = version.env
|
|
330
349
|
@config.prepares.fetch(:worker).each_with_index do |block, i|
|
|
331
350
|
resource = Resource.new(File.join(@dir, "#{label}-#{i + 1}"), label, version.root, timeout: @settings.timeout,
|
|
332
|
-
cache: File.join(@config.
|
|
351
|
+
cache: File.join(@config.state, "cache"), ruby: version.ruby, env: version.env)
|
|
333
352
|
keep(@resources, resource)
|
|
334
353
|
env = Resource.environment(label, env, block.call(resource))
|
|
335
354
|
end
|
|
@@ -851,23 +870,41 @@ module Duel
|
|
|
851
870
|
# seconds the run took. Set by #run.
|
|
852
871
|
attr_reader :rubies, :seconds
|
|
853
872
|
|
|
873
|
+
# What the run left undone when it reached its time limit
|
|
874
|
+
# (Settings#time_limit): {"unmeasured" => the benchmarks it didn't
|
|
875
|
+
# measure, "unconfirmed" => those it judged without every
|
|
876
|
+
# measurement again (see #confirm), "across" => whether it judged
|
|
877
|
+
# across the board without measuring every benchmark again (see
|
|
878
|
+
# #judge_across)}, or nil where it reached none. Set by #run.
|
|
879
|
+
def cut_short
|
|
880
|
+
return if @unmeasured.empty? && @unconfirmed.empty? && !@across_cut
|
|
881
|
+
|
|
882
|
+
{"unmeasured" => @unmeasured, "unconfirmed" => @unconfirmed, "across" => @across_cut}
|
|
883
|
+
end
|
|
884
|
+
|
|
854
885
|
def initialize(config, settings, base_root, out: $stdout)
|
|
855
886
|
@config = config
|
|
856
887
|
@settings = settings
|
|
857
888
|
@base_root = base_root
|
|
858
889
|
@out = out
|
|
859
890
|
@prepared = []
|
|
891
|
+
@started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
892
|
+
@lock = Mutex.new
|
|
893
|
+
@unmeasured = [] #: Array[String]
|
|
894
|
+
@unconfirmed = [] #: Array[String]
|
|
895
|
+
@across_cut = false
|
|
860
896
|
end
|
|
861
897
|
|
|
862
898
|
# Returns the Results.
|
|
863
899
|
def run
|
|
864
|
-
started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
900
|
+
started = @started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
865
901
|
@dir = Lane.tmpdir
|
|
866
902
|
begin
|
|
867
903
|
compare_all(started)
|
|
868
904
|
ensure
|
|
869
905
|
@prepared.reverse_each(&:stop)
|
|
870
906
|
FileUtils.rm_rf(@dir)
|
|
907
|
+
Resource.evict(File.join(@config.state, "cache"))
|
|
871
908
|
end
|
|
872
909
|
end
|
|
873
910
|
|
|
@@ -877,6 +914,27 @@ module Duel
|
|
|
877
914
|
|
|
878
915
|
def regressed_across = @across.select { |_, v| v["confirmed"] }.keys
|
|
879
916
|
|
|
917
|
+
# What a run cut short at its time limit left undone (see #cut_short),
|
|
918
|
+
# or nil where it wasn't.
|
|
919
|
+
def cut_short_note
|
|
920
|
+
return unless cut_short
|
|
921
|
+
|
|
922
|
+
undone = [("#{Runner.count(@unmeasured.size, "benchmark")} not measured" unless @unmeasured.empty?),
|
|
923
|
+
("#{Runner.count(@unconfirmed.size, "benchmark")} judged without measuring every one again" unless @unconfirmed.empty?),
|
|
924
|
+
("across the board judged without measuring again" if @across_cut)].compact
|
|
925
|
+
"the run measured for longer than #{limit_text}: #{undone.join(", ")}. Raise --time-limit, or make the slowest " \
|
|
926
|
+
"benchmarks quicker to settle"
|
|
927
|
+
end
|
|
928
|
+
|
|
929
|
+
# Whether the run has measured for longer than its time limit
|
|
930
|
+
# (Settings#time_limit). Then it starts nothing more: a benchmark it
|
|
931
|
+
# hasn't measured is an error, unmeasured, and what looked worse is
|
|
932
|
+
# judged on the measurements it has, without measuring it again. It
|
|
933
|
+
# fails what still looks worse in all of them, so a run cut short
|
|
934
|
+
# never passes a regression it would have failed. Each measurement it
|
|
935
|
+
# started runs to its end.
|
|
936
|
+
def out_of_time? = Process.clock_gettime(Process::CLOCK_MONOTONIC) - @started > @settings.time_limit
|
|
937
|
+
|
|
880
938
|
# The plugins the configuration used (Config#plugins), by name.
|
|
881
939
|
def plugins = @config.plugins
|
|
882
940
|
|
|
@@ -918,10 +976,10 @@ module Duel
|
|
|
918
976
|
side.ruby = Rubies.find(choice, side.root, side.name)
|
|
919
977
|
side.env = Rubies.environment(side.ruby)
|
|
920
978
|
@config.prepares.fetch(:version).each_with_index do |block, i|
|
|
921
|
-
resource = Resource.new(File.join(@dir, "#{side.
|
|
922
|
-
cache: File.join(@config.
|
|
979
|
+
resource = Resource.new(File.join(@dir, "#{side.label}-#{i + 1}"), side.label, side.root, timeout: @settings.timeout,
|
|
980
|
+
cache: File.join(@config.state, "cache"), ruby: side.ruby, env: side.env)
|
|
923
981
|
@prepared << resource
|
|
924
|
-
side.env = Resource.environment(side.
|
|
982
|
+
side.env = Resource.environment(side.label, side.env, block.call(resource))
|
|
925
983
|
end
|
|
926
984
|
side
|
|
927
985
|
end
|
|
@@ -944,6 +1002,14 @@ module Duel
|
|
|
944
1002
|
slower = verdicts.select { |_, v| v.fetch("verdict") == "slower" }.keys
|
|
945
1003
|
return verdicts if slower.empty?
|
|
946
1004
|
|
|
1005
|
+
if out_of_time?
|
|
1006
|
+
# (Judged on the one measurement: slower in it, so slower.)
|
|
1007
|
+
@across_cut = true
|
|
1008
|
+
@out.puts "\nSlower across the board in #{slower.join(", ")}, not measured again: the run reached its time limit (#{limit_text})"
|
|
1009
|
+
slower.each { |metric| verdicts.fetch(metric)["confirmed"] = true }
|
|
1010
|
+
return verdicts
|
|
1011
|
+
end
|
|
1012
|
+
|
|
947
1013
|
@out.puts "\nMeasuring every benchmark again, in fresh processes: slower across the board in #{slower.join(", ")}?"
|
|
948
1014
|
again = Across.new(with_lanes { |lanes| measure_all(lanes, names) }, confidence: @settings.confidence).verdicts
|
|
949
1015
|
slower.each { |metric| verdicts.fetch(metric)["confirmed"] = again.dig(metric, "verdict") == "slower" }
|
|
@@ -960,6 +1026,11 @@ module Duel
|
|
|
960
1026
|
lanes.each_index.map do |i|
|
|
961
1027
|
Thread.new do
|
|
962
1028
|
while (name, index = queue.deq)
|
|
1029
|
+
if out_of_time?
|
|
1030
|
+
results[index] = unmeasured(name)
|
|
1031
|
+
next
|
|
1032
|
+
end
|
|
1033
|
+
|
|
963
1034
|
result = results[index] = attempt(lanes.fetch(i), name)
|
|
964
1035
|
@out.print "#{line(result)}\n" # (in one write: other threads write too)
|
|
965
1036
|
next unless result.error
|
|
@@ -979,6 +1050,16 @@ module Duel
|
|
|
979
1050
|
Result.new(name).tap { |r| r.error = "#{e.class}: #{e}" }
|
|
980
1051
|
end
|
|
981
1052
|
|
|
1053
|
+
# A benchmark the run reached its time limit before it measured: an
|
|
1054
|
+
# error, so the run fails. (Each lane's thread adds to the list.)
|
|
1055
|
+
# mutant:disable -- CRuby's Array#<< is atomic anyway, and JRuby's isn't
|
|
1056
|
+
def unmeasured(name)
|
|
1057
|
+
@lock.synchronize { @unmeasured << name }
|
|
1058
|
+
Result.new(name).tap { |r| r.error = "not measured: the run reached its time limit (#{limit_text})" }
|
|
1059
|
+
end
|
|
1060
|
+
|
|
1061
|
+
def limit_text = "#{@settings.time_limit.round}s, --time-limit"
|
|
1062
|
+
|
|
982
1063
|
# Measures each suspect again, from scratch, in fresh processes, until
|
|
983
1064
|
# one measurement clears it or CONFIRM agree (and once more for each
|
|
984
1065
|
# too close to call, up to CONFIRM more: see Result#confirm).
|
|
@@ -989,6 +1070,12 @@ module Duel
|
|
|
989
1070
|
"#{CONFIRM} times each (#{CONFIRM + 1} if what's slower differs, and once more for each too close to call):"
|
|
990
1071
|
suspects.each do |result|
|
|
991
1072
|
while result.confirming?
|
|
1073
|
+
if out_of_time?
|
|
1074
|
+
@unconfirmed << result.name
|
|
1075
|
+
@out.puts format(" %-48s judged on %s: the run reached its time limit (%s)", result.name,
|
|
1076
|
+
Runner.count(result.confirmations.size + 1, "measurement"), limit_text)
|
|
1077
|
+
break
|
|
1078
|
+
end
|
|
992
1079
|
again = Lane.open(@config, @settings, @sides) { |lane| attempt(lane, result.name) }
|
|
993
1080
|
@out.puts line(again) + (result.confirm(again) ? " (too close to call: once more)" : "")
|
|
994
1081
|
end
|
|
@@ -1019,8 +1106,10 @@ module Duel
|
|
|
1019
1106
|
@out.puts format("across the board: %s %.3f [%.3f, %.3f] %s (limit %.4f, %d benchmarks)", metric, v.fetch("median"),
|
|
1020
1107
|
low, high, v.fetch("verdict"), v.fetch("limit"), v.fetch("benchmarks"))
|
|
1021
1108
|
end
|
|
1022
|
-
|
|
1109
|
+
both = @across_cut ? "slower, not measured again (the run reached its time limit)" : "slower in both measurements"
|
|
1110
|
+
regressed_across.each { |metric| @out.puts "REGRESSION across the board: #{metric} #{both}" }
|
|
1023
1111
|
errors.each { |r| @out.puts "ERROR #{r.name}: #{r.error}" }
|
|
1112
|
+
cut_short_note&.then { |note| @out.puts "TIME LIMIT: #{note}" }
|
|
1024
1113
|
regressed.each do |r|
|
|
1025
1114
|
@out.puts "REGRESSION #{r.name}: " +
|
|
1026
1115
|
(r.mixed? ? "slower in every measurement, by one of #{r.regressed.join(", ")}" : "worse #{r.regressed.join(", ")} in every measurement")
|
data/lib/duel/summary.rb
CHANGED
|
@@ -116,7 +116,7 @@ module Duel
|
|
|
116
116
|
|
|
117
117
|
# The Markdown, with the first +explained+ regressions explained.
|
|
118
118
|
def render(explained)
|
|
119
|
-
[heading, @note, compared, failures, across(confirmed: true), details(explained)].compact.join("\n\n") + "\n"
|
|
119
|
+
[heading, @note, compared, cut_short, failures, across(confirmed: true), details(explained)].compact.join("\n\n") + "\n"
|
|
120
120
|
end
|
|
121
121
|
|
|
122
122
|
# The verdict: what failed (benchmarks, or a metric across the board),
|
|
@@ -143,6 +143,9 @@ module Duel
|
|
|
143
143
|
"Compared #{@candidate} with #{@baseline}.#{why}"
|
|
144
144
|
end
|
|
145
145
|
|
|
146
|
+
# What the run left undone at its time limit, if it reached it.
|
|
147
|
+
def cut_short = @runner.cut_short_note&.then { |note| "Time limit: #{note}." }
|
|
148
|
+
|
|
146
149
|
# The benchmarks that failed: what got worse and by how much, or the
|
|
147
150
|
# error, and where the most extra CPU went, if a profile says.
|
|
148
151
|
def failures
|
|
@@ -126,8 +126,9 @@
|
|
|
126
126
|
# With c.push_base = :last_passed in .duel.rb, a push is compared with the
|
|
127
127
|
# last commit Duel passed, which a branch marks (<%= passed_branch %>/main for
|
|
128
128
|
# main), so slowdowns too small for any one push to fail add up until they
|
|
129
|
-
# fail one. This job moves the branch to a push that passed (or makes it)
|
|
130
|
-
#
|
|
129
|
+
# fail one. This job moves the branch to a push that passed (or makes it),
|
|
130
|
+
# but never back: where a newer push's job moved it first, it stays. To
|
|
131
|
+
# accept a regression, move the branch yourself (for main, git push
|
|
131
132
|
# origin main:<%= passed_branch %>/main).
|
|
132
133
|
<%= gitlab_job %>:passed:
|
|
133
134
|
image: alpine:3
|
|
@@ -138,11 +139,29 @@
|
|
|
138
139
|
variables:
|
|
139
140
|
GIT_STRATEGY: none # it runs none of the code
|
|
140
141
|
script:
|
|
141
|
-
- apk add --no-cache --quiet curl
|
|
142
|
+
- apk add --no-cache --quiet curl jq
|
|
142
143
|
- |
|
|
143
144
|
[ "$DUEL_PASSED" = true ] || exit 0
|
|
144
145
|
api="$CI_API_V4_URL/projects/$CI_PROJECT_ID"
|
|
145
146
|
token="PRIVATE-TOKEN: $DUEL_TOKEN"
|
|
146
147
|
branch="<%= passed_branch %>/$CI_COMMIT_BRANCH"
|
|
148
|
+
at=$(curl -sf -H "$token" "$api/repository/branches/$(echo "$branch" | sed 's|/|%2F|g')" | jq -r '.commit.id // empty') || at=
|
|
149
|
+
if [ -n "$at" ]; then
|
|
150
|
+
base=$(curl -sf -H "$token" "$api/repository/merge_base?refs%5B%5D=$CI_COMMIT_SHA&refs%5B%5D=$at" | jq -r '.id // empty') || base=
|
|
151
|
+
[ -n "$base" ] || { echo "Couldn't compare $branch with this push" && exit 1; }
|
|
152
|
+
# (A newer push's job moved it, or this one's did, run again.)
|
|
153
|
+
[ "$base" = "$CI_COMMIT_SHA" ] && exit 0
|
|
154
|
+
fi
|
|
147
155
|
curl -s -o /dev/null -H "$token" -X DELETE "$api/repository/branches/$(echo "$branch" | sed 's|/|%2F|g')"
|
|
148
|
-
curl -sf -o /dev/null -H "$token" --data-urlencode "branch=$branch" --data-urlencode "ref=$CI_COMMIT_SHA" "$api/repository/branches"
|
|
156
|
+
curl -sf -o /dev/null -H "$token" --data-urlencode "branch=$branch" --data-urlencode "ref=$CI_COMMIT_SHA" "$api/repository/branches" && exit 0
|
|
157
|
+
# (Git can't make a branch where another's name is a folder of its
|
|
158
|
+
# name: a branch named duel blocks duel/passed/main. If one does,
|
|
159
|
+
# this names it.)
|
|
160
|
+
prefix=$branch
|
|
161
|
+
while [ "${prefix%/*}" != "$prefix" ]; do
|
|
162
|
+
prefix=${prefix%/*}
|
|
163
|
+
if curl -sf -o /dev/null -H "$token" "$api/repository/branches/$(echo "$prefix" | sed 's|/|%2F|g')"; then
|
|
164
|
+
echo "The branch $prefix keeps Duel from making $branch, as Git can't have both. Delete it (git push origin --delete $prefix), or rename it."
|
|
165
|
+
fi
|
|
166
|
+
done
|
|
167
|
+
exit 1
|
|
@@ -214,7 +214,8 @@ jobs:
|
|
|
214
214
|
# the last commit Duel passed, which a branch marks (each branch pushed
|
|
215
215
|
# has its own), so slowdowns too small for any one push to fail add up
|
|
216
216
|
# until they fail one. This job moves the branch to a push that passed
|
|
217
|
-
# (or makes it)
|
|
217
|
+
# (or makes it), but never back: where a newer push's run moved it
|
|
218
|
+
# first, it stays. It checks nothing out and runs none of the code. To
|
|
218
219
|
# accept a regression, move the branch yourself (for main, git push
|
|
219
220
|
# origin main:<%= passed_branch %>/main).
|
|
220
221
|
passed:
|
|
@@ -230,5 +231,84 @@ jobs:
|
|
|
230
231
|
GH_TOKEN: ${{ github.token }}
|
|
231
232
|
BRANCH: <%= passed_branch %>/${{ github.ref_name }}
|
|
232
233
|
run: |
|
|
233
|
-
|
|
234
|
-
|
|
234
|
+
# (Git can't make a branch where another's name is a folder of its
|
|
235
|
+
# name: a branch named duel blocks duel/passed/main. If one does,
|
|
236
|
+
# this names it.)
|
|
237
|
+
blocked() {
|
|
238
|
+
prefix=$BRANCH
|
|
239
|
+
while [ "${prefix%/*}" != "$prefix" ]; do
|
|
240
|
+
prefix=${prefix%/*}
|
|
241
|
+
if gh api "repos/$GITHUB_REPOSITORY/git/ref/heads/$prefix" > /dev/null 2>&1; then
|
|
242
|
+
echo "::error::The branch $prefix keeps Duel from making $BRANCH, as Git can't have both. Delete it (git push origin --delete $prefix), or rename it."
|
|
243
|
+
fi
|
|
244
|
+
done
|
|
245
|
+
exit 1
|
|
246
|
+
}
|
|
247
|
+
# (Forward: GitHub moves it only where it's behind the push.)
|
|
248
|
+
gh api --method PATCH "repos/$GITHUB_REPOSITORY/git/refs/heads/$BRANCH" -f sha="$GITHUB_SHA" > /dev/null 2>&1 && exit 0
|
|
249
|
+
at=$(gh api "repos/$GITHUB_REPOSITORY/git/ref/heads/$BRANCH" --jq .object.sha 2>/dev/null) || at=
|
|
250
|
+
if [ -z "$at" ]; then
|
|
251
|
+
gh api --method POST "repos/$GITHUB_REPOSITORY/git/refs" -f ref="refs/heads/$BRANCH" -f sha="$GITHUB_SHA" > /dev/null || blocked
|
|
252
|
+
exit 0
|
|
253
|
+
fi
|
|
254
|
+
compared=$(gh api "repos/$GITHUB_REPOSITORY/compare/$GITHUB_SHA...$at" --jq .status)
|
|
255
|
+
case "$compared" in
|
|
256
|
+
# (A newer push's run moved it, or this one's did, run again.)
|
|
257
|
+
ahead | identical) ;;
|
|
258
|
+
# (History rewritten, a force push's, say: moved anyway.)
|
|
259
|
+
diverged | behind) gh api --method PATCH "repos/$GITHUB_REPOSITORY/git/refs/heads/$BRANCH" -f sha="$GITHUB_SHA" -F force=true > /dev/null ;;
|
|
260
|
+
*) echo "::error::Couldn't compare $BRANCH with this push: $compared" && exit 1 ;;
|
|
261
|
+
esac
|
|
262
|
+
|
|
263
|
+
# Duel's badge, for the README: green (perf guarded) or red (perf
|
|
264
|
+
# regressed), as the last push to the default branch that Duel compared
|
|
265
|
+
# left it. It's red where the Duel job failed or found a regression,
|
|
266
|
+
# even one that passed (with c.fail_on = :pull_requests in .duel.rb). A
|
|
267
|
+
# push with nothing to compare leaves it as it was, and so does a run
|
|
268
|
+
# for an older push than the badge's. This job writes the badge, alone,
|
|
269
|
+
# to a branch of its own, which the README shows:
|
|
270
|
+
#
|
|
271
|
+
# [](https://github.com/sferik/duel-ruby)
|
|
272
|
+
#
|
|
273
|
+
# It checks nothing out and runs none of the code.
|
|
274
|
+
badge:
|
|
275
|
+
name: Update the badge
|
|
276
|
+
needs: duel
|
|
277
|
+
if: ${{ !cancelled() && github.event_name == 'push' && github.ref_name == github.event.repository.default_branch && (needs.duel.result == 'failure' || needs.duel.outputs.status == 'passed' || needs.duel.outputs.status == 'failed') }}
|
|
278
|
+
runs-on: ubuntu-latest
|
|
279
|
+
permissions:
|
|
280
|
+
contents: write # to move the badge's branch
|
|
281
|
+
steps:
|
|
282
|
+
- name: Update the badge
|
|
283
|
+
env:
|
|
284
|
+
GH_TOKEN: ${{ github.token }}
|
|
285
|
+
BRANCH: <%= badge_branch %>
|
|
286
|
+
RESULT: ${{ needs.duel.result }}
|
|
287
|
+
STATUS: ${{ needs.duel.outputs.status }}
|
|
288
|
+
BADGES: https://raw.githubusercontent.com/sferik/duel-ruby/main/assets
|
|
289
|
+
run: |
|
|
290
|
+
# (Git can't make a branch where another's name is a folder of its
|
|
291
|
+
# name: a branch named duel blocks duel/passed/main. If one does,
|
|
292
|
+
# this names it.)
|
|
293
|
+
blocked() {
|
|
294
|
+
prefix=$BRANCH
|
|
295
|
+
while [ "${prefix%/*}" != "$prefix" ]; do
|
|
296
|
+
prefix=${prefix%/*}
|
|
297
|
+
if gh api "repos/$GITHUB_REPOSITORY/git/ref/heads/$prefix" > /dev/null 2>&1; then
|
|
298
|
+
echo "::error::The branch $prefix keeps Duel from making $BRANCH, as Git can't have both. Delete it (git push origin --delete $prefix), or rename it."
|
|
299
|
+
fi
|
|
300
|
+
done
|
|
301
|
+
exit 1
|
|
302
|
+
}
|
|
303
|
+
# (Where the badge is a newer push's, that push's run has the say.
|
|
304
|
+
# Its commit names the push.)
|
|
305
|
+
last=$(gh api "repos/$GITHUB_REPOSITORY/commits/$BRANCH" --jq .commit.message 2>/dev/null | sed -n 's/^Duel.s badge, for \([0-9a-f]*\)$/\1/p') || true
|
|
306
|
+
if [ -n "$last" ] && [ "$(gh api "repos/$GITHUB_REPOSITORY/compare/$GITHUB_SHA...$last" --jq .status 2>/dev/null)" = ahead ]; then exit 0; fi
|
|
307
|
+
badge="badge.svg"
|
|
308
|
+
if [ "$RESULT" = failure ] || [ "$STATUS" = failed ]; then badge="badge-regressed.svg"; fi
|
|
309
|
+
curl -fsSL "$BADGES/$badge" -o "$RUNNER_TEMP/badge.svg"
|
|
310
|
+
tree=$(jq -n --rawfile svg "$RUNNER_TEMP/badge.svg" '{tree: [{path: "badge.svg", mode: "100644", type: "blob", content: $svg}]}' |
|
|
311
|
+
gh api --method POST "repos/$GITHUB_REPOSITORY/git/trees" --input - --jq .sha)
|
|
312
|
+
commit=$(gh api --method POST "repos/$GITHUB_REPOSITORY/git/commits" -f message="Duel's badge, for $GITHUB_SHA" -f tree="$tree" --jq .sha)
|
|
313
|
+
gh api --method PATCH "repos/$GITHUB_REPOSITORY/git/refs/heads/$BRANCH" -f sha="$commit" -F force=true > /dev/null 2>&1 ||
|
|
314
|
+
gh api --method POST "repos/$GITHUB_REPOSITORY/git/refs" -f ref="refs/heads/$BRANCH" -f sha="$commit" > /dev/null || blocked
|
data/lib/duel/testing.rb
CHANGED
|
@@ -49,11 +49,11 @@ module Duel
|
|
|
49
49
|
|
|
50
50
|
# Runs +config+'s prepare blocks as they'd prepare a worker (+name+,
|
|
51
51
|
# running the version at +root+): its version's (Config#prepare
|
|
52
|
-
# (:version), named for the version: "
|
|
52
|
+
# (:version), named for the version: "head"), then the worker's,
|
|
53
53
|
# with their builds (Resource#cached) in +cache+ (by default, one for
|
|
54
54
|
# every test of this process). Sets the environment they give while
|
|
55
55
|
# the block runs, yields it, and stops what they started after.
|
|
56
|
-
def prepare(config, name: "
|
|
56
|
+
def prepare(config, name: "head-1", root: config.root, cache: CACHE)
|
|
57
57
|
dir = Lane.tmpdir
|
|
58
58
|
started = [] #: Array[Resource]
|
|
59
59
|
begin
|
data/lib/duel/version.rb
CHANGED
data/lib/duel/worker.rb
CHANGED
|
@@ -87,7 +87,7 @@ module Duel
|
|
|
87
87
|
# peak memory and profiles can be measured here, and whether YJIT is on.
|
|
88
88
|
def hello
|
|
89
89
|
{"benchmarks" => @benchmarks.values.map(&:to_h), "metrics" => @specs, "ruby" => RUBY_DESCRIPTION,
|
|
90
|
-
"memory" => !Native::Memory.
|
|
90
|
+
"memory" => !Native::Memory.used.nil?, "profiler" => Worker.profiler?, "yjit" => Worker.yjit?,
|
|
91
91
|
"extensions" => Worker.extensions, "gem_dirs" => Worker.gem_dirs}
|
|
92
92
|
end
|
|
93
93
|
|
data/sig/duel/config.rbs
CHANGED
|
@@ -42,6 +42,9 @@ module Duel
|
|
|
42
42
|
# Where the project is (and its config file).
|
|
43
43
|
attr_reader root: String
|
|
44
44
|
|
|
45
|
+
# Where Duel keeps what it keeps for the project: tmp/duel.
|
|
46
|
+
def state: () -> String
|
|
47
|
+
|
|
45
48
|
# A glob, from the root, of the files defining benchmarks.
|
|
46
49
|
attr_accessor benchmarks: String
|
|
47
50
|
|
data/sig/duel/init.rbs
CHANGED
|
@@ -104,6 +104,9 @@ module Duel
|
|
|
104
104
|
# The branch that marks the last commit Duel passed in a push.
|
|
105
105
|
def passed_branch: () -> String
|
|
106
106
|
|
|
107
|
+
# The branch that holds the workflow's badge.
|
|
108
|
+
def badge_branch: () -> String
|
|
109
|
+
|
|
107
110
|
# Where the GitLab CI jobs are, from the repository's root.
|
|
108
111
|
def gitlab_file: () -> String
|
|
109
112
|
|
data/sig/duel/lock.rbs
CHANGED
|
@@ -8,5 +8,16 @@ module Duel
|
|
|
8
8
|
# Runs the block holding +dir+'s lock. Fails, naming the run that
|
|
9
9
|
# holds it, if another does.
|
|
10
10
|
def self.hold: [T] (String dir) { () -> T } -> T
|
|
11
|
+
|
|
12
|
+
# The lock file at +path+, locked: the one there now.
|
|
13
|
+
def self.acquire: (String path, String dir) -> File
|
|
14
|
+
|
|
15
|
+
# The directories, +dir+ and those it's in, that aren't there yet,
|
|
16
|
+
# deepest first.
|
|
17
|
+
def self.missing: (String dir) -> Array[String]
|
|
18
|
+
|
|
19
|
+
# Removes the empty directories in +dir+, then +dir+, then those of
|
|
20
|
+
# +made+ that it's in, while they're empty.
|
|
21
|
+
def self.tidy: (String dir, Array[String] made) -> void
|
|
11
22
|
end
|
|
12
23
|
end
|