duel 0.1.1 → 0.1.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/lib/duel/lock.rb CHANGED
@@ -8,7 +8,9 @@ module Duel
8
8
  # baseline's checkout, each checking out and building over the other's.
9
9
  # A run holds an flock on tmp/duel/lock, which ends with its process
10
10
  # however that ends, so a run that crashed leaves nothing to clear. The
11
- # file says whose it is, for a run that finds it taken.
11
+ # file says whose it is, for a run that finds it taken. A run that ends
12
+ # removes it, and the directories that leaves empty: tmp/duel, and tmp
13
+ # if Duel made it.
12
14
  # @api private
13
15
  module Lock
14
16
  # An exclusive lock, or none if another holds it.
@@ -17,21 +19,63 @@ module Duel
17
19
  # Runs the block holding +dir+'s lock. Fails, naming the run that
18
20
  # holds it, if another does.
19
21
  def self.hold(dir)
22
+ made = missing(dir)
20
23
  FileUtils.mkdir_p(dir)
21
- # (Read from the start, and written at the end: there's no more.)
22
- file = File.new(File.join(dir, "lock"), "a+")
24
+ path = File.join(dir, "lock")
25
+ file = acquire(path, dir)
23
26
  begin
24
- unless file.flock(EXCLUSIVE)
25
- # (Empty for a moment, while its run writes its pid.)
26
- pid = file.read
27
- raise Error, "another duel run#{" (pid #{pid})" unless pid.empty?} is using #{dir}: wait for it to end, or stop it"
28
- end
29
27
  file.truncate(0)
30
28
  file.write(Process.pid)
31
29
  file.flush
32
30
  yield
33
31
  ensure
32
+ # (Removed while it's held, so a run that opens it after makes a new
33
+ # one. `duel --clean` may have removed it already.)
34
+ FileUtils.rm_f(path)
34
35
  file.close
36
+ tidy(dir, made)
37
+ end
38
+ end
39
+
40
+ # The lock file at +path+, locked: the one there now (a run that
41
+ # ended may have removed the one this opened, after this opened it).
42
+ def self.acquire(path, dir)
43
+ loop do
44
+ # (Read from the start, and written at the end: there's no more.)
45
+ file = File.new(path, "a+")
46
+ unless file.flock(EXCLUSIVE)
47
+ # (Empty for a moment, while its run writes its pid.)
48
+ pid = file.read
49
+ file.close
50
+ raise Error, "another duel run#{" (pid #{pid})" unless pid.empty?} is using #{dir}: wait for it to end, or stop it"
51
+ end
52
+ return file if File.exist?(path) && File.identical?(path, file)
53
+
54
+ file.close
55
+ end
56
+ end
57
+
58
+ # The directories, +dir+ and those it's in, that aren't there yet,
59
+ # deepest first.
60
+ def self.missing(dir)
61
+ made = [] #: Array[String]
62
+ path = dir
63
+ until File.exist?(path)
64
+ made << path
65
+ path = File.dirname(path)
66
+ end
67
+ made
68
+ end
69
+
70
+ # Removes the empty directories in +dir+ (cache/, say: not what's in
71
+ # them, nor in a baseline's checkout there), then +dir+, then those of
72
+ # +made+ (Duel made them), where they're empty. (One that isn't empty
73
+ # holds those it's in.)
74
+ def self.tidy(dir, made)
75
+ [*Dir.glob(File.join(dir, "*/")), dir, *made].each do |path|
76
+ Dir.rmdir(path)
77
+ rescue SystemCallError # (not empty, or gone already)
78
+ nil
35
79
  end
36
80
  end
37
81
  end
data/lib/duel/native.rb CHANGED
@@ -157,6 +157,26 @@ module Duel
157
157
  page_size = Etc.sysconf(Etc::SC_PAGESIZE) #: Integer
158
158
  Integer(File.read(statm).split.fetch(1)) * page_size
159
159
  end
160
+
161
+ # The memory a benchmark's peak is measured in (see MemorySample),
162
+ # in bytes, or nil where it can't tell: resident memory, but on
163
+ # JRuby, the JVM's heap in use.
164
+ def self.used = resident
165
+ end
166
+
167
+ # simplecov:disable JRuby's heap is the JVM's (JRuby's test run covers
168
+ # this).
169
+ if RUBY_ENGINE == "jruby"
170
+ module Memory
171
+ # The JVM takes memory for its heap ahead of time, and keeps what it
172
+ # used while it started, so resident memory barely moves as a
173
+ # benchmark's objects fill it: 24 MB more of them can show as none.
174
+ # The heap in use shows them.
175
+ HEAP = java.lang.management.ManagementFactory.getMemoryMXBean
176
+
177
+ def self.used = HEAP.getHeapMemoryUsage.getUsed
178
+ end
160
179
  end
180
+ # simplecov:enable
161
181
  end
162
182
  end
data/lib/duel/resource.rb CHANGED
@@ -42,7 +42,36 @@ module Duel
42
42
  # @api private
43
43
  CACHE = Mutex.new
44
44
 
45
- # The worker it's for ("baseline-1", say), or the version ("baseline",
45
+ # What a cache key names, without the version and the hash it ends
46
+ # with: "mysql", of mysql-0.1.2-3f9c2a1b7e5d4c6a. Builds of one name
47
+ # are of one thing, and the one a run used replaces the rest (see
48
+ # .evict).
49
+ # @api private
50
+ NAME = /\A(.+?)(?:-\d[\w.]*)?-\h{8,}\z/
51
+
52
+ # The builds this process's runs used (#cached), by path.
53
+ # @api private
54
+ USED = Set.new
55
+
56
+ # Removes from +cache+ the builds that ones this process used replace:
57
+ # of the same name (see NAME), with another version or hash. A server
58
+ # or a plugin upgraded, or a lockfile changed, left its old build
59
+ # behind otherwise: tens of megabytes of a database's files, each
60
+ # time. A key that doesn't end with a hash is never removed, nor is
61
+ # any that this process used (each version's bundle, say). (Once a
62
+ # run's workers have started: none adds to USED then.)
63
+ # @api private
64
+ def self.evict(cache)
65
+ used = USED.select { |dir| File.dirname(dir) == cache }.map { |dir| File.basename(dir) }
66
+ names = used.filter_map { |key| key[NAME, 1] }
67
+ Dir.children(cache).each do |build|
68
+ FileUtils.rm_rf(File.join(cache, build)) if !used.include?(build) && names.include?(build[NAME, 1])
69
+ end
70
+ rescue Errno::ENOENT # (no cache)
71
+ nil
72
+ end
73
+
74
+ # The worker it's for ("base-1", say), or the version ("base",
46
75
  # for Config#prepare(:version)), the directory it has of its own, the
47
76
  # tree of the version that worker runs, and the Ruby it runs on (see
48
77
  # Config#ruby).
@@ -88,11 +117,17 @@ module Duel
88
117
  # block is given, and the rest get it built. A key names what's built
89
118
  # (put a version in it), as a file name: tmp/duel/cache/+key+, which
90
119
  # `duel --clean` removes. A build that fails leaves nothing behind.
120
+ # A key that ends with a hash (and a version before it, if you like:
121
+ # mysql-0.1.2-3f9c2a1b) replaces the builds of its name when the run
122
+ # ends (see .evict): the one a run used is the one that's kept.
91
123
  def cached(key)
92
124
  raise ArgumentError, "a cache key is a file name (letters, digits, - _ .), not #{key.inspect}" unless key.match?(KEY)
93
125
 
94
126
  dir = File.join(@cache, key)
95
- CACHE.synchronize { build(dir) { |building| yield building } unless File.directory?(dir) }
127
+ CACHE.synchronize do
128
+ USED << dir
129
+ build(dir) { |building| yield building } unless File.directory?(dir)
130
+ end
96
131
  dir
97
132
  end
98
133
 
data/lib/duel/runner.rb CHANGED
@@ -27,13 +27,17 @@ module Duel
27
27
  # @api private
28
28
  YJIT_WARM = 2.0
29
29
 
30
+ # The seconds a run measures for, at most, by default (see
31
+ # Settings#time_limit and Runner#out_of_time?).
32
+ TIME_LIMIT = 1800.0
33
+
30
34
  # How a run is set up (see Duel::CLI for where each comes from).
31
35
  # @api private
32
36
  Settings = Struct.new(:base, :base_dir, :candidate_dir, :filter, :jobs, :processes, :limit, :confidence, :warm, :force,
33
- :config_path, :timeout, :ruby, :comment) do
37
+ :config_path, :timeout, :ruby, :comment, :fail_on, :pushed, :target, :time_limit) do
34
38
  # @type self: singleton(Settings)
35
39
  def self.defaults = new(filter: //, jobs: 1, processes: 3, limit: 1.02, confidence: 0.99, warm: nil, force: false,
36
- timeout: 60.0)
40
+ timeout: 60.0, time_limit: TIME_LIMIT)
37
41
  end
38
42
 
39
43
  # One version as a run has it (see Runner#prepare), +name+d "baseline"
@@ -51,6 +55,21 @@ module Duel
51
55
  end
52
56
  end
53
57
 
58
+ class Side
59
+ # What each version's workers, their directories and their sockets
60
+ # are named for (see #label): "base" and "head", as long as each
61
+ # other, so that the two versions' workers get arguments and
62
+ # environments of the same length. Where a process's stack starts
63
+ # depends on how long they are, and at some lengths, the same code
64
+ # runs the same instructions in more cycles: a version whose workers'
65
+ # were a byte longer could measure slower in every one.
66
+ LABELS = {"baseline" => "base", "candidate" => "head"}.freeze
67
+
68
+ # What its workers, and its version's resources, are named for: "base"
69
+ # or "head".
70
+ def label = LABELS.fetch(name)
71
+ end
72
+
54
73
  # A worker process (Duel::Worker) running one version of the code.
55
74
  # @api private
56
75
  class WorkerProcess
@@ -324,12 +343,12 @@ module Duel
324
343
  # A worker of +side+, with its own resources, and its version's Ruby
325
344
  # and environment.
326
345
  def spawn(side, index)
327
- label = "#{side}-#{index + 1}"
328
346
  version = @sides.fetch(side)
347
+ label = "#{version.label}-#{index + 1}"
329
348
  env = version.env
330
349
  @config.prepares.fetch(:worker).each_with_index do |block, i|
331
350
  resource = Resource.new(File.join(@dir, "#{label}-#{i + 1}"), label, version.root, timeout: @settings.timeout,
332
- cache: File.join(@config.root, "tmp", "duel", "cache"), ruby: version.ruby, env: version.env)
351
+ cache: File.join(@config.state, "cache"), ruby: version.ruby, env: version.env)
333
352
  keep(@resources, resource)
334
353
  env = Resource.environment(label, env, block.call(resource))
335
354
  end
@@ -851,23 +870,41 @@ module Duel
851
870
  # seconds the run took. Set by #run.
852
871
  attr_reader :rubies, :seconds
853
872
 
873
+ # What the run left undone when it reached its time limit
874
+ # (Settings#time_limit): {"unmeasured" => the benchmarks it didn't
875
+ # measure, "unconfirmed" => those it judged without every
876
+ # measurement again (see #confirm), "across" => whether it judged
877
+ # across the board without measuring every benchmark again (see
878
+ # #judge_across)}, or nil where it reached none. Set by #run.
879
+ def cut_short
880
+ return if @unmeasured.empty? && @unconfirmed.empty? && !@across_cut
881
+
882
+ {"unmeasured" => @unmeasured, "unconfirmed" => @unconfirmed, "across" => @across_cut}
883
+ end
884
+
854
885
  def initialize(config, settings, base_root, out: $stdout)
855
886
  @config = config
856
887
  @settings = settings
857
888
  @base_root = base_root
858
889
  @out = out
859
890
  @prepared = []
891
+ @started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
892
+ @lock = Mutex.new
893
+ @unmeasured = [] #: Array[String]
894
+ @unconfirmed = [] #: Array[String]
895
+ @across_cut = false
860
896
  end
861
897
 
862
898
  # Returns the Results.
863
899
  def run
864
- started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
900
+ started = @started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
865
901
  @dir = Lane.tmpdir
866
902
  begin
867
903
  compare_all(started)
868
904
  ensure
869
905
  @prepared.reverse_each(&:stop)
870
906
  FileUtils.rm_rf(@dir)
907
+ Resource.evict(File.join(@config.state, "cache"))
871
908
  end
872
909
  end
873
910
 
@@ -877,6 +914,27 @@ module Duel
877
914
 
878
915
  def regressed_across = @across.select { |_, v| v["confirmed"] }.keys
879
916
 
917
+ # What a run cut short at its time limit left undone (see #cut_short),
918
+ # or nil where it wasn't.
919
+ def cut_short_note
920
+ return unless cut_short
921
+
922
+ undone = [("#{Runner.count(@unmeasured.size, "benchmark")} not measured" unless @unmeasured.empty?),
923
+ ("#{Runner.count(@unconfirmed.size, "benchmark")} judged without measuring every one again" unless @unconfirmed.empty?),
924
+ ("across the board judged without measuring again" if @across_cut)].compact
925
+ "the run measured for longer than #{limit_text}: #{undone.join(", ")}. Raise --time-limit, or make the slowest " \
926
+ "benchmarks quicker to settle"
927
+ end
928
+
929
+ # Whether the run has measured for longer than its time limit
930
+ # (Settings#time_limit). Then it starts nothing more: a benchmark it
931
+ # hasn't measured is an error, unmeasured, and what looked worse is
932
+ # judged on the measurements it has, without measuring it again. It
933
+ # fails what still looks worse in all of them, so a run cut short
934
+ # never passes a regression it would have failed. Each measurement it
935
+ # started runs to its end.
936
+ def out_of_time? = Process.clock_gettime(Process::CLOCK_MONOTONIC) - @started > @settings.time_limit
937
+
880
938
  # The plugins the configuration used (Config#plugins), by name.
881
939
  def plugins = @config.plugins
882
940
 
@@ -918,10 +976,10 @@ module Duel
918
976
  side.ruby = Rubies.find(choice, side.root, side.name)
919
977
  side.env = Rubies.environment(side.ruby)
920
978
  @config.prepares.fetch(:version).each_with_index do |block, i|
921
- resource = Resource.new(File.join(@dir, "#{side.name}-#{i + 1}"), side.name, side.root, timeout: @settings.timeout,
922
- cache: File.join(@config.root, "tmp", "duel", "cache"), ruby: side.ruby, env: side.env)
979
+ resource = Resource.new(File.join(@dir, "#{side.label}-#{i + 1}"), side.label, side.root, timeout: @settings.timeout,
980
+ cache: File.join(@config.state, "cache"), ruby: side.ruby, env: side.env)
923
981
  @prepared << resource
924
- side.env = Resource.environment(side.name, side.env, block.call(resource))
982
+ side.env = Resource.environment(side.label, side.env, block.call(resource))
925
983
  end
926
984
  side
927
985
  end
@@ -944,6 +1002,14 @@ module Duel
944
1002
  slower = verdicts.select { |_, v| v.fetch("verdict") == "slower" }.keys
945
1003
  return verdicts if slower.empty?
946
1004
 
1005
+ if out_of_time?
1006
+ # (Judged on the one measurement: slower in it, so slower.)
1007
+ @across_cut = true
1008
+ @out.puts "\nSlower across the board in #{slower.join(", ")}, not measured again: the run reached its time limit (#{limit_text})"
1009
+ slower.each { |metric| verdicts.fetch(metric)["confirmed"] = true }
1010
+ return verdicts
1011
+ end
1012
+
947
1013
  @out.puts "\nMeasuring every benchmark again, in fresh processes: slower across the board in #{slower.join(", ")}?"
948
1014
  again = Across.new(with_lanes { |lanes| measure_all(lanes, names) }, confidence: @settings.confidence).verdicts
949
1015
  slower.each { |metric| verdicts.fetch(metric)["confirmed"] = again.dig(metric, "verdict") == "slower" }
@@ -960,6 +1026,11 @@ module Duel
960
1026
  lanes.each_index.map do |i|
961
1027
  Thread.new do
962
1028
  while (name, index = queue.deq)
1029
+ if out_of_time?
1030
+ results[index] = unmeasured(name)
1031
+ next
1032
+ end
1033
+
963
1034
  result = results[index] = attempt(lanes.fetch(i), name)
964
1035
  @out.print "#{line(result)}\n" # (in one write: other threads write too)
965
1036
  next unless result.error
@@ -979,6 +1050,16 @@ module Duel
979
1050
  Result.new(name).tap { |r| r.error = "#{e.class}: #{e}" }
980
1051
  end
981
1052
 
1053
+ # A benchmark the run reached its time limit before it measured: an
1054
+ # error, so the run fails. (Each lane's thread adds to the list.)
1055
+ # mutant:disable -- CRuby's Array#<< is atomic anyway, and JRuby's isn't
1056
+ def unmeasured(name)
1057
+ @lock.synchronize { @unmeasured << name }
1058
+ Result.new(name).tap { |r| r.error = "not measured: the run reached its time limit (#{limit_text})" }
1059
+ end
1060
+
1061
+ def limit_text = "#{@settings.time_limit.round}s, --time-limit"
1062
+
982
1063
  # Measures each suspect again, from scratch, in fresh processes, until
983
1064
  # one measurement clears it or CONFIRM agree (and once more for each
984
1065
  # too close to call, up to CONFIRM more: see Result#confirm).
@@ -989,6 +1070,12 @@ module Duel
989
1070
  "#{CONFIRM} times each (#{CONFIRM + 1} if what's slower differs, and once more for each too close to call):"
990
1071
  suspects.each do |result|
991
1072
  while result.confirming?
1073
+ if out_of_time?
1074
+ @unconfirmed << result.name
1075
+ @out.puts format(" %-48s judged on %s: the run reached its time limit (%s)", result.name,
1076
+ Runner.count(result.confirmations.size + 1, "measurement"), limit_text)
1077
+ break
1078
+ end
992
1079
  again = Lane.open(@config, @settings, @sides) { |lane| attempt(lane, result.name) }
993
1080
  @out.puts line(again) + (result.confirm(again) ? " (too close to call: once more)" : "")
994
1081
  end
@@ -1019,8 +1106,10 @@ module Duel
1019
1106
  @out.puts format("across the board: %s %.3f [%.3f, %.3f] %s (limit %.4f, %d benchmarks)", metric, v.fetch("median"),
1020
1107
  low, high, v.fetch("verdict"), v.fetch("limit"), v.fetch("benchmarks"))
1021
1108
  end
1022
- regressed_across.each { |metric| @out.puts "REGRESSION across the board: #{metric} slower in both measurements" }
1109
+ both = @across_cut ? "slower, not measured again (the run reached its time limit)" : "slower in both measurements"
1110
+ regressed_across.each { |metric| @out.puts "REGRESSION across the board: #{metric} #{both}" }
1023
1111
  errors.each { |r| @out.puts "ERROR #{r.name}: #{r.error}" }
1112
+ cut_short_note&.then { |note| @out.puts "TIME LIMIT: #{note}" }
1024
1113
  regressed.each do |r|
1025
1114
  @out.puts "REGRESSION #{r.name}: " +
1026
1115
  (r.mixed? ? "slower in every measurement, by one of #{r.regressed.join(", ")}" : "worse #{r.regressed.join(", ")} in every measurement")
data/lib/duel/summary.rb CHANGED
@@ -90,8 +90,10 @@ module Duel
90
90
 
91
91
  # +runner+ ran, and made +results+, comparing +baseline+ with
92
92
  # +candidate+ (each a description, in Markdown, whose commit stands
93
- # bare, so GitHub links it) as +settings+ say.
94
- def initialize(runner, results, settings:, baseline:, candidate:)
93
+ # bare, so GitHub links it) as +settings+ say, with a +note+ (in
94
+ # Markdown) under the heading, if there's one.
95
+ def initialize(runner, results, settings:, baseline:, candidate:, note: nil)
96
+ @note = note
95
97
  @runner = runner
96
98
  @results = results
97
99
  @settings = settings
@@ -114,7 +116,7 @@ module Duel
114
116
 
115
117
  # The Markdown, with the first +explained+ regressions explained.
116
118
  def render(explained)
117
- [heading, compared, failures, across(confirmed: true), details(explained)].compact.join("\n\n") + "\n"
119
+ [heading, @note, compared, cut_short, failures, across(confirmed: true), details(explained)].compact.join("\n\n") + "\n"
118
120
  end
119
121
 
120
122
  # The verdict: what failed (benchmarks, or a metric across the board),
@@ -141,6 +143,9 @@ module Duel
141
143
  "Compared #{@candidate} with #{@baseline}.#{why}"
142
144
  end
143
145
 
146
+ # What the run left undone at its time limit, if it reached it.
147
+ def cut_short = @runner.cut_short_note&.then { |note| "Time limit: #{note}." }
148
+
144
149
  # The benchmarks that failed: what got worse and by how much, or the
145
150
  # error, and where the most extra CPU went, if a profile says.
146
151
  def failures
@@ -0,0 +1,167 @@
1
+ # Duel on GitLab CI: https://github.com/sferik/duel-ruby/blob/main/docs/ci.md#run-duel-on-gitlab-ci
2
+ #
3
+ # Include it from .gitlab-ci.yml:
4
+ #
5
+ # include:
6
+ # - local: <%= gitlab_file %>
7
+ #
8
+ # It compares a merge request with the branch it targets, a push to the
9
+ # default branch with the commit before it (or as c.push_base in .duel.rb
10
+ # says, the last commit Duel passed), and on a schedule, the last release.
11
+ # The Duel job's log, and its duel.md artifact, have the report. Comments
12
+ # need project access tokens, in masked CI/CD variables:
13
+ #
14
+ # - DUEL_COMMENT_TOKEN (the Reporter role, with the api scope, and not
15
+ # protected), to comment on a merge request. Every job of a merge
16
+ # request's pipeline has it, the Duel job too, which runs the merge
17
+ # request's code, so the token can do no more than comment.
18
+ # - DUEL_TOKEN (the Developer role, with the api scope, and protected, so
19
+ # only the default branch's pipelines have it), to comment on a pushed
20
+ # commit (see c.fail_on), and to mark a push passed (see c.push_base).
21
+
22
+ <%= gitlab_job %>:
23
+ image: ruby:<%= gitlab_ruby %>
24
+ stage: test
25
+ variables:
26
+ GIT_DEPTH: 0 # the baseline comes from history, tags included
27
+ DUEL_OUTPUT: duel.env # whether to comment, and mark a push passed
28
+ rules:
29
+ - if: $CI_PIPELINE_SOURCE == "merge_request_event"
30
+ - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
31
+ variables:
32
+ DUEL_BASE: $CI_COMMIT_BEFORE_SHA
33
+ - if: $CI_PIPELINE_SOURCE == "schedule"
34
+ variables:
35
+ DUEL_BASE: last-release
36
+ # (Run by hand, it compares anyway: with nothing changed, the code
37
+ # with itself, which checks the setup.)
38
+ - if: $CI_PIPELINE_SOURCE == "web"
39
+ variables:
40
+ DUEL_FORCE: "1"
41
+ # A regression outside a merge request, where only a merge request
42
+ # fails (c.fail_on in .duel.rb), exits 3: a warning.
43
+ allow_failure:
44
+ exit_codes: [3]
45
+ before_script:
46
+ <% if working_directory -%>
47
+ - cd <%= working_directory %>
48
+ <% end -%>
49
+ # The branch a merge request targets, the last commits Duel passed,
50
+ # and the releases.
51
+ - git fetch --quiet --tags origin "+refs/heads/duel/*:refs/remotes/origin/duel/*"
52
+ - if [ -n "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" ]; then git fetch --quiet origin "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME"; fi
53
+ - bundle install
54
+ script:
55
+ - bundle exec duel --markdown duel.md
56
+ artifacts:
57
+ when: always
58
+ paths:
59
+ - <%= working_directory ? "#{working_directory}/" : "" %>duel.md
60
+ reports:
61
+ dotenv: <%= working_directory ? "#{working_directory}/" : "" %>duel.env
62
+ expire_in: 1 day
63
+
64
+ # The report, as a new comment on the merge request, when Duel says to:
65
+ # always (post), or only when its last comment's status differs from this
66
+ # run's (change), so the latest comment is never stale. It never edits or
67
+ # deletes a comment.
68
+ <%= gitlab_job %>:comment:
69
+ image: alpine:3
70
+ stage: .post
71
+ needs: [<%= gitlab_job %>]
72
+ rules:
73
+ - if: $DUEL_COMMENT_TOKEN && $CI_PIPELINE_SOURCE == "merge_request_event"
74
+ when: always
75
+ variables:
76
+ GIT_STRATEGY: none # it runs none of the code
77
+ script:
78
+ - apk add --no-cache --quiet curl jq
79
+ - |
80
+ case "$DUEL_COMMENT" in post|change) ;; *) exit 0 ;; esac
81
+ report=<%= working_directory ? "#{working_directory}/" : "" %>duel.md
82
+ [ -f "$report" ] || exit 0
83
+ api="$CI_API_V4_URL/projects/$CI_PROJECT_ID"
84
+ token="PRIVATE-TOKEN: $DUEL_COMMENT_TOKEN"
85
+ marker="<!-- <%= marker %>"
86
+ me=$(curl -sf -H "$token" "$CI_API_V4_URL/user" | jq .id)
87
+ # The first line of the last comment Duel posted here: its marker,
88
+ # with its status.
89
+ last=$(curl -sf -H "$token" "$api/merge_requests/$CI_MERGE_REQUEST_IID/notes?sort=desc&order_by=created_at&per_page=100" |
90
+ jq -r --argjson me "$me" --arg marker "$marker status=" '.[] | select(.author.id == $me) | .body | split("\n")[0] | rtrimstr("\r") | select(startswith($marker))' | head -n 1)
91
+ if [ "$DUEL_COMMENT" = post ] || { [ -n "$last" ] && [ "$last" != "$marker status=$DUEL_STATUS -->" ]; }; then
92
+ { echo "$marker status=$DUEL_STATUS -->"; cat "$report"; } | jq -Rs '{body: .}' |
93
+ curl -sf -H "$token" -H "Content-Type: application/json" --data @- "$api/merge_requests/$CI_MERGE_REQUEST_IID/notes" > /dev/null
94
+ fi
95
+
96
+ # The report, as a new comment on a pushed commit, when Duel says to, as
97
+ # for a merge request: only where a regression in a push passes (see
98
+ # c.fail_on), so the comment says what the check doesn't. Its last comment
99
+ # is the commit before's.
100
+ <%= gitlab_job %>:commit-comment:
101
+ image: alpine:3
102
+ stage: .post
103
+ needs: [<%= gitlab_job %>]
104
+ rules:
105
+ - if: $DUEL_TOKEN && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
106
+ when: always
107
+ variables:
108
+ GIT_STRATEGY: none # it runs none of the code
109
+ script:
110
+ - apk add --no-cache --quiet curl jq
111
+ - |
112
+ case "$DUEL_COMMENT" in post|change) ;; *) exit 0 ;; esac
113
+ report=<%= working_directory ? "#{working_directory}/" : "" %>duel.md
114
+ [ -f "$report" ] || exit 0
115
+ api="$CI_API_V4_URL/projects/$CI_PROJECT_ID"
116
+ token="PRIVATE-TOKEN: $DUEL_TOKEN"
117
+ marker="<!-- <%= marker %>"
118
+ me=$(curl -sf -H "$token" "$CI_API_V4_URL/user" | jq .id)
119
+ last=$(curl -sf -H "$token" "$api/repository/commits/$CI_COMMIT_BEFORE_SHA/comments?per_page=100" |
120
+ jq -r --argjson me "$me" --arg marker "$marker status=" '.[] | select(.author.id == $me) | .note | split("\n")[0] | rtrimstr("\r") | select(startswith($marker))' | tail -n 1)
121
+ if [ "$DUEL_COMMENT" = post ] || { [ -n "$last" ] && [ "$last" != "$marker status=$DUEL_STATUS -->" ]; }; then
122
+ { echo "$marker status=$DUEL_STATUS -->"; cat "$report"; } | jq -Rs '{note: .}' |
123
+ curl -sf -H "$token" -H "Content-Type: application/json" --data @- "$api/repository/commits/$CI_COMMIT_SHA/comments" > /dev/null
124
+ fi
125
+
126
+ # With c.push_base = :last_passed in .duel.rb, a push is compared with the
127
+ # last commit Duel passed, which a branch marks (<%= passed_branch %>/main for
128
+ # main), so slowdowns too small for any one push to fail add up until they
129
+ # fail one. This job moves the branch to a push that passed (or makes it),
130
+ # but never back: where a newer push's job moved it first, it stays. To
131
+ # accept a regression, move the branch yourself (for main, git push
132
+ # origin main:<%= passed_branch %>/main).
133
+ <%= gitlab_job %>:passed:
134
+ image: alpine:3
135
+ stage: .post
136
+ needs: [<%= gitlab_job %>]
137
+ rules:
138
+ - if: $DUEL_TOKEN && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
139
+ variables:
140
+ GIT_STRATEGY: none # it runs none of the code
141
+ script:
142
+ - apk add --no-cache --quiet curl jq
143
+ - |
144
+ [ "$DUEL_PASSED" = true ] || exit 0
145
+ api="$CI_API_V4_URL/projects/$CI_PROJECT_ID"
146
+ token="PRIVATE-TOKEN: $DUEL_TOKEN"
147
+ branch="<%= passed_branch %>/$CI_COMMIT_BRANCH"
148
+ at=$(curl -sf -H "$token" "$api/repository/branches/$(echo "$branch" | sed 's|/|%2F|g')" | jq -r '.commit.id // empty') || at=
149
+ if [ -n "$at" ]; then
150
+ base=$(curl -sf -H "$token" "$api/repository/merge_base?refs%5B%5D=$CI_COMMIT_SHA&refs%5B%5D=$at" | jq -r '.id // empty') || base=
151
+ [ -n "$base" ] || { echo "Couldn't compare $branch with this push" && exit 1; }
152
+ # (A newer push's job moved it, or this one's did, run again.)
153
+ [ "$base" = "$CI_COMMIT_SHA" ] && exit 0
154
+ fi
155
+ curl -s -o /dev/null -H "$token" -X DELETE "$api/repository/branches/$(echo "$branch" | sed 's|/|%2F|g')"
156
+ curl -sf -o /dev/null -H "$token" --data-urlencode "branch=$branch" --data-urlencode "ref=$CI_COMMIT_SHA" "$api/repository/branches" && exit 0
157
+ # (Git can't make a branch where another's name is a folder of its
158
+ # name: a branch named duel blocks duel/passed/main. If one does,
159
+ # this names it.)
160
+ prefix=$branch
161
+ while [ "${prefix%/*}" != "$prefix" ]; do
162
+ prefix=${prefix%/*}
163
+ if curl -sf -o /dev/null -H "$token" "$api/repository/branches/$(echo "$prefix" | sed 's|/|%2F|g')"; then
164
+ echo "The branch $prefix keeps Duel from making $branch, as Git can't have both. Delete it (git push origin --delete $prefix), or rename it."
165
+ fi
166
+ done
167
+ exit 1
@@ -37,4 +37,17 @@ Duel.configure do |c|
37
37
  # :results (passed or failed), :failures (failed, or couldn't compare),
38
38
  # or :never.
39
39
  # c.comment = :failures
40
+
41
+ # Which runs a regression fails: :every_run, or :pull_requests, for a
42
+ # project where people push to main directly (on GitHub Actions, another
43
+ # run warns and passes, and a pushed commit gets the comment instead).
44
+ # c.fail_on = :every_run
45
+
46
+ # A label that accepts a regression in a pull request (nil for none).
47
+ # c.accept_label = "duel: accept"
48
+
49
+ # What the workflow compares a push with: :before (the commit before it),
50
+ # or :last_passed (the last commit Duel passed, which a branch,
51
+ # duel/passed/main, marks), so small slowdowns add up until they fail.
52
+ # c.push_base = :before
40
53
  end
@@ -0,0 +1,59 @@
1
+ name: Perf comment<%= " (#{working_directory})" if working_directory %>
2
+
3
+ # Duel's report, as a new comment on the pull request, when Duel says to:
4
+ # always (post), or only when its last comment's status differs from the
5
+ # run's (change), so the latest comment is never stale. It never edits or
6
+ # deletes a comment. It runs once the Perf workflow has, with this
7
+ # repository's own workflow and a token that can write, so a pull request
8
+ # from a fork gets a comment too. It checks nothing out and runs none of
9
+ # the pull request's code, and of what the Duel job kept (where that code
10
+ # ran), it trusts only what that job could have said: the pull request's
11
+ # number, if it names the pull request the run compared, and the comment
12
+ # and the status, if they're ones Duel says. (The workflow must be on the
13
+ # default branch before it runs at all.)
14
+ on:
15
+ workflow_run:
16
+ workflows: ["Perf<%= " (#{working_directory})" if working_directory %>"]
17
+ types: [completed]
18
+
19
+ permissions: {}
20
+
21
+ jobs:
22
+ comment:
23
+ name: Comment
24
+ if: ${{ github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.conclusion != 'cancelled' }}
25
+ runs-on: ubuntu-latest
26
+ permissions:
27
+ actions: read # to get the report from the Perf run
28
+ pull-requests: write # to comment on the pull request with the report
29
+ steps:
30
+ - name: Get the report
31
+ id: report
32
+ continue-on-error: true # there's none when the Duel job failed before comparing
33
+ uses: actions/download-artifact@v8
34
+ with:
35
+ name: duel-report
36
+ path: ${{ runner.temp }}/report
37
+ run-id: ${{ github.event.workflow_run.id }}
38
+ github-token: ${{ github.token }}
39
+ - name: Comment on the pull request
40
+ if: steps.report.outcome == 'success'
41
+ env:
42
+ GH_TOKEN: ${{ github.token }}
43
+ HEAD_SHA: ${{ github.event.workflow_run.head_sha }}
44
+ run: |
45
+ said() { grep -xE "$1=($2)" "$RUNNER_TEMP/report/comment.env" | tail -n 1 | cut -d= -f2; }
46
+ PR=$(said pr '[0-9]+')
47
+ COMMENT=$(said comment 'post|change')
48
+ STATUS=$(said status 'passed|failed|neutral')
49
+ [ -n "$PR" ] && [ -n "$COMMENT" ] && [ -n "$STATUS" ] || exit 0
50
+ [ "$(gh api "repos/$GITHUB_REPOSITORY/pulls/$PR" --jq .head.sha)" = "$HEAD_SHA" ] || exit 0
51
+ marker="<!-- <%= marker %>"
52
+ # The first line of the last comment Duel posted here: its marker,
53
+ # with its status (none in Duel 0.1.0's, which counts as another).
54
+ last=$(gh api --paginate "repos/$GITHUB_REPOSITORY/issues/$PR/comments" \
55
+ --jq ".[] | select(.user.login == \"github-actions[bot]\") | .body | split(\"\n\")[0] | rtrimstr(\"\r\") | select(startswith(\"$marker status=\") or . == \"$marker -->\")" | tail -n 1)
56
+ if [ "$COMMENT" = post ] || { [ -n "$last" ] && [ "$last" != "$marker status=$STATUS -->" ]; }; then
57
+ { echo "$marker status=$STATUS -->"; cat "$RUNNER_TEMP/report/duel.md"; } > "$RUNNER_TEMP/comment.md"
58
+ gh api --method POST "repos/$GITHUB_REPOSITORY/issues/$PR/comments" -F body=@"$RUNNER_TEMP/comment.md" > /dev/null
59
+ fi