duel 0.1.2 → 0.1.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/lib/duel/config.rb CHANGED
@@ -31,6 +31,14 @@ module Duel
31
31
  # Where the project is (and its config file).
32
32
  attr_reader :root
33
33
 
34
+ # Where Duel keeps what it keeps for the project, tmp/duel: the lock
35
+ # that keeps two runs from sharing it (see Lock), what prepare blocks
36
+ # build once (Resource#cached), in cache/, and the baseline's checkout
37
+ # where it can't go next to the repository (see Baseline#checkout), in
38
+ # base/. A run removes the lock when it ends, and what that leaves
39
+ # empty. `duel --clean` removes the rest.
40
+ def state = File.join(root, "tmp", "duel")
41
+
34
42
  # A glob, from the root, of the files defining benchmarks.
35
43
  attr_accessor :benchmarks
36
44
 
@@ -312,18 +320,17 @@ module Duel
312
320
  # stopped when it's done with. The block returns the environment
313
321
  # variables it gets: a worker's, under its version's.
314
322
  #
315
- # A version's (named "baseline" or "candidate", its #root the
316
- # version's tree and its #ruby the Ruby it runs on) is prepared once
317
- # for a run, in the runner, before any of its workers start:
318
- # installing its bundle, say. Its environment is its workers' (and
319
- # its process benchmarks').
323
+ # A version's (named "base" or "head", its #root the version's tree
324
+ # and its #ruby the Ruby it runs on) is prepared once for a run, in
325
+ # the runner, before any of its workers start: installing its bundle,
326
+ # say. Its environment is its workers' (and its process benchmarks').
320
327
  #
321
328
  # c.prepare(:version) do |v|
322
329
  # v.cached("bundle-#{Digest::SHA256.file(File.join(v.root, "Gemfile.lock"))}") { |_| v.run("bundle", "install", chdir: v.root) }
323
330
  # {"BUNDLE_GEMFILE" => File.join(v.root, "Gemfile")}
324
331
  # end
325
332
  #
326
- # A worker's (named "baseline-1", say) is prepared by the runner as
333
+ # A worker's (named "base-1", say) is prepared by the runner as
327
334
  # the worker starts: a server of its own, say.
328
335
  #
329
336
  # c.prepare(:worker) do |w|
data/lib/duel/context.rb CHANGED
@@ -498,16 +498,17 @@ module Duel
498
498
  end
499
499
 
500
500
  # A stretch of a benchmark run for its peak memory: with the garbage
501
- # collector working as usual, the most resident memory the process
502
- # reached above where it started (after a full collection), watched
503
- # every millisecond.
501
+ # collector working as usual, the most memory the process had in use
502
+ # (Native::Memory.used: resident memory, or on JRuby, the JVM's heap)
503
+ # above where it started (after a full collection), watched every
504
+ # millisecond.
504
505
  # @api private
505
506
  class MemorySample
506
507
  attr_reader :started, :stopped, :peak
507
508
 
508
509
  def start
509
510
  @started = true
510
- GC.start(full_mark: true, immediate_sweep: true)
511
+ Native::Memory.collect
511
512
  @base = resident
512
513
  @peak = 0
513
514
  @watcher = Thread.new do
@@ -570,8 +571,8 @@ module Duel
570
571
  end
571
572
 
572
573
  def resident
573
- # (Resident memory is known: see Lane#peak_memory.)
574
- Native::Memory.resident #: Integer
574
+ # (The memory in use is known: see Lane#peak_memory.)
575
+ Native::Memory.used #: Integer
575
576
  end
576
577
 
577
578
  def watch
data/lib/duel/init.rb CHANGED
@@ -249,5 +249,9 @@ module Duel
249
249
  # What names the branches that mark the last commit Duel passed in a
250
250
  # push, before the branch pushed's name (see Baseline#passed_branch).
251
251
  def passed_branch = ["duel/passed", working_directory&.tr("/", "-")].compact.join("-")
252
+
253
+ # The branch that holds the workflow's badge: duel/badge, or for a
254
+ # project in a subdirectory, duel/badge-<its path>.
255
+ def badge_branch = ["duel/badge", working_directory&.tr("/", "-")].compact.join("-")
252
256
  end
253
257
  end
@@ -22,7 +22,7 @@ module Duel
22
22
  none = [] #: Array[value]
23
23
  nothing = {} #: report
24
24
  head(status, exit, message, settings, comment).merge("seconds" => nil, "baseline" => nil, "candidate" => nil, "plugins" => none,
25
- "benchmarks" => none, "across" => nothing, "errors" => none)
25
+ "benchmarks" => none, "across" => nothing, "errors" => none, "cut_short" => nil)
26
26
  end
27
27
 
28
28
  # What every report starts with.
@@ -30,7 +30,7 @@ module Duel
30
30
  {"schema" => SCHEMA, "duel_version" => VERSION, "status" => status, "exit" => exit, "message" => message, "comment" => comment,
31
31
  "settings" => {"confidence" => settings.confidence, "limit" => settings.limit, "filter" => settings.filter.source,
32
32
  "jobs" => settings.jobs, "processes" => settings.processes, "timeout" => settings.timeout,
33
- "warm" => settings.warm, "force" => settings.force}}
33
+ "time_limit" => settings.time_limit, "warm" => settings.warm, "force" => settings.force}}
34
34
  end
35
35
 
36
36
  # +runner+ ran, and made +results+, comparing +baseline+ with
@@ -54,7 +54,8 @@ module Duel
54
54
  "plugins" => @runner.plugins.map(&:to_s),
55
55
  "benchmarks" => @results.map { |result| benchmark(result) },
56
56
  "across" => across,
57
- "errors" => @results.select(&:error).map { |result| {"benchmark" => result.name, "message" => result.error} }
57
+ "errors" => @results.select(&:error).map { |result| {"benchmark" => result.name, "message" => result.error} },
58
+ "cut_short" => @runner.cut_short
58
59
  )
59
60
  end
60
61
 
data/lib/duel/lock.rb CHANGED
@@ -8,7 +8,9 @@ module Duel
8
8
  # baseline's checkout, each checking out and building over the other's.
9
9
  # A run holds an flock on tmp/duel/lock, which ends with its process
10
10
  # however that ends, so a run that crashed leaves nothing to clear. The
11
- # file says whose it is, for a run that finds it taken.
11
+ # file says whose it is, for a run that finds it taken. A run that ends
12
+ # removes it, and the directories that leaves empty: tmp/duel, and tmp
13
+ # if Duel made it.
12
14
  # @api private
13
15
  module Lock
14
16
  # An exclusive lock, or none if another holds it.
@@ -17,21 +19,63 @@ module Duel
17
19
  # Runs the block holding +dir+'s lock. Fails, naming the run that
18
20
  # holds it, if another does.
19
21
  def self.hold(dir)
22
+ made = missing(dir)
20
23
  FileUtils.mkdir_p(dir)
21
- # (Read from the start, and written at the end: there's no more.)
22
- file = File.new(File.join(dir, "lock"), "a+")
24
+ path = File.join(dir, "lock")
25
+ file = acquire(path, dir)
23
26
  begin
24
- unless file.flock(EXCLUSIVE)
25
- # (Empty for a moment, while its run writes its pid.)
26
- pid = file.read
27
- raise Error, "another duel run#{" (pid #{pid})" unless pid.empty?} is using #{dir}: wait for it to end, or stop it"
28
- end
29
27
  file.truncate(0)
30
28
  file.write(Process.pid)
31
29
  file.flush
32
30
  yield
33
31
  ensure
32
+ # (Removed while it's held, so a run that opens it after makes a new
33
+ # one. `duel --clean` may have removed it already.)
34
+ FileUtils.rm_f(path)
34
35
  file.close
36
+ tidy(dir, made)
37
+ end
38
+ end
39
+
40
+ # The lock file at +path+, locked: the one there now (a run that
41
+ # ended may have removed the one this opened, after this opened it).
42
+ def self.acquire(path, dir)
43
+ loop do
44
+ # (Read from the start, and written at the end: there's no more.)
45
+ file = File.new(path, "a+")
46
+ unless file.flock(EXCLUSIVE)
47
+ # (Empty for a moment, while its run writes its pid.)
48
+ pid = file.read
49
+ file.close
50
+ raise Error, "another duel run#{" (pid #{pid})" unless pid.empty?} is using #{dir}: wait for it to end, or stop it"
51
+ end
52
+ return file if File.exist?(path) && File.identical?(path, file)
53
+
54
+ file.close
55
+ end
56
+ end
57
+
58
+ # The directories, +dir+ and those it's in, that aren't there yet,
59
+ # deepest first.
60
+ def self.missing(dir)
61
+ made = [] #: Array[String]
62
+ path = dir
63
+ until File.exist?(path)
64
+ made << path
65
+ path = File.dirname(path)
66
+ end
67
+ made
68
+ end
69
+
70
+ # Removes the empty directories in +dir+ (cache/, say: not what's in
71
+ # them, nor in a baseline's checkout there), then +dir+, then those of
72
+ # +made+ (Duel made them), where they're empty. (One that isn't empty
73
+ # holds those it's in.)
74
+ def self.tidy(dir, made)
75
+ [*Dir.glob(File.join(dir, "*/")), dir, *made].each do |path|
76
+ Dir.rmdir(path)
77
+ rescue SystemCallError # (not empty, or gone already)
78
+ nil
35
79
  end
36
80
  end
37
81
  end
@@ -0,0 +1,25 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Duel
4
+ # What the runner and a worker say to each other, over the worker's
5
+ # socket: plain Hashes, Arrays, Strings, numbers, true, false and nil,
6
+ # marshaled. Not JSON: the worker would load json before it loads the
7
+ # version it measures, and in json's own repository, every baseline
8
+ # worker would load the candidate's json first, and the baseline's
9
+ # over it. Marshal is Ruby's own, with nothing to load (so a version's
10
+ # code is the first of it the worker loads), and it keeps a String's
11
+ # encoding. (Both ends are Duel's own processes, and the socket is in
12
+ # the run's own directory.)
13
+ # @api private
14
+ module Message
15
+ # Sends +message+ over +io+.
16
+ def self.write(io, message) = io.write(Marshal.dump(message))
17
+
18
+ # The next message over +io+, or nil once the other end has closed it.
19
+ def self.read(io)
20
+ Marshal.load(io)
21
+ rescue EOFError
22
+ nil
23
+ end
24
+ end
25
+ end
data/lib/duel/native.rb CHANGED
@@ -1,7 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require "etc"
4
- require "fiddle"
5
3
  require "rbconfig"
6
4
 
7
5
  module Duel
@@ -16,9 +14,19 @@ module Duel
16
14
  CPU = RbConfig::CONFIG["host_cpu"]
17
15
  UINT64 = 8 # bytes, the size of every count read
18
16
 
17
+ # Loads Fiddle, as the first C function is made (see .function): not
18
+ # as Duel loads, which in a worker is before its version (in fiddle's
19
+ # own repository, fiddle).
20
+ # mutant:disable -- the tests' process has Fiddle loaded from the start,
21
+ # so a require changes nothing there: WorkerTest's
22
+ # test_loads_no_gem_a_version_could_be_before_the_version checks it, in
23
+ # a process of its own
24
+ def self.require_fiddle = require("fiddle")
25
+
19
26
  # A C function of the process's own libraries, by name, taking and
20
27
  # returning Fiddle types (:INT, :LONG, :VOIDP...).
21
28
  def self.function(name, args, returns)
29
+ require_fiddle
22
30
  Fiddle::Function.new(Fiddle.dlopen(nil)[name], args.map { |type| Fiddle.const_get(:"TYPE_#{type}") },
23
31
  Fiddle.const_get(:"TYPE_#{returns}"))
24
32
  end
@@ -145,18 +153,57 @@ module Duel
145
153
 
146
154
  # This process's resident memory in bytes, or nil where it can't tell.
147
155
  module Memory
148
- STATM = "/proc/self/statm"
149
- # (Never false: Steep doesn't see that false || nil is nil.)
150
- MACOS = (FIDDLE && MacOS.open) || nil #: MacOS?
156
+ # (Its VmRSS line, in kB: no page size to know, as /proc/self/statm's
157
+ # pages need, which would take Etc, a gem a worker would load before
158
+ # its version.)
159
+ STATUS = "/proc/self/status"
160
+ # macOS's counter, which reads resident memory there, or nil: opened
161
+ # as resident memory is first read, as opening it loads Fiddle.
162
+ # mutant:disable -- kept only so as not to open it again: it's the
163
+ # same counter however often it's opened
164
+ def self.macos_counter
165
+ return @macos if defined?(@macos)
166
+
167
+ # (Never false: Steep doesn't see that false || nil is nil.)
168
+ @macos = (FIDDLE && MacOS.open) || nil #: MacOS?
169
+ end
151
170
 
152
171
  # From /proc (Linux), else from macOS's counter, else nil.
153
- def self.resident(statm: STATM, macos: MACOS)
154
- return macos&.resident unless File.readable?(statm)
172
+ def self.resident(status: STATUS, macos: macos_counter)
173
+ return macos&.resident unless File.readable?(status)
174
+
175
+ kb = File.read(status)[/^VmRSS:\s+(\d+) kB$/, 1] #: String
176
+ Integer(kb) * 1024
177
+ end
178
+
179
+ # The memory a benchmark's peak is measured in (see MemorySample),
180
+ # in bytes, or nil where it can't tell: resident memory, but on
181
+ # JRuby, the JVM's heap in use.
182
+ def self.used = resident
183
+
184
+ # Collects every object that's garbage, so that what's in use (see
185
+ # .used) is what's live: a full collection, swept at once.
186
+ def self.collect = GC.start(full_mark: true, immediate_sweep: true)
187
+ end
155
188
 
156
- # (The page size is always known.)
157
- page_size = Etc.sysconf(Etc::SC_PAGESIZE) #: Integer
158
- Integer(File.read(statm).split.fetch(1)) * page_size
189
+ # simplecov:disable JRuby's heap is the JVM's (JRuby's test run covers
190
+ # this).
191
+ if RUBY_ENGINE == "jruby"
192
+ module Memory
193
+ # The JVM takes memory for its heap ahead of time, and keeps what it
194
+ # used while it started, so resident memory barely moves as a
195
+ # benchmark's objects fill it: 24 MB more of them can show as none.
196
+ # The heap in use shows them.
197
+ HEAP = java.lang.management.ManagementFactory.getMemoryMXBean
198
+
199
+ def self.used = HEAP.getHeapMemoryUsage.getUsed
200
+
201
+ # JRuby's GC.start collects nothing. The JVM's own call does:
202
+ # without it, a peak was measured above garbage that the benchmark's
203
+ # objects then made room for, and came out tens of megabytes short.
204
+ def self.collect = java.lang.System.gc
159
205
  end
160
206
  end
207
+ # simplecov:enable
161
208
  end
162
209
  end
data/lib/duel/resource.rb CHANGED
@@ -42,7 +42,36 @@ module Duel
42
42
  # @api private
43
43
  CACHE = Mutex.new
44
44
 
45
- # The worker it's for ("baseline-1", say), or the version ("baseline",
45
+ # What a cache key names, without the version and the hash it ends
46
+ # with: "mysql", of mysql-0.1.2-3f9c2a1b7e5d4c6a. Builds of one name
47
+ # are of one thing, and the one a run used replaces the rest (see
48
+ # .evict).
49
+ # @api private
50
+ NAME = /\A(.+?)(?:-\d[\w.]*)?-\h{8,}\z/
51
+
52
+ # The builds this process's runs used (#cached), by path.
53
+ # @api private
54
+ USED = Set.new
55
+
56
+ # Removes from +cache+ the builds that ones this process used replace:
57
+ # of the same name (see NAME), with another version or hash. A server
58
+ # or a plugin upgraded, or a lockfile changed, left its old build
59
+ # behind otherwise: tens of megabytes of a database's files, each
60
+ # time. A key that doesn't end with a hash is never removed, nor is
61
+ # any that this process used (each version's bundle, say). (Once a
62
+ # run's workers have started: none adds to USED then.)
63
+ # @api private
64
+ def self.evict(cache)
65
+ used = USED.select { |dir| File.dirname(dir) == cache }.map { |dir| File.basename(dir) }
66
+ names = used.filter_map { |key| key[NAME, 1] }
67
+ Dir.children(cache).each do |build|
68
+ FileUtils.rm_rf(File.join(cache, build)) if !used.include?(build) && names.include?(build[NAME, 1])
69
+ end
70
+ rescue Errno::ENOENT # (no cache)
71
+ nil
72
+ end
73
+
74
+ # The worker it's for ("base-1", say), or the version ("base",
46
75
  # for Config#prepare(:version)), the directory it has of its own, the
47
76
  # tree of the version that worker runs, and the Ruby it runs on (see
48
77
  # Config#ruby).
@@ -88,11 +117,17 @@ module Duel
88
117
  # block is given, and the rest get it built. A key names what's built
89
118
  # (put a version in it), as a file name: tmp/duel/cache/+key+, which
90
119
  # `duel --clean` removes. A build that fails leaves nothing behind.
120
+ # A key that ends with a hash (and a version before it, if you like:
121
+ # mysql-0.1.2-3f9c2a1b) replaces the builds of its name when the run
122
+ # ends (see .evict): the one a run used is the one that's kept.
91
123
  def cached(key)
92
124
  raise ArgumentError, "a cache key is a file name (letters, digits, - _ .), not #{key.inspect}" unless key.match?(KEY)
93
125
 
94
126
  dir = File.join(@cache, key)
95
- CACHE.synchronize { build(dir) { |building| yield building } unless File.directory?(dir) }
127
+ CACHE.synchronize do
128
+ USED << dir
129
+ build(dir) { |building| yield building } unless File.directory?(dir)
130
+ end
96
131
  dir
97
132
  end
98
133
 
data/lib/duel/runner.rb CHANGED
@@ -1,7 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require "fileutils"
4
- require "json"
5
4
  require "rbconfig"
6
5
  require "socket"
7
6
  require "tmpdir"
@@ -9,6 +8,7 @@ require_relative "../duel"
9
8
  require_relative "stats"
10
9
  require_relative "profile"
11
10
  require_relative "across"
11
+ require_relative "message"
12
12
  require_relative "resource"
13
13
  require_relative "rubies"
14
14
  require_relative "process_sample"
@@ -27,13 +27,17 @@ module Duel
27
27
  # @api private
28
28
  YJIT_WARM = 2.0
29
29
 
30
+ # The seconds a run measures for, at most, by default (see
31
+ # Settings#time_limit and Runner#out_of_time?).
32
+ TIME_LIMIT = 1800.0
33
+
30
34
  # How a run is set up (see Duel::CLI for where each comes from).
31
35
  # @api private
32
36
  Settings = Struct.new(:base, :base_dir, :candidate_dir, :filter, :jobs, :processes, :limit, :confidence, :warm, :force,
33
- :config_path, :timeout, :ruby, :comment, :fail_on, :pushed, :target) do
37
+ :config_path, :timeout, :ruby, :comment, :fail_on, :pushed, :target, :time_limit) do
34
38
  # @type self: singleton(Settings)
35
39
  def self.defaults = new(filter: //, jobs: 1, processes: 3, limit: 1.02, confidence: 0.99, warm: nil, force: false,
36
- timeout: 60.0)
40
+ timeout: 60.0, time_limit: TIME_LIMIT)
37
41
  end
38
42
 
39
43
  # One version as a run has it (see Runner#prepare), +name+d "baseline"
@@ -51,6 +55,21 @@ module Duel
51
55
  end
52
56
  end
53
57
 
58
+ class Side
59
+ # What each version's workers, their directories and their sockets
60
+ # are named for (see #label): "base" and "head", as long as each
61
+ # other, so that the two versions' workers get arguments and
62
+ # environments of the same length. Where a process's stack starts
63
+ # depends on how long they are, and at some lengths, the same code
64
+ # runs the same instructions in more cycles: a version whose workers'
65
+ # were a byte longer could measure slower in every one.
66
+ LABELS = {"baseline" => "base", "candidate" => "head"}.freeze
67
+
68
+ # What its workers, and its version's resources, are named for: "base"
69
+ # or "head".
70
+ def label = LABELS.fetch(name)
71
+ end
72
+
54
73
  # A worker process (Duel::Worker) running one version of the code.
55
74
  # @api private
56
75
  class WorkerProcess
@@ -66,9 +85,10 @@ module Duel
66
85
  attr_reader :label, :benchmarks, :metrics, :memory, :profiler, :yjit
67
86
 
68
87
  # The Ruby it runs on (RUBY_DESCRIPTION), the native extensions it
69
- # loaded (see Worker.extensions), and the directories gems are
70
- # installed in (see Worker.gem_dirs).
71
- attr_reader :description, :extensions, :gem_dirs
88
+ # loaded (see Worker.extensions), those it loaded twice (see
89
+ # Worker.twice), and the directories gems are installed in (see
90
+ # Worker.gem_dirs).
91
+ attr_reader :description, :extensions, :twice, :gem_dirs
72
92
 
73
93
  # Where a worker on +ruby+ loads Duel from, ahead of anything else
74
94
  # (a version's own bundle, which may not have Duel, or an older one):
@@ -109,13 +129,15 @@ module Duel
109
129
  @yjit = hello.fetch("yjit")
110
130
  @description = hello.fetch("ruby")
111
131
  @extensions = hello.fetch("extensions")
132
+ @twice = hello.fetch("twice")
112
133
  @gem_dirs = hello.fetch("gem_dirs")
113
134
  end
114
135
 
115
136
  # Sends a request, and returns the reply, or raises Failed with its
116
137
  # error.
117
138
  def request(op, name, **args)
118
- @socket.puts JSON.generate({"op" => op, "name" => name, **args})
139
+ # (By name: +args+ come by keyword, as Symbols.)
140
+ Message.write(@socket, {"op" => op, "name" => name, **args.transform_keys(&:to_s)})
119
141
  reply = read
120
142
  raise Failed, "#{@label}: #{reply.fetch("error")}\n #{reply.fetch("backtrace").join("\n ")}" if reply.key?("error")
121
143
 
@@ -166,8 +188,7 @@ module Duel
166
188
 
167
189
  def read
168
190
  hung("answer") unless @socket.wait_readable(@timeout)
169
- line = @socket.gets or raise Failed, "#{@label} worker exited"
170
- JSON.parse(line)
191
+ Message.read(@socket) or raise Failed, "#{@label} worker exited"
171
192
  end
172
193
 
173
194
  # Kills a worker that didn't +what+ within the timeout, and fails.
@@ -324,12 +345,12 @@ module Duel
324
345
  # A worker of +side+, with its own resources, and its version's Ruby
325
346
  # and environment.
326
347
  def spawn(side, index)
327
- label = "#{side}-#{index + 1}"
328
348
  version = @sides.fetch(side)
349
+ label = "#{version.label}-#{index + 1}"
329
350
  env = version.env
330
351
  @config.prepares.fetch(:worker).each_with_index do |block, i|
331
352
  resource = Resource.new(File.join(@dir, "#{label}-#{i + 1}"), label, version.root, timeout: @settings.timeout,
332
- cache: File.join(@config.root, "tmp", "duel", "cache"), ruby: version.ruby, env: version.env)
353
+ cache: File.join(@config.state, "cache"), ruby: version.ruby, env: version.env)
333
354
  keep(@resources, resource)
334
355
  env = Resource.environment(label, env, block.call(resource))
335
356
  end
@@ -347,10 +368,17 @@ module Duel
347
368
  # be of its code. An installed gem's extension (in the directories
348
369
  # gems are installed in, by the worker that loaded it) is in no
349
370
  # version's tree, even in a bundle there (vendor/bundle, say, which
350
- # both versions may load from).
371
+ # both versions may load from). And fails if a version's workers
372
+ # loaded one twice (see Worker.twice): the candidate's, that something
373
+ # required before Duel loaded the version, and then its own over it.
351
374
  def check_builds
352
375
  workers = {"baseline" => @bases.first, "candidate" => @candidates.first}
353
376
  workers.each do |side, worker|
377
+ worker.twice.each do |feature, paths|
378
+ raise Failed, "the #{side} loaded #{feature} twice, from #{paths.join(" and ")}: something required it before " \
379
+ "Duel loaded the #{side}, and both stay loaded. Require what loads it in a block that runs once the version " \
380
+ "has loaded (c.in_worker, in .duel.rb), not as .duel.rb or a plugin loads"
381
+ end
354
382
  other = (side == "baseline") ? "candidate" : "baseline"
355
383
  theirs = workers.fetch(other).then { |w| w.extensions.select { |_, path| inside?(path, other, w) } }
356
384
  worker.extensions.each do |feature, path|
@@ -851,23 +879,41 @@ module Duel
851
879
  # seconds the run took. Set by #run.
852
880
  attr_reader :rubies, :seconds
853
881
 
882
+ # What the run left undone when it reached its time limit
883
+ # (Settings#time_limit): {"unmeasured" => the benchmarks it didn't
884
+ # measure, "unconfirmed" => those it judged without every
885
+ # measurement again (see #confirm), "across" => whether it judged
886
+ # across the board without measuring every benchmark again (see
887
+ # #judge_across)}, or nil where it reached none. Set by #run.
888
+ def cut_short
889
+ return if @unmeasured.empty? && @unconfirmed.empty? && !@across_cut
890
+
891
+ {"unmeasured" => @unmeasured, "unconfirmed" => @unconfirmed, "across" => @across_cut}
892
+ end
893
+
854
894
  def initialize(config, settings, base_root, out: $stdout)
855
895
  @config = config
856
896
  @settings = settings
857
897
  @base_root = base_root
858
898
  @out = out
859
899
  @prepared = []
900
+ @started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
901
+ @lock = Mutex.new
902
+ @unmeasured = [] #: Array[String]
903
+ @unconfirmed = [] #: Array[String]
904
+ @across_cut = false
860
905
  end
861
906
 
862
907
  # Returns the Results.
863
908
  def run
864
- started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
909
+ started = @started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
865
910
  @dir = Lane.tmpdir
866
911
  begin
867
912
  compare_all(started)
868
913
  ensure
869
914
  @prepared.reverse_each(&:stop)
870
915
  FileUtils.rm_rf(@dir)
916
+ Resource.evict(File.join(@config.state, "cache"))
871
917
  end
872
918
  end
873
919
 
@@ -877,6 +923,27 @@ module Duel
877
923
 
878
924
  def regressed_across = @across.select { |_, v| v["confirmed"] }.keys
879
925
 
926
+ # What a run cut short at its time limit left undone (see #cut_short),
927
+ # or nil where it wasn't.
928
+ def cut_short_note
929
+ return unless cut_short
930
+
931
+ undone = [("#{Runner.count(@unmeasured.size, "benchmark")} not measured" unless @unmeasured.empty?),
932
+ ("#{Runner.count(@unconfirmed.size, "benchmark")} judged without measuring every one again" unless @unconfirmed.empty?),
933
+ ("across the board judged without measuring again" if @across_cut)].compact
934
+ "the run measured for longer than #{limit_text}: #{undone.join(", ")}. Raise --time-limit, or make the slowest " \
935
+ "benchmarks quicker to settle"
936
+ end
937
+
938
+ # Whether the run has measured for longer than its time limit
939
+ # (Settings#time_limit). Then it starts nothing more: a benchmark it
940
+ # hasn't measured is an error, unmeasured, and what looked worse is
941
+ # judged on the measurements it has, without measuring it again. It
942
+ # fails what still looks worse in all of them, so a run cut short
943
+ # never passes a regression it would have failed. Each measurement it
944
+ # started runs to its end.
945
+ def out_of_time? = Process.clock_gettime(Process::CLOCK_MONOTONIC) - @started > @settings.time_limit
946
+
880
947
  # The plugins the configuration used (Config#plugins), by name.
881
948
  def plugins = @config.plugins
882
949
 
@@ -918,10 +985,10 @@ module Duel
918
985
  side.ruby = Rubies.find(choice, side.root, side.name)
919
986
  side.env = Rubies.environment(side.ruby)
920
987
  @config.prepares.fetch(:version).each_with_index do |block, i|
921
- resource = Resource.new(File.join(@dir, "#{side.name}-#{i + 1}"), side.name, side.root, timeout: @settings.timeout,
922
- cache: File.join(@config.root, "tmp", "duel", "cache"), ruby: side.ruby, env: side.env)
988
+ resource = Resource.new(File.join(@dir, "#{side.label}-#{i + 1}"), side.label, side.root, timeout: @settings.timeout,
989
+ cache: File.join(@config.state, "cache"), ruby: side.ruby, env: side.env)
923
990
  @prepared << resource
924
- side.env = Resource.environment(side.name, side.env, block.call(resource))
991
+ side.env = Resource.environment(side.label, side.env, block.call(resource))
925
992
  end
926
993
  side
927
994
  end
@@ -944,6 +1011,14 @@ module Duel
944
1011
  slower = verdicts.select { |_, v| v.fetch("verdict") == "slower" }.keys
945
1012
  return verdicts if slower.empty?
946
1013
 
1014
+ if out_of_time?
1015
+ # (Judged on the one measurement: slower in it, so slower.)
1016
+ @across_cut = true
1017
+ @out.puts "\nSlower across the board in #{slower.join(", ")}, not measured again: the run reached its time limit (#{limit_text})"
1018
+ slower.each { |metric| verdicts.fetch(metric)["confirmed"] = true }
1019
+ return verdicts
1020
+ end
1021
+
947
1022
  @out.puts "\nMeasuring every benchmark again, in fresh processes: slower across the board in #{slower.join(", ")}?"
948
1023
  again = Across.new(with_lanes { |lanes| measure_all(lanes, names) }, confidence: @settings.confidence).verdicts
949
1024
  slower.each { |metric| verdicts.fetch(metric)["confirmed"] = again.dig(metric, "verdict") == "slower" }
@@ -960,6 +1035,11 @@ module Duel
960
1035
  lanes.each_index.map do |i|
961
1036
  Thread.new do
962
1037
  while (name, index = queue.deq)
1038
+ if out_of_time?
1039
+ results[index] = unmeasured(name)
1040
+ next
1041
+ end
1042
+
963
1043
  result = results[index] = attempt(lanes.fetch(i), name)
964
1044
  @out.print "#{line(result)}\n" # (in one write: other threads write too)
965
1045
  next unless result.error
@@ -979,6 +1059,16 @@ module Duel
979
1059
  Result.new(name).tap { |r| r.error = "#{e.class}: #{e}" }
980
1060
  end
981
1061
 
1062
+ # A benchmark the run reached its time limit before it measured: an
1063
+ # error, so the run fails. (Each lane's thread adds to the list.)
1064
+ # mutant:disable -- CRuby's Array#<< is atomic anyway, and JRuby's isn't
1065
+ def unmeasured(name)
1066
+ @lock.synchronize { @unmeasured << name }
1067
+ Result.new(name).tap { |r| r.error = "not measured: the run reached its time limit (#{limit_text})" }
1068
+ end
1069
+
1070
+ def limit_text = "#{@settings.time_limit.round}s, --time-limit"
1071
+
982
1072
  # Measures each suspect again, from scratch, in fresh processes, until
983
1073
  # one measurement clears it or CONFIRM agree (and once more for each
984
1074
  # too close to call, up to CONFIRM more: see Result#confirm).
@@ -989,6 +1079,12 @@ module Duel
989
1079
  "#{CONFIRM} times each (#{CONFIRM + 1} if what's slower differs, and once more for each too close to call):"
990
1080
  suspects.each do |result|
991
1081
  while result.confirming?
1082
+ if out_of_time?
1083
+ @unconfirmed << result.name
1084
+ @out.puts format(" %-48s judged on %s: the run reached its time limit (%s)", result.name,
1085
+ Runner.count(result.confirmations.size + 1, "measurement"), limit_text)
1086
+ break
1087
+ end
992
1088
  again = Lane.open(@config, @settings, @sides) { |lane| attempt(lane, result.name) }
993
1089
  @out.puts line(again) + (result.confirm(again) ? " (too close to call: once more)" : "")
994
1090
  end
@@ -1019,8 +1115,10 @@ module Duel
1019
1115
  @out.puts format("across the board: %s %.3f [%.3f, %.3f] %s (limit %.4f, %d benchmarks)", metric, v.fetch("median"),
1020
1116
  low, high, v.fetch("verdict"), v.fetch("limit"), v.fetch("benchmarks"))
1021
1117
  end
1022
- regressed_across.each { |metric| @out.puts "REGRESSION across the board: #{metric} slower in both measurements" }
1118
+ both = @across_cut ? "slower, not measured again (the run reached its time limit)" : "slower in both measurements"
1119
+ regressed_across.each { |metric| @out.puts "REGRESSION across the board: #{metric} #{both}" }
1023
1120
  errors.each { |r| @out.puts "ERROR #{r.name}: #{r.error}" }
1121
+ cut_short_note&.then { |note| @out.puts "TIME LIMIT: #{note}" }
1024
1122
  regressed.each do |r|
1025
1123
  @out.puts "REGRESSION #{r.name}: " +
1026
1124
  (r.mixed? ? "slower in every measurement, by one of #{r.regressed.join(", ")}" : "worse #{r.regressed.join(", ")} in every measurement")