duel 0.1.8 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +146 -1
  3. data/README.md +35 -0
  4. data/docs/ci.md +96 -3
  5. data/docs/configuration.md +35 -1
  6. data/docs/how-it-works.md +14 -8
  7. data/docs/plugins.md +16 -0
  8. data/lib/duel/across.rb +1 -6
  9. data/lib/duel/baseline.rb +8 -7
  10. data/lib/duel/cli.rb +45 -19
  11. data/lib/duel/comment.rb +71 -10
  12. data/lib/duel/config.rb +113 -2
  13. data/lib/duel/context.rb +33 -34
  14. data/lib/duel/control.rb +52 -0
  15. data/lib/duel/init.rb +142 -18
  16. data/lib/duel/lane.rb +734 -0
  17. data/lib/duel/lock.rb +34 -4
  18. data/lib/duel/memory.rb +14 -5
  19. data/lib/duel/metric.rb +97 -26
  20. data/lib/duel/native.rb +199 -32
  21. data/lib/duel/platform.rb +31 -0
  22. data/lib/duel/process_sample.rb +74 -49
  23. data/lib/duel/resource.rb +71 -42
  24. data/lib/duel/result.rb +165 -0
  25. data/lib/duel/rubies.rb +21 -3
  26. data/lib/duel/runner.rb +13 -1039
  27. data/lib/duel/spawn.rb +327 -0
  28. data/lib/duel/summary.rb +10 -10
  29. data/lib/duel/templates/perf-caller.yml.erb +43 -0
  30. data/lib/duel/templates/perf-comment-caller.yml.erb +42 -0
  31. data/lib/duel/templates/perf-comment.yml.erb +45 -3
  32. data/lib/duel/templates/perf.yml.erb +143 -7
  33. data/lib/duel/testing.rb +3 -0
  34. data/lib/duel/version.rb +1 -1
  35. data/lib/duel/worker.rb +13 -4
  36. data/lib/duel/worker_process.rb +142 -0
  37. data/lib/duel.rb +27 -3
  38. data/sig/duel/cli.rbs +8 -0
  39. data/sig/duel/comment.rbs +16 -1
  40. data/sig/duel/config.rbs +52 -2
  41. data/sig/duel/context.rbs +4 -31
  42. data/sig/duel/control.rbs +27 -0
  43. data/sig/duel/init.rbs +56 -0
  44. data/sig/duel/lane.rbs +202 -0
  45. data/sig/duel/lock.rbs +8 -1
  46. data/sig/duel/memory.rbs +1 -1
  47. data/sig/duel/metric.rbs +53 -10
  48. data/sig/duel/native.rbs +109 -9
  49. data/sig/duel/platform.rbs +13 -0
  50. data/sig/duel/process_sample.rbs +7 -9
  51. data/sig/duel/resource.rbs +28 -9
  52. data/sig/duel/result.rbs +164 -0
  53. data/sig/duel/rubies.rbs +4 -0
  54. data/sig/duel/runner.rbs +3 -460
  55. data/sig/duel/spawn.rbs +111 -0
  56. data/sig/duel/summary.rbs +3 -2
  57. data/sig/duel/worker_process.rbs +90 -0
  58. data/sig/duel.rbs +17 -3
  59. metadata +121 -107
data/sig/duel/native.rbs CHANGED
@@ -9,16 +9,14 @@ module Duel
9
9
  end
10
10
 
11
11
  FIDDLE: bool
12
- OS: String
13
- CPU: String
14
12
  UINT64: Integer
15
13
 
16
- # A C function of the process's own libraries, by name, taking and
17
- # returning Fiddle types (:INT, :LONG, :VOIDP...).
18
14
  # Loads Fiddle, as the first C function is made.
19
15
  def self.require_fiddle: () -> bool
20
16
 
21
- def self.function: (String name, Array[Symbol] args, Symbol returns) -> _Function
17
+ # A C function of the process's own libraries (or of +library+), by
18
+ # name, taking and returning Fiddle types (:INT, :LONG, :VOIDP...).
19
+ def self.function: (String name, Array[Symbol] args, Symbol returns, ?library: String?) -> _Function
22
20
 
23
21
  # Instructions retired in user space, from the CPU's counters.
24
22
  class Counter
@@ -68,6 +66,9 @@ module Duel
68
66
 
69
67
  def self.open: (?platform: String, ?rusage: _Function?) -> MacOS?
70
68
 
69
+ # The same call, for resident memory alone.
70
+ def self.memory: (?platform: String, ?rusage: _Function?) -> MacOS?
71
+
71
72
  def initialize: (_Function rusage) -> void
72
73
 
73
74
  def read: () -> Integer
@@ -79,6 +80,101 @@ module Duel
79
80
  def field: (Integer offset) -> Integer
80
81
  end
81
82
 
83
+ # Windows: this process's working set, and the memory available, from
84
+ # kernel32.
85
+ class Windows
86
+ KERNEL32: String
87
+ CURRENT_PROCESS: Integer
88
+ COUNTERS: Integer
89
+ PEAK_WORKING_SET: Integer
90
+ WORKING_SET: Integer
91
+ STATUS: Integer
92
+ AVAILABLE: Integer
93
+
94
+ @memory_info: _Function
95
+ @handle: Integer
96
+ # The struct K32GetProcessMemoryInfo fills in, in place.
97
+ @buffer: String
98
+
99
+ def self.open: (?platform: String, ?memory_info: _Function?) -> Windows?
100
+
101
+ # The bytes of physical memory available, or nil.
102
+ def self.available: (?platform: String, ?status: _Function?) -> Integer?
103
+
104
+ def initialize: (_Function memory_info, ?handle: Integer) -> void
105
+
106
+ # The bytes in its working set, or nil.
107
+ def resident: () -> Integer?
108
+
109
+ # The most bytes its working set has held, or nil.
110
+ def peak: () -> Integer?
111
+
112
+ # The CPU cycles charged to this thread or this process, read
113
+ # without allocating.
114
+ class Cycles
115
+ CURRENT_THREAD: Integer
116
+ QUERIES: Hash[:thread | :process, [String, Integer]]
117
+
118
+ @handle: Integer
119
+ @query: _Function
120
+ # Where the call writes the count.
121
+ @count: Fiddle::Pointer
122
+ @address: Integer
123
+
124
+ def initialize: (:thread | :process scope, ?query: _Function?, ?handle: Integer?) -> void
125
+
126
+ # The count, as a Float.
127
+ def read: () -> Float
128
+ end
129
+
130
+ # A child process, by a handle of its own: what it took, once it
131
+ # has ended.
132
+ class Spawned
133
+ QUERY: Integer
134
+
135
+ @handle: Integer
136
+ @cycles: Cycles | _Reader
137
+ @memory: Windows
138
+ @close: _Function
139
+
140
+ interface _Reader
141
+ def read: () -> Float
142
+ end
143
+
144
+ # +pid+'s, or nil where it can't be opened.
145
+ def self.open: (Integer pid, ?open_process: _Function?) -> Spawned?
146
+
147
+ def initialize: (Integer handle, ?cycles: (Cycles | _Reader)?, ?memory_info: _Function?, ?close: _Function?) -> void
148
+
149
+ def cycles: () -> Float
150
+
151
+ def peak: () -> Integer?
152
+
153
+ def close: () -> Integer
154
+ end
155
+
156
+ private
157
+
158
+ def counter: (Integer offset) -> Integer?
159
+ end
160
+
161
+ # macOS: the calling thread's instructions, from thread_selfcounts.
162
+ class MacOSThread < Counter
163
+ THSC_CPI: Integer
164
+ SIZE: Integer
165
+
166
+ @selfcounts: _Function
167
+ # The struct thread_selfcounts fills in, in place.
168
+ @buffer: String
169
+
170
+ def self.open: (?platform: String, ?selfcounts: _Function?) -> MacOSThread?
171
+
172
+ def initialize: (_Function selfcounts) -> void
173
+
174
+ # The count, or nil where the call fails.
175
+ def read: () -> Integer?
176
+ end
177
+
82
178
  # A child process of this one, waited for.
83
179
  module Child
84
180
  RUSAGE_SIZE: Integer
@@ -88,17 +184,21 @@ module Duel
88
184
  # Waits for the child +pid+: its exit status (or nil), the signal
89
185
  # that killed it (or nil), and its peak memory in bytes (or nil).
90
186
  def self.wait: (Integer pid, ?fiddle: bool, ?platform: String, ?wait4: _Function?) -> [Integer?, Integer?, Integer?]
187
+
188
+ # How a child that was waited for ended, from its +status+.
189
+ def self.ended: (Process::Status status) -> [Integer?, Integer?, nil]
91
190
  end
92
191
 
93
192
  # This process's resident memory in bytes, or nil where it can't tell.
94
193
  module Memory
95
194
  STATUS: String
96
- self.@macos: MacOS?
195
+ self.@reader: (MacOS | Windows)?
97
196
 
98
- # macOS's counter, opened as resident memory is first read, or nil.
99
- def self.macos_counter: () -> MacOS?
197
+ # macOS's counter, or Windows's reader, opened as resident memory is
198
+ # first read, or nil.
199
+ def self.opened_reader: () -> (MacOS | Windows)?
100
200
 
101
- def self.resident: (?status: String, ?macos: MacOS?) -> Integer?
201
+ def self.resident: (?status: String, ?reader: (MacOS | Windows)?) -> Integer?
102
202
 
103
203
  def self.used: () -> Integer?
104
204
 
@@ -0,0 +1,13 @@
1
+ module Duel
2
+ # What this process runs on, for what differs by it.
3
+ # @api private
4
+ module Platform
5
+ OS: String
6
+ CPU: String
7
+ WINDOWS_OS: Regexp
8
+ WINDOWS: bool
9
+ CRUBY: bool
10
+ JRUBY: bool
11
+ JRUBY_ON_WINDOWS: bool
12
+ end
13
+ end
@@ -9,19 +9,18 @@ module Duel
9
9
  # What the process of its own reports of a run.
10
10
  type ended = { "cpu" => Integer, "peak" => Integer?, "failed" => String? }
11
11
 
12
- # The lines of its output a failure shows.
13
- LOG_LINES: Integer
14
12
 
15
13
  @label: String
16
14
  @env: Hash[String, String?]
17
15
  @chdir: String
18
16
  @log: String
19
17
  @timeout: Float
20
- @pid: Integer?
18
+ @child: Spawn::Child?
21
19
  @requests: IO
22
20
  @replies: IO
21
+ @starter: Spawn::starter
23
22
 
24
- def initialize: (String label, env: Hash[String, String?], chdir: String, log: String, timeout: Float) -> void
23
+ def initialize: (String label, env: Hash[String, String?], chdir: String, log: String, timeout: Float, ?starter: Spawn::starter) -> void
25
24
 
26
25
  # The fastest of +samples+: the least CPU time and peak memory.
27
26
  def self.fastest: (Array[sampled] samples) -> sampled
@@ -30,20 +29,19 @@ module Duel
30
29
  def run: (Array[String] command) -> sampled
31
30
 
32
31
  # Ends the Ruby process of its own, if it started.
33
- def stop: () -> void
32
+ def stop: (?wait: Numeric) -> void
34
33
 
35
34
  # Runs a process for each line of +input+, and reports each on
36
35
  # +output+.
37
- def self.main: (String spec, ?input: IO, ?output: IO) -> void
36
+ def self.main: (?input: IO, ?output: IO) -> void
38
37
 
39
38
  # Runs +command+ once, and returns what it took.
40
- def self.once: (Array[String] command, Hash[String, String?] env, String chdir, String log) -> ended
39
+ def self.once: (Array[String] command, Hash[String, String?] env, String chdir, String log, ?windows: bool, ?starter: Spawn::starter) -> ended
41
40
 
42
41
  # The CPU time of the children waited for so far, in nanoseconds.
43
42
  def self.cpu: () -> Integer
44
43
 
45
- # How a process that failed ended.
46
- def self.why: (Integer? code, Integer? signal) -> String
44
+ STOP_WAIT: Integer
47
45
 
48
46
  private
49
47
 
@@ -12,9 +12,17 @@ module Duel
12
12
  # @api private
13
13
  STOP_WAIT: Integer
14
14
 
15
- # The lines of its log a failure shows.
15
+ # A process #spawn started: its program, its log, the signal that
16
+ # stops it, and its Spawn::Child.
16
17
  # @api private
17
- LOG_LINES: Integer
18
+ class Spawned < Data
19
+ attr_reader program: String
20
+ attr_reader log: String
21
+ attr_reader stop_signal: String
22
+ attr_reader child: Spawn::Child
23
+
24
+ def self.new: (program: String, log: String, stop_signal: String, child: Spawn::Child) -> instance
25
+ end
18
26
 
19
27
  # What a cache key may be.
20
28
  # @api private
@@ -31,12 +39,19 @@ module Duel
31
39
  @env: Hash[String, String?]
32
40
  @timeout: Float
33
41
  @cache: String
34
- # Each process's program, log and stop signal, by pid.
35
- @processes: Hash[Integer, [String, String, String]]
42
+ # The processes #spawn started.
43
+ @processes: Array[Spawned]
44
+ @starter: Spawn::starter
36
45
  @stops: Array[^() -> void]
37
46
 
38
47
  # +env+ with +vars+ too: two prepare blocks that set a variable differently
39
48
  # fail.
49
+ # Runs each prepare block, given a Resource of its own (kept by the
50
+ # block given), and returns +env+ with what each returned.
51
+ # @api private
52
+ def self.prepare: (Array[Config::prepare] blocks, String dir, String name, String root, env: Hash[String, String?],
53
+ ruby: String, timeout: Float, cache: String, ?so_far: bool) { (Resource) -> void } -> Hash[String, String?]
54
+
40
55
  # @api private
41
56
  def self.environment: (String name, Hash[String, String?] env, Hash[String, String?] vars) -> Hash[String, String?]
42
57
 
@@ -66,18 +81,23 @@ module Duel
66
81
 
67
82
  # @api private
68
83
  def initialize: (String dir, String name, String root, timeout: Float, cache: String, ?ruby: String,
69
- ?env: Hash[String, String?]) -> void
84
+ ?env: Hash[String, String?], ?starter: Spawn::starter) -> void
70
85
 
71
86
  # A path in its directory.
72
87
  def path: (String file) -> String
73
88
 
89
+ # A port of 127.0.0.1's that's free, for a server of its own to listen
90
+ # on.
91
+ def port: () -> Integer
92
+
74
93
  # A directory built once, by the block, for every worker of this run
75
94
  # and later ones. The first that asks for +key+ builds it.
76
95
  def cached: (String key) { (String dir) -> void } -> String
77
96
 
78
97
  # Starts a process (as Process.spawn takes it), its output added to a
79
98
  # log in the directory (+log+.log: by default, its program's name), and
80
- # stopped with the resource by +stop_signal+. Returns its pid.
99
+ # stopped with the resource by +stop_signal+ (on Windows, killed).
100
+ # Returns its pid.
81
101
  def spawn: (*untyped command, ?log: String?, ?stop_signal: String, **untyped options) -> Integer
82
102
 
83
103
  # Runs a command to its end, logged as #spawn's is. Fails, showing its
@@ -98,6 +118,7 @@ module Duel
98
118
  # @api private
99
119
  def stop: () -> void
100
120
 
121
+
101
122
  private
102
123
 
103
124
  # Builds +dir+ in a directory of its own, then renames it into place.
@@ -106,9 +127,7 @@ module Duel
106
127
  # The log a command's output goes to.
107
128
  def log_path: (Array[untyped] command, String? name) -> String
108
129
 
109
- def start: (Array[untyped] command, String log, Hash[Symbol, untyped] options) -> Integer
110
-
111
- def end_process: (Integer pid, String signal) -> void
130
+ def start: (Array[untyped] command, String log, Hash[Symbol, untyped] options) -> Spawn::Child
112
131
 
113
132
  def monotonic: () -> Float
114
133
  end
@@ -0,0 +1,164 @@
1
+ module Duel
2
+ # What one comparison of a benchmark found (Testing::Comparison#results
3
+ # has them).
4
+ class Result
5
+ # A ratio metric's verdict, and each version's (the median of its
6
+ # samples).
7
+ type timed = { "median" => Float, "limit" => Float, "across" => bool, "unit" => String?, "plural" => String?, "interval" => Array[Float]?, "pairs" => Integer,
8
+ ?"processes" => Integer, "verdict" => String, "baseline" => Metric::value, "candidate" => Metric::value }
9
+
10
+ # Each worker's median of a measure judged by process, by version,
11
+ # and its lane's number.
12
+ type processes = { "lane" => Integer, "baseline" => Array[Metric::value], "candidate" => Array[Metric::value] }
13
+
14
+ # A counted metric's fewest, on each version, in +n+ iterations.
15
+ type counted = { "baseline" => Metric::value, "candidate" => Metric::value, ?"more" => Metric::value?, "n" => Integer,
16
+ "tolerance" => Metric::value, "allowance" => Metric::value, "unit" => String?, "plural" => String? }
17
+
18
+ # Where a regression's time went (see Profile#to_h), or why there's no
19
+ # telling.
20
+ # @api private
21
+ type profile = Profile::report | { "unavailable" => String }
22
+
23
+ # Peak memory grows by more than MEMORY_RATIO, and by MEMORY_FLOOR
24
+ # bytes at least, before it's worse.
25
+ # @api private
26
+ MEMORY_RATIO: Float
27
+
28
+ # @api private
29
+ MEMORY_FLOOR: Integer
30
+
31
+ # What a sample says when no metric said anything.
32
+ # @api private
33
+ NOTHING_SAID: Hash[String, String]
34
+
35
+ # The verdicts that say a timing isn't slower.
36
+ # @api private
37
+ CLEARED: Array[String]
38
+
39
+ @name: String
40
+ @timed: Hash[String, timed]
41
+ @counted: Hash[String, counted]
42
+ @details: Hash[String, Hash[String, String]]
43
+ @confirmations: Array[Result]
44
+ @undecided: Array[Result]
45
+ @processes: Hash[String, processes]
46
+ @n: Integer?
47
+ @seconds: Float?
48
+ @skipped: String?
49
+ @error: String?
50
+ @memory: Hash[String, Integer]?
51
+ @profile: profile?
52
+
53
+ # The benchmark's name.
54
+ attr_reader name: String
55
+
56
+ # Each ratio measure's verdict and medians, by name.
57
+ attr_reader timed: Hash[String, timed]
58
+
59
+ # Each counted measure's fewest on each version, by name.
60
+ attr_reader counted: Hash[String, counted]
61
+
62
+ # What metrics said of the latest samples (Metric#details): by
63
+ # measure, by version.
64
+ attr_reader details: Hash[String, Hash[String, String]]
65
+
66
+ # The measurements that confirmed it was worse.
67
+ attr_reader confirmations: Array[Result]
68
+
69
+ # The measurements again that were too close to call.
70
+ # @api private
71
+ attr_reader undecided: Array[Result]
72
+
73
+ # Each worker's median of each measure judged by process, by metric.
74
+ # @api private
75
+ attr_reader processes: Hash[String, processes]
76
+
77
+ # Its iterations a sample.
78
+ attr_reader n: Integer?
79
+
80
+ # @api private
81
+ attr_writer n: Integer?
82
+
83
+ # The seconds it took.
84
+ attr_reader seconds: Float?
85
+
86
+ # @api private
87
+ attr_writer seconds: Float?
88
+
89
+ # Why it was skipped (the baseline couldn't run it), or nil.
90
+ attr_reader skipped: String?
91
+
92
+ # @api private
93
+ attr_writer skipped: String?
94
+
95
+ # How it failed, or nil.
96
+ attr_reader error: String?
97
+
98
+ # @api private
99
+ attr_writer error: String?
100
+
101
+ # Each version's peak memory, in bytes.
102
+ attr_reader memory: Hash[String, Integer]?
103
+
104
+ # @api private
105
+ attr_writer memory: Hash[String, Integer]?
106
+
107
+ # Where a regression's time went.
108
+ # @api private
109
+ attr_accessor profile: profile?
110
+
111
+ # @api private
112
+ def initialize: (String name) -> void
113
+
114
+ # Keeps what metrics said of a pair of samples.
115
+ # @api private
116
+ def note: (Worker::sampled base, Worker::sampled candidate, ?String? measure) -> void
117
+
118
+ # The metrics on which the candidate did worse.
119
+ # @api private
120
+ def worse: () -> Array[String]
121
+
122
+ # The timed metrics on which the candidate was slower.
123
+ # @api private
124
+ def slower: () -> Array[String]
125
+
126
+ # Whether its timings say slower: one is, and none says it isn't.
127
+ # @api private
128
+ def slower_in_time?: () -> bool
129
+
130
+ # Whether it's to be measured again.
131
+ # @api private
132
+ def confirming?: () -> bool
133
+
134
+ # Keeps a measurement again as a confirmation, or sets it aside
135
+ # where it's too close to call. Whether it was set aside.
136
+ # @api private
137
+ def confirm: (Result again) -> bool
138
+
139
+ # Whether its peak memory grew by too much.
140
+ # @api private
141
+ def more_memory?: () -> bool
142
+
143
+ # Worse on a metric in the first measurement and in every confirmation,
144
+ # or slower in time in all of them (see #mixed?).
145
+ def regressed: () -> Array[String]
146
+
147
+ # Whether it regressed in time on one timed metric in some
148
+ # measurements and on another in the rest.
149
+ # @api private
150
+ def mixed?: () -> bool
151
+
152
+ # Whether it regressed on nothing, and didn't fail.
153
+ def passed?: () -> bool
154
+
155
+ # Whether a timed measure's verdict was too close to the limit to settle.
156
+ # @api private
157
+ def unsettled?: () -> bool
158
+
159
+ private
160
+
161
+ # The first measurement and its confirmations.
162
+ def measurements: () -> Array[Result]
163
+ end
164
+ end
data/sig/duel/rubies.rbs CHANGED
@@ -33,6 +33,10 @@ module Duel
33
33
  # Whether +ruby+ is the runner's own.
34
34
  def self.runner?: (String ruby) -> bool
35
35
 
36
+ # The command that starts +ruby+ with +env+: +ruby+, or on JRuby on
37
+ # Windows, the JVM's own command.
38
+ def self.launch: (String ruby, String engine, Hash[String, String?] env, ?windows: bool) -> Array[String]
39
+
36
40
  # What a version's processes on +ruby+ get in their environment.
37
41
  def self.environment: (String ruby, ?path: String?) -> Hash[String, String?]
38
42