duel 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/AGENT_PROMPT.md +51 -27
- data/CHANGELOG.md +247 -1
- data/README.md +30 -14
- data/docs/benchmarks.md +28 -24
- data/docs/ci.md +328 -362
- data/docs/configuration.md +40 -37
- data/docs/how-it-works.md +23 -10
- data/docs/json.md +21 -25
- data/docs/plugins.md +40 -22
- data/docs/troubleshooting.md +3 -2
- data/lib/duel/across.rb +18 -18
- data/lib/duel/badge.rb +174 -0
- data/lib/duel/baseline.rb +38 -15
- data/lib/duel/cli.rb +175 -107
- data/lib/duel/comment.rb +80 -71
- data/lib/duel/config.rb +145 -63
- data/lib/duel/context.rb +29 -24
- data/lib/duel/forge.rb +87 -0
- data/lib/duel/init.rb +44 -87
- data/lib/duel/json_report.rb +34 -25
- data/lib/duel/lane.rb +107 -61
- data/lib/duel/lock.rb +1 -1
- data/lib/duel/mark_passed.rb +132 -0
- data/lib/duel/metric.rb +28 -27
- data/lib/duel/native.rb +7 -5
- data/lib/duel/platform.rb +4 -0
- data/lib/duel/process_sample.rb +9 -6
- data/lib/duel/rake_task.rb +4 -5
- data/lib/duel/resource.rb +10 -11
- data/lib/duel/result.rb +22 -14
- data/lib/duel/rubies.rb +6 -2
- data/lib/duel/runner.rb +27 -15
- data/lib/duel/spawn.rb +29 -4
- data/lib/duel/stats.rb +14 -12
- data/lib/duel/summary.rb +31 -29
- data/lib/duel/templates/benchmark_duel.rb.erb +2 -2
- data/lib/duel/templates/duel.gitlab-ci.yml.erb +18 -54
- data/lib/duel/templates/duel.rb.erb +7 -7
- data/lib/duel/templates/perf-caller.yml.erb +22 -5
- data/lib/duel/templates/perf-comment.yml.erb +11 -11
- data/lib/duel/templates/perf-push-jobs.yml.erb +127 -0
- data/lib/duel/templates/perf-push.yml.erb +50 -0
- data/lib/duel/templates/perf.yml.erb +113 -335
- data/lib/duel/testing.rb +20 -25
- data/lib/duel/version.rb +1 -1
- data/lib/duel/worker.rb +16 -9
- data/lib/duel/worker_process.rb +6 -6
- data/lib/duel.rb +88 -33
- data/sig/duel/across.rbs +4 -0
- data/sig/duel/badge.rbs +51 -0
- data/sig/duel/baseline.rbs +7 -2
- data/sig/duel/cli.rbs +31 -20
- data/sig/duel/comment.rbs +14 -15
- data/sig/duel/config.rbs +71 -29
- data/sig/duel/context.rbs +4 -4
- data/sig/duel/forge.rbs +42 -0
- data/sig/duel/init.rbs +8 -26
- data/sig/duel/json_report.rbs +7 -6
- data/sig/duel/lane.rbs +29 -6
- data/sig/duel/mark_passed.rbs +39 -0
- data/sig/duel/metric.rbs +10 -10
- data/sig/duel/native.rbs +3 -0
- data/sig/duel/platform.rbs +1 -0
- data/sig/duel/process_sample.rbs +3 -0
- data/sig/duel/resource.rbs +3 -3
- data/sig/duel/result.rbs +12 -5
- data/sig/duel/rubies.rbs +3 -0
- data/sig/duel/runner.rbs +23 -16
- data/sig/duel/spawn.rbs +7 -0
- data/sig/duel/stats.rbs +4 -1
- data/sig/duel/summary.rbs +8 -0
- data/sig/duel/testing.rbs +2 -2
- data/sig/duel/worker.rbs +6 -5
- data/sig/duel/worker_process.rbs +3 -3
- data/sig/duel.rbs +50 -31
- metadata +9 -1
data/sig/duel/resource.rbs
CHANGED
|
@@ -87,8 +87,8 @@ module Duel
|
|
|
87
87
|
def path: (String file) -> String
|
|
88
88
|
|
|
89
89
|
# A port of 127.0.0.1's that's free, for a server of its own to listen
|
|
90
|
-
# on.
|
|
91
|
-
def
|
|
90
|
+
# on: another each call.
|
|
91
|
+
def free_port: () -> Integer
|
|
92
92
|
|
|
93
93
|
# A directory built once, by the block, for every worker of this run
|
|
94
94
|
# and later ones. The first that asks for +key+ builds it.
|
|
@@ -112,7 +112,7 @@ module Duel
|
|
|
112
112
|
def at_stop: () { () -> void } -> void
|
|
113
113
|
|
|
114
114
|
# Fails, saying it's this resource that did, with the end of +log+.
|
|
115
|
-
def
|
|
115
|
+
def fail!: (String message, ?log: String?) -> bot
|
|
116
116
|
|
|
117
117
|
# Stops it.
|
|
118
118
|
# @api private
|
data/sig/duel/result.rbs
CHANGED
|
@@ -2,7 +2,7 @@ module Duel
|
|
|
2
2
|
# What one comparison of a benchmark found (Testing::Comparison#results
|
|
3
3
|
# has them).
|
|
4
4
|
class Result
|
|
5
|
-
# A
|
|
5
|
+
# A timed metric's verdict, and each version's (the median of its
|
|
6
6
|
# samples).
|
|
7
7
|
type timed = { "median" => Float, "limit" => Float, "across" => bool, "unit" => String?, "plural" => String?, "interval" => Array[Float]?, "pairs" => Integer,
|
|
8
8
|
?"processes" => Integer, "verdict" => String, "baseline" => Metric::value, "candidate" => Metric::value }
|
|
@@ -13,7 +13,7 @@ module Duel
|
|
|
13
13
|
|
|
14
14
|
# A counted metric's fewest, on each version, in +n+ iterations.
|
|
15
15
|
type counted = { "baseline" => Metric::value, "candidate" => Metric::value, ?"more" => Metric::value?, "n" => Integer,
|
|
16
|
-
"
|
|
16
|
+
"per_iteration" => Metric::value, "per_sample" => Metric::value, "unit" => String?, "plural" => String? }
|
|
17
17
|
|
|
18
18
|
# Where a regression's time went (see Profile#to_h), or why there's no
|
|
19
19
|
# telling.
|
|
@@ -41,7 +41,7 @@ module Duel
|
|
|
41
41
|
@counted: Hash[String, counted]
|
|
42
42
|
@details: Hash[String, Hash[String, String]]
|
|
43
43
|
@confirmations: Array[Result]
|
|
44
|
-
@
|
|
44
|
+
@close_calls: Array[Result]
|
|
45
45
|
@processes: Hash[String, processes]
|
|
46
46
|
@n: Integer?
|
|
47
47
|
@seconds: Float?
|
|
@@ -53,7 +53,7 @@ module Duel
|
|
|
53
53
|
# The benchmark's name.
|
|
54
54
|
attr_reader name: String
|
|
55
55
|
|
|
56
|
-
# Each
|
|
56
|
+
# Each timed measure's verdict and medians, by name.
|
|
57
57
|
attr_reader timed: Hash[String, timed]
|
|
58
58
|
|
|
59
59
|
# Each counted measure's fewest on each version, by name.
|
|
@@ -68,7 +68,7 @@ module Duel
|
|
|
68
68
|
|
|
69
69
|
# The measurements again that were too close to call.
|
|
70
70
|
# @api private
|
|
71
|
-
attr_reader
|
|
71
|
+
attr_reader close_calls: Array[Result]
|
|
72
72
|
|
|
73
73
|
# Each worker's median of each measure judged by process, by metric.
|
|
74
74
|
# @api private
|
|
@@ -108,6 +108,10 @@ module Duel
|
|
|
108
108
|
# @api private
|
|
109
109
|
attr_accessor profile: profile?
|
|
110
110
|
|
|
111
|
+
# A counted measure's count an iteration: [baseline's, candidate's].
|
|
112
|
+
# @api private
|
|
113
|
+
def self.per_iteration: (counted c) -> [Float, Float]
|
|
114
|
+
|
|
111
115
|
# @api private
|
|
112
116
|
def initialize: (String name) -> void
|
|
113
117
|
|
|
@@ -160,5 +164,8 @@ module Duel
|
|
|
160
164
|
|
|
161
165
|
# The first measurement and its confirmations.
|
|
162
166
|
def measurements: () -> Array[Result]
|
|
167
|
+
|
|
168
|
+
# The metrics it was worse on in every measurement.
|
|
169
|
+
def agreed: () -> Array[String]
|
|
163
170
|
end
|
|
164
171
|
end
|
data/sig/duel/rubies.rbs
CHANGED
data/sig/duel/runner.rbs
CHANGED
|
@@ -12,9 +12,17 @@ module Duel
|
|
|
12
12
|
JIT_WARM: Float
|
|
13
13
|
|
|
14
14
|
# Seconds each benchmark runs before it's measured where a worker has
|
|
15
|
-
# YJIT on, unless --warm says.
|
|
15
|
+
# YJIT or ZJIT on, unless --warm says.
|
|
16
16
|
# @api private
|
|
17
|
-
|
|
17
|
+
CRUBY_JIT_WARM: Float
|
|
18
|
+
|
|
19
|
+
# CRuby's JITs, by the variable that turns each on in a worker.
|
|
20
|
+
# @api private
|
|
21
|
+
JIT_VARIABLES: Hash[String, String]
|
|
22
|
+
|
|
23
|
+
# The values of a JIT's variable that turn it on.
|
|
24
|
+
# @api private
|
|
25
|
+
JIT_ON: Array[String]
|
|
18
26
|
|
|
19
27
|
# The seconds a run measures for, at most, by default.
|
|
20
28
|
# @api private
|
|
@@ -34,7 +42,7 @@ module Duel
|
|
|
34
42
|
|
|
35
43
|
# How a run is set up (see Duel::CLI for where each comes from).
|
|
36
44
|
# @api private
|
|
37
|
-
class Settings < Struct[String | Regexp | Integer | Float | bool | Config::ruby |
|
|
45
|
+
class Settings < Struct[String | Regexp | Integer | Float | bool | Config::ruby | nil]
|
|
38
46
|
attr_accessor base: String?
|
|
39
47
|
attr_accessor base_dir: String?
|
|
40
48
|
attr_accessor candidate_dir: String?
|
|
@@ -51,13 +59,8 @@ module Duel
|
|
|
51
59
|
attr_accessor time_limit: Float
|
|
52
60
|
# Each version's Ruby (--ruby), over Config#ruby.
|
|
53
61
|
attr_accessor ruby: Config::ruby
|
|
54
|
-
# When the workflow comments on a pull request (--comment), over
|
|
55
|
-
# Config#comment.
|
|
56
|
-
attr_accessor comment: Config::comment?
|
|
57
|
-
# Which runs a regression fails (--fail-on), over Config#fail_on.
|
|
58
|
-
attr_accessor fail_on: Config::fail_on?
|
|
59
62
|
# The branch pushed, where #base is the push's, the commit before it,
|
|
60
|
-
#
|
|
63
|
+
# as its event says: see Baseline#ref.
|
|
61
64
|
attr_accessor pushed: String?
|
|
62
65
|
# The branch a pull request targets, on GitHub Actions
|
|
63
66
|
# (GITHUB_BASE_REF), on origin: see Baseline#main_branch.
|
|
@@ -66,20 +69,22 @@ module Duel
|
|
|
66
69
|
# a pull request with Config#accept_label (DUEL_ACCEPTED): see
|
|
67
70
|
# Baseline#last_release.
|
|
68
71
|
attr_accessor accepted: String?
|
|
72
|
+
# On a schedule on GitHub Actions, the repository (GITHUB_REPOSITORY)
|
|
73
|
+
# Duel finds what was accepted in, without DUEL_ACCEPTED: see
|
|
74
|
+
# Baseline#merged_accepted.
|
|
75
|
+
attr_accessor repository: String?
|
|
69
76
|
# Of jobs and processes, those Duel may lower to fit the memory
|
|
70
77
|
# available (see Runner#fit).
|
|
71
78
|
attr_accessor fit: Array[:jobs | :processes]
|
|
72
79
|
|
|
73
80
|
def self.new: (?String? base, ?String? base_dir, ?String? candidate_dir, ?Regexp filter, ?Integer jobs,
|
|
74
81
|
?Integer processes, ?Float limit, ?Float confidence, ?Float? warm, ?bool force, ?String? config_path,
|
|
75
|
-
?Float timeout, ?Config::ruby ruby, ?
|
|
76
|
-
?
|
|
77
|
-
?String? target, ?Float time_limit, ?String? accepted, ?Array[:jobs | :processes] fit) -> instance
|
|
82
|
+
?Float timeout, ?Config::ruby ruby, ?String? pushed,
|
|
83
|
+
?String? target, ?Float time_limit, ?String? accepted, ?String? repository, ?Array[:jobs | :processes] fit) -> instance
|
|
78
84
|
| (?base: String?, ?base_dir: String?, ?candidate_dir: String?, ?filter: Regexp, ?jobs: Integer,
|
|
79
85
|
?processes: Integer, ?limit: Float, ?confidence: Float, ?warm: Float?, ?force: bool,
|
|
80
|
-
?config_path: String?, ?timeout: Float, ?ruby: Config::ruby, ?
|
|
81
|
-
?
|
|
82
|
-
?target: String?, ?time_limit: Float, ?accepted: String?, ?fit: Array[:jobs | :processes]) -> instance
|
|
86
|
+
?config_path: String?, ?timeout: Float, ?ruby: Config::ruby, ?pushed: String?,
|
|
87
|
+
?target: String?, ?time_limit: Float, ?accepted: String?, ?repository: String?, ?fit: Array[:jobs | :processes]) -> instance
|
|
83
88
|
|
|
84
89
|
def self.defaults: () -> Settings
|
|
85
90
|
end
|
|
@@ -183,8 +188,10 @@ module Duel
|
|
|
183
188
|
|
|
184
189
|
private
|
|
185
190
|
|
|
191
|
+
def monotonic: () -> Float
|
|
192
|
+
|
|
186
193
|
# What #run does.
|
|
187
|
-
def compare_all: (
|
|
194
|
+
def compare_all: () -> Array[Result]
|
|
188
195
|
|
|
189
196
|
# Each version, prepared once for the run.
|
|
190
197
|
def prepare: () -> Hash[String, Side]
|
data/sig/duel/spawn.rbs
CHANGED
|
@@ -81,6 +81,9 @@ module Duel
|
|
|
81
81
|
# The end of the log at +path+, for a failure to show.
|
|
82
82
|
def self.tail: (String path) -> String
|
|
83
83
|
|
|
84
|
+
# +message+, then the end of the log at +path+, if it has any.
|
|
85
|
+
def self.with_tail: (String message, String? path) -> String
|
|
86
|
+
|
|
84
87
|
# The log at +path+, opened for a process's output.
|
|
85
88
|
def self.log: (String path, String mode) -> File
|
|
86
89
|
|
|
@@ -88,6 +91,10 @@ module Duel
|
|
|
88
91
|
# program found.
|
|
89
92
|
def self.open3: (Symbol name, *untyped args, ?jruby_on_windows: bool, **untyped options) ?{ (*untyped) -> untyped } -> untyped
|
|
90
93
|
|
|
94
|
+
# A command's environment (none, if it doesn't start with one), and
|
|
95
|
+
# the rest.
|
|
96
|
+
def self.environment_and: (Array[untyped] command) -> [Hash[String, String?], Array[untyped]]
|
|
97
|
+
|
|
91
98
|
# +command+ as Windows runs it, its program found.
|
|
92
99
|
def self.program: (Array[String] command, ?path: String, ?exts: Array[String]) -> Array[String]
|
|
93
100
|
|
data/sig/duel/stats.rbs
CHANGED
|
@@ -3,7 +3,7 @@ module Duel
|
|
|
3
3
|
# interval for the median of paired ratios, and the decision it supports.
|
|
4
4
|
# @api private
|
|
5
5
|
module Stats
|
|
6
|
-
type verdict = :slower | :faster | :same | :
|
|
6
|
+
type verdict = :slower | :faster | :same | :unsettled
|
|
7
7
|
|
|
8
8
|
# The 1-based ranks [lower, upper] bounding a +confidence+ interval for
|
|
9
9
|
# the median of +n+ samples, or nil when +n+ is too small for one.
|
|
@@ -36,6 +36,9 @@ module Duel
|
|
|
36
36
|
def self.difference_interval: (Array[Metric::value] bases, Array[Metric::value] candidates, Float confidence) -> [Metric::value, Metric::value]?
|
|
37
37
|
|
|
38
38
|
# Each of +candidates+ over each of +bases+, sorted.
|
|
39
|
+
# The rth least and most of the sorted values the block gives, or nil.
|
|
40
|
+
def self.rank_interval: [T] (Array[Metric::value] bases, Array[Metric::value] candidates, Float confidence) { () -> Array[T] } -> [T, T]?
|
|
41
|
+
|
|
39
42
|
def self.cross_ratios: (Array[Metric::value] bases, Array[Metric::value] candidates) -> Array[Float]
|
|
40
43
|
|
|
41
44
|
def self.median: (Array[Float] samples) -> Float
|
data/sig/duel/summary.rbs
CHANGED
|
@@ -33,7 +33,9 @@ module Duel
|
|
|
33
33
|
@settings: Settings
|
|
34
34
|
@baseline: String
|
|
35
35
|
@candidate: String
|
|
36
|
+
@verdicts: Hash[Result, String]
|
|
36
37
|
@regressed: Array[Result]
|
|
38
|
+
@skipped: Integer
|
|
37
39
|
@failed: Array[Result]
|
|
38
40
|
|
|
39
41
|
# +text+ as code, in Markdown.
|
|
@@ -107,6 +109,12 @@ module Duel
|
|
|
107
109
|
# Each counted measure that changed, an iteration.
|
|
108
110
|
def counts: (Result result) -> String
|
|
109
111
|
|
|
112
|
+
# How a counted measure changed: each version's count, an iteration.
|
|
113
|
+
def count_change: (String metric, Float base, Float candidate) -> String
|
|
114
|
+
|
|
115
|
+
# How peak memory changed: each version's, in MB.
|
|
116
|
+
def memory_change: (Result result) -> String
|
|
117
|
+
|
|
110
118
|
# The first +explained+ regressions, explained.
|
|
111
119
|
def regressions: (Integer explained) -> Array[String]
|
|
112
120
|
|
data/sig/duel/testing.rbs
CHANGED
|
@@ -54,7 +54,7 @@ module Duel
|
|
|
54
54
|
# of +n+ iterations of a benchmark would, with its hooks (each
|
|
55
55
|
# iteration inside around_iteration hooks, if it has them). The
|
|
56
56
|
# block gets the benchmark's Context.
|
|
57
|
-
def measure: (Config config, ?n: Integer, ?
|
|
57
|
+
def measure: (Config config, ?n: Integer, ?judge: judged | Array[judged], **untyped hooks) { (Context context) -> untyped } -> Measurement
|
|
58
58
|
|
|
59
59
|
# Runs the block with +env+ set, and the environment as it was after.
|
|
60
60
|
# @api private
|
|
@@ -77,7 +77,7 @@ module Duel
|
|
|
77
77
|
|
|
78
78
|
# A stand-in Sample of +n+ iterations of a benchmark, measuring
|
|
79
79
|
# nothing.
|
|
80
|
-
def sample: (?n: Integer, ?name: String, ?
|
|
80
|
+
def sample: (?n: Integer, ?name: String, ?judge: judged | Array[judged]) -> Sample
|
|
81
81
|
|
|
82
82
|
# A stand-in reading of running totals, and what was read with them.
|
|
83
83
|
def reading: [T, C] (?Snapshot::totals[T] totals, ?C? context) -> Snapshot::Reading[T, C?]
|
data/sig/duel/worker.rbs
CHANGED
|
@@ -4,7 +4,7 @@ module Duel
|
|
|
4
4
|
# @api private
|
|
5
5
|
class Worker
|
|
6
6
|
type hello = { "benchmarks" => Array[Definition::summary], "metrics" => Array[Metric::spec], "ruby" => String,
|
|
7
|
-
"memory" => bool, "profiler" => bool, "
|
|
7
|
+
"memory" => bool, "profiler" => bool, "jit" => String?, "extensions" => Hash[String, String],
|
|
8
8
|
"twice" => Hash[String, Array[String]], "gem_dirs" => Array[String] }
|
|
9
9
|
|
|
10
10
|
type ok = { "ok" => true }
|
|
@@ -45,11 +45,12 @@ module Duel
|
|
|
45
45
|
# Whether StackProf is installed, for profiles: on CRuby only.
|
|
46
46
|
def self.profiler?: (?String engine) -> bool
|
|
47
47
|
|
|
48
|
-
#
|
|
49
|
-
|
|
48
|
+
# CRuby's JITs, as RubyVM names them.
|
|
49
|
+
JITS: Array[String]
|
|
50
50
|
|
|
51
|
-
#
|
|
52
|
-
|
|
51
|
+
# Which of CRuby's JITs is on ("yjit" or "zjit"), for a longer
|
|
52
|
+
# warm-up, or nil.
|
|
53
|
+
def self.jit: (?engine: String) -> String?
|
|
53
54
|
|
|
54
55
|
def initialize: (?Array[singleton(Metric)] plugin_metrics, ?Hash[String, Hash[Symbol, untyped]] measures,
|
|
55
56
|
?hooks: Hash[Symbol, Array[hook]], ?before: Array[String]) -> void
|
data/sig/duel/worker_process.rbs
CHANGED
|
@@ -30,7 +30,7 @@ module Duel
|
|
|
30
30
|
@metrics: Array[Metric::spec]
|
|
31
31
|
@memory: bool
|
|
32
32
|
@profiler: bool
|
|
33
|
-
@
|
|
33
|
+
@jit: String?
|
|
34
34
|
|
|
35
35
|
attr_reader label: String
|
|
36
36
|
|
|
@@ -42,7 +42,7 @@ module Duel
|
|
|
42
42
|
|
|
43
43
|
attr_reader profiler: bool
|
|
44
44
|
|
|
45
|
-
attr_reader
|
|
45
|
+
attr_reader jit: String?
|
|
46
46
|
|
|
47
47
|
# The Ruby it runs on (RUBY_DESCRIPTION).
|
|
48
48
|
attr_reader description: String
|
|
@@ -78,7 +78,7 @@ module Duel
|
|
|
78
78
|
|
|
79
79
|
private
|
|
80
80
|
|
|
81
|
-
def accept: (UNIXServer | TCPServer server, String
|
|
81
|
+
def accept: (UNIXServer | TCPServer server, String address) -> (UNIXSocket | TCPSocket)
|
|
82
82
|
|
|
83
83
|
def read: () -> reply
|
|
84
84
|
|
data/sig/duel.rbs
CHANGED
|
@@ -11,18 +11,16 @@ module Duel
|
|
|
11
11
|
class Failed < Error
|
|
12
12
|
end
|
|
13
13
|
|
|
14
|
-
# What a benchmark
|
|
15
|
-
|
|
16
|
-
type only = :count | :timing
|
|
14
|
+
# What a benchmark can be judged on (see DSL#duel, and JUDGED).
|
|
15
|
+
type judged = :timed | :counted | :peak_memory
|
|
17
16
|
|
|
18
17
|
# A benchmark's body, run in its Context: given the iterations to do
|
|
19
18
|
# when it takes a parameter, else run once an iteration.
|
|
20
19
|
# @api private
|
|
21
20
|
type body = ^() [self: Context] -> void | ^(Integer n) [self: Context] -> void
|
|
22
21
|
|
|
23
|
-
# A hook (see HOOKS): a block, or a lambda, given the
|
|
24
|
-
#
|
|
25
|
-
# (around_iteration), or nothing (after_iterations). A file's, or a
|
|
22
|
+
# A hook (see HOOKS): a block, or a lambda, given the Iteration to run
|
|
23
|
+
# (around_iteration), or nothing (the rest). A file's, or a
|
|
26
24
|
# benchmark's own, runs in each benchmark's Context.
|
|
27
25
|
type hook = ^(?untyped) [self: Context] -> untyped
|
|
28
26
|
|
|
@@ -30,18 +28,18 @@ module Duel
|
|
|
30
28
|
# @api private
|
|
31
29
|
type hooks = Hash[Symbol, hook]
|
|
32
30
|
|
|
33
|
-
# A benchmark (see DSL#duel), defined in +file+,
|
|
34
|
-
# file's, and its own) it gets. Or one of a
|
|
35
|
-
# DSL#duel_process). What a metric sees of it
|
|
36
|
-
# name
|
|
37
|
-
class Definition < Struct[String |
|
|
31
|
+
# A benchmark (see DSL#duel), defined in +file+, judged on what +judge+
|
|
32
|
+
# says, whose hooks (its file's, and its own) it gets. Or one of a
|
|
33
|
+
# whole process (see DSL#duel_process). What a metric sees of it
|
|
34
|
+
# (Sample#benchmark): its name and what it's judged on.
|
|
35
|
+
class Definition < Struct[String | Array[judged] | body | hooks | Definition::process | nil]
|
|
38
36
|
# A process benchmark's process: its command, or Ruby code.
|
|
39
37
|
# @api private
|
|
40
|
-
type process = { "command" => Array[String]?, "
|
|
38
|
+
type process = { "command" => Array[String]?, "code" => String?, "runs" => Integer? }
|
|
41
39
|
|
|
42
40
|
# What the runner learns of it (see #to_h).
|
|
43
41
|
# @api private
|
|
44
|
-
type summary = { "name" => String, "
|
|
42
|
+
type summary = { "name" => String, "judge" => Array[String], "process" => process? }
|
|
45
43
|
|
|
46
44
|
# Its name, as DSL#duel was given it.
|
|
47
45
|
attr_reader name: String
|
|
@@ -49,18 +47,20 @@ module Duel
|
|
|
49
47
|
# @api private
|
|
50
48
|
attr_writer name: String
|
|
51
49
|
|
|
52
|
-
# What it's judged on (DSL#duel's
|
|
53
|
-
|
|
54
|
-
attr_reader only: only?
|
|
50
|
+
# What it's judged on (DSL#duel's judge:), in JUDGED's order.
|
|
51
|
+
attr_reader judge: Array[judged]
|
|
55
52
|
|
|
56
53
|
# @api private
|
|
57
|
-
attr_writer
|
|
54
|
+
attr_writer judge: Array[judged]
|
|
58
55
|
|
|
59
|
-
# Whether
|
|
60
|
-
|
|
56
|
+
# Whether it's judged on its timings.
|
|
57
|
+
def timed?: () -> bool
|
|
61
58
|
|
|
62
|
-
#
|
|
63
|
-
|
|
59
|
+
# Whether it's judged on its counts.
|
|
60
|
+
def counted?: () -> bool
|
|
61
|
+
|
|
62
|
+
# Whether it's judged on its peak memory too.
|
|
63
|
+
def peak_memory?: () -> bool
|
|
64
64
|
|
|
65
65
|
# @api private
|
|
66
66
|
attr_accessor file: String
|
|
@@ -78,7 +78,7 @@ module Duel
|
|
|
78
78
|
attr_accessor process: process?
|
|
79
79
|
|
|
80
80
|
# @api private
|
|
81
|
-
def self.new: (String name,
|
|
81
|
+
def self.new: (String name, Array[judged] judge, String file, body? body, ?hooks? hooks, ?process? process) -> instance
|
|
82
82
|
|
|
83
83
|
# Whose CPU time it takes: the benchmark thread's, or the process's.
|
|
84
84
|
# @api private
|
|
@@ -93,9 +93,15 @@ module Duel
|
|
|
93
93
|
# @api private
|
|
94
94
|
COUNTS_ONLY: Integer
|
|
95
95
|
|
|
96
|
-
# What a benchmark
|
|
96
|
+
# What a benchmark can be judged on, in order.
|
|
97
|
+
JUDGED: Array[judged]
|
|
98
|
+
|
|
99
|
+
# What a benchmark is judged on, unless it says.
|
|
100
|
+
JUDGE: Array[judged]
|
|
101
|
+
|
|
102
|
+
# What a process benchmark is judged on.
|
|
97
103
|
# @api private
|
|
98
|
-
|
|
104
|
+
PROCESS_JUDGE: Array[judged]
|
|
99
105
|
|
|
100
106
|
# The hooks, by name, in the order a benchmark's life runs them.
|
|
101
107
|
HOOKS: Array[Symbol]
|
|
@@ -208,9 +214,18 @@ module Duel
|
|
|
208
214
|
# Whether a server that a plugin starts for a worker can listen on a
|
|
209
215
|
# Unix socket that a worker can reach: not on Windows, nor where Ruby
|
|
210
216
|
# has none. Where it can't, the server listens on a port (see
|
|
211
|
-
# Resource#
|
|
217
|
+
# Resource#free_port).
|
|
212
218
|
def self.unix_sockets?: () -> bool
|
|
213
219
|
|
|
220
|
+
# Whether +program+ is a path, with a / or a \ in it, not a name to
|
|
221
|
+
# look for on the PATH.
|
|
222
|
+
def self.path?: (String program) -> bool
|
|
223
|
+
|
|
224
|
+
# A plugin's server program: +program+ if it's a path, expanded, or
|
|
225
|
+
# the one by that name in +path+'s directories, as it is or ending with
|
|
226
|
+
# +exe+. Nil if there's none.
|
|
227
|
+
def self.find_program: (String program, ?String path, ?exe: String) -> String?
|
|
228
|
+
|
|
214
229
|
# Whether Duel runs on Windows: for a plugin to do what it does
|
|
215
230
|
# differently there.
|
|
216
231
|
def self.windows?: () -> bool
|
|
@@ -247,7 +262,11 @@ module Duel
|
|
|
247
262
|
# iterations to do. Its own hooks (see HOOKS) run after its file's:
|
|
248
263
|
# an around_iteration one runs around each iteration, given it to
|
|
249
264
|
# run, and only the iteration is measured.
|
|
250
|
-
def duel: (String name, ?
|
|
265
|
+
def duel: (String name, ?judge: judged | Array[judged], **hook hooks) { (?Integer n) [self: Context] -> void } -> void
|
|
266
|
+
|
|
267
|
+
# What a benchmark is judged on, given judge:, checked.
|
|
268
|
+
# @api private
|
|
269
|
+
def self.judged: (String name, untyped judge) -> Array[judged]
|
|
251
270
|
|
|
252
271
|
# A benchmark's own hooks, checked.
|
|
253
272
|
# @api private
|
|
@@ -256,7 +275,7 @@ module Duel
|
|
|
256
275
|
# Defines a benchmark of a whole process: a command (as Process.spawn
|
|
257
276
|
# takes it), or Ruby code, run fresh for each version in every sample
|
|
258
277
|
# (the fastest of +runs+).
|
|
259
|
-
def duel_process: (String name, *String command, ?
|
|
278
|
+
def duel_process: (String name, *String command, ?code: String?, ?runs: Integer?, **untyped options) -> void
|
|
260
279
|
|
|
261
280
|
# En garde.
|
|
262
281
|
# @api private
|
|
@@ -268,12 +287,12 @@ module Duel
|
|
|
268
287
|
# The file's hooks (see HOOKS), for every benchmark in it, run in its
|
|
269
288
|
# context: setup once in each worker before a benchmark's samples,
|
|
270
289
|
# teardown after them, and before_sample before every sample, not
|
|
271
|
-
# measured
|
|
272
|
-
def setup: () { (
|
|
290
|
+
# measured. None takes a parameter.
|
|
291
|
+
def setup: () { () [self: Context] -> void } -> void
|
|
273
292
|
|
|
274
|
-
def teardown: () { (
|
|
293
|
+
def teardown: () { () [self: Context] -> void } -> void
|
|
275
294
|
|
|
276
|
-
def before_sample: () { (
|
|
295
|
+
def before_sample: () { () [self: Context] -> void } -> void
|
|
277
296
|
|
|
278
297
|
# Runs the block around each iteration of every benchmark in the
|
|
279
298
|
# file, given the iteration to run: only the iteration is measured.
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: duel
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.3.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Erik Berlin
|
|
@@ -48,16 +48,19 @@ files:
|
|
|
48
48
|
- exe/duel
|
|
49
49
|
- lib/duel.rb
|
|
50
50
|
- lib/duel/across.rb
|
|
51
|
+
- lib/duel/badge.rb
|
|
51
52
|
- lib/duel/baseline.rb
|
|
52
53
|
- lib/duel/cli.rb
|
|
53
54
|
- lib/duel/comment.rb
|
|
54
55
|
- lib/duel/config.rb
|
|
55
56
|
- lib/duel/context.rb
|
|
56
57
|
- lib/duel/control.rb
|
|
58
|
+
- lib/duel/forge.rb
|
|
57
59
|
- lib/duel/init.rb
|
|
58
60
|
- lib/duel/json_report.rb
|
|
59
61
|
- lib/duel/lane.rb
|
|
60
62
|
- lib/duel/lock.rb
|
|
63
|
+
- lib/duel/mark_passed.rb
|
|
61
64
|
- lib/duel/memory.rb
|
|
62
65
|
- lib/duel/message.rb
|
|
63
66
|
- lib/duel/metric.rb
|
|
@@ -79,6 +82,8 @@ files:
|
|
|
79
82
|
- lib/duel/templates/perf-caller.yml.erb
|
|
80
83
|
- lib/duel/templates/perf-comment-caller.yml.erb
|
|
81
84
|
- lib/duel/templates/perf-comment.yml.erb
|
|
85
|
+
- lib/duel/templates/perf-push-jobs.yml.erb
|
|
86
|
+
- lib/duel/templates/perf-push.yml.erb
|
|
82
87
|
- lib/duel/templates/perf.yml.erb
|
|
83
88
|
- lib/duel/testing.rb
|
|
84
89
|
- lib/duel/version.rb
|
|
@@ -86,16 +91,19 @@ files:
|
|
|
86
91
|
- lib/duel/worker_process.rb
|
|
87
92
|
- sig/duel.rbs
|
|
88
93
|
- sig/duel/across.rbs
|
|
94
|
+
- sig/duel/badge.rbs
|
|
89
95
|
- sig/duel/baseline.rbs
|
|
90
96
|
- sig/duel/cli.rbs
|
|
91
97
|
- sig/duel/comment.rbs
|
|
92
98
|
- sig/duel/config.rbs
|
|
93
99
|
- sig/duel/context.rbs
|
|
94
100
|
- sig/duel/control.rbs
|
|
101
|
+
- sig/duel/forge.rbs
|
|
95
102
|
- sig/duel/init.rbs
|
|
96
103
|
- sig/duel/json_report.rbs
|
|
97
104
|
- sig/duel/lane.rbs
|
|
98
105
|
- sig/duel/lock.rbs
|
|
106
|
+
- sig/duel/mark_passed.rbs
|
|
99
107
|
- sig/duel/memory.rbs
|
|
100
108
|
- sig/duel/message.rbs
|
|
101
109
|
- sig/duel/metric.rbs
|