duel 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/AGENT_PROMPT.md +211 -0
- data/CHANGELOG.md +281 -0
- data/LICENSE +21 -0
- data/README.md +354 -0
- data/docs/README.md +23 -0
- data/docs/benchmarks.md +289 -0
- data/docs/ci.md +141 -0
- data/docs/configuration.md +429 -0
- data/docs/how-it-works.md +207 -0
- data/docs/json.md +306 -0
- data/docs/plugins.md +408 -0
- data/docs/troubleshooting.md +105 -0
- data/exe/duel +6 -0
- data/lib/duel/across.rb +48 -0
- data/lib/duel/baseline.rb +166 -0
- data/lib/duel/cli.rb +264 -0
- data/lib/duel/config.rb +343 -0
- data/lib/duel/context.rb +630 -0
- data/lib/duel/init.rb +211 -0
- data/lib/duel/json_report.rb +107 -0
- data/lib/duel/lock.rb +38 -0
- data/lib/duel/metric.rb +496 -0
- data/lib/duel/native.rb +162 -0
- data/lib/duel/process_sample.rb +144 -0
- data/lib/duel/profile.rb +116 -0
- data/lib/duel/rake_task.rb +25 -0
- data/lib/duel/resource.rb +209 -0
- data/lib/duel/rubies.rb +149 -0
- data/lib/duel/runner.rb +1114 -0
- data/lib/duel/stats.rb +73 -0
- data/lib/duel/summary.rb +237 -0
- data/lib/duel/templates/benchmark_duel.rb.erb +34 -0
- data/lib/duel/templates/duel.rb.erb +35 -0
- data/lib/duel/templates/perf.yml.erb +97 -0
- data/lib/duel/testing.rb +241 -0
- data/lib/duel/version.rb +5 -0
- data/lib/duel/worker.rb +211 -0
- data/lib/duel.rb +413 -0
- data/sig/duel/across.rbs +21 -0
- data/sig/duel/baseline.rbs +61 -0
- data/sig/duel/cli.rbs +95 -0
- data/sig/duel/config.rbs +181 -0
- data/sig/duel/context.rbs +387 -0
- data/sig/duel/init.rbs +97 -0
- data/sig/duel/json_report.rbs +47 -0
- data/sig/duel/lock.rbs +12 -0
- data/sig/duel/metric.rbs +351 -0
- data/sig/duel/native.rbs +98 -0
- data/sig/duel/process_sample.rbs +54 -0
- data/sig/duel/profile.rbs +64 -0
- data/sig/duel/rake_task.rbs +9 -0
- data/sig/duel/resource.rbs +106 -0
- data/sig/duel/rubies.rbs +45 -0
- data/sig/duel/runner.rbs +530 -0
- data/sig/duel/stats.rbs +26 -0
- data/sig/duel/summary.rbs +87 -0
- data/sig/duel/testing.rbs +91 -0
- data/sig/duel/version.rbs +3 -0
- data/sig/duel/worker.rbs +74 -0
- data/sig/duel.rbs +276 -0
- data/sig/manifest.yaml +4 -0
- metadata +124 -0
data/lib/duel/metric.rb
ADDED
|
@@ -0,0 +1,496 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "native"
|
|
4
|
+
|
|
5
|
+
module Duel
|
|
6
|
+
# Something measured around every sample of every benchmark. A subclass
|
|
7
|
+
# (Config#metric adds it) names what it measures, and how each is
|
|
8
|
+
# judged, in +measures+: {"name" => settings}, each of these, any left
|
|
9
|
+
# out as by default:
|
|
10
|
+
#
|
|
11
|
+
# - +kind+: :ratio (noisy: judged on paired ratios with a confidence
|
|
12
|
+
# interval) or :counted (exact: any increase beyond what's allowed is
|
|
13
|
+
# a regression, counted in separate samples).
|
|
14
|
+
# - +unit+: what it's in, as reports show it: a time unit ("ns", "µs",
|
|
15
|
+
# "ms", "s"), shown as time (and giving +unit_ns+), or anything else,
|
|
16
|
+
# shown after the number, or nil for none. A word comes with its
|
|
17
|
+
# plural, [singular, plural] (%w[row rows]: 1 row, 2 rows), and a symbol ("B",
|
|
18
|
+
# "KB") is one String, the same for any number.
|
|
19
|
+
# - +floor+: for a ratio metric, the least a sample must measure (in
|
|
20
|
+
# its own unit) on either version for it to be judged. Below it, it's
|
|
21
|
+
# mostly noise.
|
|
22
|
+
# - +cpu_floor+: for a ratio metric measured in time (see +unit_ns+),
|
|
23
|
+
# the least share of the sample's CPU time it must be on either
|
|
24
|
+
# version to be judged (0.05: 5%): a floor that scales with the work.
|
|
25
|
+
# - +limit+: for a ratio metric, its own ratio above which it's slower
|
|
26
|
+
# (nil: the run's, 1.02 by default).
|
|
27
|
+
# - +unit_ns+: for a ratio metric, nanoseconds per unit, for comparing
|
|
28
|
+
# it with CPU time when sizing samples.
|
|
29
|
+
# - +sizes_samples+: for a ratio metric, whether samples are made long
|
|
30
|
+
# enough for it to settle, as they are for CPU time (false for one
|
|
31
|
+
# that's noisy whatever the length, or not in step with the work).
|
|
32
|
+
# - +across+: for a ratio metric, whether it's judged across the board
|
|
33
|
+
# too: false for one too noisy for so small a margin.
|
|
34
|
+
# - +tolerance+: for a counted metric, the increase per iteration that
|
|
35
|
+
# still isn't one (0: one more is).
|
|
36
|
+
# - +allowance+: for a counted metric, the increase per sample that
|
|
37
|
+
# still isn't one, whatever its length (noise that doesn't grow with
|
|
38
|
+
# the iterations).
|
|
39
|
+
#
|
|
40
|
+
# By default it measures one thing, named for the class, counted. One
|
|
41
|
+
# that measures several (from one reading of a server's statistics,
|
|
42
|
+
# say) returns a Hash of them, by name, from #stop_sample.
|
|
43
|
+
# +available?+ says whether it can measure here at all.
|
|
44
|
+
#
|
|
45
|
+
# An instance lives as long as its worker: made once the worker's
|
|
46
|
+
# blocks (Config#in_worker) have run, and opened (#open: what it reads
|
|
47
|
+
# set up, a subscription, say), and closed (#close) when the worker
|
|
48
|
+
# ends. Each sample calls #start_sample with the Sample (its benchmark
|
|
49
|
+
# and its iterations, n), then #stop_sample, which returns the
|
|
50
|
+
# measurement (nil: none this time). Around an +untimed+ block, and
|
|
51
|
+
# around what a benchmark's around_iteration hooks do outside each
|
|
52
|
+
# iteration, it's #pause_sample (by default, #stop_sample) and
|
|
53
|
+
# #resume_sample (by default, #start_sample). Metrics start in the
|
|
54
|
+
# order they were added and stop in reverse, around the built-in ones,
|
|
55
|
+
# so a plugin's own work (reading a server's statistics, say) isn't
|
|
56
|
+
# counted in allocations or CPU.
|
|
57
|
+
#
|
|
58
|
+
# One that's costly to read (+costly?+: a query of a server's
|
|
59
|
+
# statistics, say) isn't paused around each iteration of a benchmark
|
|
60
|
+
# with around_iteration hooks, but goes on measuring what they do too:
|
|
61
|
+
# what that adds, an iteration, is learned in each worker, and taken
|
|
62
|
+
# away (see Harness#leak).
|
|
63
|
+
class Metric
|
|
64
|
+
# Every setting there is (see Metric), as #settings resolves them, and
|
|
65
|
+
# running totals' calibrate and Config#measure's skip.
|
|
66
|
+
# @api private
|
|
67
|
+
SETTINGS = %i[kind unit floor cpu_floor limit unit_ns sizes_samples across tolerance allowance calibrate skip].freeze
|
|
68
|
+
|
|
69
|
+
# A unit that's a word (or words), which needs its plural given: one
|
|
70
|
+
# that isn't (a symbol: "B", "KB", "op/s") is the same for any number.
|
|
71
|
+
# @api private
|
|
72
|
+
WORD = /\A[a-z]+( [a-z]+)*\z/
|
|
73
|
+
|
|
74
|
+
# The units that are time, and the nanoseconds in each: a measure in
|
|
75
|
+
# one is shown as time (and its unit_ns follows from it).
|
|
76
|
+
# @api private
|
|
77
|
+
TIME_UNITS = {"ns" => 1, "µs" => 1_000, "ms" => 1_000_000, "s" => 1_000_000_000}.freeze
|
|
78
|
+
|
|
79
|
+
# No measure's settings changed.
|
|
80
|
+
# @api private
|
|
81
|
+
UNCHANGED = {} #: Hash[String, Hash[Symbol, untyped]]
|
|
82
|
+
|
|
83
|
+
# No change to a measure's settings.
|
|
84
|
+
# @api private
|
|
85
|
+
NO_CHANGE = {} #: Hash[Symbol, untyped]
|
|
86
|
+
|
|
87
|
+
class << self
|
|
88
|
+
# Whether it can measure here at all (false leaves it out): by
|
|
89
|
+
# default, it can.
|
|
90
|
+
def available? = true
|
|
91
|
+
|
|
92
|
+
# Whether reading it is costly (a round trip to a server, say), so
|
|
93
|
+
# it isn't paused around each iteration of a benchmark with
|
|
94
|
+
# around_iteration hooks (see Metric): by default, it isn't.
|
|
95
|
+
def costly? = false
|
|
96
|
+
|
|
97
|
+
# The metric's name in reports: by default, its class's, in snake
|
|
98
|
+
# case ("QueryCount" is "query_count").
|
|
99
|
+
def metric_name = name.split("::").last.gsub(/([a-z\d])([A-Z])/, '\1_\2').downcase
|
|
100
|
+
|
|
101
|
+
# What it measures, and how each is judged: {name => settings}, by
|
|
102
|
+
# default one, named metric_name, counted. The one place a metric
|
|
103
|
+
# says so.
|
|
104
|
+
def measures = {metric_name => {}}
|
|
105
|
+
|
|
106
|
+
# Fails unless +settings+ (a measure's, named +name+) are settings
|
|
107
|
+
# there are, with a kind there is and a unit that's one (a word with
|
|
108
|
+
# its plural): a typo isn't left to mean nothing.
|
|
109
|
+
# @api private
|
|
110
|
+
def check(name, settings)
|
|
111
|
+
unknown = settings.keys - SETTINGS
|
|
112
|
+
raise Error, "#{name}: no setting #{unknown.join(", ")} (settings: #{SETTINGS.join(", ")})" unless unknown.empty?
|
|
113
|
+
unless %i[ratio counted].include?(settings.fetch(:kind, :counted))
|
|
114
|
+
raise Error, "#{name}: kind is :ratio or :counted, not #{settings.fetch(:kind).inspect}"
|
|
115
|
+
end
|
|
116
|
+
unit = settings.fetch(:unit, nil)
|
|
117
|
+
if unit.instance_of?(String) && unit.match?(WORD) && !TIME_UNITS.key?(unit)
|
|
118
|
+
raise Error, "#{name}: a word as a unit needs its plural too: unit: [#{unit.inspect}, plural]"
|
|
119
|
+
end
|
|
120
|
+
return if unit.nil? || unit.instance_of?(String) || (unit.instance_of?(Array) && unit.size == 2 && unit.all?(String))
|
|
121
|
+
|
|
122
|
+
raise Error, "#{name}: unit is a String or [singular, plural], not #{unit.inspect}"
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
# Whether +more+ (what a count grew by, over +n+ iterations) is
|
|
126
|
+
# still within what's allowed: +tolerance+ an iteration and
|
|
127
|
+
# +allowance+ a sample. Duel judges counts so, and so can a metric, to
|
|
128
|
+
# tell whether it's worth explaining one (see #within?).
|
|
129
|
+
def within?(more, n, tolerance:, allowance:) = more <= allowance + (tolerance * n)
|
|
130
|
+
|
|
131
|
+
# Each measure's settings, by name, every one of them filled in (as
|
|
132
|
+
# declared, and the rest as by default): what a plugin's tests can
|
|
133
|
+
# check. (+overrides+, by name, are Config#measure's: @api private.)
|
|
134
|
+
def settings(overrides = UNCHANGED)
|
|
135
|
+
measures.to_h do |name, declared|
|
|
136
|
+
given = declared.merge(overrides.fetch(name, NO_CHANGE)) #: Hash[Symbol, untyped]
|
|
137
|
+
[name, resolve(given)]
|
|
138
|
+
end
|
|
139
|
+
end
|
|
140
|
+
|
|
141
|
+
# A measure's settings: those +given+, and the rest as by default (a
|
|
142
|
+
# time unit gives unit_ns, unless it's given, and a symbol is its own
|
|
143
|
+
# plural).
|
|
144
|
+
# @api private
|
|
145
|
+
def resolve(given)
|
|
146
|
+
unit, plural = given.fetch(:unit, nil)
|
|
147
|
+
{kind: given.fetch(:kind, :counted), unit: unit, plural: plural || unit, floor: given.fetch(:floor, 0),
|
|
148
|
+
cpu_floor: given.fetch(:cpu_floor, 0),
|
|
149
|
+
limit: given.fetch(:limit, nil), unit_ns: given.fetch(:unit_ns) { TIME_UNITS.fetch(unit, 1) },
|
|
150
|
+
sizes_samples: given.fetch(:sizes_samples, true), across: given.fetch(:across, true),
|
|
151
|
+
tolerance: given.fetch(:tolerance, 0), allowance: given.fetch(:allowance, 0)}
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
# Each measure's kind, :ratio or :counted, by name.
|
|
155
|
+
# @api private
|
|
156
|
+
def kinds = settings.transform_values { |settings| settings.fetch(:kind) }
|
|
157
|
+
|
|
158
|
+
# Its measures as the runner learns them, with +overrides+ (see
|
|
159
|
+
# Config#measure) by name.
|
|
160
|
+
# @api private
|
|
161
|
+
def specs(overrides = UNCHANGED)
|
|
162
|
+
settings(overrides).map do |name, spec|
|
|
163
|
+
{"name" => name, "kind" => spec.fetch(:kind).to_s, "unit" => spec.fetch(:unit), "plural" => spec.fetch(:plural),
|
|
164
|
+
"floor" => spec.fetch(:floor),
|
|
165
|
+
"cpu_floor" => spec.fetch(:cpu_floor), "limit" => spec.fetch(:limit), "unit_ns" => spec.fetch(:unit_ns),
|
|
166
|
+
"sizes_samples" => spec.fetch(:sizes_samples), "across" => spec.fetch(:across), "tolerance" => spec.fetch(:tolerance),
|
|
167
|
+
"allowance" => spec.fetch(:allowance)}
|
|
168
|
+
end
|
|
169
|
+
end
|
|
170
|
+
end
|
|
171
|
+
|
|
172
|
+
# Each measure's settings, by name, with Config#measure's changes (as
|
|
173
|
+
# the worker it's in has them).
|
|
174
|
+
def settings = @settings ||= self.class.settings
|
|
175
|
+
|
|
176
|
+
# @api private
|
|
177
|
+
def settings=(settings)
|
|
178
|
+
@kinds = @first_measure = @only_measure = nil
|
|
179
|
+
@settings = settings
|
|
180
|
+
end
|
|
181
|
+
|
|
182
|
+
# Each measure's kind, by name, as #settings has it, and the measure
|
|
183
|
+
# that a measurement of one (not a Hash) is, the first: how a sample
|
|
184
|
+
# files what it measured (asked of every stretch, so worked out once).
|
|
185
|
+
# @api private
|
|
186
|
+
def kinds = @kinds ||= settings.transform_values { |settings| settings.fetch(:kind) }
|
|
187
|
+
|
|
188
|
+
# @api private
|
|
189
|
+
def first_measure = @first_measure ||= settings.keys.first
|
|
190
|
+
|
|
191
|
+
# The setting +key+ (:allowance, say) of +measure+: by default, of the
|
|
192
|
+
# one it measures (one that measures several names it).
|
|
193
|
+
def setting(key, measure = only_measure) = settings.fetch(measure).fetch(key)
|
|
194
|
+
|
|
195
|
+
# Whether +more+ (what +measure+'s count grew by, over +n+
|
|
196
|
+
# iterations) is still within its tolerance and allowance, as Duel
|
|
197
|
+
# judges it (see Metric.within?): whether it's worth explaining.
|
|
198
|
+
def within?(more, n, measure = only_measure)
|
|
199
|
+
Metric.within?(more, n, tolerance: setting(:tolerance, measure), allowance: setting(:allowance, measure))
|
|
200
|
+
end
|
|
201
|
+
|
|
202
|
+
# Sets up what it reads (a subscription, say), once in each worker,
|
|
203
|
+
# before it measures anything.
|
|
204
|
+
def open
|
|
205
|
+
end
|
|
206
|
+
|
|
207
|
+
# Begins measuring a sample (a Duel::Sample: its benchmark, n).
|
|
208
|
+
def start_sample(sample)
|
|
209
|
+
end
|
|
210
|
+
|
|
211
|
+
# Ends it, and returns the measurement, a Hash of them, or nil.
|
|
212
|
+
def stop_sample
|
|
213
|
+
end
|
|
214
|
+
|
|
215
|
+
# Pauses it, for an +untimed+ block, and returns what it measured so
|
|
216
|
+
# far.
|
|
217
|
+
def pause_sample = stop_sample
|
|
218
|
+
|
|
219
|
+
# Resumes it after an +untimed+ block.
|
|
220
|
+
def resume_sample(sample) = start_sample(sample)
|
|
221
|
+
|
|
222
|
+
# What it would say of the sample it last stopped, to explain a
|
|
223
|
+
# regression in it ("100 String, 50 Hash", say): text, text by measure
|
|
224
|
+
# name, or nil. Asked once a sample, right after #stop_sample (not
|
|
225
|
+
# after #pause_sample).
|
|
226
|
+
def details
|
|
227
|
+
end
|
|
228
|
+
|
|
229
|
+
# Lets go of what #open set up, when its worker ends. Metrics close
|
|
230
|
+
# last first.
|
|
231
|
+
def close
|
|
232
|
+
end
|
|
233
|
+
|
|
234
|
+
private
|
|
235
|
+
|
|
236
|
+
# The one measure it measures. One that measures several (or none)
|
|
237
|
+
# fails. Found once (see #settings=): #within? may be asked every
|
|
238
|
+
# sample, and allocates nothing.
|
|
239
|
+
def only_measure
|
|
240
|
+
@only_measure ||= begin
|
|
241
|
+
name, *others = settings.keys
|
|
242
|
+
raise Error, "#{self.class} doesn't measure one thing (#{settings.keys.join(", ")}): name which" unless name && others.empty?
|
|
243
|
+
|
|
244
|
+
name
|
|
245
|
+
end
|
|
246
|
+
end
|
|
247
|
+
end
|
|
248
|
+
|
|
249
|
+
# A metric of running totals (see Config#metric): its block reads them,
|
|
250
|
+
# a number for one measure or a Hash of them by name (nil for one it
|
|
251
|
+
# can't read this time), and a sample measures what they grew by.
|
|
252
|
+
# Reading them can itself add to what's counted (a query that reads a
|
|
253
|
+
# server's statistics is one more query), so each worker reads them
|
|
254
|
+
# twice in a row, CALIBRATION times, and the least each counted total
|
|
255
|
+
# grew by is taken from every sample's: unless a measure says
|
|
256
|
+
# calibrate: false (its reading costs nothing, or a varying amount its
|
|
257
|
+
# block takes away itself). A ratio one is left alone: its reading's
|
|
258
|
+
# cost isn't steady enough to take away.
|
|
259
|
+
#
|
|
260
|
+
# Config#metric makes one, and returns it (to take .open, .close,
|
|
261
|
+
# .details and .costly, the block forms of a Metric's #open, #close,
|
|
262
|
+
# #details and .costly?). It isn't made or subclassed otherwise.
|
|
263
|
+
class Snapshot < Metric
|
|
264
|
+
# The pairs of readings each worker takes to learn what reading costs.
|
|
265
|
+
# @api private
|
|
266
|
+
CALIBRATION = 3
|
|
267
|
+
|
|
268
|
+
# One reading: the totals, by name, and what else the block read with
|
|
269
|
+
# them (+context+: whatever its details block needs, or nil). Its
|
|
270
|
+
# writers, and making one, are @api private.
|
|
271
|
+
Reading = Struct.new(:totals, :context)
|
|
272
|
+
|
|
273
|
+
class Reading
|
|
274
|
+
# The total named +name+: one this reading has (it fails for one it
|
|
275
|
+
# couldn't read, or one there isn't).
|
|
276
|
+
def total(name) = totals[name] || raise(Error, "no total #{name} in this reading (#{totals.compact.keys.join(", ")})")
|
|
277
|
+
end
|
|
278
|
+
|
|
279
|
+
class << self
|
|
280
|
+
attr_reader :measures
|
|
281
|
+
|
|
282
|
+
# A Snapshot for +measures+ ({name => settings}), read by +read+.
|
|
283
|
+
# @api private
|
|
284
|
+
def define(measures, read)
|
|
285
|
+
raise ArgumentError, "a metric of running totals needs a block to read them" unless read
|
|
286
|
+
raise ArgumentError, "a metric of running totals needs a name for what it measures" if measures.empty?
|
|
287
|
+
|
|
288
|
+
measures.each { |name, settings| check(name, settings) }
|
|
289
|
+
snapshot = Class.new(self) #: singleton(Snapshot)
|
|
290
|
+
snapshot.reading(measures, read)
|
|
291
|
+
end
|
|
292
|
+
|
|
293
|
+
# Sets what it measures and how it reads them, and returns it.
|
|
294
|
+
# @api private
|
|
295
|
+
def reading(measures, read)
|
|
296
|
+
@measures = measures
|
|
297
|
+
@read = read
|
|
298
|
+
@costly = false
|
|
299
|
+
self
|
|
300
|
+
end
|
|
301
|
+
|
|
302
|
+
# Whether reading it is costly (see Metric.costly? and .costly).
|
|
303
|
+
def costly? = @costly
|
|
304
|
+
|
|
305
|
+
# Says that reading it is costly (a query of a server's statistics,
|
|
306
|
+
# say): it isn't paused around each iteration of a benchmark with
|
|
307
|
+
# around_iteration hooks, and what they add to it is taken away
|
|
308
|
+
# instead (see Metric). Returns the metric.
|
|
309
|
+
#
|
|
310
|
+
# c.metric("postgres_queries") { queries }.costly
|
|
311
|
+
def costly
|
|
312
|
+
@costly = true
|
|
313
|
+
self
|
|
314
|
+
end
|
|
315
|
+
|
|
316
|
+
# Explains its samples by the block (see #details), given each one's
|
|
317
|
+
# stretches and the sample: what Config#metric returns takes it.
|
|
318
|
+
# Returns the metric.
|
|
319
|
+
#
|
|
320
|
+
# c.metric("sql_queries") { [queries, statements] }.details { |stretches, sample| ... }
|
|
321
|
+
def details(&explainer)
|
|
322
|
+
@explainer = explainer
|
|
323
|
+
self
|
|
324
|
+
end
|
|
325
|
+
|
|
326
|
+
# Runs the block in each worker as the metric is opened (see
|
|
327
|
+
# Metric#open), before it reads anything: to set up what it reads
|
|
328
|
+
# (subscribing to notifications, say). Returns the metric.
|
|
329
|
+
#
|
|
330
|
+
# c.metric("sql_queries") { counter.queries }.open { counter.subscribe }.close { counter.unsubscribe }
|
|
331
|
+
def open(&opener)
|
|
332
|
+
@opener = opener
|
|
333
|
+
self
|
|
334
|
+
end
|
|
335
|
+
|
|
336
|
+
# Runs the block when each worker ends (see Metric#close): to let go
|
|
337
|
+
# of what .open set up. Returns the metric.
|
|
338
|
+
def close(&closer)
|
|
339
|
+
@closer = closer
|
|
340
|
+
self
|
|
341
|
+
end
|
|
342
|
+
|
|
343
|
+
# What explains a sample (see .details), opens the metric in a
|
|
344
|
+
# worker (.open) and closes it (.close), or nil.
|
|
345
|
+
# @api private
|
|
346
|
+
attr_reader :explainer, :opener, :closer
|
|
347
|
+
|
|
348
|
+
# A Reading: the totals, by name (those it measures, or it fails,
|
|
349
|
+
# naming them), and their context. The block returns the totals (a
|
|
350
|
+
# number, for one), or [totals, context].
|
|
351
|
+
# @api private
|
|
352
|
+
def read
|
|
353
|
+
# (A number or a Hash, alone, is the totals, without a context.)
|
|
354
|
+
totals, context = @read.call
|
|
355
|
+
# (A Hash of them, or one: as the block's type says.)
|
|
356
|
+
totals = (totals.is_a?(Hash) ? totals : {measures.keys.first => totals}) #: totals[Metric::value]
|
|
357
|
+
return Reading.new(totals, context) if totals.keys.sort == measures.keys.sort
|
|
358
|
+
|
|
359
|
+
raise Error, "running totals of #{measures.keys.join(", ")}: read #{totals.keys.join(", ")}"
|
|
360
|
+
end
|
|
361
|
+
end
|
|
362
|
+
|
|
363
|
+
# Runs its .open block, if it has one, then learns what reading its
|
|
364
|
+
# counted totals costs.
|
|
365
|
+
def open
|
|
366
|
+
self.class.opener&.call
|
|
367
|
+
calibrated = self.class.measures.filter_map do |name, settings|
|
|
368
|
+
name if settings.fetch(:kind, :counted) == :counted && settings.fetch(:calibrate, true)
|
|
369
|
+
end
|
|
370
|
+
@overhead = calibrated.to_h { |name| [name, Array.new(CALIBRATION) { grew(self.class.read.totals, name) }.min] }
|
|
371
|
+
end
|
|
372
|
+
|
|
373
|
+
# A sample begins: its first stretch (see #details).
|
|
374
|
+
def start_sample(sample)
|
|
375
|
+
@sample = sample
|
|
376
|
+
@stretches = [] #: Snapshot::stretches[Metric::value, untyped]
|
|
377
|
+
@started = self.class.read
|
|
378
|
+
end
|
|
379
|
+
|
|
380
|
+
# Another stretch of the same sample, after an untimed block.
|
|
381
|
+
def resume_sample(_sample) = @started = self.class.read
|
|
382
|
+
|
|
383
|
+
def stop_sample
|
|
384
|
+
stopped = self.class.read
|
|
385
|
+
@stretches << [@started, stopped]
|
|
386
|
+
stopped.totals.to_h do |name, total|
|
|
387
|
+
started = @started.totals.fetch(name)
|
|
388
|
+
[name, (total - started - @overhead.fetch(name, 0) if total && started)]
|
|
389
|
+
end
|
|
390
|
+
end
|
|
391
|
+
|
|
392
|
+
# What its +details+ block says of the sample it last stopped, given
|
|
393
|
+
# each stretch of it measured ([started, stopped] Readings: one, unless
|
|
394
|
+
# untimed blocks paused it) and the Sample (its n, say).
|
|
395
|
+
def details = self.class.explainer&.call(@stretches, @sample)
|
|
396
|
+
|
|
397
|
+
# Runs its .close block, if it has one.
|
|
398
|
+
def close = self.class.closer&.call
|
|
399
|
+
|
|
400
|
+
private
|
|
401
|
+
|
|
402
|
+
# What +name+'s total grew by in a reading after +before+ (nothing,
|
|
403
|
+
# where it can't be read).
|
|
404
|
+
def grew(before, name)
|
|
405
|
+
after = self.class.read.totals.fetch(name)
|
|
406
|
+
started = before.fetch(name)
|
|
407
|
+
(after && started) ? after - started : 0
|
|
408
|
+
end
|
|
409
|
+
end
|
|
410
|
+
|
|
411
|
+
# CPU time in nanoseconds: the benchmark thread's, or the process's.
|
|
412
|
+
# @api private
|
|
413
|
+
module Clock
|
|
414
|
+
def self.thread = Process.clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID, :nanosecond)
|
|
415
|
+
|
|
416
|
+
def self.process = Process.clock_gettime(Process::CLOCK_PROCESS_CPUTIME_ID, :nanosecond)
|
|
417
|
+
end
|
|
418
|
+
|
|
419
|
+
# simplecov:disable JRuby has no thread CPU clock of Ruby's, but the JVM's (JRuby's
|
|
420
|
+
# test run covers these).
|
|
421
|
+
if RUBY_ENGINE == "jruby"
|
|
422
|
+
module Clock
|
|
423
|
+
def self.thread = java.lang.management.ManagementFactory.getThreadMXBean.getCurrentThreadCpuTime
|
|
424
|
+
|
|
425
|
+
def self.process = java.lang.management.ManagementFactory.getOperatingSystemMXBean.getProcessCpuTime
|
|
426
|
+
end
|
|
427
|
+
end
|
|
428
|
+
# simplecov:enable
|
|
429
|
+
|
|
430
|
+
# The built-in metrics: their measures are named ("cpu",
|
|
431
|
+
# "instructions", "allocations"), as Config#measure takes them.
|
|
432
|
+
# @api private
|
|
433
|
+
module Metrics
|
|
434
|
+
# CPU time (see Definition#cpu), in nanoseconds: what other
|
|
435
|
+
# processes on the machine don't inflate. Always measured last, inside
|
|
436
|
+
# every other metric.
|
|
437
|
+
class CPU < Metric
|
|
438
|
+
def self.measures = {"cpu" => {kind: :ratio, unit: "ns", floor: 300_000}}
|
|
439
|
+
|
|
440
|
+
def start_sample(sample)
|
|
441
|
+
@clock = sample.benchmark.cpu
|
|
442
|
+
@started = Clock.public_send(@clock)
|
|
443
|
+
end
|
|
444
|
+
|
|
445
|
+
def stop_sample = Clock.public_send(@clock) - @started
|
|
446
|
+
end
|
|
447
|
+
|
|
448
|
+
# Instructions retired in user space, from the CPU's own counters:
|
|
449
|
+
# nearly the same from run to run, so it catches slowdowns too small
|
|
450
|
+
# for CPU time to settle (with a limit of its own, 1%). Where the
|
|
451
|
+
# counters are: Linux (the benchmark thread's, unless a virtual
|
|
452
|
+
# machine hides them, so not for benchmarks with threads of their
|
|
453
|
+
# own) and macOS (the whole process's). It misses what instructions
|
|
454
|
+
# don't show, like cache misses: CPU time is still judged.
|
|
455
|
+
class Instructions < Metric
|
|
456
|
+
def self.measures = {"instructions" => {kind: :ratio, floor: 100_000, sizes_samples: false, limit: 1.01}}
|
|
457
|
+
|
|
458
|
+
def self.available? = !counter.nil?
|
|
459
|
+
|
|
460
|
+
# This process's counter (see Native::Counter.open), or nil.
|
|
461
|
+
def self.counter
|
|
462
|
+
return @counter if instance_variable_defined?(:@counter)
|
|
463
|
+
|
|
464
|
+
@counter = Native::Counter.open
|
|
465
|
+
end
|
|
466
|
+
|
|
467
|
+
def initialize
|
|
468
|
+
# (There's one: it's made only where it's available?.)
|
|
469
|
+
@counter = self.class.counter #: Native::Counter
|
|
470
|
+
end
|
|
471
|
+
|
|
472
|
+
def start_sample(sample)
|
|
473
|
+
@counting = sample.benchmark.cpu == :thread || @counter.scope == "process"
|
|
474
|
+
@started = @counter.read if @counting
|
|
475
|
+
end
|
|
476
|
+
|
|
477
|
+
def stop_sample = (@counter.read - @started if @counting)
|
|
478
|
+
end
|
|
479
|
+
|
|
480
|
+
# Ruby objects allocated (CRuby). The same code's count can differ by
|
|
481
|
+
# a few from run to run (ids of random lengths, say), so it's one
|
|
482
|
+
# more per two iterations, not one more, that's a regression.
|
|
483
|
+
class Allocations < Metric
|
|
484
|
+
def self.measures = {"allocations" => {tolerance: 0.5}}
|
|
485
|
+
|
|
486
|
+
# mutant:disable -- mutant runs on CRuby, where this is always true
|
|
487
|
+
def self.available? = RUBY_ENGINE == "ruby"
|
|
488
|
+
|
|
489
|
+
def start_sample(_sample)
|
|
490
|
+
@started = GC.stat(:total_allocated_objects)
|
|
491
|
+
end
|
|
492
|
+
|
|
493
|
+
def stop_sample = GC.stat(:total_allocated_objects) - @started
|
|
494
|
+
end
|
|
495
|
+
end
|
|
496
|
+
end
|
data/lib/duel/native.rb
ADDED
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "etc"
|
|
4
|
+
require "fiddle"
|
|
5
|
+
require "rbconfig"
|
|
6
|
+
|
|
7
|
+
module Duel
|
|
8
|
+
# What the operating system can tell about this process, through Fiddle:
|
|
9
|
+
# instructions retired (hardware counters) and resident memory. Each
|
|
10
|
+
# answers nil where it can't, as on JRuby, where Fiddle's calls can
|
|
11
|
+
# bring the JVM down.
|
|
12
|
+
# @api private
|
|
13
|
+
module Native
|
|
14
|
+
FIDDLE = RUBY_ENGINE == "ruby"
|
|
15
|
+
OS = RbConfig::CONFIG["host_os"]
|
|
16
|
+
CPU = RbConfig::CONFIG["host_cpu"]
|
|
17
|
+
UINT64 = 8 # bytes, the size of every count read
|
|
18
|
+
|
|
19
|
+
# A C function of the process's own libraries, by name, taking and
|
|
20
|
+
# returning Fiddle types (:INT, :LONG, :VOIDP...).
|
|
21
|
+
def self.function(name, args, returns)
|
|
22
|
+
Fiddle::Function.new(Fiddle.dlopen(nil)[name], args.map { |type| Fiddle.const_get(:"TYPE_#{type}") },
|
|
23
|
+
Fiddle.const_get(:"TYPE_#{returns}"))
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
# Instructions retired in user space, from the CPU's counters.
|
|
27
|
+
class Counter
|
|
28
|
+
# The best counter here: the benchmark thread's (Linux), the
|
|
29
|
+
# process's (macOS), or nil.
|
|
30
|
+
def self.open(fiddle: FIDDLE) = (Linux.open || MacOS.open if fiddle)
|
|
31
|
+
|
|
32
|
+
# "thread" or "process": whose instructions it counts.
|
|
33
|
+
attr_reader :scope
|
|
34
|
+
|
|
35
|
+
def initialize(scope) = @scope = scope
|
|
36
|
+
|
|
37
|
+
def read = raise(NotImplementedError)
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# Linux: perf_event_open(2), counting this thread's instructions in
|
|
41
|
+
# user space. Unavailable in many virtual machines (no counters), and
|
|
42
|
+
# where kernel.perf_event_paranoid forbids it.
|
|
43
|
+
class Linux < Counter
|
|
44
|
+
SYSCALL = {"x86_64" => 298, "aarch64" => 241}.freeze
|
|
45
|
+
PERF_TYPE_HARDWARE = 0
|
|
46
|
+
PERF_COUNT_HW_INSTRUCTIONS = 1
|
|
47
|
+
# exclude_kernel and exclude_hv: user space only, which needs no
|
|
48
|
+
# privileges.
|
|
49
|
+
FLAGS = (1 << 5) | (1 << 6)
|
|
50
|
+
PERF_FLAG_FD_CLOEXEC = 8
|
|
51
|
+
ATTR_SIZE = 64 # PERF_ATTR_SIZE_VER0
|
|
52
|
+
|
|
53
|
+
def self.open(platform: OS, cpu: CPU, syscall: nil)
|
|
54
|
+
number = SYSCALL[cpu]
|
|
55
|
+
return unless platform.include?("linux") && number
|
|
56
|
+
|
|
57
|
+
syscall ||= Native.function("syscall", %i[LONG VOIDP INT INT INT LONG], :LONG)
|
|
58
|
+
attr = [PERF_TYPE_HARDWARE, ATTR_SIZE, PERF_COUNT_HW_INSTRUCTIONS, 0, 0, 0, FLAGS, 0, 0, 0].pack("LLQQQQQLLQ")
|
|
59
|
+
fd = syscall.call(number, attr, 0, -1, -1, PERF_FLAG_FD_CLOEXEC)
|
|
60
|
+
new(IO.for_fd(fd)) unless fd.negative?
|
|
61
|
+
rescue Fiddle::DLError # no such function
|
|
62
|
+
nil
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def initialize(io)
|
|
66
|
+
super("thread")
|
|
67
|
+
@io = io
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def read = @io.pread(UINT64, 0).unpack1("Q")
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
# macOS: proc_pid_rusage(RUSAGE_INFO_V4), the process's instructions
|
|
74
|
+
# and resident memory, no privileges needed.
|
|
75
|
+
class MacOS < Counter
|
|
76
|
+
RUSAGE_INFO_V4 = 4
|
|
77
|
+
SIZE = 296 # sizeof(struct rusage_info_v4)
|
|
78
|
+
RESIDENT = 64 # offsetof(ri_resident_size)
|
|
79
|
+
INSTRUCTIONS = 248 # offsetof(ri_instructions)
|
|
80
|
+
|
|
81
|
+
def self.open(platform: OS, rusage: nil)
|
|
82
|
+
return unless platform.include?("darwin")
|
|
83
|
+
|
|
84
|
+
rusage ||= Native.function("proc_pid_rusage", %i[INT INT VOIDP], :INT)
|
|
85
|
+
counter = new(rusage)
|
|
86
|
+
counter if counter.read.positive?
|
|
87
|
+
rescue Fiddle::DLError # no such function
|
|
88
|
+
nil
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
def initialize(rusage)
|
|
92
|
+
super("process")
|
|
93
|
+
@rusage = rusage
|
|
94
|
+
# (A String of its own: Fiddle passes its bytes, for the call to
|
|
95
|
+
# fill in place.)
|
|
96
|
+
@buffer = "\0".b * SIZE
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
def read = field(INSTRUCTIONS)
|
|
100
|
+
|
|
101
|
+
def resident = field(RESIDENT)
|
|
102
|
+
|
|
103
|
+
private
|
|
104
|
+
|
|
105
|
+
def field(offset)
|
|
106
|
+
@rusage.call(Process.pid, RUSAGE_INFO_V4, @buffer)
|
|
107
|
+
@buffer.unpack1("Q", offset: offset)
|
|
108
|
+
end
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
# A child process of this one, waited for.
|
|
112
|
+
module Child
|
|
113
|
+
# wait4(2)'s struct rusage: its size, and where its ru_maxrss is (on
|
|
114
|
+
# Linux and macOS, 64-bit).
|
|
115
|
+
RUSAGE_SIZE = 144
|
|
116
|
+
MAXRSS = 32
|
|
117
|
+
# A wait a signal interrupted: waited again.
|
|
118
|
+
EINTR = Errno::EINTR::Errno
|
|
119
|
+
|
|
120
|
+
# Waits for the child +pid+, and returns how it ended (its exit
|
|
121
|
+
# status, or nil, and the signal that killed it, or nil) and the most
|
|
122
|
+
# resident memory it reached (its children's too), in bytes, or nil
|
|
123
|
+
# where that can't be told (without Fiddle).
|
|
124
|
+
def self.wait(pid, fiddle: FIDDLE, platform: OS, wait4: nil)
|
|
125
|
+
unless fiddle
|
|
126
|
+
status = Process.wait2(pid).last
|
|
127
|
+
return [status.exitstatus, status.termsig, nil]
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
wait4 ||= Native.function("wait4", %i[INT VOIDP INT VOIDP], :INT)
|
|
131
|
+
status = "\0".b * 4
|
|
132
|
+
rusage = "\0".b * RUSAGE_SIZE
|
|
133
|
+
until wait4.call(pid, status, 0, rusage) == pid
|
|
134
|
+
raise SystemCallError.new("wait4", Fiddle.last_error) unless Fiddle.last_error == EINTR
|
|
135
|
+
end
|
|
136
|
+
# (As WIFEXITED, WEXITSTATUS and WTERMSIG take it apart: what wait4
|
|
137
|
+
# gives is a process's that exited or was killed, in 16 bits.)
|
|
138
|
+
raw = status.unpack1("l") #: Integer
|
|
139
|
+
signal = raw & 0x7f
|
|
140
|
+
# (ru_maxrss is in kilobytes on Linux, bytes on macOS.)
|
|
141
|
+
[(raw >> 8 if signal.zero?), (signal unless signal.zero?),
|
|
142
|
+
rusage.unpack1("q", offset: MAXRSS) * (platform.include?("darwin") ? 1 : 1024)]
|
|
143
|
+
end
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
# This process's resident memory in bytes, or nil where it can't tell.
|
|
147
|
+
module Memory
|
|
148
|
+
STATM = "/proc/self/statm"
|
|
149
|
+
# (Never false: Steep doesn't see that false || nil is nil.)
|
|
150
|
+
MACOS = (FIDDLE && MacOS.open) || nil #: MacOS?
|
|
151
|
+
|
|
152
|
+
# From /proc (Linux), else from macOS's counter, else nil.
|
|
153
|
+
def self.resident(statm: STATM, macos: MACOS)
|
|
154
|
+
return macos&.resident unless File.readable?(statm)
|
|
155
|
+
|
|
156
|
+
# (The page size is always known.)
|
|
157
|
+
page_size = Etc.sysconf(Etc::SC_PAGESIZE) #: Integer
|
|
158
|
+
Integer(File.read(statm).split.fetch(1)) * page_size
|
|
159
|
+
end
|
|
160
|
+
end
|
|
161
|
+
end
|
|
162
|
+
end
|