duel 0.1.8 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +146 -1
  3. data/README.md +35 -0
  4. data/docs/ci.md +96 -3
  5. data/docs/configuration.md +35 -1
  6. data/docs/how-it-works.md +14 -8
  7. data/docs/plugins.md +16 -0
  8. data/lib/duel/across.rb +1 -6
  9. data/lib/duel/baseline.rb +8 -7
  10. data/lib/duel/cli.rb +45 -19
  11. data/lib/duel/comment.rb +71 -10
  12. data/lib/duel/config.rb +113 -2
  13. data/lib/duel/context.rb +33 -34
  14. data/lib/duel/control.rb +52 -0
  15. data/lib/duel/init.rb +142 -18
  16. data/lib/duel/lane.rb +734 -0
  17. data/lib/duel/lock.rb +34 -4
  18. data/lib/duel/memory.rb +14 -5
  19. data/lib/duel/metric.rb +97 -26
  20. data/lib/duel/native.rb +199 -32
  21. data/lib/duel/platform.rb +31 -0
  22. data/lib/duel/process_sample.rb +74 -49
  23. data/lib/duel/resource.rb +71 -42
  24. data/lib/duel/result.rb +165 -0
  25. data/lib/duel/rubies.rb +21 -3
  26. data/lib/duel/runner.rb +13 -1039
  27. data/lib/duel/spawn.rb +327 -0
  28. data/lib/duel/summary.rb +10 -10
  29. data/lib/duel/templates/perf-caller.yml.erb +43 -0
  30. data/lib/duel/templates/perf-comment-caller.yml.erb +42 -0
  31. data/lib/duel/templates/perf-comment.yml.erb +45 -3
  32. data/lib/duel/templates/perf.yml.erb +143 -7
  33. data/lib/duel/testing.rb +3 -0
  34. data/lib/duel/version.rb +1 -1
  35. data/lib/duel/worker.rb +13 -4
  36. data/lib/duel/worker_process.rb +142 -0
  37. data/lib/duel.rb +27 -3
  38. data/sig/duel/cli.rbs +8 -0
  39. data/sig/duel/comment.rbs +16 -1
  40. data/sig/duel/config.rbs +52 -2
  41. data/sig/duel/context.rbs +4 -31
  42. data/sig/duel/control.rbs +27 -0
  43. data/sig/duel/init.rbs +56 -0
  44. data/sig/duel/lane.rbs +202 -0
  45. data/sig/duel/lock.rbs +8 -1
  46. data/sig/duel/memory.rbs +1 -1
  47. data/sig/duel/metric.rbs +53 -10
  48. data/sig/duel/native.rbs +109 -9
  49. data/sig/duel/platform.rbs +13 -0
  50. data/sig/duel/process_sample.rbs +7 -9
  51. data/sig/duel/resource.rbs +28 -9
  52. data/sig/duel/result.rbs +164 -0
  53. data/sig/duel/rubies.rbs +4 -0
  54. data/sig/duel/runner.rbs +3 -460
  55. data/sig/duel/spawn.rbs +111 -0
  56. data/sig/duel/summary.rbs +3 -2
  57. data/sig/duel/worker_process.rbs +90 -0
  58. data/sig/duel.rbs +17 -3
  59. metadata +121 -107
@@ -1,3 +1,101 @@
1
+ <% if reusable? -%>
2
+ name: Duel
3
+
4
+ # Duel's Perf workflow, reusable: a project's own Perf workflow (as
5
+ # `duel init --reusable` writes it) calls it, with its triggers and its
6
+ # concurrency, and passes on what .duel.rb sets that it needs as its
7
+ # inputs. Its jobs run as the Perf workflow's that `duel init` writes
8
+ # do, and the caller grants what they need (contents: write to mark a
9
+ # push passed, comment on it, or keep the badge, and pull-requests: read
10
+ # for the labels), which each job narrows. See Duel's docs/ci.md.
11
+ on:
12
+ workflow_call:
13
+ inputs:
14
+ ruby-version:
15
+ description: The Ruby the Duel job runs (default, as .ruby-version names it)
16
+ type: string
17
+ default: default
18
+ version-file:
19
+ description: Whether each version runs on the Ruby its .ruby-version names (c.ruby = :version_file)
20
+ type: boolean
21
+ default: false
22
+ yjit:
23
+ description: Whether the benchmarks run with YJIT on (RUBY_YJIT_ENABLE=1)
24
+ type: boolean
25
+ default: false
26
+ runs-on:
27
+ description: The runner the Duel job runs on (a larger one, say, for steadier timings), Linux or macOS
28
+ type: string
29
+ default: ubuntu-latest
30
+ timeout-minutes:
31
+ description: The minutes the Duel job may take before it's cancelled
32
+ type: number
33
+ default: 60
34
+ working-directory:
35
+ description: The project's directory, from the repository's root
36
+ type: string
37
+ default: .
38
+ gemfile:
39
+ description: The Gemfile of the bundle Duel runs in (BUNDLE_GEMFILE), from the project's directory, or none for the project's own
40
+ type: string
41
+ default: ""
42
+ submodules:
43
+ description: Whether to check out the repository's submodules too (true, or recursive), as actions/checkout takes it
44
+ type: string
45
+ default: "false"
46
+ setup:
47
+ description: Shell commands that set up what else the tests need (packages, services), before Ruby's
48
+ type: string
49
+ default: ""
50
+ report:
51
+ description: A name for this call's report (yjit, say), where the workflow calls this one more than once. The Perf comment workflow comments with every call's, each under its name
52
+ type: string
53
+ default: ""
54
+ accept-label:
55
+ description: The label that accepts a regression (c.accept_label), or none
56
+ type: string
57
+ default: ""
58
+ release-tags:
59
+ description: The glob release tags match (c.release_tags)
60
+ type: string
61
+ default: v*
62
+ comment-on-pushes:
63
+ description: Whether to comment on a pushed commit (c.fail_on = :pull_requests)
64
+ type: boolean
65
+ default: false
66
+ mark-passed:
67
+ description: Whether to mark a push passed (c.push_base = :last_passed)
68
+ type: boolean
69
+ default: false
70
+ badge:
71
+ description: Whether to keep a badge for the README
72
+ type: boolean
73
+ default: false
74
+ passed-branch:
75
+ description: What names the branches that mark the last commit Duel passed
76
+ type: string
77
+ default: duel/passed
78
+ badge-branch:
79
+ description: The branch that holds the badge
80
+ type: string
81
+ default: duel/badge
82
+ marker:
83
+ description: What marks Duel's comments
84
+ type: string
85
+ default: duel
86
+ # What Duel said, for the caller's own jobs (a notice, say).
87
+ outputs:
88
+ comment:
89
+ description: Whether Duel said to comment (post or change), or nothing
90
+ value: ${{ jobs.duel.outputs.comment }}
91
+ status:
92
+ description: The run's status (passed, failed or neutral), or nothing where Duel didn't get to compare
93
+ value: ${{ jobs.duel.outputs.status }}
94
+ passed:
95
+ description: Whether to mark a push passed (true), or nothing
96
+ value: ${{ jobs.duel.outputs.passed }}
97
+
98
+ <% else -%>
1
99
  name: Perf<%= " (#{working_directory})" if working_directory %>
2
100
 
3
101
  on:
@@ -16,6 +114,7 @@ concurrency:
16
114
  group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.run_id }}
17
115
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
18
116
 
117
+ <% end -%>
19
118
  # The Duel job runs the branch's code (its benchmarks too), so it only
20
119
  # reads. Commenting on a commit is the Comment on the commit job's, which
21
120
  # runs none of it, and commenting on a pull request is the Perf comment
@@ -26,8 +125,14 @@ permissions:
26
125
 
27
126
  jobs:
28
127
  duel:
128
+ <% if reusable? -%>
129
+ name: Duel${{ inputs.report != '' && format(' ({0})', inputs.report) || '' }}
130
+ runs-on: ${{ inputs.runs-on }}
131
+ timeout-minutes: ${{ inputs.timeout-minutes }}
132
+ <% else -%>
29
133
  name: Duel
30
134
  runs-on: ubuntu-latest
135
+ <% end -%>
31
136
  # Whether to comment (post, change or none), the run's status (passed,
32
137
  # failed or neutral), and whether to mark a push passed (true, or
33
138
  # nothing), from Duel.
@@ -47,6 +152,9 @@ jobs:
47
152
  with:
48
153
  fetch-depth: 0 # the baseline comes from history, tags included
49
154
  persist-credentials: false
155
+ <% if reusable? -%>
156
+ submodules: ${{ inputs.submodules }}
157
+ <% end -%>
50
158
  # The labels of the pull request (or for a push, of the one that
51
159
  # merged it), for c.accept_label in .duel.rb, as a JSON array. They're
52
160
  # read as the job runs, and adding the label (or taking it off) runs
@@ -58,7 +166,7 @@ jobs:
58
166
  <% end -%>
59
167
  - name: Find the pull request's labels
60
168
  id: labels
61
- if: github.event_name == 'pull_request' || github.event_name == 'merge_group' || github.event_name == 'push'<% if accept_label %> || github.event_name == 'schedule'<% end %>
169
+ if: github.event_name == 'pull_request' || github.event_name == 'merge_group' || github.event_name == 'push'<% if accept_label %><%= reusable? ? " || (github.event_name == 'schedule' && inputs.accept-label != '')" : " || github.event_name == 'schedule'" %><% end %>
62
170
  continue-on-error: true # (without them, nothing's accepted)
63
171
  env:
64
172
  GH_TOKEN: ${{ github.token }}
@@ -92,7 +200,29 @@ jobs:
92
200
  labels=$(gh api "repos/$GITHUB_REPOSITORY/commits/$GITHUB_SHA/pulls" --jq '[.[0].labels[]?.name]')
93
201
  fi
94
202
  echo "labels=$labels" >> "$GITHUB_OUTPUT"
203
+ <% if reusable? -%>
204
+ # What else the tests need set up (packages, services), as the
205
+ # caller says.
206
+ - name: Set up
207
+ if: inputs.setup != ''
208
+ env:
209
+ SETUP: ${{ inputs.setup }}
210
+ run: eval "$SETUP"
211
+ # The bundle Duel and both versions' workers run in, where the caller
212
+ # names one of its own (a Gemfile for Duel alone, say): an absolute
213
+ # path, as the baseline's checkout is elsewhere.
214
+ - name: Use the caller's Gemfile
215
+ if: inputs.gemfile != ''
216
+ env:
217
+ GEMFILE: ${{ inputs.gemfile }}
218
+ run: echo "BUNDLE_GEMFILE=$PWD/$GEMFILE" >> "$GITHUB_ENV"
219
+ # YJIT, where the caller says to, for every worker of both versions.
220
+ - name: Turn on YJIT
221
+ if: inputs.yjit
222
+ run: echo "RUBY_YJIT_ENABLE=1" >> "$GITHUB_ENV"
223
+ <% else -%>
95
224
  # (And whatever else your tests need set up: packages, services.)
225
+ <% end -%>
96
226
  <% if version_file -%>
97
227
  # Each version runs on the Ruby its own .ruby-version names (see
98
228
  # .duel.rb). When the baseline's names another Ruby than this code's,
@@ -108,6 +238,9 @@ jobs:
108
238
  # RELEASE_TAGS to match.)
109
239
  - name: Find the baseline's Ruby
110
240
  id: baseline
241
+ <% if reusable? -%>
242
+ if: inputs.version-file
243
+ <% end -%>
111
244
  env:
112
245
  BEFORE: ${{ github.event.before }}
113
246
  QUEUE_BASE: ${{ github.event.merge_group.base_sha }}
@@ -197,7 +330,7 @@ jobs:
197
330
  if: ${{ !cancelled() && (github.event_name == 'pull_request' || github.event_name == 'push') }}
198
331
  uses: actions/upload-artifact@v7
199
332
  with:
200
- name: duel-report
333
+ name: duel-report<%= "${{ inputs.report != '' && format('-{0}', inputs.report) || '' }}" if reusable? %>
201
334
  path: ${{ runner.temp }}/duel/
202
335
  if-no-files-found: ignore # none when Duel didn't get to compare
203
336
  retention-days: 1
@@ -212,7 +345,7 @@ jobs:
212
345
  commit-comment:
213
346
  name: Comment on the commit
214
347
  needs: duel
215
- if: ${{ !cancelled() && github.event_name == 'push' && (needs.duel.outputs.comment == 'post' || needs.duel.outputs.comment == 'change') }}
348
+ if: ${{ <%= "inputs.comment-on-pushes && " if reusable? %>!cancelled() && github.event_name == 'push' && (needs.duel.outputs.comment == 'post' || needs.duel.outputs.comment == 'change') }}
216
349
  runs-on: ubuntu-latest
217
350
  permissions:
218
351
  contents: write # to comment on the commit with the report
@@ -222,7 +355,7 @@ jobs:
222
355
  continue-on-error: true # there's none when the Duel job failed before comparing
223
356
  uses: actions/download-artifact@v8
224
357
  with:
225
- name: duel-report
358
+ name: duel-report<%= "${{ inputs.report != '' && format('-{0}', inputs.report) || '' }}" if reusable? %>
226
359
  path: ${{ runner.temp }}/report
227
360
  - name: Comment on the commit
228
361
  if: steps.report.outcome == 'success'
@@ -230,8 +363,11 @@ jobs:
230
363
  GH_TOKEN: ${{ github.token }}
231
364
  COMMENT: ${{ needs.duel.outputs.comment }}
232
365
  STATUS: ${{ needs.duel.outputs.status }}
366
+ <% if reusable? -%>
367
+ MARKER: ${{ inputs.marker }}
368
+ <% end -%>
233
369
  run: |
234
- marker="<!-- <%= marker %>"
370
+ marker="<!-- <%= reusable? ? "$MARKER" : marker %>"
235
371
  last=$(gh api --paginate "repos/$GITHUB_REPOSITORY/comments" \
236
372
  --jq ".[] | select(.user.login == \"github-actions[bot]\") | .body | split(\"\n\")[0] | rtrimstr(\"\r\") | select(startswith(\"$marker status=\"))" | tail -n 1)
237
373
  if [ "$COMMENT" = post ] || { [ -n "$last" ] && [ "$last" != "$marker status=$STATUS -->" ]; }; then
@@ -259,7 +395,7 @@ jobs:
259
395
  concurrency:
260
396
  group: ${{ github.workflow }}-passed-${{ github.ref_name }}
261
397
  cancel-in-progress: false
262
- if: ${{ !cancelled() && github.event_name == 'push' && needs.duel.outputs.passed == 'true' }}
398
+ if: ${{ <%= "inputs.mark-passed && " if reusable? %>!cancelled() && github.event_name == 'push' && needs.duel.outputs.passed == 'true' }}
263
399
  runs-on: ubuntu-latest
264
400
  permissions:
265
401
  contents: write # to move the branch
@@ -330,7 +466,7 @@ jobs:
330
466
  concurrency:
331
467
  group: ${{ github.workflow }}-badge
332
468
  cancel-in-progress: false
333
- if: ${{ !cancelled() && github.event_name == 'push' && github.ref_name == github.event.repository.default_branch && (needs.duel.result == 'failure' || needs.duel.outputs.status == 'passed' || needs.duel.outputs.status == 'failed') }}
469
+ if: ${{ <%= "inputs.badge && " if reusable? %>!cancelled() && github.event_name == 'push' && github.ref_name == github.event.repository.default_branch && (needs.duel.result == 'failure' || needs.duel.outputs.status == 'passed' || needs.duel.outputs.status == 'failed') }}
334
470
  runs-on: ubuntu-latest
335
471
  permissions:
336
472
  contents: write # to move the badge's branch
data/lib/duel/testing.rb CHANGED
@@ -217,6 +217,9 @@ module Duel
217
217
  run = Settings.defaults
218
218
  run.config_path = config_path if File.exist?(config_path)
219
219
  run.processes = 2
220
+ # (.duel.rb's run settings, c.limit say, but where one's given, and
221
+ # its processes: a test's are few.)
222
+ config.run_settings.except(:processes).each { |name, value| run[name] = value }
220
223
  settings.each { |setting, value| run[setting] = value }
221
224
  out = StringIO.new
222
225
  runner = Runner.new(config, run, base, out: out)
data/lib/duel/version.rb CHANGED
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Duel
4
- VERSION = "0.1.8"
4
+ VERSION = "0.2.0"
5
5
  end
data/lib/duel/worker.rb CHANGED
@@ -5,6 +5,7 @@ require "socket"
5
5
  require_relative "../duel"
6
6
  require_relative "context"
7
7
  require_relative "message"
8
+ require_relative "control"
8
9
  require_relative "profile"
9
10
 
10
11
  module Duel
@@ -12,7 +13,7 @@ module Duel
12
13
  # benchmarks, and answers the runner's requests. Started (see
13
14
  # WorkerProcess) with Worker.main(VERSION_ROOT, CANDIDATE_ROOT, CONTROL
14
15
  # [, CONFIG]), it gives its main object the DSL, loads the benchmarks,
15
- # connects to the runner's Unix socket at CONTROL, sends what it has
16
+ # connects to the runner at CONTROL (see Control), sends what it has
16
17
  # (see #hello), and answers each request with a reply (see Message).
17
18
  # (Not over stdin: a JRuby started by JRuby sees its stdin pipe
18
19
  # closed.)
@@ -21,8 +22,16 @@ module Duel
21
22
  # The built-in metrics, outermost first: allocations (CRuby's objects,
22
23
  # or JRuby's bytes) inside instructions, whose counter allocates as
23
24
  # it's read (a sample's own allocations are none but the benchmark's),
24
- # and CPU time innermost.
25
- BUILT_IN = [Metrics::Instructions, Metrics::Allocations, Metrics::AllocatedBytes, Metrics::CPU].freeze
25
+ # and CPU time innermost. But on JRuby on Windows, JRuby's bytes are
26
+ # innermost, inside CPU time, whose readings of Windows's cycles
27
+ # (through Fiddle: see CycleClock) allocate there.
28
+ # simplecov:disable JRuby on Windows's (its test run covers it).
29
+ BUILT_IN = if Platform::JRUBY_ON_WINDOWS
30
+ [Metrics::Instructions, Metrics::Allocations, Metrics::CPU, Metrics::AllocatedBytes].freeze
31
+ else
32
+ [Metrics::Instructions, Metrics::Allocations, Metrics::AllocatedBytes, Metrics::CPU].freeze
33
+ end
34
+ # simplecov:enable
26
35
 
27
36
  def self.main(version_root, candidate_root, control, config_path = nil)
28
37
  config = config_path ? Duel.load_config(config_path) : (Duel.config = Config.new(candidate_root))
@@ -40,7 +49,7 @@ module Duel
40
49
  GC.start
41
50
  worker = new(config.metrics, config.measures, hooks: config.hooks, before: before)
42
51
  begin
43
- socket = UNIXSocket.new(control)
52
+ socket = Control.connect(control)
44
53
  Message.write(socket, worker.hello)
45
54
  while (request = Message.read(socket))
46
55
  Message.write(socket, worker.handle(request))
@@ -0,0 +1,142 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../duel"
4
+ require_relative "control"
5
+ require_relative "message"
6
+ require_relative "process_sample"
7
+ require_relative "rubies"
8
+ require_relative "spawn"
9
+
10
+ module Duel
11
+ # A worker process (Duel::Worker) running one version of the code.
12
+ # @api private
13
+ class WorkerProcess
14
+ LIB = File.expand_path("..", __dir__)
15
+ # Seconds between looks at a worker that's loading.
16
+ POLL = 0.01
17
+ # Seconds a worker may take to end, once its connection is closed,
18
+ # before it's killed.
19
+ STOP_WAIT = 10
20
+ # The gems that are Duel's: Duel, and its plugins (duel-redis, say).
21
+ DUEL_GEM = /\Aduel(-|\z)/
22
+
23
+ attr_reader :label, :benchmarks, :metrics, :memory, :profiler, :yjit
24
+
25
+ # The Ruby it runs on (RUBY_DESCRIPTION), the native extensions it
26
+ # loaded (see Worker.extensions), those it loaded twice (see
27
+ # Worker.twice), and the directories gems are installed in (see
28
+ # Worker.gem_dirs).
29
+ attr_reader :description, :extensions, :twice, :gem_dirs
30
+
31
+ # Where a worker on +ruby+ loads Duel from, ahead of anything else
32
+ # (a version's own bundle, which may not have Duel, or an older one):
33
+ # the runner's Duel, and every Duel gem (+specs+) the runner loaded
34
+ # (its plugins), so both versions are measured with the same. On the
35
+ # runner's own Ruby, Duel's dependencies (Fiddle) too: on another,
36
+ # their builds wouldn't load.
37
+ def self.load_path(ruby, specs: Gem.loaded_specs)
38
+ duel = specs.select { |name, _| name.match?(DUEL_GEM) }.values
39
+ none = [] #: Array[Gem::BasicSpecification]
40
+ dependencies = Rubies.runner?(ruby) ? specs["duel"]&.runtime_dependencies.to_a.filter_map { |d| specs[d.name] } : none
41
+ [LIB, *(duel + dependencies).flat_map(&:full_require_paths)].uniq
42
+ end
43
+
44
+ # +timeout+: the seconds it may take to load, or to answer a request,
45
+ # before it's taken for hung (and killed). It runs on +ruby+ (whose
46
+ # engine is +engine+), with +env+, and on JRuby, the JVM's options for
47
+ # a worker (see Rubies.worker_environment), not given to the processes
48
+ # it starts for a process benchmark (see #run): their boot is measured
49
+ # as it is. It connects over a Unix socket, unless +unix+ is false
50
+ # (see Control).
51
+ def initialize(label, root, candidate_root, dir, config_path:, env:, timeout:, ruby:, engine:, unix: Control::UNIX)
52
+ @label = label
53
+ @timeout = timeout
54
+ @root = root
55
+ @dir = dir
56
+ @env = env
57
+ @ruby = ruby
58
+ @engine = engine
59
+ server, control = Control.listen(File.join(dir, "#{label}.sock"), unix: unix)
60
+ # (A nil config_path is left out. And the worker gets no other
61
+ # worker's connection: TruffleRuby's sockets aren't close-on-exec,
62
+ # and one left open in another worker would keep it from ending.)
63
+ worker_env = Rubies.worker_environment(env, engine)
64
+ @child = Spawn::STARTER.started(worker_env, [*Rubies.launch(ruby, engine, worker_env),
65
+ *WorkerProcess.load_path(ruby).flat_map { |lib| ["-I", lib] }, "-r", "duel/worker", "-e", "Duel::Worker.main(*ARGV)", root,
66
+ candidate_root, control, *[config_path].compact], close_others: true)
67
+ @socket = accept(server, control)
68
+ hello = read
69
+ @benchmarks = hello.fetch("benchmarks").to_h { |b| [b.fetch("name"), b] }
70
+ @metrics = hello.fetch("metrics")
71
+ @memory = hello.fetch("memory")
72
+ @profiler = hello.fetch("profiler")
73
+ @yjit = hello.fetch("yjit")
74
+ @description = hello.fetch("ruby")
75
+ @extensions = hello.fetch("extensions")
76
+ @twice = hello.fetch("twice")
77
+ @gem_dirs = hello.fetch("gem_dirs")
78
+ end
79
+
80
+ # Sends a request, and returns the reply, or raises Failed with its
81
+ # error.
82
+ def request(op, name, **args)
83
+ # (By name: +args+ come by keyword, as Symbols.)
84
+ Message.write(@socket, {"op" => op, "name" => name, **args.transform_keys(&:to_s)})
85
+ reply = read
86
+ raise Failed, "#{@label}: #{reply.fetch("error")}\n #{reply.fetch("backtrace").join("\n ")}" if reply.key?("error")
87
+
88
+ reply
89
+ end
90
+
91
+ def sample(name, n, runs: 1, count: false) = request("sample", name, n: n, runs: runs, count: count)
92
+
93
+ # A run of a process benchmark (see DSL#duel_process): +process+, a
94
+ # fresh process, started as this worker was (on its Ruby, in its
95
+ # version's tree, with its environment), and measured (see
96
+ # ProcessSample#run).
97
+ def run(process)
98
+ # (Code, without a command.)
99
+ code = process.fetch("ruby") #: String
100
+ command = process.fetch("command") || [*(@launch ||= Rubies.launch(@ruby, @engine, @env)), "-I", File.join(@root, "lib"), "-e", code]
101
+ runs = @runs ||= ProcessSample.new(@label, env: @env, chdir: @root, log: File.join(@dir, "#{@label}.log"), timeout: @timeout)
102
+ runs.run(command)
103
+ end
104
+
105
+ # Closing the connection ends the worker. One still running +wait+
106
+ # seconds later is killed.
107
+ def stop(wait: STOP_WAIT)
108
+ @runs&.stop
109
+ @socket.close
110
+ @child.stop(wait)
111
+ end
112
+
113
+ private
114
+
115
+ # The worker's connection, once it has loaded (or Failed if it died,
116
+ # or took too long, first).
117
+ def accept(server, path)
118
+ deadline = monotonic + @timeout
119
+ until server.wait_readable(POLL)
120
+ raise Failed, "#{@label} worker exited while loading" unless @child.alive?
121
+ hung("load") if monotonic > deadline
122
+ end
123
+ server.accept
124
+ ensure
125
+ server.close
126
+ Control.remove(path)
127
+ end
128
+
129
+ def read
130
+ hung("answer") unless @socket.wait_readable(@timeout)
131
+ Message.read(@socket) or raise Failed, "#{@label} worker exited"
132
+ end
133
+
134
+ # Kills a worker that didn't +what+ within the timeout, and fails.
135
+ def hung(what)
136
+ @child.kill
137
+ raise Failed, "#{@label} worker didn't #{what} within #{@timeout}s (killed: a benchmark that hangs?)"
138
+ end
139
+
140
+ def monotonic = Process.clock_gettime(Process::CLOCK_MONOTONIC)
141
+ end
142
+ end
data/lib/duel.rb CHANGED
@@ -1,8 +1,10 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require_relative "duel/version"
4
+ require_relative "duel/platform"
4
5
  require_relative "duel/config"
5
6
  require_relative "duel/metric"
7
+ require_relative "duel/control"
6
8
 
7
9
  # Performance regression guards: every benchmark runs on two versions of
8
10
  # a codebase side by side, the baseline and the candidate, and the
@@ -13,6 +15,11 @@ module Duel
13
15
  # that didn't start (Failed).
14
16
  class Error < StandardError; end
15
17
 
18
+ # What Duel raises when something a run needs fails: a resource that
19
+ # didn't start (Resource#failed), two that set a variable differently,
20
+ # a worker that died or hung.
21
+ class Failed < Error; end
22
+
16
23
  # The hooks, which run code around a benchmark's samples: in .duel.rb
17
24
  # (Config#setup, say), at the top of a file of benchmarks (DSL#setup),
18
25
  # or as an option of one benchmark (DSL#duel's setup:). Where several
@@ -137,9 +144,10 @@ module Duel
137
144
  end
138
145
  end
139
146
 
140
- # Loads a config file (.duel.rb): a fresh configuration, rooted where
141
- # the file is.
142
- # @api private
147
+ # Loads a config file (.duel.rb) and returns its configuration: a
148
+ # fresh one, rooted where the file is, which is Duel.config from then
149
+ # on (what the file's Duel.configure sets up). For a tool that reads a
150
+ # project's settings, as a plugin's generator does (duel-rails's).
143
151
  def load_config(path)
144
152
  path = File.expand_path(path)
145
153
  self.config = Config.new(File.dirname(path))
@@ -226,6 +234,22 @@ module Duel
226
234
  # @api private
227
235
  def plugins = @plugins ||= {} #: Hash[Symbol, Module & _Plugin]
228
236
 
237
+ # Whether Duel runs on Windows: for a plugin to do what it does
238
+ # differently there (start its server another way, say). For a
239
+ # plugin's default:
240
+ #
241
+ # def self.configure(config, pg_ctl: Duel.windows?)
242
+ def windows? = Platform::WINDOWS
243
+
244
+ # Whether a server that a plugin starts for a worker can listen on a
245
+ # Unix socket (in the worker's directory) that a worker can reach: not
246
+ # on Windows, nor where Ruby has none (see Control.unix_sockets?).
247
+ # Where it can't, the server listens on a port (see Resource#port).
248
+ # For a plugin's default:
249
+ #
250
+ # def self.configure(config, tcp: !Duel.unix_sockets?)
251
+ def unix_sockets? = Control.unix_sockets?(windows: windows?)
252
+
229
253
  # The file of benchmarks that called the DSL method running (see
230
254
  # DSL.add), if one is: the file that what it defines is filed
231
255
  # under, so it gets that file's hooks, not those of the file the
data/sig/duel/cli.rbs CHANGED
@@ -56,6 +56,8 @@ module Duel
56
56
  @json: String?
57
57
  @settings: Settings?
58
58
  @config: Config?
59
+ # The Config::RUN_SETTINGS options given.
60
+ @given: Array[Symbol]
59
61
  @passed: bool?
60
62
  @note: String?
61
63
  @accepted: bool?
@@ -188,6 +190,12 @@ module Duel
188
190
  # The environment's variables that are set.
189
191
  def variables: () -> _Variables
190
192
 
193
+ # Sets a Config::RUN_SETTINGS setting to an option's +value+.
194
+ def given: (Settings settings, Symbol name, Float | Integer value) -> void
195
+
196
+ # Sets a setting, a number of processes not fitted to memory.
197
+ def set: (Settings settings, Symbol name, Float | Integer value) -> void
198
+
191
199
  def parser: (Settings settings) -> OptionParser
192
200
  end
193
201
  end
data/sig/duel/comment.rbs CHANGED
@@ -14,6 +14,12 @@ module Duel
14
14
  # What Duel says in comment.env, as each line must have it.
15
15
  SAID: Hash[String, Regexp]
16
16
 
17
+ # The characters a comment of reports has room for.
18
+ ROOM: Integer
19
+
20
+ # A report's, in a comment of them, where its Duel job failed first.
21
+ NONE: String
22
+
17
23
  # The comments on a pull request, a page at a time.
18
24
  COMMENTS: String
19
25
 
@@ -41,7 +47,16 @@ module Duel
41
47
 
42
48
  # Comments, if Duel said to, and the pull request's head is still the
43
49
  # run's.
44
- def comment: (String report) -> Integer
50
+ def comment: (String dir) -> Integer
51
+
52
+ # The reports' directories in +dir+.
53
+ def reports: (String dir) -> Array[String]
54
+
55
+ # What each report's +values+ say together for +name+, or nil.
56
+ def merged: (String name, Array[String?] values) -> String?
57
+
58
+ # The comment's Markdown, of +reports+.
59
+ def body: (Array[String] reports) -> String
45
60
 
46
61
  # Posts +body+ as a new comment, or nil.
47
62
  def post: (String pr, String body) -> untyped
data/sig/duel/config.rbs CHANGED
@@ -32,6 +32,12 @@ module Duel
32
32
  @fail_on: fail_on
33
33
  @push_base: push_base
34
34
  @accept_label: String?
35
+ @limit: Float?
36
+ @confidence: Float?
37
+ @processes: Integer?
38
+ @timeout: Float?
39
+ @time_limit: Float?
40
+ @warm: Float?
35
41
  @metrics: Array[singleton(Metric)]
36
42
  @in_workers: Array[^(Stops stops) -> void]
37
43
  @hooks: Hash[Symbol, Array[hook]]
@@ -46,6 +52,10 @@ module Duel
46
52
  # Where Duel keeps what it keeps for the project: tmp/duel.
47
53
  def state: () -> String
48
54
 
55
+ # Where what a resource builds once is kept: tmp/duel/cache.
56
+ # @api private
57
+ def cache: () -> String
58
+
49
59
  # A glob, from the root, of the files defining benchmarks.
50
60
  attr_accessor benchmarks: String
51
61
 
@@ -90,6 +100,48 @@ module Duel
90
100
  # Sets #fail_on, or fails for a value it doesn't take.
91
101
  def fail_on=: (untyped runs) -> untyped
92
102
 
103
+ # The settings of a run that .duel.rb can set too: for each, what it
104
+ # takes, and whether a value is that.
105
+ # @api private
106
+ RUN_SETTINGS: Hash[Symbol, [String, ^(Numeric) -> bool]]
107
+
108
+ # The ratio above which a timing is slower (--limit), or nil.
109
+ attr_reader limit: Float?
110
+ def limit=: (Numeric? ratio) -> void
111
+
112
+ # The confidence of every interval (--confidence), or nil.
113
+ attr_reader confidence: Float?
114
+ def confidence=: (Numeric? level) -> void
115
+
116
+ # The worker processes of each version (--processes), or nil.
117
+ attr_reader processes: Integer?
118
+ def processes=: (Integer? count) -> void
119
+
120
+ # The most seconds a worker may take (--timeout), or nil.
121
+ attr_reader timeout: Float?
122
+ def timeout=: (Numeric? seconds) -> void
123
+
124
+ # The most seconds a run measures for (--time-limit), or nil.
125
+ attr_reader time_limit: Float?
126
+ def time_limit=: (Numeric? seconds) -> void
127
+
128
+ # The seconds each benchmark warms up for (--warm), or nil.
129
+ attr_reader warm: Float?
130
+ def warm=: (Numeric? seconds) -> void
131
+
132
+ # +value+ for the RUN_SETTINGS +name+d, as that takes it, or nil, or
133
+ # fails saying what to give.
134
+ # @api private
135
+ def self.setting: (Symbol name, Numeric? value) -> untyped
136
+
137
+ # What to give instead of +value+, or nil if it's one that takes.
138
+ # @api private
139
+ def self.problem: (Symbol name, Numeric value) -> String?
140
+
141
+ # The RUN_SETTINGS .duel.rb set, by name.
142
+ # @api private
143
+ def run_settings: () -> Hash[Symbol, Float | Integer]
144
+
93
145
  # The name of a label that accepts a regression in a pull request, or
94
146
  # nil (the default) for none.
95
147
  attr_reader accept_label: String?
@@ -165,8 +217,6 @@ module Duel
165
217
  @options: Hash[Symbol, untyped]
166
218
  @value: untyped
167
219
 
168
- attr_reader options: Hash[Symbol, untyped]
169
-
170
220
  attr_accessor value: untyped
171
221
 
172
222
  def initialize: (Hash[Symbol, untyped] options) -> void