duel 0.1.8 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +146 -1
- data/README.md +35 -0
- data/docs/ci.md +96 -3
- data/docs/configuration.md +35 -1
- data/docs/how-it-works.md +14 -8
- data/docs/plugins.md +16 -0
- data/lib/duel/across.rb +1 -6
- data/lib/duel/baseline.rb +8 -7
- data/lib/duel/cli.rb +45 -19
- data/lib/duel/comment.rb +71 -10
- data/lib/duel/config.rb +113 -2
- data/lib/duel/context.rb +33 -34
- data/lib/duel/control.rb +52 -0
- data/lib/duel/init.rb +142 -18
- data/lib/duel/lane.rb +734 -0
- data/lib/duel/lock.rb +34 -4
- data/lib/duel/memory.rb +14 -5
- data/lib/duel/metric.rb +97 -26
- data/lib/duel/native.rb +199 -32
- data/lib/duel/platform.rb +31 -0
- data/lib/duel/process_sample.rb +74 -49
- data/lib/duel/resource.rb +71 -42
- data/lib/duel/result.rb +165 -0
- data/lib/duel/rubies.rb +21 -3
- data/lib/duel/runner.rb +13 -1039
- data/lib/duel/spawn.rb +327 -0
- data/lib/duel/summary.rb +10 -10
- data/lib/duel/templates/perf-caller.yml.erb +43 -0
- data/lib/duel/templates/perf-comment-caller.yml.erb +42 -0
- data/lib/duel/templates/perf-comment.yml.erb +45 -3
- data/lib/duel/templates/perf.yml.erb +143 -7
- data/lib/duel/testing.rb +3 -0
- data/lib/duel/version.rb +1 -1
- data/lib/duel/worker.rb +13 -4
- data/lib/duel/worker_process.rb +142 -0
- data/lib/duel.rb +27 -3
- data/sig/duel/cli.rbs +8 -0
- data/sig/duel/comment.rbs +16 -1
- data/sig/duel/config.rbs +52 -2
- data/sig/duel/context.rbs +4 -31
- data/sig/duel/control.rbs +27 -0
- data/sig/duel/init.rbs +56 -0
- data/sig/duel/lane.rbs +202 -0
- data/sig/duel/lock.rbs +8 -1
- data/sig/duel/memory.rbs +1 -1
- data/sig/duel/metric.rbs +53 -10
- data/sig/duel/native.rbs +109 -9
- data/sig/duel/platform.rbs +13 -0
- data/sig/duel/process_sample.rbs +7 -9
- data/sig/duel/resource.rbs +28 -9
- data/sig/duel/result.rbs +164 -0
- data/sig/duel/rubies.rbs +4 -0
- data/sig/duel/runner.rbs +3 -460
- data/sig/duel/spawn.rbs +111 -0
- data/sig/duel/summary.rbs +3 -2
- data/sig/duel/worker_process.rbs +90 -0
- data/sig/duel.rbs +17 -3
- metadata +121 -107
|
@@ -1,3 +1,101 @@
|
|
|
1
|
+
<% if reusable? -%>
|
|
2
|
+
name: Duel
|
|
3
|
+
|
|
4
|
+
# Duel's Perf workflow, reusable: a project's own Perf workflow (as
|
|
5
|
+
# `duel init --reusable` writes it) calls it, with its triggers and its
|
|
6
|
+
# concurrency, and passes on what .duel.rb sets that it needs as its
|
|
7
|
+
# inputs. Its jobs run as the Perf workflow's that `duel init` writes
|
|
8
|
+
# do, and the caller grants what they need (contents: write to mark a
|
|
9
|
+
# push passed, comment on it, or keep the badge, and pull-requests: read
|
|
10
|
+
# for the labels), which each job narrows. See Duel's docs/ci.md.
|
|
11
|
+
on:
|
|
12
|
+
workflow_call:
|
|
13
|
+
inputs:
|
|
14
|
+
ruby-version:
|
|
15
|
+
description: The Ruby the Duel job runs (default, as .ruby-version names it)
|
|
16
|
+
type: string
|
|
17
|
+
default: default
|
|
18
|
+
version-file:
|
|
19
|
+
description: Whether each version runs on the Ruby its .ruby-version names (c.ruby = :version_file)
|
|
20
|
+
type: boolean
|
|
21
|
+
default: false
|
|
22
|
+
yjit:
|
|
23
|
+
description: Whether the benchmarks run with YJIT on (RUBY_YJIT_ENABLE=1)
|
|
24
|
+
type: boolean
|
|
25
|
+
default: false
|
|
26
|
+
runs-on:
|
|
27
|
+
description: The runner the Duel job runs on (a larger one, say, for steadier timings), Linux or macOS
|
|
28
|
+
type: string
|
|
29
|
+
default: ubuntu-latest
|
|
30
|
+
timeout-minutes:
|
|
31
|
+
description: The minutes the Duel job may take before it's cancelled
|
|
32
|
+
type: number
|
|
33
|
+
default: 60
|
|
34
|
+
working-directory:
|
|
35
|
+
description: The project's directory, from the repository's root
|
|
36
|
+
type: string
|
|
37
|
+
default: .
|
|
38
|
+
gemfile:
|
|
39
|
+
description: The Gemfile of the bundle Duel runs in (BUNDLE_GEMFILE), from the project's directory, or none for the project's own
|
|
40
|
+
type: string
|
|
41
|
+
default: ""
|
|
42
|
+
submodules:
|
|
43
|
+
description: Whether to check out the repository's submodules too (true, or recursive), as actions/checkout takes it
|
|
44
|
+
type: string
|
|
45
|
+
default: "false"
|
|
46
|
+
setup:
|
|
47
|
+
description: Shell commands that set up what else the tests need (packages, services), before Ruby's
|
|
48
|
+
type: string
|
|
49
|
+
default: ""
|
|
50
|
+
report:
|
|
51
|
+
description: A name for this call's report (yjit, say), where the workflow calls this one more than once. The Perf comment workflow comments with every call's, each under its name
|
|
52
|
+
type: string
|
|
53
|
+
default: ""
|
|
54
|
+
accept-label:
|
|
55
|
+
description: The label that accepts a regression (c.accept_label), or none
|
|
56
|
+
type: string
|
|
57
|
+
default: ""
|
|
58
|
+
release-tags:
|
|
59
|
+
description: The glob release tags match (c.release_tags)
|
|
60
|
+
type: string
|
|
61
|
+
default: v*
|
|
62
|
+
comment-on-pushes:
|
|
63
|
+
description: Whether to comment on a pushed commit (c.fail_on = :pull_requests)
|
|
64
|
+
type: boolean
|
|
65
|
+
default: false
|
|
66
|
+
mark-passed:
|
|
67
|
+
description: Whether to mark a push passed (c.push_base = :last_passed)
|
|
68
|
+
type: boolean
|
|
69
|
+
default: false
|
|
70
|
+
badge:
|
|
71
|
+
description: Whether to keep a badge for the README
|
|
72
|
+
type: boolean
|
|
73
|
+
default: false
|
|
74
|
+
passed-branch:
|
|
75
|
+
description: What names the branches that mark the last commit Duel passed
|
|
76
|
+
type: string
|
|
77
|
+
default: duel/passed
|
|
78
|
+
badge-branch:
|
|
79
|
+
description: The branch that holds the badge
|
|
80
|
+
type: string
|
|
81
|
+
default: duel/badge
|
|
82
|
+
marker:
|
|
83
|
+
description: What marks Duel's comments
|
|
84
|
+
type: string
|
|
85
|
+
default: duel
|
|
86
|
+
# What Duel said, for the caller's own jobs (a notice, say).
|
|
87
|
+
outputs:
|
|
88
|
+
comment:
|
|
89
|
+
description: Whether Duel said to comment (post or change), or nothing
|
|
90
|
+
value: ${{ jobs.duel.outputs.comment }}
|
|
91
|
+
status:
|
|
92
|
+
description: The run's status (passed, failed or neutral), or nothing where Duel didn't get to compare
|
|
93
|
+
value: ${{ jobs.duel.outputs.status }}
|
|
94
|
+
passed:
|
|
95
|
+
description: Whether to mark a push passed (true), or nothing
|
|
96
|
+
value: ${{ jobs.duel.outputs.passed }}
|
|
97
|
+
|
|
98
|
+
<% else -%>
|
|
1
99
|
name: Perf<%= " (#{working_directory})" if working_directory %>
|
|
2
100
|
|
|
3
101
|
on:
|
|
@@ -16,6 +114,7 @@ concurrency:
|
|
|
16
114
|
group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.run_id }}
|
|
17
115
|
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
|
18
116
|
|
|
117
|
+
<% end -%>
|
|
19
118
|
# The Duel job runs the branch's code (its benchmarks too), so it only
|
|
20
119
|
# reads. Commenting on a commit is the Comment on the commit job's, which
|
|
21
120
|
# runs none of it, and commenting on a pull request is the Perf comment
|
|
@@ -26,8 +125,14 @@ permissions:
|
|
|
26
125
|
|
|
27
126
|
jobs:
|
|
28
127
|
duel:
|
|
128
|
+
<% if reusable? -%>
|
|
129
|
+
name: Duel${{ inputs.report != '' && format(' ({0})', inputs.report) || '' }}
|
|
130
|
+
runs-on: ${{ inputs.runs-on }}
|
|
131
|
+
timeout-minutes: ${{ inputs.timeout-minutes }}
|
|
132
|
+
<% else -%>
|
|
29
133
|
name: Duel
|
|
30
134
|
runs-on: ubuntu-latest
|
|
135
|
+
<% end -%>
|
|
31
136
|
# Whether to comment (post, change or none), the run's status (passed,
|
|
32
137
|
# failed or neutral), and whether to mark a push passed (true, or
|
|
33
138
|
# nothing), from Duel.
|
|
@@ -47,6 +152,9 @@ jobs:
|
|
|
47
152
|
with:
|
|
48
153
|
fetch-depth: 0 # the baseline comes from history, tags included
|
|
49
154
|
persist-credentials: false
|
|
155
|
+
<% if reusable? -%>
|
|
156
|
+
submodules: ${{ inputs.submodules }}
|
|
157
|
+
<% end -%>
|
|
50
158
|
# The labels of the pull request (or for a push, of the one that
|
|
51
159
|
# merged it), for c.accept_label in .duel.rb, as a JSON array. They're
|
|
52
160
|
# read as the job runs, and adding the label (or taking it off) runs
|
|
@@ -58,7 +166,7 @@ jobs:
|
|
|
58
166
|
<% end -%>
|
|
59
167
|
- name: Find the pull request's labels
|
|
60
168
|
id: labels
|
|
61
|
-
if: github.event_name == 'pull_request' || github.event_name == 'merge_group' || github.event_name == 'push'<% if accept_label
|
|
169
|
+
if: github.event_name == 'pull_request' || github.event_name == 'merge_group' || github.event_name == 'push'<% if accept_label %><%= reusable? ? " || (github.event_name == 'schedule' && inputs.accept-label != '')" : " || github.event_name == 'schedule'" %><% end %>
|
|
62
170
|
continue-on-error: true # (without them, nothing's accepted)
|
|
63
171
|
env:
|
|
64
172
|
GH_TOKEN: ${{ github.token }}
|
|
@@ -92,7 +200,29 @@ jobs:
|
|
|
92
200
|
labels=$(gh api "repos/$GITHUB_REPOSITORY/commits/$GITHUB_SHA/pulls" --jq '[.[0].labels[]?.name]')
|
|
93
201
|
fi
|
|
94
202
|
echo "labels=$labels" >> "$GITHUB_OUTPUT"
|
|
203
|
+
<% if reusable? -%>
|
|
204
|
+
# What else the tests need set up (packages, services), as the
|
|
205
|
+
# caller says.
|
|
206
|
+
- name: Set up
|
|
207
|
+
if: inputs.setup != ''
|
|
208
|
+
env:
|
|
209
|
+
SETUP: ${{ inputs.setup }}
|
|
210
|
+
run: eval "$SETUP"
|
|
211
|
+
# The bundle Duel and both versions' workers run in, where the caller
|
|
212
|
+
# names one of its own (a Gemfile for Duel alone, say): an absolute
|
|
213
|
+
# path, as the baseline's checkout is elsewhere.
|
|
214
|
+
- name: Use the caller's Gemfile
|
|
215
|
+
if: inputs.gemfile != ''
|
|
216
|
+
env:
|
|
217
|
+
GEMFILE: ${{ inputs.gemfile }}
|
|
218
|
+
run: echo "BUNDLE_GEMFILE=$PWD/$GEMFILE" >> "$GITHUB_ENV"
|
|
219
|
+
# YJIT, where the caller says to, for every worker of both versions.
|
|
220
|
+
- name: Turn on YJIT
|
|
221
|
+
if: inputs.yjit
|
|
222
|
+
run: echo "RUBY_YJIT_ENABLE=1" >> "$GITHUB_ENV"
|
|
223
|
+
<% else -%>
|
|
95
224
|
# (And whatever else your tests need set up: packages, services.)
|
|
225
|
+
<% end -%>
|
|
96
226
|
<% if version_file -%>
|
|
97
227
|
# Each version runs on the Ruby its own .ruby-version names (see
|
|
98
228
|
# .duel.rb). When the baseline's names another Ruby than this code's,
|
|
@@ -108,6 +238,9 @@ jobs:
|
|
|
108
238
|
# RELEASE_TAGS to match.)
|
|
109
239
|
- name: Find the baseline's Ruby
|
|
110
240
|
id: baseline
|
|
241
|
+
<% if reusable? -%>
|
|
242
|
+
if: inputs.version-file
|
|
243
|
+
<% end -%>
|
|
111
244
|
env:
|
|
112
245
|
BEFORE: ${{ github.event.before }}
|
|
113
246
|
QUEUE_BASE: ${{ github.event.merge_group.base_sha }}
|
|
@@ -197,7 +330,7 @@ jobs:
|
|
|
197
330
|
if: ${{ !cancelled() && (github.event_name == 'pull_request' || github.event_name == 'push') }}
|
|
198
331
|
uses: actions/upload-artifact@v7
|
|
199
332
|
with:
|
|
200
|
-
name: duel-report
|
|
333
|
+
name: duel-report<%= "${{ inputs.report != '' && format('-{0}', inputs.report) || '' }}" if reusable? %>
|
|
201
334
|
path: ${{ runner.temp }}/duel/
|
|
202
335
|
if-no-files-found: ignore # none when Duel didn't get to compare
|
|
203
336
|
retention-days: 1
|
|
@@ -212,7 +345,7 @@ jobs:
|
|
|
212
345
|
commit-comment:
|
|
213
346
|
name: Comment on the commit
|
|
214
347
|
needs: duel
|
|
215
|
-
if: ${{
|
|
348
|
+
if: ${{ <%= "inputs.comment-on-pushes && " if reusable? %>!cancelled() && github.event_name == 'push' && (needs.duel.outputs.comment == 'post' || needs.duel.outputs.comment == 'change') }}
|
|
216
349
|
runs-on: ubuntu-latest
|
|
217
350
|
permissions:
|
|
218
351
|
contents: write # to comment on the commit with the report
|
|
@@ -222,7 +355,7 @@ jobs:
|
|
|
222
355
|
continue-on-error: true # there's none when the Duel job failed before comparing
|
|
223
356
|
uses: actions/download-artifact@v8
|
|
224
357
|
with:
|
|
225
|
-
name: duel-report
|
|
358
|
+
name: duel-report<%= "${{ inputs.report != '' && format('-{0}', inputs.report) || '' }}" if reusable? %>
|
|
226
359
|
path: ${{ runner.temp }}/report
|
|
227
360
|
- name: Comment on the commit
|
|
228
361
|
if: steps.report.outcome == 'success'
|
|
@@ -230,8 +363,11 @@ jobs:
|
|
|
230
363
|
GH_TOKEN: ${{ github.token }}
|
|
231
364
|
COMMENT: ${{ needs.duel.outputs.comment }}
|
|
232
365
|
STATUS: ${{ needs.duel.outputs.status }}
|
|
366
|
+
<% if reusable? -%>
|
|
367
|
+
MARKER: ${{ inputs.marker }}
|
|
368
|
+
<% end -%>
|
|
233
369
|
run: |
|
|
234
|
-
marker="<!-- <%= marker %>"
|
|
370
|
+
marker="<!-- <%= reusable? ? "$MARKER" : marker %>"
|
|
235
371
|
last=$(gh api --paginate "repos/$GITHUB_REPOSITORY/comments" \
|
|
236
372
|
--jq ".[] | select(.user.login == \"github-actions[bot]\") | .body | split(\"\n\")[0] | rtrimstr(\"\r\") | select(startswith(\"$marker status=\"))" | tail -n 1)
|
|
237
373
|
if [ "$COMMENT" = post ] || { [ -n "$last" ] && [ "$last" != "$marker status=$STATUS -->" ]; }; then
|
|
@@ -259,7 +395,7 @@ jobs:
|
|
|
259
395
|
concurrency:
|
|
260
396
|
group: ${{ github.workflow }}-passed-${{ github.ref_name }}
|
|
261
397
|
cancel-in-progress: false
|
|
262
|
-
if: ${{
|
|
398
|
+
if: ${{ <%= "inputs.mark-passed && " if reusable? %>!cancelled() && github.event_name == 'push' && needs.duel.outputs.passed == 'true' }}
|
|
263
399
|
runs-on: ubuntu-latest
|
|
264
400
|
permissions:
|
|
265
401
|
contents: write # to move the branch
|
|
@@ -330,7 +466,7 @@ jobs:
|
|
|
330
466
|
concurrency:
|
|
331
467
|
group: ${{ github.workflow }}-badge
|
|
332
468
|
cancel-in-progress: false
|
|
333
|
-
if: ${{
|
|
469
|
+
if: ${{ <%= "inputs.badge && " if reusable? %>!cancelled() && github.event_name == 'push' && github.ref_name == github.event.repository.default_branch && (needs.duel.result == 'failure' || needs.duel.outputs.status == 'passed' || needs.duel.outputs.status == 'failed') }}
|
|
334
470
|
runs-on: ubuntu-latest
|
|
335
471
|
permissions:
|
|
336
472
|
contents: write # to move the badge's branch
|
data/lib/duel/testing.rb
CHANGED
|
@@ -217,6 +217,9 @@ module Duel
|
|
|
217
217
|
run = Settings.defaults
|
|
218
218
|
run.config_path = config_path if File.exist?(config_path)
|
|
219
219
|
run.processes = 2
|
|
220
|
+
# (.duel.rb's run settings, c.limit say, but where one's given, and
|
|
221
|
+
# its processes: a test's are few.)
|
|
222
|
+
config.run_settings.except(:processes).each { |name, value| run[name] = value }
|
|
220
223
|
settings.each { |setting, value| run[setting] = value }
|
|
221
224
|
out = StringIO.new
|
|
222
225
|
runner = Runner.new(config, run, base, out: out)
|
data/lib/duel/version.rb
CHANGED
data/lib/duel/worker.rb
CHANGED
|
@@ -5,6 +5,7 @@ require "socket"
|
|
|
5
5
|
require_relative "../duel"
|
|
6
6
|
require_relative "context"
|
|
7
7
|
require_relative "message"
|
|
8
|
+
require_relative "control"
|
|
8
9
|
require_relative "profile"
|
|
9
10
|
|
|
10
11
|
module Duel
|
|
@@ -12,7 +13,7 @@ module Duel
|
|
|
12
13
|
# benchmarks, and answers the runner's requests. Started (see
|
|
13
14
|
# WorkerProcess) with Worker.main(VERSION_ROOT, CANDIDATE_ROOT, CONTROL
|
|
14
15
|
# [, CONFIG]), it gives its main object the DSL, loads the benchmarks,
|
|
15
|
-
# connects to the runner
|
|
16
|
+
# connects to the runner at CONTROL (see Control), sends what it has
|
|
16
17
|
# (see #hello), and answers each request with a reply (see Message).
|
|
17
18
|
# (Not over stdin: a JRuby started by JRuby sees its stdin pipe
|
|
18
19
|
# closed.)
|
|
@@ -21,8 +22,16 @@ module Duel
|
|
|
21
22
|
# The built-in metrics, outermost first: allocations (CRuby's objects,
|
|
22
23
|
# or JRuby's bytes) inside instructions, whose counter allocates as
|
|
23
24
|
# it's read (a sample's own allocations are none but the benchmark's),
|
|
24
|
-
# and CPU time innermost.
|
|
25
|
-
|
|
25
|
+
# and CPU time innermost. But on JRuby on Windows, JRuby's bytes are
|
|
26
|
+
# innermost, inside CPU time, whose readings of Windows's cycles
|
|
27
|
+
# (through Fiddle: see CycleClock) allocate there.
|
|
28
|
+
# simplecov:disable JRuby on Windows's (its test run covers it).
|
|
29
|
+
BUILT_IN = if Platform::JRUBY_ON_WINDOWS
|
|
30
|
+
[Metrics::Instructions, Metrics::Allocations, Metrics::CPU, Metrics::AllocatedBytes].freeze
|
|
31
|
+
else
|
|
32
|
+
[Metrics::Instructions, Metrics::Allocations, Metrics::AllocatedBytes, Metrics::CPU].freeze
|
|
33
|
+
end
|
|
34
|
+
# simplecov:enable
|
|
26
35
|
|
|
27
36
|
def self.main(version_root, candidate_root, control, config_path = nil)
|
|
28
37
|
config = config_path ? Duel.load_config(config_path) : (Duel.config = Config.new(candidate_root))
|
|
@@ -40,7 +49,7 @@ module Duel
|
|
|
40
49
|
GC.start
|
|
41
50
|
worker = new(config.metrics, config.measures, hooks: config.hooks, before: before)
|
|
42
51
|
begin
|
|
43
|
-
socket =
|
|
52
|
+
socket = Control.connect(control)
|
|
44
53
|
Message.write(socket, worker.hello)
|
|
45
54
|
while (request = Message.read(socket))
|
|
46
55
|
Message.write(socket, worker.handle(request))
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../duel"
|
|
4
|
+
require_relative "control"
|
|
5
|
+
require_relative "message"
|
|
6
|
+
require_relative "process_sample"
|
|
7
|
+
require_relative "rubies"
|
|
8
|
+
require_relative "spawn"
|
|
9
|
+
|
|
10
|
+
module Duel
|
|
11
|
+
# A worker process (Duel::Worker) running one version of the code.
|
|
12
|
+
# @api private
|
|
13
|
+
class WorkerProcess
|
|
14
|
+
LIB = File.expand_path("..", __dir__)
|
|
15
|
+
# Seconds between looks at a worker that's loading.
|
|
16
|
+
POLL = 0.01
|
|
17
|
+
# Seconds a worker may take to end, once its connection is closed,
|
|
18
|
+
# before it's killed.
|
|
19
|
+
STOP_WAIT = 10
|
|
20
|
+
# The gems that are Duel's: Duel, and its plugins (duel-redis, say).
|
|
21
|
+
DUEL_GEM = /\Aduel(-|\z)/
|
|
22
|
+
|
|
23
|
+
attr_reader :label, :benchmarks, :metrics, :memory, :profiler, :yjit
|
|
24
|
+
|
|
25
|
+
# The Ruby it runs on (RUBY_DESCRIPTION), the native extensions it
|
|
26
|
+
# loaded (see Worker.extensions), those it loaded twice (see
|
|
27
|
+
# Worker.twice), and the directories gems are installed in (see
|
|
28
|
+
# Worker.gem_dirs).
|
|
29
|
+
attr_reader :description, :extensions, :twice, :gem_dirs
|
|
30
|
+
|
|
31
|
+
# Where a worker on +ruby+ loads Duel from, ahead of anything else
|
|
32
|
+
# (a version's own bundle, which may not have Duel, or an older one):
|
|
33
|
+
# the runner's Duel, and every Duel gem (+specs+) the runner loaded
|
|
34
|
+
# (its plugins), so both versions are measured with the same. On the
|
|
35
|
+
# runner's own Ruby, Duel's dependencies (Fiddle) too: on another,
|
|
36
|
+
# their builds wouldn't load.
|
|
37
|
+
def self.load_path(ruby, specs: Gem.loaded_specs)
|
|
38
|
+
duel = specs.select { |name, _| name.match?(DUEL_GEM) }.values
|
|
39
|
+
none = [] #: Array[Gem::BasicSpecification]
|
|
40
|
+
dependencies = Rubies.runner?(ruby) ? specs["duel"]&.runtime_dependencies.to_a.filter_map { |d| specs[d.name] } : none
|
|
41
|
+
[LIB, *(duel + dependencies).flat_map(&:full_require_paths)].uniq
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
# +timeout+: the seconds it may take to load, or to answer a request,
|
|
45
|
+
# before it's taken for hung (and killed). It runs on +ruby+ (whose
|
|
46
|
+
# engine is +engine+), with +env+, and on JRuby, the JVM's options for
|
|
47
|
+
# a worker (see Rubies.worker_environment), not given to the processes
|
|
48
|
+
# it starts for a process benchmark (see #run): their boot is measured
|
|
49
|
+
# as it is. It connects over a Unix socket, unless +unix+ is false
|
|
50
|
+
# (see Control).
|
|
51
|
+
def initialize(label, root, candidate_root, dir, config_path:, env:, timeout:, ruby:, engine:, unix: Control::UNIX)
|
|
52
|
+
@label = label
|
|
53
|
+
@timeout = timeout
|
|
54
|
+
@root = root
|
|
55
|
+
@dir = dir
|
|
56
|
+
@env = env
|
|
57
|
+
@ruby = ruby
|
|
58
|
+
@engine = engine
|
|
59
|
+
server, control = Control.listen(File.join(dir, "#{label}.sock"), unix: unix)
|
|
60
|
+
# (A nil config_path is left out. And the worker gets no other
|
|
61
|
+
# worker's connection: TruffleRuby's sockets aren't close-on-exec,
|
|
62
|
+
# and one left open in another worker would keep it from ending.)
|
|
63
|
+
worker_env = Rubies.worker_environment(env, engine)
|
|
64
|
+
@child = Spawn::STARTER.started(worker_env, [*Rubies.launch(ruby, engine, worker_env),
|
|
65
|
+
*WorkerProcess.load_path(ruby).flat_map { |lib| ["-I", lib] }, "-r", "duel/worker", "-e", "Duel::Worker.main(*ARGV)", root,
|
|
66
|
+
candidate_root, control, *[config_path].compact], close_others: true)
|
|
67
|
+
@socket = accept(server, control)
|
|
68
|
+
hello = read
|
|
69
|
+
@benchmarks = hello.fetch("benchmarks").to_h { |b| [b.fetch("name"), b] }
|
|
70
|
+
@metrics = hello.fetch("metrics")
|
|
71
|
+
@memory = hello.fetch("memory")
|
|
72
|
+
@profiler = hello.fetch("profiler")
|
|
73
|
+
@yjit = hello.fetch("yjit")
|
|
74
|
+
@description = hello.fetch("ruby")
|
|
75
|
+
@extensions = hello.fetch("extensions")
|
|
76
|
+
@twice = hello.fetch("twice")
|
|
77
|
+
@gem_dirs = hello.fetch("gem_dirs")
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
# Sends a request, and returns the reply, or raises Failed with its
|
|
81
|
+
# error.
|
|
82
|
+
def request(op, name, **args)
|
|
83
|
+
# (By name: +args+ come by keyword, as Symbols.)
|
|
84
|
+
Message.write(@socket, {"op" => op, "name" => name, **args.transform_keys(&:to_s)})
|
|
85
|
+
reply = read
|
|
86
|
+
raise Failed, "#{@label}: #{reply.fetch("error")}\n #{reply.fetch("backtrace").join("\n ")}" if reply.key?("error")
|
|
87
|
+
|
|
88
|
+
reply
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
def sample(name, n, runs: 1, count: false) = request("sample", name, n: n, runs: runs, count: count)
|
|
92
|
+
|
|
93
|
+
# A run of a process benchmark (see DSL#duel_process): +process+, a
|
|
94
|
+
# fresh process, started as this worker was (on its Ruby, in its
|
|
95
|
+
# version's tree, with its environment), and measured (see
|
|
96
|
+
# ProcessSample#run).
|
|
97
|
+
def run(process)
|
|
98
|
+
# (Code, without a command.)
|
|
99
|
+
code = process.fetch("ruby") #: String
|
|
100
|
+
command = process.fetch("command") || [*(@launch ||= Rubies.launch(@ruby, @engine, @env)), "-I", File.join(@root, "lib"), "-e", code]
|
|
101
|
+
runs = @runs ||= ProcessSample.new(@label, env: @env, chdir: @root, log: File.join(@dir, "#{@label}.log"), timeout: @timeout)
|
|
102
|
+
runs.run(command)
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
# Closing the connection ends the worker. One still running +wait+
|
|
106
|
+
# seconds later is killed.
|
|
107
|
+
def stop(wait: STOP_WAIT)
|
|
108
|
+
@runs&.stop
|
|
109
|
+
@socket.close
|
|
110
|
+
@child.stop(wait)
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
private
|
|
114
|
+
|
|
115
|
+
# The worker's connection, once it has loaded (or Failed if it died,
|
|
116
|
+
# or took too long, first).
|
|
117
|
+
def accept(server, path)
|
|
118
|
+
deadline = monotonic + @timeout
|
|
119
|
+
until server.wait_readable(POLL)
|
|
120
|
+
raise Failed, "#{@label} worker exited while loading" unless @child.alive?
|
|
121
|
+
hung("load") if monotonic > deadline
|
|
122
|
+
end
|
|
123
|
+
server.accept
|
|
124
|
+
ensure
|
|
125
|
+
server.close
|
|
126
|
+
Control.remove(path)
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
def read
|
|
130
|
+
hung("answer") unless @socket.wait_readable(@timeout)
|
|
131
|
+
Message.read(@socket) or raise Failed, "#{@label} worker exited"
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
# Kills a worker that didn't +what+ within the timeout, and fails.
|
|
135
|
+
def hung(what)
|
|
136
|
+
@child.kill
|
|
137
|
+
raise Failed, "#{@label} worker didn't #{what} within #{@timeout}s (killed: a benchmark that hangs?)"
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
def monotonic = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
141
|
+
end
|
|
142
|
+
end
|
data/lib/duel.rb
CHANGED
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require_relative "duel/version"
|
|
4
|
+
require_relative "duel/platform"
|
|
4
5
|
require_relative "duel/config"
|
|
5
6
|
require_relative "duel/metric"
|
|
7
|
+
require_relative "duel/control"
|
|
6
8
|
|
|
7
9
|
# Performance regression guards: every benchmark runs on two versions of
|
|
8
10
|
# a codebase side by side, the baseline and the candidate, and the
|
|
@@ -13,6 +15,11 @@ module Duel
|
|
|
13
15
|
# that didn't start (Failed).
|
|
14
16
|
class Error < StandardError; end
|
|
15
17
|
|
|
18
|
+
# What Duel raises when something a run needs fails: a resource that
|
|
19
|
+
# didn't start (Resource#failed), two that set a variable differently,
|
|
20
|
+
# a worker that died or hung.
|
|
21
|
+
class Failed < Error; end
|
|
22
|
+
|
|
16
23
|
# The hooks, which run code around a benchmark's samples: in .duel.rb
|
|
17
24
|
# (Config#setup, say), at the top of a file of benchmarks (DSL#setup),
|
|
18
25
|
# or as an option of one benchmark (DSL#duel's setup:). Where several
|
|
@@ -137,9 +144,10 @@ module Duel
|
|
|
137
144
|
end
|
|
138
145
|
end
|
|
139
146
|
|
|
140
|
-
# Loads a config file (.duel.rb)
|
|
141
|
-
# the file is.
|
|
142
|
-
#
|
|
147
|
+
# Loads a config file (.duel.rb) and returns its configuration: a
|
|
148
|
+
# fresh one, rooted where the file is, which is Duel.config from then
|
|
149
|
+
# on (what the file's Duel.configure sets up). For a tool that reads a
|
|
150
|
+
# project's settings, as a plugin's generator does (duel-rails's).
|
|
143
151
|
def load_config(path)
|
|
144
152
|
path = File.expand_path(path)
|
|
145
153
|
self.config = Config.new(File.dirname(path))
|
|
@@ -226,6 +234,22 @@ module Duel
|
|
|
226
234
|
# @api private
|
|
227
235
|
def plugins = @plugins ||= {} #: Hash[Symbol, Module & _Plugin]
|
|
228
236
|
|
|
237
|
+
# Whether Duel runs on Windows: for a plugin to do what it does
|
|
238
|
+
# differently there (start its server another way, say). For a
|
|
239
|
+
# plugin's default:
|
|
240
|
+
#
|
|
241
|
+
# def self.configure(config, pg_ctl: Duel.windows?)
|
|
242
|
+
def windows? = Platform::WINDOWS
|
|
243
|
+
|
|
244
|
+
# Whether a server that a plugin starts for a worker can listen on a
|
|
245
|
+
# Unix socket (in the worker's directory) that a worker can reach: not
|
|
246
|
+
# on Windows, nor where Ruby has none (see Control.unix_sockets?).
|
|
247
|
+
# Where it can't, the server listens on a port (see Resource#port).
|
|
248
|
+
# For a plugin's default:
|
|
249
|
+
#
|
|
250
|
+
# def self.configure(config, tcp: !Duel.unix_sockets?)
|
|
251
|
+
def unix_sockets? = Control.unix_sockets?(windows: windows?)
|
|
252
|
+
|
|
229
253
|
# The file of benchmarks that called the DSL method running (see
|
|
230
254
|
# DSL.add), if one is: the file that what it defines is filed
|
|
231
255
|
# under, so it gets that file's hooks, not those of the file the
|
data/sig/duel/cli.rbs
CHANGED
|
@@ -56,6 +56,8 @@ module Duel
|
|
|
56
56
|
@json: String?
|
|
57
57
|
@settings: Settings?
|
|
58
58
|
@config: Config?
|
|
59
|
+
# The Config::RUN_SETTINGS options given.
|
|
60
|
+
@given: Array[Symbol]
|
|
59
61
|
@passed: bool?
|
|
60
62
|
@note: String?
|
|
61
63
|
@accepted: bool?
|
|
@@ -188,6 +190,12 @@ module Duel
|
|
|
188
190
|
# The environment's variables that are set.
|
|
189
191
|
def variables: () -> _Variables
|
|
190
192
|
|
|
193
|
+
# Sets a Config::RUN_SETTINGS setting to an option's +value+.
|
|
194
|
+
def given: (Settings settings, Symbol name, Float | Integer value) -> void
|
|
195
|
+
|
|
196
|
+
# Sets a setting, a number of processes not fitted to memory.
|
|
197
|
+
def set: (Settings settings, Symbol name, Float | Integer value) -> void
|
|
198
|
+
|
|
191
199
|
def parser: (Settings settings) -> OptionParser
|
|
192
200
|
end
|
|
193
201
|
end
|
data/sig/duel/comment.rbs
CHANGED
|
@@ -14,6 +14,12 @@ module Duel
|
|
|
14
14
|
# What Duel says in comment.env, as each line must have it.
|
|
15
15
|
SAID: Hash[String, Regexp]
|
|
16
16
|
|
|
17
|
+
# The characters a comment of reports has room for.
|
|
18
|
+
ROOM: Integer
|
|
19
|
+
|
|
20
|
+
# A report's, in a comment of them, where its Duel job failed first.
|
|
21
|
+
NONE: String
|
|
22
|
+
|
|
17
23
|
# The comments on a pull request, a page at a time.
|
|
18
24
|
COMMENTS: String
|
|
19
25
|
|
|
@@ -41,7 +47,16 @@ module Duel
|
|
|
41
47
|
|
|
42
48
|
# Comments, if Duel said to, and the pull request's head is still the
|
|
43
49
|
# run's.
|
|
44
|
-
def comment: (String
|
|
50
|
+
def comment: (String dir) -> Integer
|
|
51
|
+
|
|
52
|
+
# The reports' directories in +dir+.
|
|
53
|
+
def reports: (String dir) -> Array[String]
|
|
54
|
+
|
|
55
|
+
# What each report's +values+ say together for +name+, or nil.
|
|
56
|
+
def merged: (String name, Array[String?] values) -> String?
|
|
57
|
+
|
|
58
|
+
# The comment's Markdown, of +reports+.
|
|
59
|
+
def body: (Array[String] reports) -> String
|
|
45
60
|
|
|
46
61
|
# Posts +body+ as a new comment, or nil.
|
|
47
62
|
def post: (String pr, String body) -> untyped
|
data/sig/duel/config.rbs
CHANGED
|
@@ -32,6 +32,12 @@ module Duel
|
|
|
32
32
|
@fail_on: fail_on
|
|
33
33
|
@push_base: push_base
|
|
34
34
|
@accept_label: String?
|
|
35
|
+
@limit: Float?
|
|
36
|
+
@confidence: Float?
|
|
37
|
+
@processes: Integer?
|
|
38
|
+
@timeout: Float?
|
|
39
|
+
@time_limit: Float?
|
|
40
|
+
@warm: Float?
|
|
35
41
|
@metrics: Array[singleton(Metric)]
|
|
36
42
|
@in_workers: Array[^(Stops stops) -> void]
|
|
37
43
|
@hooks: Hash[Symbol, Array[hook]]
|
|
@@ -46,6 +52,10 @@ module Duel
|
|
|
46
52
|
# Where Duel keeps what it keeps for the project: tmp/duel.
|
|
47
53
|
def state: () -> String
|
|
48
54
|
|
|
55
|
+
# Where what a resource builds once is kept: tmp/duel/cache.
|
|
56
|
+
# @api private
|
|
57
|
+
def cache: () -> String
|
|
58
|
+
|
|
49
59
|
# A glob, from the root, of the files defining benchmarks.
|
|
50
60
|
attr_accessor benchmarks: String
|
|
51
61
|
|
|
@@ -90,6 +100,48 @@ module Duel
|
|
|
90
100
|
# Sets #fail_on, or fails for a value it doesn't take.
|
|
91
101
|
def fail_on=: (untyped runs) -> untyped
|
|
92
102
|
|
|
103
|
+
# The settings of a run that .duel.rb can set too: for each, what it
|
|
104
|
+
# takes, and whether a value is that.
|
|
105
|
+
# @api private
|
|
106
|
+
RUN_SETTINGS: Hash[Symbol, [String, ^(Numeric) -> bool]]
|
|
107
|
+
|
|
108
|
+
# The ratio above which a timing is slower (--limit), or nil.
|
|
109
|
+
attr_reader limit: Float?
|
|
110
|
+
def limit=: (Numeric? ratio) -> void
|
|
111
|
+
|
|
112
|
+
# The confidence of every interval (--confidence), or nil.
|
|
113
|
+
attr_reader confidence: Float?
|
|
114
|
+
def confidence=: (Numeric? level) -> void
|
|
115
|
+
|
|
116
|
+
# The worker processes of each version (--processes), or nil.
|
|
117
|
+
attr_reader processes: Integer?
|
|
118
|
+
def processes=: (Integer? count) -> void
|
|
119
|
+
|
|
120
|
+
# The most seconds a worker may take (--timeout), or nil.
|
|
121
|
+
attr_reader timeout: Float?
|
|
122
|
+
def timeout=: (Numeric? seconds) -> void
|
|
123
|
+
|
|
124
|
+
# The most seconds a run measures for (--time-limit), or nil.
|
|
125
|
+
attr_reader time_limit: Float?
|
|
126
|
+
def time_limit=: (Numeric? seconds) -> void
|
|
127
|
+
|
|
128
|
+
# The seconds each benchmark warms up for (--warm), or nil.
|
|
129
|
+
attr_reader warm: Float?
|
|
130
|
+
def warm=: (Numeric? seconds) -> void
|
|
131
|
+
|
|
132
|
+
# +value+ for the RUN_SETTINGS +name+d, as that takes it, or nil, or
|
|
133
|
+
# fails saying what to give.
|
|
134
|
+
# @api private
|
|
135
|
+
def self.setting: (Symbol name, Numeric? value) -> untyped
|
|
136
|
+
|
|
137
|
+
# What to give instead of +value+, or nil if it's one that takes.
|
|
138
|
+
# @api private
|
|
139
|
+
def self.problem: (Symbol name, Numeric value) -> String?
|
|
140
|
+
|
|
141
|
+
# The RUN_SETTINGS .duel.rb set, by name.
|
|
142
|
+
# @api private
|
|
143
|
+
def run_settings: () -> Hash[Symbol, Float | Integer]
|
|
144
|
+
|
|
93
145
|
# The name of a label that accepts a regression in a pull request, or
|
|
94
146
|
# nil (the default) for none.
|
|
95
147
|
attr_reader accept_label: String?
|
|
@@ -165,8 +217,6 @@ module Duel
|
|
|
165
217
|
@options: Hash[Symbol, untyped]
|
|
166
218
|
@value: untyped
|
|
167
219
|
|
|
168
|
-
attr_reader options: Hash[Symbol, untyped]
|
|
169
|
-
|
|
170
220
|
attr_accessor value: untyped
|
|
171
221
|
|
|
172
222
|
def initialize: (Hash[Symbol, untyped] options) -> void
|