duel 0.1.7 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. checksums.yaml +4 -4
  2. data/AGENT_PROMPT.md +2 -2
  3. data/CHANGELOG.md +203 -1
  4. data/README.md +41 -1
  5. data/docs/ci.md +170 -76
  6. data/docs/configuration.md +38 -2
  7. data/docs/how-it-works.md +18 -9
  8. data/docs/plugins.md +23 -0
  9. data/lib/duel/across.rb +1 -6
  10. data/lib/duel/baseline.rb +8 -7
  11. data/lib/duel/cli.rb +96 -28
  12. data/lib/duel/comment.rb +296 -0
  13. data/lib/duel/config.rb +124 -4
  14. data/lib/duel/context.rb +33 -34
  15. data/lib/duel/control.rb +52 -0
  16. data/lib/duel/init.rb +255 -20
  17. data/lib/duel/lane.rb +734 -0
  18. data/lib/duel/lock.rb +34 -4
  19. data/lib/duel/memory.rb +14 -5
  20. data/lib/duel/metric.rb +97 -26
  21. data/lib/duel/native.rb +199 -32
  22. data/lib/duel/platform.rb +31 -0
  23. data/lib/duel/process_sample.rb +74 -49
  24. data/lib/duel/resource.rb +71 -42
  25. data/lib/duel/result.rb +165 -0
  26. data/lib/duel/rubies.rb +21 -3
  27. data/lib/duel/runner.rb +33 -1044
  28. data/lib/duel/spawn.rb +327 -0
  29. data/lib/duel/summary.rb +21 -15
  30. data/lib/duel/templates/duel.gitlab-ci.yml.erb +8 -19
  31. data/lib/duel/templates/perf-caller.yml.erb +43 -0
  32. data/lib/duel/templates/perf-comment-caller.yml.erb +42 -0
  33. data/lib/duel/templates/perf-comment.yml.erb +61 -44
  34. data/lib/duel/templates/perf.yml.erb +143 -7
  35. data/lib/duel/testing.rb +3 -0
  36. data/lib/duel/version.rb +1 -1
  37. data/lib/duel/worker.rb +13 -4
  38. data/lib/duel/worker_process.rb +142 -0
  39. data/lib/duel.rb +35 -3
  40. data/sig/duel/cli.rbs +19 -0
  41. data/sig/duel/comment.rbs +97 -0
  42. data/sig/duel/config.rbs +57 -2
  43. data/sig/duel/context.rbs +4 -31
  44. data/sig/duel/control.rbs +27 -0
  45. data/sig/duel/init.rbs +95 -1
  46. data/sig/duel/lane.rbs +202 -0
  47. data/sig/duel/lock.rbs +8 -1
  48. data/sig/duel/memory.rbs +1 -1
  49. data/sig/duel/metric.rbs +53 -10
  50. data/sig/duel/native.rbs +109 -9
  51. data/sig/duel/platform.rbs +13 -0
  52. data/sig/duel/process_sample.rbs +7 -9
  53. data/sig/duel/resource.rbs +28 -9
  54. data/sig/duel/result.rbs +164 -0
  55. data/sig/duel/rubies.rbs +4 -0
  56. data/sig/duel/runner.rbs +6 -460
  57. data/sig/duel/spawn.rbs +111 -0
  58. data/sig/duel/summary.rbs +6 -3
  59. data/sig/duel/worker_process.rbs +90 -0
  60. data/sig/duel.rbs +17 -3
  61. metadata +121 -105
@@ -1,16 +1,46 @@
1
+ <% if reusable? -%>
2
+ name: Duel comment
3
+
4
+ # Duel's Perf comment workflow, reusable: a project's own Perf comment
5
+ # workflow (as `duel init --reusable` writes it) calls it, triggered as
6
+ # that workflow is (once the Perf workflow has run, and where there's a
7
+ # label that accepts a regression, when it's added or taken off), and
8
+ # passes on what .duel.rb sets that it needs as its inputs. Its jobs run
9
+ # as the Perf comment workflow's that `duel init` writes do, and the
10
+ # caller grants what they need (actions: read, or write to run the Duel
11
+ # job again, and pull-requests: write), which each job narrows. See
12
+ # Duel's docs/ci.md.
13
+ on:
14
+ workflow_call:
15
+ inputs:
16
+ accept-label:
17
+ description: The label that accepts a regression (c.accept_label), or none
18
+ type: string
19
+ default: ""
20
+ perf-workflow:
21
+ description: The name of the Perf workflow, whose runs it comments on
22
+ type: string
23
+ default: Perf
24
+ marker:
25
+ description: What marks Duel's comments
26
+ type: string
27
+ default: duel
28
+
29
+ <% else -%>
1
30
  name: Perf comment<%= " (#{working_directory})" if working_directory %>
2
31
 
3
32
  # Duel's report, as a new comment on the pull request, when Duel says to:
4
33
  # always (post), or only when its last comment's status differs from the
5
34
  # run's (change), so the latest comment is never stale. It never edits or
6
- # deletes a comment. It runs once the Perf workflow has, with this
7
- # repository's own workflow and a token that can write, so a pull request
8
- # from a fork gets a comment too. It checks nothing out and runs none of
9
- # the pull request's code, and of what the Duel job kept (where that code
10
- # ran), it trusts only what that job could have said: the pull request's
11
- # number, if it names the pull request the run compared, and the comment
12
- # and the status, if they're ones Duel says. A fork's report, which the
13
- # fork's code wrote, is shown as text.
35
+ # deletes a comment, but hides Duel's earlier ones as outdated, so the
36
+ # newest is the one in view. It runs once the Perf workflow has, with
37
+ # this repository's own workflow and a token that can write, so a pull
38
+ # request from a fork gets a comment too. It checks nothing out and runs
39
+ # none of the pull request's code, only `duel comment` (see Duel's
40
+ # docs/ci.md), which trusts only what the Duel job could have said in
41
+ # what it kept: the pull request's number, if it names the pull request
42
+ # the run compared, and the comment and the status, if they're ones Duel
43
+ # says. Its report, a fork's too, is the comment, as Markdown.
14
44
  <% if accept_label -%>
15
45
  #
16
46
  # And when the label that accepts a regression (c.accept_label in
@@ -32,6 +62,7 @@ on:
32
62
  types: [labeled, unlabeled]
33
63
  <% end -%>
34
64
 
65
+ <% end -%>
35
66
  permissions: {}
36
67
 
37
68
  jobs:
@@ -48,56 +79,39 @@ jobs:
48
79
  continue-on-error: true # there's none when the Duel job failed before comparing
49
80
  uses: actions/download-artifact@v8
50
81
  with:
82
+ <% if reusable? -%>
83
+ # (Every call's, where the Perf workflow called Duel's more than
84
+ # once, each in a directory of its own, which make one comment.)
85
+ pattern: duel-report*
86
+ <% else -%>
51
87
  name: duel-report
88
+ <% end -%>
52
89
  path: ${{ runner.temp }}/report
53
90
  run-id: ${{ github.event.workflow_run.id }}
54
91
  github-token: ${{ github.token }}
92
+ - uses: ruby/setup-ruby@v1
93
+ if: steps.report.outcome == 'success'
94
+ with:
95
+ ruby-version: ruby
96
+ # Duel's own code, from RubyGems (none of the pull request's), so a
97
+ # fix to how it comments comes with a release of Duel.
55
98
  - name: Comment on the pull request
56
99
  if: steps.report.outcome == 'success'
57
100
  env:
58
101
  GH_TOKEN: ${{ github.token }}
59
102
  HEAD_SHA: ${{ github.event.workflow_run.head_sha }}
103
+ <% if reusable? -%>
104
+ MARKER: ${{ inputs.marker }}
105
+ <% end -%>
60
106
  run: |
61
- said() { grep -xE "$1=($2)" "$RUNNER_TEMP/report/comment.env" | tail -n 1 | cut -d= -f2; }
62
- PR=$(said pr '[0-9]+')
63
- COMMENT=$(said comment 'post|change')
64
- STATUS=$(said status 'passed|failed|neutral')
65
- [ -n "$PR" ] && [ -n "$COMMENT" ] && [ -n "$STATUS" ] || exit 0
66
- # (The pull request's head commit, and the repository it's from.)
67
- head=$(gh api "repos/$GITHUB_REPOSITORY/pulls/$PR" --jq '"\(.head.sha) \(.head.repo.full_name)"')
68
- [ "${head%% *}" = "$HEAD_SHA" ] || exit 0
69
- # The report, as Markdown for a branch of this repository (whose
70
- # author can push here anyway), but for a fork's, as text: its run
71
- # ran the fork's code, which could have written anything in it. In
72
- # a code block, fenced with more backticks than any run of them in
73
- # the report, it can't mention anyone, link anywhere or show an
74
- # image. It's cut short where a comment couldn't hold it, and loses
75
- # any NUL bytes, which GitHub wouldn't take, and grep reads it as
76
- # text whatever's in it, so it finds every run of backticks.
77
- report() {
78
- if [ "${head#* }" = "$GITHUB_REPOSITORY" ]; then cat "$RUNNER_TEMP/report/duel.md" && return; fi
79
- text="$RUNNER_TEMP/report.txt"
80
- LC_ALL=C tr -d '\000' < "$RUNNER_TEMP/report/duel.md" | head -c 60000 > "$text"
81
- fence=$(LC_ALL=C grep -ao '``*' "$text" | awk '{ if (length > n) n = length } END { n = (n < 3) ? 3 : n + 1; while (n-- > 0) printf "`" }')
82
- printf 'From a fork, so shown as text:\n\n%s\n' "$fence"
83
- cat "$text"
84
- printf '\n%s\n' "$fence"
85
- }
86
- marker="<!-- <%= marker %>"
87
- # The first line of the last comment Duel posted here: its marker,
88
- # with its status (none in Duel 0.1.0's, which counts as another).
89
- last=$(gh api --paginate "repos/$GITHUB_REPOSITORY/issues/$PR/comments" \
90
- --jq ".[] | select(.user.login == \"github-actions[bot]\") | .body | split(\"\n\")[0] | rtrimstr(\"\r\") | select(startswith(\"$marker status=\") or . == \"$marker -->\")" | tail -n 1)
91
- if [ "$COMMENT" = post ] || { [ -n "$last" ] && [ "$last" != "$marker status=$STATUS -->" ]; }; then
92
- { echo "$marker status=$STATUS -->"; report; } > "$RUNNER_TEMP/comment.md"
93
- gh api --method POST "repos/$GITHUB_REPOSITORY/issues/$PR/comments" -F body=@"$RUNNER_TEMP/comment.md" > /dev/null
94
- fi
107
+ gem install duel --version '~> <%= Duel::Comment::SINCE %>' --no-document
108
+ duel comment --report "$RUNNER_TEMP/report" --head "$HEAD_SHA" --marker <%= reusable? ? '"$MARKER"' : "'#{marker.gsub("'") { "'\\''" }}'" %>
95
109
  <% if accept_label -%>
96
110
 
97
111
  again:
98
112
  name: Run Duel again
99
113
  if: >-
100
- github.event_name == 'pull_request_target' && github.event.label.name == '<%= accept_label.gsub("'", "''") %>'
114
+ github.event_name == 'pull_request_target' && <%= reusable? ? "inputs.accept-label != '' && github.event.label.name == inputs.accept-label" : "github.event.label.name == '#{accept_label.gsub("'", "''")}'" %>
101
115
  runs-on: ubuntu-latest
102
116
  permissions:
103
117
  actions: write # to run the Perf workflow's jobs again
@@ -108,10 +122,13 @@ jobs:
108
122
  GH_REPO: ${{ github.repository }}
109
123
  HEAD_SHA: ${{ github.event.pull_request.head.sha }}
110
124
  ACTION: ${{ github.event.action }}
125
+ <% if reusable? -%>
126
+ PERF_WORKFLOW: ${{ inputs.perf-workflow }}
127
+ <% end -%>
111
128
  run: |
112
129
  # The newest Perf run of the pull request's head commit, once it's
113
130
  # done (one that's running may have read the labels already).
114
- id=$(gh run list --workflow "Perf<%= " (#{working_directory})" if working_directory %>" --event pull_request --commit "$HEAD_SHA" --limit 1 \
131
+ id=$(gh run list --workflow "<%= reusable? ? "$PERF_WORKFLOW" : "Perf#{" (#{working_directory})" if working_directory}" %>" --event pull_request --commit "$HEAD_SHA" --limit 1 \
115
132
  --json databaseId --jq '.[0].databaseId // empty')
116
133
  [ -n "$id" ] || exit 0
117
134
  gh run watch "$id" --interval 30 > /dev/null || true
@@ -1,3 +1,101 @@
1
+ <% if reusable? -%>
2
+ name: Duel
3
+
4
+ # Duel's Perf workflow, reusable: a project's own Perf workflow (as
5
+ # `duel init --reusable` writes it) calls it, with its triggers and its
6
+ # concurrency, and passes on what .duel.rb sets that it needs as its
7
+ # inputs. Its jobs run as the Perf workflow's that `duel init` writes
8
+ # do, and the caller grants what they need (contents: write to mark a
9
+ # push passed, comment on it, or keep the badge, and pull-requests: read
10
+ # for the labels), which each job narrows. See Duel's docs/ci.md.
11
+ on:
12
+ workflow_call:
13
+ inputs:
14
+ ruby-version:
15
+ description: The Ruby the Duel job runs (default, as .ruby-version names it)
16
+ type: string
17
+ default: default
18
+ version-file:
19
+ description: Whether each version runs on the Ruby its .ruby-version names (c.ruby = :version_file)
20
+ type: boolean
21
+ default: false
22
+ yjit:
23
+ description: Whether the benchmarks run with YJIT on (RUBY_YJIT_ENABLE=1)
24
+ type: boolean
25
+ default: false
26
+ runs-on:
27
+ description: The runner the Duel job runs on (a larger one, say, for steadier timings), Linux or macOS
28
+ type: string
29
+ default: ubuntu-latest
30
+ timeout-minutes:
31
+ description: The minutes the Duel job may take before it's cancelled
32
+ type: number
33
+ default: 60
34
+ working-directory:
35
+ description: The project's directory, from the repository's root
36
+ type: string
37
+ default: .
38
+ gemfile:
39
+ description: The Gemfile of the bundle Duel runs in (BUNDLE_GEMFILE), from the project's directory, or none for the project's own
40
+ type: string
41
+ default: ""
42
+ submodules:
43
+ description: Whether to check out the repository's submodules too (true, or recursive), as actions/checkout takes it
44
+ type: string
45
+ default: "false"
46
+ setup:
47
+ description: Shell commands that set up what else the tests need (packages, services), before Ruby's
48
+ type: string
49
+ default: ""
50
+ report:
51
+ description: A name for this call's report (yjit, say), where the workflow calls this one more than once. The Perf comment workflow comments with every call's, each under its name
52
+ type: string
53
+ default: ""
54
+ accept-label:
55
+ description: The label that accepts a regression (c.accept_label), or none
56
+ type: string
57
+ default: ""
58
+ release-tags:
59
+ description: The glob release tags match (c.release_tags)
60
+ type: string
61
+ default: v*
62
+ comment-on-pushes:
63
+ description: Whether to comment on a pushed commit (c.fail_on = :pull_requests)
64
+ type: boolean
65
+ default: false
66
+ mark-passed:
67
+ description: Whether to mark a push passed (c.push_base = :last_passed)
68
+ type: boolean
69
+ default: false
70
+ badge:
71
+ description: Whether to keep a badge for the README
72
+ type: boolean
73
+ default: false
74
+ passed-branch:
75
+ description: What names the branches that mark the last commit Duel passed
76
+ type: string
77
+ default: duel/passed
78
+ badge-branch:
79
+ description: The branch that holds the badge
80
+ type: string
81
+ default: duel/badge
82
+ marker:
83
+ description: What marks Duel's comments
84
+ type: string
85
+ default: duel
86
+ # What Duel said, for the caller's own jobs (a notice, say).
87
+ outputs:
88
+ comment:
89
+ description: Whether Duel said to comment (post or change), or nothing
90
+ value: ${{ jobs.duel.outputs.comment }}
91
+ status:
92
+ description: The run's status (passed, failed or neutral), or nothing where Duel didn't get to compare
93
+ value: ${{ jobs.duel.outputs.status }}
94
+ passed:
95
+ description: Whether to mark a push passed (true), or nothing
96
+ value: ${{ jobs.duel.outputs.passed }}
97
+
98
+ <% else -%>
1
99
  name: Perf<%= " (#{working_directory})" if working_directory %>
2
100
 
3
101
  on:
@@ -16,6 +114,7 @@ concurrency:
16
114
  group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.run_id }}
17
115
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
18
116
 
117
+ <% end -%>
19
118
  # The Duel job runs the branch's code (its benchmarks too), so it only
20
119
  # reads. Commenting on a commit is the Comment on the commit job's, which
21
120
  # runs none of it, and commenting on a pull request is the Perf comment
@@ -26,8 +125,14 @@ permissions:
26
125
 
27
126
  jobs:
28
127
  duel:
128
+ <% if reusable? -%>
129
+ name: Duel${{ inputs.report != '' && format(' ({0})', inputs.report) || '' }}
130
+ runs-on: ${{ inputs.runs-on }}
131
+ timeout-minutes: ${{ inputs.timeout-minutes }}
132
+ <% else -%>
29
133
  name: Duel
30
134
  runs-on: ubuntu-latest
135
+ <% end -%>
31
136
  # Whether to comment (post, change or none), the run's status (passed,
32
137
  # failed or neutral), and whether to mark a push passed (true, or
33
138
  # nothing), from Duel.
@@ -47,6 +152,9 @@ jobs:
47
152
  with:
48
153
  fetch-depth: 0 # the baseline comes from history, tags included
49
154
  persist-credentials: false
155
+ <% if reusable? -%>
156
+ submodules: ${{ inputs.submodules }}
157
+ <% end -%>
50
158
  # The labels of the pull request (or for a push, of the one that
51
159
  # merged it), for c.accept_label in .duel.rb, as a JSON array. They're
52
160
  # read as the job runs, and adding the label (or taking it off) runs
@@ -58,7 +166,7 @@ jobs:
58
166
  <% end -%>
59
167
  - name: Find the pull request's labels
60
168
  id: labels
61
- if: github.event_name == 'pull_request' || github.event_name == 'merge_group' || github.event_name == 'push'<% if accept_label %> || github.event_name == 'schedule'<% end %>
169
+ if: github.event_name == 'pull_request' || github.event_name == 'merge_group' || github.event_name == 'push'<% if accept_label %><%= reusable? ? " || (github.event_name == 'schedule' && inputs.accept-label != '')" : " || github.event_name == 'schedule'" %><% end %>
62
170
  continue-on-error: true # (without them, nothing's accepted)
63
171
  env:
64
172
  GH_TOKEN: ${{ github.token }}
@@ -92,7 +200,29 @@ jobs:
92
200
  labels=$(gh api "repos/$GITHUB_REPOSITORY/commits/$GITHUB_SHA/pulls" --jq '[.[0].labels[]?.name]')
93
201
  fi
94
202
  echo "labels=$labels" >> "$GITHUB_OUTPUT"
203
+ <% if reusable? -%>
204
+ # What else the tests need set up (packages, services), as the
205
+ # caller says.
206
+ - name: Set up
207
+ if: inputs.setup != ''
208
+ env:
209
+ SETUP: ${{ inputs.setup }}
210
+ run: eval "$SETUP"
211
+ # The bundle Duel and both versions' workers run in, where the caller
212
+ # names one of its own (a Gemfile for Duel alone, say): an absolute
213
+ # path, as the baseline's checkout is elsewhere.
214
+ - name: Use the caller's Gemfile
215
+ if: inputs.gemfile != ''
216
+ env:
217
+ GEMFILE: ${{ inputs.gemfile }}
218
+ run: echo "BUNDLE_GEMFILE=$PWD/$GEMFILE" >> "$GITHUB_ENV"
219
+ # YJIT, where the caller says to, for every worker of both versions.
220
+ - name: Turn on YJIT
221
+ if: inputs.yjit
222
+ run: echo "RUBY_YJIT_ENABLE=1" >> "$GITHUB_ENV"
223
+ <% else -%>
95
224
  # (And whatever else your tests need set up: packages, services.)
225
+ <% end -%>
96
226
  <% if version_file -%>
97
227
  # Each version runs on the Ruby its own .ruby-version names (see
98
228
  # .duel.rb). When the baseline's names another Ruby than this code's,
@@ -108,6 +238,9 @@ jobs:
108
238
  # RELEASE_TAGS to match.)
109
239
  - name: Find the baseline's Ruby
110
240
  id: baseline
241
+ <% if reusable? -%>
242
+ if: inputs.version-file
243
+ <% end -%>
111
244
  env:
112
245
  BEFORE: ${{ github.event.before }}
113
246
  QUEUE_BASE: ${{ github.event.merge_group.base_sha }}
@@ -197,7 +330,7 @@ jobs:
197
330
  if: ${{ !cancelled() && (github.event_name == 'pull_request' || github.event_name == 'push') }}
198
331
  uses: actions/upload-artifact@v7
199
332
  with:
200
- name: duel-report
333
+ name: duel-report<%= "${{ inputs.report != '' && format('-{0}', inputs.report) || '' }}" if reusable? %>
201
334
  path: ${{ runner.temp }}/duel/
202
335
  if-no-files-found: ignore # none when Duel didn't get to compare
203
336
  retention-days: 1
@@ -212,7 +345,7 @@ jobs:
212
345
  commit-comment:
213
346
  name: Comment on the commit
214
347
  needs: duel
215
- if: ${{ !cancelled() && github.event_name == 'push' && (needs.duel.outputs.comment == 'post' || needs.duel.outputs.comment == 'change') }}
348
+ if: ${{ <%= "inputs.comment-on-pushes && " if reusable? %>!cancelled() && github.event_name == 'push' && (needs.duel.outputs.comment == 'post' || needs.duel.outputs.comment == 'change') }}
216
349
  runs-on: ubuntu-latest
217
350
  permissions:
218
351
  contents: write # to comment on the commit with the report
@@ -222,7 +355,7 @@ jobs:
222
355
  continue-on-error: true # there's none when the Duel job failed before comparing
223
356
  uses: actions/download-artifact@v8
224
357
  with:
225
- name: duel-report
358
+ name: duel-report<%= "${{ inputs.report != '' && format('-{0}', inputs.report) || '' }}" if reusable? %>
226
359
  path: ${{ runner.temp }}/report
227
360
  - name: Comment on the commit
228
361
  if: steps.report.outcome == 'success'
@@ -230,8 +363,11 @@ jobs:
230
363
  GH_TOKEN: ${{ github.token }}
231
364
  COMMENT: ${{ needs.duel.outputs.comment }}
232
365
  STATUS: ${{ needs.duel.outputs.status }}
366
+ <% if reusable? -%>
367
+ MARKER: ${{ inputs.marker }}
368
+ <% end -%>
233
369
  run: |
234
- marker="<!-- <%= marker %>"
370
+ marker="<!-- <%= reusable? ? "$MARKER" : marker %>"
235
371
  last=$(gh api --paginate "repos/$GITHUB_REPOSITORY/comments" \
236
372
  --jq ".[] | select(.user.login == \"github-actions[bot]\") | .body | split(\"\n\")[0] | rtrimstr(\"\r\") | select(startswith(\"$marker status=\"))" | tail -n 1)
237
373
  if [ "$COMMENT" = post ] || { [ -n "$last" ] && [ "$last" != "$marker status=$STATUS -->" ]; }; then
@@ -259,7 +395,7 @@ jobs:
259
395
  concurrency:
260
396
  group: ${{ github.workflow }}-passed-${{ github.ref_name }}
261
397
  cancel-in-progress: false
262
- if: ${{ !cancelled() && github.event_name == 'push' && needs.duel.outputs.passed == 'true' }}
398
+ if: ${{ <%= "inputs.mark-passed && " if reusable? %>!cancelled() && github.event_name == 'push' && needs.duel.outputs.passed == 'true' }}
263
399
  runs-on: ubuntu-latest
264
400
  permissions:
265
401
  contents: write # to move the branch
@@ -330,7 +466,7 @@ jobs:
330
466
  concurrency:
331
467
  group: ${{ github.workflow }}-badge
332
468
  cancel-in-progress: false
333
- if: ${{ !cancelled() && github.event_name == 'push' && github.ref_name == github.event.repository.default_branch && (needs.duel.result == 'failure' || needs.duel.outputs.status == 'passed' || needs.duel.outputs.status == 'failed') }}
469
+ if: ${{ <%= "inputs.badge && " if reusable? %>!cancelled() && github.event_name == 'push' && github.ref_name == github.event.repository.default_branch && (needs.duel.result == 'failure' || needs.duel.outputs.status == 'passed' || needs.duel.outputs.status == 'failed') }}
334
470
  runs-on: ubuntu-latest
335
471
  permissions:
336
472
  contents: write # to move the badge's branch
data/lib/duel/testing.rb CHANGED
@@ -217,6 +217,9 @@ module Duel
217
217
  run = Settings.defaults
218
218
  run.config_path = config_path if File.exist?(config_path)
219
219
  run.processes = 2
220
+ # (.duel.rb's run settings, c.limit say, but where one's given, and
221
+ # its processes: a test's are few.)
222
+ config.run_settings.except(:processes).each { |name, value| run[name] = value }
220
223
  settings.each { |setting, value| run[setting] = value }
221
224
  out = StringIO.new
222
225
  runner = Runner.new(config, run, base, out: out)
data/lib/duel/version.rb CHANGED
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Duel
4
- VERSION = "0.1.7"
4
+ VERSION = "0.2.0"
5
5
  end
data/lib/duel/worker.rb CHANGED
@@ -5,6 +5,7 @@ require "socket"
5
5
  require_relative "../duel"
6
6
  require_relative "context"
7
7
  require_relative "message"
8
+ require_relative "control"
8
9
  require_relative "profile"
9
10
 
10
11
  module Duel
@@ -12,7 +13,7 @@ module Duel
12
13
  # benchmarks, and answers the runner's requests. Started (see
13
14
  # WorkerProcess) with Worker.main(VERSION_ROOT, CANDIDATE_ROOT, CONTROL
14
15
  # [, CONFIG]), it gives its main object the DSL, loads the benchmarks,
15
- # connects to the runner's Unix socket at CONTROL, sends what it has
16
+ # connects to the runner at CONTROL (see Control), sends what it has
16
17
  # (see #hello), and answers each request with a reply (see Message).
17
18
  # (Not over stdin: a JRuby started by JRuby sees its stdin pipe
18
19
  # closed.)
@@ -21,8 +22,16 @@ module Duel
21
22
  # The built-in metrics, outermost first: allocations (CRuby's objects,
22
23
  # or JRuby's bytes) inside instructions, whose counter allocates as
23
24
  # it's read (a sample's own allocations are none but the benchmark's),
24
- # and CPU time innermost.
25
- BUILT_IN = [Metrics::Instructions, Metrics::Allocations, Metrics::AllocatedBytes, Metrics::CPU].freeze
25
+ # and CPU time innermost. But on JRuby on Windows, JRuby's bytes are
26
+ # innermost, inside CPU time, whose readings of Windows's cycles
27
+ # (through Fiddle: see CycleClock) allocate there.
28
+ # simplecov:disable JRuby on Windows's (its test run covers it).
29
+ BUILT_IN = if Platform::JRUBY_ON_WINDOWS
30
+ [Metrics::Instructions, Metrics::Allocations, Metrics::CPU, Metrics::AllocatedBytes].freeze
31
+ else
32
+ [Metrics::Instructions, Metrics::Allocations, Metrics::AllocatedBytes, Metrics::CPU].freeze
33
+ end
34
+ # simplecov:enable
26
35
 
27
36
  def self.main(version_root, candidate_root, control, config_path = nil)
28
37
  config = config_path ? Duel.load_config(config_path) : (Duel.config = Config.new(candidate_root))
@@ -40,7 +49,7 @@ module Duel
40
49
  GC.start
41
50
  worker = new(config.metrics, config.measures, hooks: config.hooks, before: before)
42
51
  begin
43
- socket = UNIXSocket.new(control)
52
+ socket = Control.connect(control)
44
53
  Message.write(socket, worker.hello)
45
54
  while (request = Message.read(socket))
46
55
  Message.write(socket, worker.handle(request))
@@ -0,0 +1,142 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../duel"
4
+ require_relative "control"
5
+ require_relative "message"
6
+ require_relative "process_sample"
7
+ require_relative "rubies"
8
+ require_relative "spawn"
9
+
10
+ module Duel
11
+ # A worker process (Duel::Worker) running one version of the code.
12
+ # @api private
13
+ class WorkerProcess
14
+ LIB = File.expand_path("..", __dir__)
15
+ # Seconds between looks at a worker that's loading.
16
+ POLL = 0.01
17
+ # Seconds a worker may take to end, once its connection is closed,
18
+ # before it's killed.
19
+ STOP_WAIT = 10
20
+ # The gems that are Duel's: Duel, and its plugins (duel-redis, say).
21
+ DUEL_GEM = /\Aduel(-|\z)/
22
+
23
+ attr_reader :label, :benchmarks, :metrics, :memory, :profiler, :yjit
24
+
25
+ # The Ruby it runs on (RUBY_DESCRIPTION), the native extensions it
26
+ # loaded (see Worker.extensions), those it loaded twice (see
27
+ # Worker.twice), and the directories gems are installed in (see
28
+ # Worker.gem_dirs).
29
+ attr_reader :description, :extensions, :twice, :gem_dirs
30
+
31
+ # Where a worker on +ruby+ loads Duel from, ahead of anything else
32
+ # (a version's own bundle, which may not have Duel, or an older one):
33
+ # the runner's Duel, and every Duel gem (+specs+) the runner loaded
34
+ # (its plugins), so both versions are measured with the same. On the
35
+ # runner's own Ruby, Duel's dependencies (Fiddle) too: on another,
36
+ # their builds wouldn't load.
37
+ def self.load_path(ruby, specs: Gem.loaded_specs)
38
+ duel = specs.select { |name, _| name.match?(DUEL_GEM) }.values
39
+ none = [] #: Array[Gem::BasicSpecification]
40
+ dependencies = Rubies.runner?(ruby) ? specs["duel"]&.runtime_dependencies.to_a.filter_map { |d| specs[d.name] } : none
41
+ [LIB, *(duel + dependencies).flat_map(&:full_require_paths)].uniq
42
+ end
43
+
44
+ # +timeout+: the seconds it may take to load, or to answer a request,
45
+ # before it's taken for hung (and killed). It runs on +ruby+ (whose
46
+ # engine is +engine+), with +env+, and on JRuby, the JVM's options for
47
+ # a worker (see Rubies.worker_environment), not given to the processes
48
+ # it starts for a process benchmark (see #run): their boot is measured
49
+ # as it is. It connects over a Unix socket, unless +unix+ is false
50
+ # (see Control).
51
+ def initialize(label, root, candidate_root, dir, config_path:, env:, timeout:, ruby:, engine:, unix: Control::UNIX)
52
+ @label = label
53
+ @timeout = timeout
54
+ @root = root
55
+ @dir = dir
56
+ @env = env
57
+ @ruby = ruby
58
+ @engine = engine
59
+ server, control = Control.listen(File.join(dir, "#{label}.sock"), unix: unix)
60
+ # (A nil config_path is left out. And the worker gets no other
61
+ # worker's connection: TruffleRuby's sockets aren't close-on-exec,
62
+ # and one left open in another worker would keep it from ending.)
63
+ worker_env = Rubies.worker_environment(env, engine)
64
+ @child = Spawn::STARTER.started(worker_env, [*Rubies.launch(ruby, engine, worker_env),
65
+ *WorkerProcess.load_path(ruby).flat_map { |lib| ["-I", lib] }, "-r", "duel/worker", "-e", "Duel::Worker.main(*ARGV)", root,
66
+ candidate_root, control, *[config_path].compact], close_others: true)
67
+ @socket = accept(server, control)
68
+ hello = read
69
+ @benchmarks = hello.fetch("benchmarks").to_h { |b| [b.fetch("name"), b] }
70
+ @metrics = hello.fetch("metrics")
71
+ @memory = hello.fetch("memory")
72
+ @profiler = hello.fetch("profiler")
73
+ @yjit = hello.fetch("yjit")
74
+ @description = hello.fetch("ruby")
75
+ @extensions = hello.fetch("extensions")
76
+ @twice = hello.fetch("twice")
77
+ @gem_dirs = hello.fetch("gem_dirs")
78
+ end
79
+
80
+ # Sends a request, and returns the reply, or raises Failed with its
81
+ # error.
82
+ def request(op, name, **args)
83
+ # (By name: +args+ come by keyword, as Symbols.)
84
+ Message.write(@socket, {"op" => op, "name" => name, **args.transform_keys(&:to_s)})
85
+ reply = read
86
+ raise Failed, "#{@label}: #{reply.fetch("error")}\n #{reply.fetch("backtrace").join("\n ")}" if reply.key?("error")
87
+
88
+ reply
89
+ end
90
+
91
+ def sample(name, n, runs: 1, count: false) = request("sample", name, n: n, runs: runs, count: count)
92
+
93
+ # A run of a process benchmark (see DSL#duel_process): +process+, a
94
+ # fresh process, started as this worker was (on its Ruby, in its
95
+ # version's tree, with its environment), and measured (see
96
+ # ProcessSample#run).
97
+ def run(process)
98
+ # (Code, without a command.)
99
+ code = process.fetch("ruby") #: String
100
+ command = process.fetch("command") || [*(@launch ||= Rubies.launch(@ruby, @engine, @env)), "-I", File.join(@root, "lib"), "-e", code]
101
+ runs = @runs ||= ProcessSample.new(@label, env: @env, chdir: @root, log: File.join(@dir, "#{@label}.log"), timeout: @timeout)
102
+ runs.run(command)
103
+ end
104
+
105
+ # Closing the connection ends the worker. One still running +wait+
106
+ # seconds later is killed.
107
+ def stop(wait: STOP_WAIT)
108
+ @runs&.stop
109
+ @socket.close
110
+ @child.stop(wait)
111
+ end
112
+
113
+ private
114
+
115
+ # The worker's connection, once it has loaded (or Failed if it died,
116
+ # or took too long, first).
117
+ def accept(server, path)
118
+ deadline = monotonic + @timeout
119
+ until server.wait_readable(POLL)
120
+ raise Failed, "#{@label} worker exited while loading" unless @child.alive?
121
+ hung("load") if monotonic > deadline
122
+ end
123
+ server.accept
124
+ ensure
125
+ server.close
126
+ Control.remove(path)
127
+ end
128
+
129
+ def read
130
+ hung("answer") unless @socket.wait_readable(@timeout)
131
+ Message.read(@socket) or raise Failed, "#{@label} worker exited"
132
+ end
133
+
134
+ # Kills a worker that didn't +what+ within the timeout, and fails.
135
+ def hung(what)
136
+ @child.kill
137
+ raise Failed, "#{@label} worker didn't #{what} within #{@timeout}s (killed: a benchmark that hangs?)"
138
+ end
139
+
140
+ def monotonic = Process.clock_gettime(Process::CLOCK_MONOTONIC)
141
+ end
142
+ end