rspec-signal 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +68 -0
- data/LICENSE +21 -0
- data/README.md +672 -0
- data/exe/rspec-signal +11 -0
- data/exe/rspec-signal-parallel +52 -0
- data/lib/rspec/signal/backtrace/classifier.rb +88 -0
- data/lib/rspec/signal/backtrace/frame.rb +31 -0
- data/lib/rspec/signal/backtrace/parser.rb +61 -0
- data/lib/rspec/signal/backtrace/reducer.rb +244 -0
- data/lib/rspec/signal/cluster.rb +68 -0
- data/lib/rspec/signal/clusterer.rb +49 -0
- data/lib/rspec/signal/configuration.rb +144 -0
- data/lib/rspec/signal/failure.rb +62 -0
- data/lib/rspec/signal/failure_builder.rb +224 -0
- data/lib/rspec/signal/fingerprint.rb +56 -0
- data/lib/rspec/signal/formatter.rb +207 -0
- data/lib/rspec/signal/group.rb +61 -0
- data/lib/rspec/signal/grouper.rb +28 -0
- data/lib/rspec/signal/html_summary.rb +224 -0
- data/lib/rspec/signal/integrations/capybara.rb +102 -0
- data/lib/rspec/signal/message.rb +161 -0
- data/lib/rspec/signal/parallel_merger.rb +125 -0
- data/lib/rspec/signal/parallel_run.rb +56 -0
- data/lib/rspec/signal/project.rb +158 -0
- data/lib/rspec/signal/redactor.rb +91 -0
- data/lib/rspec/signal/report.rb +88 -0
- data/lib/rspec/signal/reporters/full_output.rb +38 -0
- data/lib/rspec/signal/reporters/json_report.rb +20 -0
- data/lib/rspec/signal/reporters/markdown.rb +271 -0
- data/lib/rspec/signal/reporters/related_failures.rb +101 -0
- data/lib/rspec/signal/symptom.rb +22 -0
- data/lib/rspec/signal/symptoms/exception_class.rb +46 -0
- data/lib/rspec/signal/symptoms/http_status.rb +98 -0
- data/lib/rspec/signal/symptoms/record.rb +56 -0
- data/lib/rspec/signal/symptoms/route.rb +45 -0
- data/lib/rspec/signal/symptoms/ruby_error.rb +55 -0
- data/lib/rspec/signal/symptoms/selector.rb +72 -0
- data/lib/rspec/signal/symptoms.rb +42 -0
- data/lib/rspec/signal/version.rb +7 -0
- data/lib/rspec/signal/writer.rb +88 -0
- data/lib/rspec/signal.rb +156 -0
- data/lib/rspec-signal.rb +3 -0
- metadata +117 -0
|
@@ -0,0 +1,271 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module RSpec
|
|
4
|
+
module Signal
|
|
5
|
+
module Reporters
|
|
6
|
+
# Renders the primary artifact: a compact, model-neutral Markdown report.
|
|
7
|
+
#
|
|
8
|
+
# Deliberately contains no instructions to the reader. It is a diagnostic
|
|
9
|
+
# document, not a prompt, so it works the same pasted into any assistant or
|
|
10
|
+
# read by a human.
|
|
11
|
+
class Markdown
|
|
12
|
+
FENCE = "```"
|
|
13
|
+
MAX_RERUN_ARGUMENTS = 10
|
|
14
|
+
|
|
15
|
+
DIAGNOSTIC_LABELS = {
|
|
16
|
+
url: "URL", path: "Path", title: "Page title", status_code: "Status",
|
|
17
|
+
driver: "Driver", console: "Console", screenshot: "Screenshot", saved_page: "Saved HTML"
|
|
18
|
+
}.freeze
|
|
19
|
+
|
|
20
|
+
def initialize(report, config)
|
|
21
|
+
@report = report
|
|
22
|
+
@config = config
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def render
|
|
26
|
+
sections = [header]
|
|
27
|
+
sections << outside_examples_notice
|
|
28
|
+
# Themes before the inventory: what an agent needs first is the
|
|
29
|
+
# possibility that thirty-five signatures are five problems.
|
|
30
|
+
sections << related_section
|
|
31
|
+
sections << index if @report.group_count > 1
|
|
32
|
+
sections.concat(rendered_groups)
|
|
33
|
+
sections << footer
|
|
34
|
+
"#{sections.compact.join("\n\n").rstrip}\n"
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
private
|
|
38
|
+
|
|
39
|
+
def header
|
|
40
|
+
lines = ["# RSpec Signal", "", headline]
|
|
41
|
+
lines << reduction_line if @report.total_frames.positive?
|
|
42
|
+
lines << "" << meta_line
|
|
43
|
+
lines << "" << conventions
|
|
44
|
+
lines.join("\n")
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def headline
|
|
48
|
+
parts = [quantity(@report.example_count, "example"), quantity(@report.failure_count, "failure")]
|
|
49
|
+
parts << "#{number(@report.pending_count)} pending" if @report.pending_count.positive?
|
|
50
|
+
parts << "#{number(@report.group_count)} distinct #{plural(@report.group_count, "signature")}"
|
|
51
|
+
parts << "#{number(@report.cluster_count)} related #{plural(@report.cluster_count, "cluster")}" \
|
|
52
|
+
if @report.cluster_count.positive?
|
|
53
|
+
parts << outside_examples_count if @report.errors_outside_examples.positive?
|
|
54
|
+
"**#{parts.join(" | ")}**"
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
def outside_examples_count = "#{quantity(@report.errors_outside_examples, "error")} outside examples"
|
|
58
|
+
|
|
59
|
+
def quantity(value, word) = "#{number(value)} #{plural(value, word)}"
|
|
60
|
+
|
|
61
|
+
def reduction_line
|
|
62
|
+
"Backtraces reduced from #{number(@report.total_frames)} to " \
|
|
63
|
+
"#{number(@report.kept_frames)} frames " \
|
|
64
|
+
"(#{number(@report.omitted_frames)} framework/library #{plural(@report.omitted_frames, "frame")} omitted)."
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
def meta_line
|
|
68
|
+
bits = []
|
|
69
|
+
bits << "seed `#{@report.seed}`" if @report.seed_used?
|
|
70
|
+
bits << "#{@report.duration.round(2)}s" if @report.duration
|
|
71
|
+
@report.environment.each { |name, version| bits << "#{name} #{version}" }
|
|
72
|
+
bits << "rspec-signal #{VERSION}"
|
|
73
|
+
bits.join(" | ")
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def conventions
|
|
77
|
+
"Trace frames are innermost first. Project paths are relative to the repository root; " \
|
|
78
|
+
"third-party frames appear as `gem/path.rb:line`."
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
def index
|
|
82
|
+
rows = groups.each_with_index.map do |group, position|
|
|
83
|
+
"| #{position + 1} | #{group.size} | `#{group.exception_class}` | " \
|
|
84
|
+
"#{escape(group.fingerprint.culprit)} | #{escape(one_line(group.message.headline(90)))} |"
|
|
85
|
+
end
|
|
86
|
+
["## Signatures", "",
|
|
87
|
+
"| # | Examples | Exception | Raised in | Message |",
|
|
88
|
+
"|--:|---------:|-----------|-----------|---------|",
|
|
89
|
+
*rows].join("\n")
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
def related_section
|
|
93
|
+
RelatedFailures.new(@report.clusters, signature_positions, @config).render
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
# Cluster members point back at the numbered sections below, so the
|
|
97
|
+
# reader can go straight from a symptom to the failures carrying it.
|
|
98
|
+
def signature_positions
|
|
99
|
+
@signature_positions ||= @report.groups.each_with_index.to_h { |group, i| [group.fingerprint.digest, i + 1] }
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
# A `before(:suite)` blow-up produces no failed examples at all. RSpec
|
|
103
|
+
# only reports those through its message stream, which a formatter
|
|
104
|
+
# cannot capture without swallowing every other message, so say plainly
|
|
105
|
+
# what happened and where to look.
|
|
106
|
+
def outside_examples_notice
|
|
107
|
+
return nil unless @report.errors_outside_examples.positive?
|
|
108
|
+
|
|
109
|
+
count = @report.errors_outside_examples
|
|
110
|
+
["## Errors outside examples", "",
|
|
111
|
+
"#{number(count)} #{plural(count, "error")} occurred outside of any example " \
|
|
112
|
+
"(a `before(:suite)` hook, a spec file that failed to load, or similar). " \
|
|
113
|
+
"rspec-signal cannot capture their detail; the full text is in RSpec's own output."].join("\n")
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
def rendered_groups
|
|
117
|
+
rendered = groups.each_with_index.map { |group, position| group_section(group, position + 1) }
|
|
118
|
+
if truncated_groups.positive?
|
|
119
|
+
rendered << "_#{truncated_groups} further #{plural(truncated_groups, "signature")} " \
|
|
120
|
+
"not rendered (see `signal.json`)._"
|
|
121
|
+
end
|
|
122
|
+
rendered
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
def group_section(group, position)
|
|
126
|
+
sections = [
|
|
127
|
+
heading(group, position),
|
|
128
|
+
fenced(group.message.body(max_lines: @config.max_message_lines,
|
|
129
|
+
max_diff_lines: @config.max_diff_lines)),
|
|
130
|
+
locators(group).join("\n"),
|
|
131
|
+
labelled("Trace", fenced(trace_lines(group.representative))),
|
|
132
|
+
diagnostics_section(group.representative),
|
|
133
|
+
labelled("Rerun", fenced(rerun_commands(group), "bash")),
|
|
134
|
+
affected_section(group)
|
|
135
|
+
]
|
|
136
|
+
sections.compact.join("\n\n")
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
def heading(group, position)
|
|
140
|
+
["## #{position}. #{group.exception_class}#{count_suffix(group)}", "",
|
|
141
|
+
"> #{one_line(group.representative.description)}"].join("\n")
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
def labelled(label, body) = "**#{label}**\n\n#{body}"
|
|
145
|
+
|
|
146
|
+
# The representative on its own, plus the whole signature when that is a
|
|
147
|
+
# different command and short enough to be worth typing.
|
|
148
|
+
def rerun_commands(group)
|
|
149
|
+
commands = ["bundle exec rspec #{group.representative.rerun}"]
|
|
150
|
+
arguments = group.failures.map(&:rerun).uniq
|
|
151
|
+
if arguments.size > 1 && arguments.size <= MAX_RERUN_ARGUMENTS
|
|
152
|
+
commands << "bundle exec rspec #{arguments.join(" ")}"
|
|
153
|
+
end
|
|
154
|
+
commands
|
|
155
|
+
end
|
|
156
|
+
|
|
157
|
+
def count_suffix(group)
|
|
158
|
+
return "" if group.size == 1
|
|
159
|
+
|
|
160
|
+
" -- #{group.size} examples"
|
|
161
|
+
end
|
|
162
|
+
|
|
163
|
+
def locators(group)
|
|
164
|
+
failure = group.representative
|
|
165
|
+
seen = [failure.spec_location]
|
|
166
|
+
lines = ["- Example `#{failure.spec_location}`"]
|
|
167
|
+
|
|
168
|
+
app_context = group.fingerprint.app_context
|
|
169
|
+
if app_context && !seen.include?(app_context)
|
|
170
|
+
lines << "- Your code `#{app_context}`"
|
|
171
|
+
seen << app_context
|
|
172
|
+
end
|
|
173
|
+
|
|
174
|
+
culprit = group.fingerprint.culprit
|
|
175
|
+
if culprit && !seen.include?(culprit)
|
|
176
|
+
gem_name = culprit_frame(failure)&.gem_name
|
|
177
|
+
lines << "- Raised in `#{culprit}`#{" (#{gem_name})" if gem_name}"
|
|
178
|
+
end
|
|
179
|
+
|
|
180
|
+
failure.shared_group_locations.first(3).each do |location|
|
|
181
|
+
lines << "- Via shared example group #{location}"
|
|
182
|
+
end
|
|
183
|
+
|
|
184
|
+
if failure.reduced.fallback?
|
|
185
|
+
lines << "- No first-party frames in this backtrace; innermost frames shown instead"
|
|
186
|
+
end
|
|
187
|
+
|
|
188
|
+
lines
|
|
189
|
+
end
|
|
190
|
+
|
|
191
|
+
# The same frame the fingerprint calls the culprit, so the gem label
|
|
192
|
+
# beside a location always belongs to that location.
|
|
193
|
+
def culprit_frame(failure) = failure.frames.reject(&:framework?).first
|
|
194
|
+
|
|
195
|
+
def trace_lines(failure)
|
|
196
|
+
entries = failure.reduced.entries.map(&:to_s)
|
|
197
|
+
return ["#{failure.spec_location} (no backtrace available)"] if entries.empty?
|
|
198
|
+
|
|
199
|
+
entries
|
|
200
|
+
end
|
|
201
|
+
|
|
202
|
+
def diagnostics_section(failure)
|
|
203
|
+
items = failure.diagnostics.reject { |_, value| value.nil? || value.to_s.empty? }
|
|
204
|
+
return nil if items.empty?
|
|
205
|
+
|
|
206
|
+
rows = items.map { |key, value| "- #{humanize(key)}: #{format_diagnostic(value)}" }
|
|
207
|
+
labelled("Browser state", rows.join("\n"))
|
|
208
|
+
end
|
|
209
|
+
|
|
210
|
+
def format_diagnostic(value)
|
|
211
|
+
return value.map { |item| "`#{one_line(item)}`" }.join("; ") if value.is_a?(Array)
|
|
212
|
+
|
|
213
|
+
text = one_line(value)
|
|
214
|
+
text.length > 300 ? "`#{text[0, 297]}...`" : "`#{text}`"
|
|
215
|
+
end
|
|
216
|
+
|
|
217
|
+
def affected_section(group)
|
|
218
|
+
others = group.failures.reject { |failure| failure.equal?(group.representative) }
|
|
219
|
+
return nil if others.empty?
|
|
220
|
+
|
|
221
|
+
grouped = others.group_by(&:spec_location)
|
|
222
|
+
shown = grouped.first(@config.max_affected_examples)
|
|
223
|
+
rows = shown.map { |location, failures| affected_row(location, failures) }
|
|
224
|
+
hidden = grouped.size - shown.size
|
|
225
|
+
rows << "... and #{hidden} more #{plural(hidden, "location")}" if hidden.positive?
|
|
226
|
+
|
|
227
|
+
labelled("Also failing identically (#{others.size})", fenced(rows))
|
|
228
|
+
end
|
|
229
|
+
|
|
230
|
+
# Parameterised examples all share one `it` line. Listing that line once
|
|
231
|
+
# with a count says the same thing in a fraction of the space.
|
|
232
|
+
def affected_row(location, failures)
|
|
233
|
+
return "#{location} #{one_line(failures.first.description)}" if failures.size == 1
|
|
234
|
+
|
|
235
|
+
"#{location} (#{failures.size} examples)"
|
|
236
|
+
end
|
|
237
|
+
|
|
238
|
+
def footer
|
|
239
|
+
notes = ["---", "",
|
|
240
|
+
"Generated by [rspec-signal](https://github.com/SilenceDogood1984/rspec-signal). " \
|
|
241
|
+
"Backtrace frames from the test runner, loader and CLI are removed; " \
|
|
242
|
+
"first-party frames and the library frames adjacent to them are kept."]
|
|
243
|
+
notes << "Review this file before sharing it: it can contain application data." if @config.redact?
|
|
244
|
+
notes.join("\n")
|
|
245
|
+
end
|
|
246
|
+
|
|
247
|
+
def groups
|
|
248
|
+
@groups ||= @config.max_groups ? @report.groups.first(@config.max_groups) : @report.groups
|
|
249
|
+
end
|
|
250
|
+
|
|
251
|
+
def truncated_groups = @report.group_count - groups.size
|
|
252
|
+
|
|
253
|
+
def fenced(lines, language = "text")
|
|
254
|
+
[FENCE + language, *Array(lines), FENCE].join("\n")
|
|
255
|
+
end
|
|
256
|
+
|
|
257
|
+
def one_line(text) = text.to_s.gsub(/\s+/, " ").strip
|
|
258
|
+
|
|
259
|
+
def escape(text) = text.to_s.gsub("|", "\\|")
|
|
260
|
+
|
|
261
|
+
def humanize(key)
|
|
262
|
+
DIAGNOSTIC_LABELS.fetch(key.to_sym) { key.to_s.tr("_", " ").capitalize }
|
|
263
|
+
end
|
|
264
|
+
|
|
265
|
+
def plural(count, word) = count == 1 ? word : "#{word}s"
|
|
266
|
+
|
|
267
|
+
def number(value) = value.to_s.reverse.scan(/\d{1,3}/).join(",").reverse
|
|
268
|
+
end
|
|
269
|
+
end
|
|
270
|
+
end
|
|
271
|
+
end
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module RSpec
|
|
4
|
+
module Signal
|
|
5
|
+
module Reporters
|
|
6
|
+
# The "Related failures" section of the Markdown report.
|
|
7
|
+
#
|
|
8
|
+
# The second, looser grouping layer. A signature says "these failures are
|
|
9
|
+
# the same failure"; a cluster says only "these share one strong
|
|
10
|
+
# diagnostic symptom, so look at them together". The section says so in
|
|
11
|
+
# those words, because the difference is the whole point: a reader who
|
|
12
|
+
# takes a cluster for a root cause has been misled.
|
|
13
|
+
#
|
|
14
|
+
# Kept to a few lines per cluster. This layer exists to save the reader
|
|
15
|
+
# from reading thirty-five signature sections, and it would be a poor
|
|
16
|
+
# trade if it cost thirty-five sections' worth of prose to do it.
|
|
17
|
+
class RelatedFailures
|
|
18
|
+
MAX_SYMPTOMS = 4
|
|
19
|
+
MAX_SIGNATURES = 8
|
|
20
|
+
|
|
21
|
+
PREAMBLE = "Failures sharing one diagnostic symptom across more than one signature. " \
|
|
22
|
+
"Weaker than a signature: a likely common cause, not a proven identical failure. " \
|
|
23
|
+
"The signatures below remain authoritative."
|
|
24
|
+
|
|
25
|
+
# @param clusters [Array<Cluster>]
|
|
26
|
+
# @param signature_positions [Hash{String => Integer}] digest => index in the report
|
|
27
|
+
def initialize(clusters, signature_positions, config)
|
|
28
|
+
@clusters = clusters
|
|
29
|
+
@signature_positions = signature_positions
|
|
30
|
+
@config = config
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
# @return [String, nil] nil when nothing relates, so the section vanishes
|
|
34
|
+
def render
|
|
35
|
+
return nil if shown.empty?
|
|
36
|
+
|
|
37
|
+
blocks = shown.each_with_index.map { |cluster, position| block(cluster, position + 1) }
|
|
38
|
+
blocks << truncation_note if hidden.positive?
|
|
39
|
+
[["## Related failures", "", PREAMBLE].join("\n"), *blocks].join("\n\n")
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
private
|
|
43
|
+
|
|
44
|
+
def shown
|
|
45
|
+
@shown ||= @config.max_clusters ? @clusters.first(@config.max_clusters) : @clusters
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def hidden = @clusters.size - shown.size
|
|
49
|
+
|
|
50
|
+
def truncation_note
|
|
51
|
+
"_#{hidden} further related #{plural(hidden, "cluster")} not rendered (see `signal.json`)._"
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def block(cluster, position)
|
|
55
|
+
lines = [heading(cluster, position), ""]
|
|
56
|
+
# Only worth a line when the symptom took more than one wording: with
|
|
57
|
+
# one, the heading already said it.
|
|
58
|
+
lines << "- Symptoms: #{symptoms(cluster)}" if cluster.symptom_counts.size > 1
|
|
59
|
+
lines << "- Specs: #{specs(cluster)}"
|
|
60
|
+
lines << "- Signatures: #{signatures(cluster)}"
|
|
61
|
+
lines.join("\n")
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
def heading(cluster, position)
|
|
65
|
+
"### R#{position}. #{sentence(cluster.label)} -- " \
|
|
66
|
+
"#{cluster.size} #{plural(cluster.size, "example")} across " \
|
|
67
|
+
"#{cluster.signature_count} #{plural(cluster.signature_count, "signature")}"
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def symptoms(cluster)
|
|
71
|
+
listing(cluster.symptom_counts.keys, MAX_SYMPTOMS, cluster.symptom_counts) do |detail, count|
|
|
72
|
+
"`#{one_line(detail)}`#{" (#{count})" if count}"
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def specs(cluster)
|
|
77
|
+
listing(cluster.spec_files, @config.max_cluster_specs) { |path, _| "`#{path}`" }
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
def signatures(cluster)
|
|
81
|
+
positions = cluster.signatures.filter_map { |digest| @signature_positions[digest] }.sort
|
|
82
|
+
listing(positions, MAX_SIGNATURES) { |position, _| "##{position}" }
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
def listing(items, limit, counts = nil)
|
|
86
|
+
visible = items.first(limit)
|
|
87
|
+
rendered = visible.map { |item| yield(item, counts&.fetch(item, nil)) }
|
|
88
|
+
remaining = items.size - visible.size
|
|
89
|
+
rendered << "and #{remaining} more" if remaining.positive?
|
|
90
|
+
rendered.join(", ")
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
def one_line(text) = text.to_s.gsub(/\s+/, " ").strip
|
|
94
|
+
|
|
95
|
+
def sentence(text) = text.to_s.sub(/\A[a-z]/, &:upcase)
|
|
96
|
+
|
|
97
|
+
def plural(count, word) = count == 1 ? word : "#{word}s"
|
|
98
|
+
end
|
|
99
|
+
end
|
|
100
|
+
end
|
|
101
|
+
end
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module RSpec
|
|
4
|
+
module Signal
|
|
5
|
+
# One strong diagnostic characteristic pulled out of a failure message.
|
|
6
|
+
#
|
|
7
|
+
# Symptoms are what the looser "related failures" layer clusters on, and
|
|
8
|
+
# they are deliberately narrow. Every extractor is anchored on a specific
|
|
9
|
+
# phrase produced by a specific library -- an HTTP status mismatch, a
|
|
10
|
+
# missing selector, a route that does not exist -- never on two messages
|
|
11
|
+
# merely resembling each other. A failure that matches nothing has no
|
|
12
|
+
# symptom and joins no cluster, which is the safe direction to be wrong in.
|
|
13
|
+
#
|
|
14
|
+
# kind :http_status, :selector, ... -- which extractor fired
|
|
15
|
+
# key the cluster identity; equal keys mean "same symptom"
|
|
16
|
+
# label the cluster heading, already human-readable
|
|
17
|
+
# detail this failure's own wording of the symptom, for the symptom list
|
|
18
|
+
Symptom = Struct.new(:kind, :key, :label, :detail, keyword_init: true) do
|
|
19
|
+
def to_h = { kind: kind, key: key, label: label, detail: detail }
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
end
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module RSpec
|
|
4
|
+
module Signal
|
|
5
|
+
module Symptoms
|
|
6
|
+
# The last resort: cluster on the exception class itself.
|
|
7
|
+
#
|
|
8
|
+
# This is the one that would wreck the report if it were applied
|
|
9
|
+
# generally, so it is applied narrowly. It fires only for a *namespaced*
|
|
10
|
+
# class -- `PG::ConnectionBad`, `Errno::ECONNREFUSED` -- which is
|
|
11
|
+
# specific enough that seeing it twice is a fact worth reporting. Bare
|
|
12
|
+
# classes like RuntimeError and ArgumentError say nothing about cause and
|
|
13
|
+
# are refused outright, and so is every class an earlier, finer extractor
|
|
14
|
+
# is responsible for, so that a Capybara lookup with an unparseable
|
|
15
|
+
# message cannot fall through and drag unrelated selectors in with it.
|
|
16
|
+
module ExceptionClass
|
|
17
|
+
GENERIC = %w[
|
|
18
|
+
RuntimeError StandardError Exception ScriptError ArgumentError TypeError NameError
|
|
19
|
+
NoMethodError NotImplementedError IndexError KeyError RangeError IOError EOFError
|
|
20
|
+
FrozenError LocalJumpError StopIteration ZeroDivisionError SystemStackError
|
|
21
|
+
LoadError SyntaxError SecurityError ThreadError FiberError
|
|
22
|
+
].freeze
|
|
23
|
+
|
|
24
|
+
# Owned by an earlier extractor; never eligible here.
|
|
25
|
+
CLAIMED = %w[
|
|
26
|
+
RSpec::
|
|
27
|
+
Capybara::ElementNotFound Capybara::ExpectationNotMet
|
|
28
|
+
ActiveRecord::RecordNotFound ActiveRecord::RecordInvalid
|
|
29
|
+
ActionController::RoutingError ActionController::UrlGenerationError
|
|
30
|
+
].freeze
|
|
31
|
+
|
|
32
|
+
module_function
|
|
33
|
+
|
|
34
|
+
def call(failure, _text)
|
|
35
|
+
name = failure.exception_class.to_s
|
|
36
|
+
return nil unless name.include?("::")
|
|
37
|
+
return nil if GENERIC.include?(name)
|
|
38
|
+
return nil if CLAIMED.any? { |prefix| name.start_with?(prefix) }
|
|
39
|
+
|
|
40
|
+
Symptom.new(kind: :exception_class, key: "exception:#{name}",
|
|
41
|
+
label: "`#{name}` raised in several places", detail: name)
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
end
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module RSpec
|
|
4
|
+
module Signal
|
|
5
|
+
module Symptoms
|
|
6
|
+
# An HTTP status the application returned but the spec did not expect.
|
|
7
|
+
#
|
|
8
|
+
# Clustered on the *actual* status, because that is the fact with a shared
|
|
9
|
+
# cause: eight specs that each wanted something different and all got 404
|
|
10
|
+
# are eight symptoms of one broken route. The expected side survives only
|
|
11
|
+
# as this failure's detail line, so a cluster can still show that some
|
|
12
|
+
# examples wanted 200 and others wanted a redirect.
|
|
13
|
+
module HttpStatus
|
|
14
|
+
# One side of a status mismatch: `404`, `:not_found`, `:ok (200)`.
|
|
15
|
+
CODE = /:?[\w-]+(?: \(\d{3}\))?/
|
|
16
|
+
|
|
17
|
+
PATTERNS = [
|
|
18
|
+
# rspec-rails `have_http_status(200)` / `have_http_status(:ok)`
|
|
19
|
+
/expected the response to have status code (?<expected>#{CODE}) *,? *but it was (?<actual>#{CODE})/i,
|
|
20
|
+
# rspec-rails `have_http_status(:success)` / `:redirect` / `:error` / `:missing`
|
|
21
|
+
/expected the response to have an? (?<expected>\w+) status code \([^)]*\) *,? *but it was (?<actual>#{CODE})/i, # rubocop:disable Layout/LineLength
|
|
22
|
+
# Rails' own `assert_response`
|
|
23
|
+
/expected response to be a <(?<expected>[^>]+)>, *but was a <(?<actual>[^>]+)>/i,
|
|
24
|
+
# The shorthand several house matchers use
|
|
25
|
+
/expected(?: the)? response status (?:to be )?(?<expected>:?[\w-]+) *,? *(?:but )?got:? *(?<actual>:?[\w-]+)/i
|
|
26
|
+
].freeze
|
|
27
|
+
|
|
28
|
+
# Enough of the standard set to name the common ones. An unrecognised
|
|
29
|
+
# code still clusters, it just does not get a friendly name.
|
|
30
|
+
NAMES = {
|
|
31
|
+
"200" => "OK", "201" => "Created", "204" => "No Content", "301" => "Moved Permanently",
|
|
32
|
+
"302" => "Found", "304" => "Not Modified", "400" => "Bad Request", "401" => "Unauthorized",
|
|
33
|
+
"403" => "Forbidden", "404" => "Not Found", "406" => "Not Acceptable", "409" => "Conflict",
|
|
34
|
+
"422" => "Unprocessable Entity", "429" => "Too Many Requests",
|
|
35
|
+
"500" => "Internal Server Error", "502" => "Bad Gateway", "503" => "Service Unavailable"
|
|
36
|
+
}.freeze
|
|
37
|
+
|
|
38
|
+
CODES = {
|
|
39
|
+
"ok" => "200", "created" => "201", "no_content" => "204", "moved_permanently" => "301",
|
|
40
|
+
"found" => "302", "not_modified" => "304", "bad_request" => "400", "unauthorized" => "401",
|
|
41
|
+
"forbidden" => "403", "not_found" => "404", "not_acceptable" => "406", "conflict" => "409",
|
|
42
|
+
"unprocessable_entity" => "422", "unprocessable_content" => "422", "too_many_requests" => "429",
|
|
43
|
+
"internal_server_error" => "500", "bad_gateway" => "502", "service_unavailable" => "503"
|
|
44
|
+
}.freeze
|
|
45
|
+
|
|
46
|
+
module_function
|
|
47
|
+
|
|
48
|
+
GENERIC_EXPECTATION = /expected:\s*(?<expected>\d{3})\s+got:\s*(?<actual>\d{3})/i
|
|
49
|
+
STATUS_EXPRESSION = /(?:response\s*\.\s*status|response\s*\.\s*status_code|response\s*\[\s*:status\s*\])/i
|
|
50
|
+
|
|
51
|
+
def call(failure, text)
|
|
52
|
+
match = PATTERNS.filter_map { |pattern| pattern.match(text) }.first
|
|
53
|
+
match ||= generic_expectation(failure, text)
|
|
54
|
+
return nil unless match
|
|
55
|
+
|
|
56
|
+
actual = code(match[:actual])
|
|
57
|
+
return nil unless actual
|
|
58
|
+
|
|
59
|
+
Symptom.new(kind: :http_status, key: "http-status:#{actual}", label: label(actual),
|
|
60
|
+
detail: "expected #{token(match[:expected])}, got #{actual}")
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
# RSpec's generic equality matcher does not mention HTTP in its
|
|
64
|
+
# expected/got lines. Only accept it when the captured failing
|
|
65
|
+
# expression explicitly reads a response status, avoiding arbitrary
|
|
66
|
+
# three-digit numeric comparisons.
|
|
67
|
+
def generic_expectation(failure, text)
|
|
68
|
+
return nil unless STATUS_EXPRESSION.match?(failure.message.text.to_s)
|
|
69
|
+
|
|
70
|
+
GENERIC_EXPECTATION.match(text)
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
def label(actual)
|
|
74
|
+
name = NAMES[actual]
|
|
75
|
+
"unexpected #{actual}#{" (#{name})" if name} responses"
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
# The numeric code, whether it arrived as a number, a symbol, or both.
|
|
79
|
+
def code(raw)
|
|
80
|
+
text = raw.to_s
|
|
81
|
+
return ::Regexp.last_match(1) if text =~ /(\d{3})/
|
|
82
|
+
|
|
83
|
+
CODES[text.delete_prefix(":").downcase]
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
# How to word one side of the mismatch: a code where there is one, and
|
|
87
|
+
# otherwise the word the matcher used -- `redirect`, `success`.
|
|
88
|
+
def token(raw)
|
|
89
|
+
text = raw.to_s.strip
|
|
90
|
+
return ::Regexp.last_match(1) if text =~ /(\d{3})/
|
|
91
|
+
|
|
92
|
+
bare = text.delete_prefix(":").downcase
|
|
93
|
+
CODES[bare] || bare
|
|
94
|
+
end
|
|
95
|
+
end
|
|
96
|
+
end
|
|
97
|
+
end
|
|
98
|
+
end
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module RSpec
|
|
4
|
+
module Signal
|
|
5
|
+
module Symptoms
|
|
6
|
+
# ActiveRecord shapes: a model that has no rows, a validation that keeps
|
|
7
|
+
# failing, a column or table the schema does not have.
|
|
8
|
+
#
|
|
9
|
+
# The model, the validation sentence and the missing identifier are the
|
|
10
|
+
# cluster keys. A `RecordNotFound` for `User` and one for `Order` are two
|
|
11
|
+
# different problems and stay apart.
|
|
12
|
+
module Record
|
|
13
|
+
NOT_FOUND = /Couldn't find (?<model>[A-Z]\w*(?:::\w+)*)/
|
|
14
|
+
VALIDATION = %r{Validation failed: (?<detail>.{1,120}?)(?=\s{2,}|\s+Caused by\b|\s+Failure/Error\b|\z)}
|
|
15
|
+
MISSING = /(?<what>column|relation|table)\s+["'`]?(?<name>[\w."]+?)["'`]?\s+does not exist/i
|
|
16
|
+
SQLITE = /no such (?<what>column|table):\s*(?<name>[\w.]+)/i
|
|
17
|
+
|
|
18
|
+
module_function
|
|
19
|
+
|
|
20
|
+
def call(_failure, text)
|
|
21
|
+
not_found(text) || schema(text) || validation(text)
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def not_found(text)
|
|
25
|
+
match = NOT_FOUND.match(text)
|
|
26
|
+
return nil unless match
|
|
27
|
+
|
|
28
|
+
model = match[:model]
|
|
29
|
+
Symptom.new(kind: :record_not_found, key: "record-not-found:#{model}",
|
|
30
|
+
label: "missing `#{model}` records", detail: "no #{model} found")
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def schema(text)
|
|
34
|
+
match = MISSING.match(text) || SQLITE.match(text)
|
|
35
|
+
return nil unless match
|
|
36
|
+
|
|
37
|
+
what = match[:what].downcase
|
|
38
|
+
name = match[:name].delete('"')
|
|
39
|
+
Symptom.new(kind: :schema, key: "schema:#{what}:#{name}",
|
|
40
|
+
label: "missing database #{what} `#{name}`", detail: "#{what} #{name} does not exist")
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def validation(text)
|
|
44
|
+
match = VALIDATION.match(text)
|
|
45
|
+
return nil unless match
|
|
46
|
+
|
|
47
|
+
detail = match[:detail].strip.sub(/[.,]\z/, "")
|
|
48
|
+
return nil if detail.empty?
|
|
49
|
+
|
|
50
|
+
Symptom.new(kind: :validation, key: "validation:#{detail.downcase}",
|
|
51
|
+
label: "validation failure `#{detail}`", detail: detail)
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
end
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module RSpec
|
|
4
|
+
module Signal
|
|
5
|
+
module Symptoms
|
|
6
|
+
# A route that does not exist -- the usual thing sitting behind a suite
|
|
7
|
+
# full of unexplained 404s.
|
|
8
|
+
module Route
|
|
9
|
+
NO_ROUTE = /No route matches (?<target>\[[A-Z]+\]\s*"[^"]*"|\{[^}]*\})/
|
|
10
|
+
# `reader_progress_path` after the route was renamed or removed.
|
|
11
|
+
HELPER = /undefined (?:local variable or method|method) [`'"](?<helper>\w+_(?:path|url))['"`]/
|
|
12
|
+
|
|
13
|
+
module_function
|
|
14
|
+
|
|
15
|
+
def call(_failure, text)
|
|
16
|
+
match = NO_ROUTE.match(text)
|
|
17
|
+
return helper(text) unless match
|
|
18
|
+
|
|
19
|
+
target = normalize(match[:target])
|
|
20
|
+
Symptom.new(kind: :route, key: "route:#{target}",
|
|
21
|
+
label: "no route matches `#{target}`", detail: "no route matches #{target}")
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def helper(text)
|
|
25
|
+
match = HELPER.match(text)
|
|
26
|
+
return nil unless match
|
|
27
|
+
|
|
28
|
+
name = match[:helper]
|
|
29
|
+
Symptom.new(kind: :route_helper, key: "route-helper:#{name}",
|
|
30
|
+
label: "undefined route helper `#{name}`", detail: "undefined `#{name}`")
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
# `/readers/48213/progress` and `/readers/91055/progress` are the same
|
|
34
|
+
# missing route, so identifier-shaped segments become `:id`.
|
|
35
|
+
def normalize(target)
|
|
36
|
+
target.to_s
|
|
37
|
+
.gsub(%r{/\d+(?=/|"|\z)}, "/:id")
|
|
38
|
+
.gsub(%r{/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}(?=/|"|\z)}i, "/:id")
|
|
39
|
+
.gsub(/\s+/, " ")
|
|
40
|
+
.strip
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
end
|