jitcatch 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE +21 -0
- data/README.ja.md +1052 -0
- data/README.md +529 -0
- data/bin/jitcatch +16 -0
- data/lib/jitcatch/assessor.rb +502 -0
- data/lib/jitcatch/changes.rb +1084 -0
- data/lib/jitcatch/cli.rb +201 -0
- data/lib/jitcatch/diff_parser.rb +59 -0
- data/lib/jitcatch/factories.rb +164 -0
- data/lib/jitcatch/failure.rb +86 -0
- data/lib/jitcatch/framework.rb +110 -0
- data/lib/jitcatch/git_repo.rb +180 -0
- data/lib/jitcatch/inflections.rb +142 -0
- data/lib/jitcatch/minitest_parser.rb +110 -0
- data/lib/jitcatch/minitest_report.rb +54 -0
- data/lib/jitcatch/readers.rb +907 -0
- data/lib/jitcatch/render.rb +712 -0
- data/lib/jitcatch/routes.rb +288 -0
- data/lib/jitcatch/rspec.rb +9 -0
- data/lib/jitcatch/rspec_parser.rb +122 -0
- data/lib/jitcatch/runner.rb +905 -0
- data/lib/jitcatch/source_index.rb +1382 -0
- data/lib/jitcatch/template.rb +38 -0
- data/lib/jitcatch/version.rb +5 -0
- data/lib/jitcatch.rb +96 -0
- data/skills/jitcatch/SKILL.md +244 -0
- metadata +73 -0
|
@@ -0,0 +1,712 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module JitCatch
|
|
4
|
+
module Render
|
|
5
|
+
module_function
|
|
6
|
+
|
|
7
|
+
LONG_LINE = 100
|
|
8
|
+
WINDOW = 30
|
|
9
|
+
DIFF_KEYS = 4
|
|
10
|
+
# The element diff keeps a table of this many cells.
|
|
11
|
+
MAX_ELEMENT_PAIRS = 250_000
|
|
12
|
+
|
|
13
|
+
# The readers a context shows unless `--all-readers` asks for every one.
|
|
14
|
+
READERS_SHOWN = 10
|
|
15
|
+
|
|
16
|
+
def context_text(doc, readers_limit: READERS_SHOWN)
|
|
17
|
+
lines = []
|
|
18
|
+
lines << "jitcatch context"
|
|
19
|
+
lines << "version #{doc["version"]}"
|
|
20
|
+
lines << "base #{doc["base"]}"
|
|
21
|
+
lines << "head #{doc["head"]}"
|
|
22
|
+
lines.concat(warning_lines(doc))
|
|
23
|
+
lines << "existing_tests #{doc["existing_tests_via"]}"
|
|
24
|
+
if doc["intent"].empty?
|
|
25
|
+
lines << "intent: (none)"
|
|
26
|
+
else
|
|
27
|
+
lines << "intent:"
|
|
28
|
+
doc["intent"].each { |line| lines << " #{line}" }
|
|
29
|
+
end
|
|
30
|
+
lines << "files:"
|
|
31
|
+
if doc["files"].empty?
|
|
32
|
+
lines << " (none)"
|
|
33
|
+
else
|
|
34
|
+
doc["files"].each { |file| lines << " #{file["status"]} #{file["file"]}" }
|
|
35
|
+
end
|
|
36
|
+
lines.concat(dependency_lines(Array(doc["dependencies"])))
|
|
37
|
+
lines.concat(constant_lines(doc["constants"]))
|
|
38
|
+
lines.concat(method_lines(doc["methods"], reader_counts(doc["readers"])))
|
|
39
|
+
lines.concat(reader_lines(shown_readers(doc["readers"], readers_limit))) if doc["readers"]
|
|
40
|
+
lines.concat(entrypoint_lines(doc["entrypoints"]))
|
|
41
|
+
skipped = Array(doc["factories_skipped"])
|
|
42
|
+
unless skipped.empty?
|
|
43
|
+
lines << "factories_skipped:"
|
|
44
|
+
skipped.each { |factory| lines << " #{factory["file"]}:#{factory["line"]} factory#{factory["name"] && " :#{factory["name"]}"}: #{factory["reason"]}" }
|
|
45
|
+
end
|
|
46
|
+
unless doc["parse_errors"].empty?
|
|
47
|
+
lines << "parse_errors:"
|
|
48
|
+
doc["parse_errors"].each { |err| lines << " #{err["ref"]} #{err["file"]} #{err["message"]}" }
|
|
49
|
+
end
|
|
50
|
+
finish(lines)
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def context_json(doc, readers_limit: READERS_SHOWN)
|
|
54
|
+
doc = doc.merge("readers" => shown_readers(doc["readers"], readers_limit)) if doc["readers"]
|
|
55
|
+
{ "command" => "context" }.merge(doc)
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
def reader_counts(readers)
|
|
59
|
+
Array(readers && readers["values"]).select { |value| value["id"] }.to_h { |value| [value["id"], value["readers"]] }
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
# The highest-scored rows, and how many the limit left out.
|
|
63
|
+
def shown_readers(readers, limit)
|
|
64
|
+
rows = limit ? readers["rows"].first(limit) : readers["rows"]
|
|
65
|
+
readers.merge("rows" => rows, "omitted" => readers["rows"].size - rows.size)
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
# Rows go under the value they read, values in the order the change
|
|
69
|
+
# lists them; a value keeps its line when the limit shows none of its rows.
|
|
70
|
+
def reader_lines(readers)
|
|
71
|
+
read = readers["values"].select { |value| value["readers"].positive? }
|
|
72
|
+
return ["readers: (none)", " not seen: #{readers["not_seen"].join("; ")}"] if read.empty?
|
|
73
|
+
|
|
74
|
+
total = readers["rows"].size + readers["omitted"]
|
|
75
|
+
more = readers["omitted"].positive? ? ", showing #{readers["rows"].size}; --all-readers shows all" : ""
|
|
76
|
+
lines = ["readers: #{total} for #{read.size} changed #{read.size == 1 ? "value" : "values"}#{more}"]
|
|
77
|
+
read.each do |value|
|
|
78
|
+
id = value["id"] ? " #{value["id"]}" : ""
|
|
79
|
+
lines << " #{value["value"]} (#{value["change"]}) #{value["kind"]}#{id} #{value["rev"][0, 7]}:#{value["file"]}:#{value["line"]} readers=#{value["readers"]}"
|
|
80
|
+
readers["rows"].select { |row| row["value"] == value["value"] }.each do |row|
|
|
81
|
+
lines << " #{row["score"]} #{row["mechanism"]} #{row["rev"][0, 7]}:#{row["file"]}:#{row["line"]}"
|
|
82
|
+
lines << " note: this line is itself part of BASE..HEAD" if row["edited_by_change"]
|
|
83
|
+
lines << " rank: #{row["rank"]}"
|
|
84
|
+
lines << " why: #{row["why"]}"
|
|
85
|
+
lines << " also: #{row["also"].join(", ")}" unless row["also"].empty?
|
|
86
|
+
lines << " test: #{row["test"]}"
|
|
87
|
+
end
|
|
88
|
+
end
|
|
89
|
+
lines << " not seen: #{readers["not_seen"].join("; ")}"
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
def run_text(doc)
|
|
93
|
+
lines = []
|
|
94
|
+
lines.concat(untested_lines(doc)) if Assessor.untested_run?(doc)
|
|
95
|
+
lines << "jitcatch run"
|
|
96
|
+
lines << "version #{doc["version"]}"
|
|
97
|
+
lines << "base #{doc["base"]}"
|
|
98
|
+
lines << "head #{doc["head"]}"
|
|
99
|
+
lines.concat(warning_lines(doc))
|
|
100
|
+
lines.concat(no_table_warning_lines(doc))
|
|
101
|
+
lines << "reruns #{doc["reruns"]}"
|
|
102
|
+
lines << "timeout_s #{doc["timeout_s"]}"
|
|
103
|
+
lines << "setup #{doc["setup"]}" if doc["setup"]
|
|
104
|
+
lines << "serial" if doc["serial"]
|
|
105
|
+
lines << "numbered_databases #{doc["numbered_databases"].join(" ")}" unless doc["numbered_databases"].to_a.empty?
|
|
106
|
+
lines << "env #{doc["env"].join(" ")}" unless doc["env"].to_a.empty?
|
|
107
|
+
lines << "env_unset #{doc["env_unset"].join(" ")} (a note, not an error: the shell does not set #{them(doc["env_unset"])}, so each side ran with the code's own default)" unless doc["env_unset"].to_a.empty?
|
|
108
|
+
doc.fetch("bundle", {}).each { |side, bundle| lines << "bundle #{side} #{bundle}" unless bundle == "none" }
|
|
109
|
+
lines << "test_files_omitted #{doc["test_files_omitted"]}" unless doc["test_files_omitted"].to_i.zero?
|
|
110
|
+
lines << "tests:"
|
|
111
|
+
if doc["tests"].empty?
|
|
112
|
+
lines << " (none)"
|
|
113
|
+
else
|
|
114
|
+
doc["tests"].each do |test|
|
|
115
|
+
lines << " #{test_line(test)}"
|
|
116
|
+
lines.concat(rerun_lines(test)) if test["disposition"] == "flaky"
|
|
117
|
+
lines.concat(timeout_lines(test, doc))
|
|
118
|
+
lines.concat(base_failure_lines(test, doc))
|
|
119
|
+
lines.concat(server_error_lines(test, doc))
|
|
120
|
+
end
|
|
121
|
+
end
|
|
122
|
+
lines << counts_line(doc["counts"])
|
|
123
|
+
lines.concat(assess_body(doc))
|
|
124
|
+
finish(lines)
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
def run_json(doc)
|
|
128
|
+
doc
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
def setup_failed_text(doc)
|
|
132
|
+
load = doc["stage"] == "load"
|
|
133
|
+
command = doc["stage"] == "setup_command"
|
|
134
|
+
lines = [if load
|
|
135
|
+
"jitcatch: every candidate file failed to load on both sides, so no row can tell base from head"
|
|
136
|
+
elsif command
|
|
137
|
+
"jitcatch: --setup failed, so no test ran"
|
|
138
|
+
else
|
|
139
|
+
"jitcatch: setup failed, so no test ran; a test cannot pass or fail on a side that cannot load"
|
|
140
|
+
end]
|
|
141
|
+
lines << "base #{doc["base"]}"
|
|
142
|
+
lines << "head #{doc["head"]}"
|
|
143
|
+
lines << "ruby #{doc.dig("ruby", "version")} #{doc.dig("ruby", "path")}"
|
|
144
|
+
doc["setup_failed"].each do |failure|
|
|
145
|
+
lines << "#{load ? "load" : "setup"} #{failure["side"]} failed: #{failure["command"]}"
|
|
146
|
+
failure["stderr"].each { |line| lines << " #{line}" }
|
|
147
|
+
names = failure["not_passed"]
|
|
148
|
+
lines << " not passed: #{names.join(" ")} (add #{env_flags(names)})" unless names.empty?
|
|
149
|
+
database = Array(failure["database_not_passed"])
|
|
150
|
+
lines << " the database refused the connection, and jitcatch did not pass #{database.join(" ")} from your shell: add #{env_flags(database)}" unless database.empty?
|
|
151
|
+
end
|
|
152
|
+
lines << if command
|
|
153
|
+
"Run that command in a checkout of that side, with its TEST_ENV_NUMBER and the variables --env passes, fix what it says, and run jitcatch again."
|
|
154
|
+
elsif load
|
|
155
|
+
rails = " A file git ignores, such as config/master.key, is not in the checkouts jitcatch makes; pass its value with --env, such as RAILS_MASTER_KEY." if doc["rails_app"]
|
|
156
|
+
"Run that command from the repository root with that side checked out, fix what it says (often a database, a credential, or a service the app needs at boot), and run jitcatch again.#{rails}"
|
|
157
|
+
else
|
|
158
|
+
"Run that command with that Ruby in a checkout of that side, fix what it says (often `bundle install` there; a bundle installed for another Ruby reads as gems not installed), and run jitcatch again."
|
|
159
|
+
end
|
|
160
|
+
finish(lines)
|
|
161
|
+
end
|
|
162
|
+
|
|
163
|
+
def assess_text(doc)
|
|
164
|
+
lines = []
|
|
165
|
+
lines << "jitcatch assess"
|
|
166
|
+
lines << "version #{doc["version"]}"
|
|
167
|
+
lines << "base #{doc["base"]}"
|
|
168
|
+
lines << "head #{doc["head"]}"
|
|
169
|
+
lines.concat(warning_lines(doc))
|
|
170
|
+
lines << counts_line(doc["counts"])
|
|
171
|
+
lines.concat(assess_body(doc))
|
|
172
|
+
finish(lines)
|
|
173
|
+
end
|
|
174
|
+
|
|
175
|
+
def assess_json(doc)
|
|
176
|
+
{
|
|
177
|
+
"command" => "assess",
|
|
178
|
+
"version" => doc["version"],
|
|
179
|
+
"base" => doc["base"],
|
|
180
|
+
"head" => doc["head"],
|
|
181
|
+
"warnings" => Array(doc["warnings"]),
|
|
182
|
+
"counts" => doc["counts"],
|
|
183
|
+
"candidates" => candidate_rows(doc)
|
|
184
|
+
}
|
|
185
|
+
end
|
|
186
|
+
|
|
187
|
+
def dump(obj)
|
|
188
|
+
JSON.pretty_generate(obj) + "\n"
|
|
189
|
+
end
|
|
190
|
+
|
|
191
|
+
def warning_lines(doc)
|
|
192
|
+
Array(doc["warnings"]).map { |warning| "warning: #{warning}" }
|
|
193
|
+
end
|
|
194
|
+
|
|
195
|
+
def dependency_lines(dependencies)
|
|
196
|
+
return [] if dependencies.empty?
|
|
197
|
+
|
|
198
|
+
["dependencies:", *dependencies.map { |dep| " #{dep["gem"]} #{dep["old"] || "(none)"} -> #{dep["new"] || "(none)"}" }]
|
|
199
|
+
end
|
|
200
|
+
|
|
201
|
+
def constant_lines(constants)
|
|
202
|
+
if constants.empty?
|
|
203
|
+
["constants: (none)"]
|
|
204
|
+
else
|
|
205
|
+
lines = ["constants:"]
|
|
206
|
+
constants.each do |const|
|
|
207
|
+
moved = const["moved_from"] ? " moved_from=#{const["moved_from"]}" : ""
|
|
208
|
+
lines << " [#{const["id"]}] #{const["change"]} #{const["name"]} #{const["file"]}:#{const["start_line"]}#{moved}"
|
|
209
|
+
lines << " old: #{const["old_source"] || "(none)"}"
|
|
210
|
+
lines << " new: #{const["new_source"] || "(none)"}"
|
|
211
|
+
read_by = const["read_by"].empty? ? "(none)" : const["read_by"].join(", ")
|
|
212
|
+
lines << " read_by: #{read_by}"
|
|
213
|
+
end
|
|
214
|
+
lines
|
|
215
|
+
end
|
|
216
|
+
end
|
|
217
|
+
|
|
218
|
+
def method_lines(methods, readers = {})
|
|
219
|
+
if methods.empty?
|
|
220
|
+
["methods: (none)"]
|
|
221
|
+
else
|
|
222
|
+
lines = ["methods:"]
|
|
223
|
+
methods.each do |method|
|
|
224
|
+
via = method["via_constant"] ? " via=#{method["via_constant"]}" : ""
|
|
225
|
+
via += " moved_from=#{method["moved_from"]}" if method["moved_from"]
|
|
226
|
+
lines << " [#{method["id"]}] #{method["change"]} #{method["name"]} #{method["file"]}:#{method["start_line"]}-#{method["end_line"]} signature=#{method["signature"]} visibility=#{method["visibility"]}#{via}"
|
|
227
|
+
lines << " callsites: #{format_callsites(method, readers.fetch(method["id"], 0))}"
|
|
228
|
+
unless method["reachable_from"].empty?
|
|
229
|
+
lines << " reachable_from: #{method["reachable_from"].join(", ")}"
|
|
230
|
+
end
|
|
231
|
+
unless method["dynamic_callers"].to_a.empty?
|
|
232
|
+
lines << " dynamic_callers: #{method["dynamic_callers"].map { |caller| "#{caller["from"]} (#{caller["name"]})" }.join(", ")}"
|
|
233
|
+
end
|
|
234
|
+
lines << " built_by: #{method["builders"].join(", ")}" unless method["builders"].to_a.empty?
|
|
235
|
+
lines << " run_by: #{method["run_by"].join(", ")} (#{method["callbacks"].join(", ")})" unless method["run_by"].to_a.empty?
|
|
236
|
+
lines << " tests: #{format_tests(method["existing_tests"])}"
|
|
237
|
+
unless method["entry_point_tests"].empty?
|
|
238
|
+
hits = method["entry_point_tests"].map { |hit| "#{hit["file"]} via #{hit["entry_points"].join(", ")}" }
|
|
239
|
+
lines << " entry_point_tests: #{hits.join("; ")}"
|
|
240
|
+
end
|
|
241
|
+
lines.concat(source_lines(method))
|
|
242
|
+
end
|
|
243
|
+
lines
|
|
244
|
+
end
|
|
245
|
+
end
|
|
246
|
+
|
|
247
|
+
def entrypoint_lines(entrypoints)
|
|
248
|
+
return ["entrypoints: (none)"] if entrypoints.empty?
|
|
249
|
+
|
|
250
|
+
lines = []
|
|
251
|
+
entrypoints.each do |entry|
|
|
252
|
+
lines << "entrypoints #{entry["class"]}:"
|
|
253
|
+
entry["methods"].each { |method| lines << " #{method["name"]} #{method["role"]}" }
|
|
254
|
+
lines << " (+#{entry["omitted"]} more)" if entry["omitted"].to_i.positive?
|
|
255
|
+
end
|
|
256
|
+
lines
|
|
257
|
+
end
|
|
258
|
+
|
|
259
|
+
# A method nothing calls by name can still be read: say so where an
|
|
260
|
+
# empty list would read as nothing reaching it.
|
|
261
|
+
def format_callsites(method, readers = 0)
|
|
262
|
+
return "(none)" if method["callsites"].empty? && readers.zero?
|
|
263
|
+
return "(none by name; #{readers} under readers:)" if method["callsites"].empty?
|
|
264
|
+
|
|
265
|
+
# A template's call site is made from the template, which its path already names.
|
|
266
|
+
text = method["callsites"].map { |site|
|
|
267
|
+
place = "#{site["file"]}:#{site["line"]}"
|
|
268
|
+
site["enclosing"] == site["file"] ? place : "#{place} #{site["enclosing"]}"
|
|
269
|
+
}.join("; ")
|
|
270
|
+
omitted = method["callsites_omitted"].to_i
|
|
271
|
+
omitted.positive? ? "#{text} (+#{omitted} more)" : text
|
|
272
|
+
end
|
|
273
|
+
|
|
274
|
+
def format_tests(tests)
|
|
275
|
+
return "(none)" if tests.empty?
|
|
276
|
+
|
|
277
|
+
tests.map { |test|
|
|
278
|
+
if test["via"] == "coverage"
|
|
279
|
+
"#{test["file"]} coverage #{test["test"]}"
|
|
280
|
+
elsif %w[builder callback route].include?(test["via"])
|
|
281
|
+
"#{test["file"]} via #{test["calls"].join(", ")}"
|
|
282
|
+
else
|
|
283
|
+
"#{test["file"]} grep"
|
|
284
|
+
end
|
|
285
|
+
}.join("; ")
|
|
286
|
+
end
|
|
287
|
+
|
|
288
|
+
def source_lines(method)
|
|
289
|
+
old_src = method["old_source"]
|
|
290
|
+
new_src = method["new_source"]
|
|
291
|
+
if old_src == new_src
|
|
292
|
+
[" source:", *Text.indent(old_src, 6).split("\n")]
|
|
293
|
+
else
|
|
294
|
+
lines = []
|
|
295
|
+
lines << " old:"
|
|
296
|
+
lines.concat(Text.indent(old_src, 6).split("\n"))
|
|
297
|
+
lines << " new:"
|
|
298
|
+
lines.concat(Text.indent(new_src, 6).split("\n"))
|
|
299
|
+
lines
|
|
300
|
+
end
|
|
301
|
+
end
|
|
302
|
+
|
|
303
|
+
def test_line(test)
|
|
304
|
+
method = test["method_id"] ? " #{test["method_id"]}" : ""
|
|
305
|
+
place = test["line"] ? "#{test["file"]}:#{test["line"]}" : test["file"]
|
|
306
|
+
"#{test["id"]} #{test["disposition"]} #{test["name"]} #{place}#{method}"
|
|
307
|
+
end
|
|
308
|
+
|
|
309
|
+
# A flaky row says which run moved and how: the outcome, or only the
|
|
310
|
+
# message, as a random id in a printed record does. Base reruns only
|
|
311
|
+
# when head failed the same way each time.
|
|
312
|
+
def rerun_lines(test)
|
|
313
|
+
head = test["head"] || {}
|
|
314
|
+
head_lines = Text.message_lines(head["message"])
|
|
315
|
+
reruns = Array(test["reruns"]).each_with_index.map do |rerun, index|
|
|
316
|
+
# The first line that moved says why the row is flaky.
|
|
317
|
+
moved = Text.message_lines(rerun["message"]).find { |line| !head_lines.include?(line) }
|
|
318
|
+
same = rerun.values_at("status", "error_class") == head.values_at("status", "error_class") && moved
|
|
319
|
+
outcome = [rerun["status"], rerun["error_class"], ("(same outcome, message differs)" if same), moved || rerun["message_first_line"]]
|
|
320
|
+
cut_line(" rerun #{index + 1}: #{outcome.reject { |part| part.to_s.empty? }.join(" ")}")
|
|
321
|
+
end
|
|
322
|
+
base = Array(test["base_reruns"]).each_with_index.map do |rerun, index|
|
|
323
|
+
outcome = [rerun["status"], rerun["error_class"], rerun["message_first_line"]].reject { |part| part.to_s.empty? }
|
|
324
|
+
cut_line(" base rerun #{index + 1}: #{outcome.join(" ")}")
|
|
325
|
+
end
|
|
326
|
+
reruns + base
|
|
327
|
+
end
|
|
328
|
+
|
|
329
|
+
# The ways RSpec's `eq` on a status, rspec-rails' have_http_status, and
|
|
330
|
+
# Minitest's assert_response say a response was a 500.
|
|
331
|
+
SERVER_ERROR = /\A\s*got: \[?500\b|\binternal_server_error \(500\)|<500: Internal Server Error>/
|
|
332
|
+
|
|
333
|
+
# A Rails app rescues the error behind a 500 and logs it, so the
|
|
334
|
+
# failure message alone never names the cause.
|
|
335
|
+
def server_error_lines(test, doc)
|
|
336
|
+
return [] unless doc["rails_app"] && test["disposition"] == "reported"
|
|
337
|
+
return [] unless test.dig("head", "message").to_s.lines.any? { |line| SERVER_ERROR.match?(line) }
|
|
338
|
+
|
|
339
|
+
[" hint: head answered 500, and a Rails app writes the error behind it to log/test.log; run the row's rerun line and read that log"]
|
|
340
|
+
end
|
|
341
|
+
|
|
342
|
+
# A row that failed on base is no candidate, so nothing else in the
|
|
343
|
+
# text says what went wrong there.
|
|
344
|
+
def base_failure_lines(test, doc)
|
|
345
|
+
base = test["base"]
|
|
346
|
+
return [] unless test["disposition"] == "fail_on_base" && base && base["status"] != "timeout"
|
|
347
|
+
|
|
348
|
+
first, *rest = message_lines(base)
|
|
349
|
+
outcome = [base["status"], base["error_class"], first].reject { |part| part.to_s.empty? }
|
|
350
|
+
lines = [" base: #{outcome.join(" ")}", *rest.map { |line| " #{line}" }]
|
|
351
|
+
return lines unless Assessor.no_table?(base)
|
|
352
|
+
|
|
353
|
+
databases = base_databases(doc)
|
|
354
|
+
into = databases.empty? ? "its test database" : databases.join(" and ")
|
|
355
|
+
lines << " hint: base found no table, so this row checked nothing; load base's schema into #{into}"
|
|
356
|
+
end
|
|
357
|
+
|
|
358
|
+
# Only the numbered databases have names the reader can load a schema
|
|
359
|
+
# into; the URL itself can hold a password, so it is never printed.
|
|
360
|
+
def untested_lines(doc)
|
|
361
|
+
causes = Assessor::UNTESTED_CAUSES.keys & doc["tests"].flat_map { |test| Assessor.untested_causes(test, doc["timeout_s"]) }
|
|
362
|
+
size = doc["tests"].size
|
|
363
|
+
hidden = "#{doc["untested"]} of #{size} #{causes.join(" or ")}"
|
|
364
|
+
lines = [if doc["untested"] == size
|
|
365
|
+
"jitcatch: no row ran to a result on both sides (#{hidden}), so this run cannot tell base from head"
|
|
366
|
+
else
|
|
367
|
+
"jitcatch: no row compared base with head (#{hidden}, and #{size - doc["untested"]} failed on base), so this run cannot tell base from head"
|
|
368
|
+
end]
|
|
369
|
+
lines << "jitcatch: load #{schema_advice(doc)}" if causes.include?("found no table")
|
|
370
|
+
lines
|
|
371
|
+
end
|
|
372
|
+
|
|
373
|
+
# A run that compared some rows says nothing on its first line, so the
|
|
374
|
+
# rows the database hid would read as rows that checked something.
|
|
375
|
+
def no_table_warning_lines(doc)
|
|
376
|
+
return [] if Assessor.untested_run?(doc)
|
|
377
|
+
|
|
378
|
+
count = Array(doc["tests"]).count { |test| Assessor.untested_causes(test, doc["timeout_s"]).include?("found no table") }
|
|
379
|
+
return [] if count.zero?
|
|
380
|
+
|
|
381
|
+
rows = count == 1 ? "1 of #{doc["tests"].size} rows found no table on a side, so it checked" : "#{count} of #{doc["tests"].size} rows found no table on a side, so they checked"
|
|
382
|
+
["warning: #{rows} nothing; load #{schema_advice(doc)}"]
|
|
383
|
+
end
|
|
384
|
+
|
|
385
|
+
# Each side needs its own schema, which only a command run in each
|
|
386
|
+
# checkout loads: `db:prepare` also loads seeds into a database it
|
|
387
|
+
# makes, and `db:schema:load` alone fails on one nobody made.
|
|
388
|
+
def schema_advice(doc)
|
|
389
|
+
numbered = doc["numbered_databases"].to_a
|
|
390
|
+
into = numbered.empty? ? "each side's test database" : numbered.join(" and ")
|
|
391
|
+
load = "`--setup 'bin/rails db:create db:schema:load'`"
|
|
392
|
+
step = if doc["setup"]
|
|
393
|
+
"`--setup` ran `#{doc["setup"]}`, yet the rows found no table; pass a command that loads each side's schema, as #{load} does"
|
|
394
|
+
else
|
|
395
|
+
"run jitcatch again with #{load}, which loads each side's schema into its own database"
|
|
396
|
+
end
|
|
397
|
+
"the schema into #{into}: #{step}, or call `ActiveRecord::Migration.maintain_test_schema!` in the test helper"
|
|
398
|
+
end
|
|
399
|
+
|
|
400
|
+
# Base takes the odd TEST_ENV_NUMBER of each pair of checkouts.
|
|
401
|
+
def base_databases(doc)
|
|
402
|
+
doc["numbered_databases"].to_a.each_slice(2).map(&:first)
|
|
403
|
+
end
|
|
404
|
+
|
|
405
|
+
# A child that waits on the network or on a command an environment
|
|
406
|
+
# variable names often lacks a name the parent has, which a longer
|
|
407
|
+
# timeout does not fix, so the row names those first: a name --env
|
|
408
|
+
# passed that the shell did not set, then a name jitcatch did not pass.
|
|
409
|
+
# Only a Rails app loads db/schema.rb in each new checkout.
|
|
410
|
+
def timeout_lines(test, doc)
|
|
411
|
+
sides = %w[base head].select { |side| test.dig(side, "status") == "timeout" }
|
|
412
|
+
return [] if sides.empty?
|
|
413
|
+
|
|
414
|
+
base_seconds = test.dig("base", "seconds")
|
|
415
|
+
if Assessor.head_only_timeout?(test, doc["timeout_s"])
|
|
416
|
+
return [" hint: base passed in #{base_seconds}s; head did not finish in #{doc["timeout_s"]}s, so head may loop or wait forever"]
|
|
417
|
+
end
|
|
418
|
+
|
|
419
|
+
unset = Array(doc["env_unset"])
|
|
420
|
+
not_passed = Array(doc["env_not_passed"])
|
|
421
|
+
missing = [
|
|
422
|
+
("#{env_flags(unset)} passed nothing, because #{unset.join(" ")} #{unset.one? ? "is" : "are"} not set, so set #{them(unset)} if the file needs #{them(unset)}" unless unset.empty?),
|
|
423
|
+
("jitcatch did not pass #{not_passed.join(" ")}, so add #{env_flags(not_passed)} if the file needs #{them(not_passed)}" unless not_passed.empty?)
|
|
424
|
+
].compact
|
|
425
|
+
wait = missing.empty? ? "raise --timeout if the file needs longer" : "#{missing.join("; ")}, then raise --timeout if it needs longer"
|
|
426
|
+
rails = ", as a Rails app's first run in a new checkout can (it loads db/schema.rb)" if doc["rails_app"]
|
|
427
|
+
took = ", and base passed in #{base_seconds}s" if base_seconds
|
|
428
|
+
[" hint: #{sides.join(" and ")} timed out after #{doc["timeout_s"]}s#{took}; #{wait}#{rails}"]
|
|
429
|
+
end
|
|
430
|
+
|
|
431
|
+
def env_flags(names)
|
|
432
|
+
names.map { |name| "--env #{name}" }.join(" ")
|
|
433
|
+
end
|
|
434
|
+
|
|
435
|
+
def them(names)
|
|
436
|
+
names.one? ? "it" : "them"
|
|
437
|
+
end
|
|
438
|
+
|
|
439
|
+
# The rerun line runs in the reader's shell, so it names the variables
|
|
440
|
+
# the run took from jitcatch's, and no value: a DATABASE_URL can hold a
|
|
441
|
+
# password, and a report is pasted where others read it.
|
|
442
|
+
def rerun_env_line(doc)
|
|
443
|
+
names = Array(doc["env"])
|
|
444
|
+
return nil if names.empty?
|
|
445
|
+
|
|
446
|
+
unset = Array(doc["env_unset"])
|
|
447
|
+
" rerun env: reads #{names.join(" ")} from your shell, as jitcatch run did#{"; #{unset.join(" ")} #{unset.one? ? "was" : "were"} unset, so the run used the code's own default" unless unset.empty?}"
|
|
448
|
+
end
|
|
449
|
+
|
|
450
|
+
def counts_line(counts)
|
|
451
|
+
"counts generated=#{counts["generated"]} no_signal=#{counts["no_signal"]} fail_on_base=#{counts["fail_on_base"]} candidates=#{counts["candidates"]} flaky=#{counts["flaky"]} discarded=#{counts["discarded"]} reported=#{counts["reported"]}"
|
|
452
|
+
end
|
|
453
|
+
|
|
454
|
+
def assess_body(doc)
|
|
455
|
+
rows = candidate_rows(doc)
|
|
456
|
+
lines = ["assess:"]
|
|
457
|
+
env_line = rerun_env_line(doc)
|
|
458
|
+
if rows.empty?
|
|
459
|
+
lines << " (none)"
|
|
460
|
+
return lines
|
|
461
|
+
end
|
|
462
|
+
names = Array(doc["methods"]).to_h { |item| [item["id"], item["name"]] }
|
|
463
|
+
rows.each do |row|
|
|
464
|
+
method = row["method_id"] || "-"
|
|
465
|
+
lines << " #{row["candidate_id"]} #{row["name"]} base=#{row["base_status"]} head=#{row["head_status"]} #{method} #{row["tags"].join(",")}"
|
|
466
|
+
lines << " #{guess_line(row, names, doc)}" if row["method_ids"].size > 1
|
|
467
|
+
lines << " #{out_of_date_line(row["out_of_date"])}" if row["out_of_date"]
|
|
468
|
+
if (dynamic = row["dynamic_call"])
|
|
469
|
+
lines << " related: none, #{dynamic["from"]} sends #{dynamic["name"]} to a receiver the source does not name"
|
|
470
|
+
end
|
|
471
|
+
row["message_lines"].each { |line| lines << " #{line}" }
|
|
472
|
+
lines << " cause: #{cause_line(row["cause"])}" if row["cause"]
|
|
473
|
+
lines << " rerun: #{row["rerun"]}" if row["rerun"]
|
|
474
|
+
lines << env_line if row["rerun"] && env_line
|
|
475
|
+
end
|
|
476
|
+
lines
|
|
477
|
+
end
|
|
478
|
+
|
|
479
|
+
# An id alone sends the reader to the context brief to find what it
|
|
480
|
+
# names. With no pick, no frame of head's failure is in a changed
|
|
481
|
+
# method, so the value can come from a file the brief lists no method
|
|
482
|
+
# for, such as a view.
|
|
483
|
+
def guess_line(row, names, doc)
|
|
484
|
+
pool = row["method_ids"].map { |id| "#{names[id] || id} (#{id})" }
|
|
485
|
+
if row["method_id"]
|
|
486
|
+
return "related guess: #{names[row["method_id"]] || row["method_id"]} (#{row["method_id"]}), ranked first of #{pool.size}: #{pool.join(", ")}"
|
|
487
|
+
end
|
|
488
|
+
|
|
489
|
+
shas = %w[base head].map { |side| doc[side].to_s[0, 12] }.join(" ")
|
|
490
|
+
"related guess: none; the test reaches #{pool.size} changed methods equally and no frame of head's failure is in one: #{pool.join(", ")}. " \
|
|
491
|
+
"Next: find what makes head's value in them, or in a changed file that holds no method, such as a view: git diff --stat #{shas}"
|
|
492
|
+
end
|
|
493
|
+
|
|
494
|
+
OUT_OF_DATE = { "removed" => "was removed", "parameters" => "changed its parameters", "definition" => "changed its definition" }.freeze
|
|
495
|
+
|
|
496
|
+
def out_of_date_line(stale)
|
|
497
|
+
at = stale["call"] ? "the call at #{stale["call"]}" : "the call"
|
|
498
|
+
"candidate is out of date with head: #{stale["name"]} (#{stale["id"]}) #{OUT_OF_DATE.fetch(stale["change"])}; update #{at} to head's code, or drop the test"
|
|
499
|
+
end
|
|
500
|
+
|
|
501
|
+
def candidate_rows(doc)
|
|
502
|
+
doc["tests"].select { |test| test["classification"] == "candidate" }.sort_by { |test| test["id"] }.map { |test|
|
|
503
|
+
{
|
|
504
|
+
"candidate_id" => test["candidate_id"],
|
|
505
|
+
"id" => test["id"],
|
|
506
|
+
"file" => test["file"],
|
|
507
|
+
"line" => test["line"],
|
|
508
|
+
"name" => test["name"],
|
|
509
|
+
"rerun" => test["rerun"],
|
|
510
|
+
"base_status" => test.dig("base", "status"),
|
|
511
|
+
"head_status" => test.dig("head", "status"),
|
|
512
|
+
"method_id" => test["method_id"],
|
|
513
|
+
"method_ids" => Array(test["method_ids"]),
|
|
514
|
+
"dynamic_call" => test["dynamic_call"],
|
|
515
|
+
"out_of_date" => test["out_of_date"],
|
|
516
|
+
"tags" => test["tags"],
|
|
517
|
+
"disposition" => test["disposition"],
|
|
518
|
+
"message_first_line" => test.dig("head", "message_first_line").to_s,
|
|
519
|
+
"message" => test.dig("head", "message").to_s,
|
|
520
|
+
"message_lines" => message_lines(test["head"]),
|
|
521
|
+
"cause" => test.dig("head", "cause")
|
|
522
|
+
}
|
|
523
|
+
}
|
|
524
|
+
end
|
|
525
|
+
|
|
526
|
+
def cause_line(cause)
|
|
527
|
+
cause && [cause["error_class"], Failure.first_line(cause["message"])].reject(&:empty?).join(": ")
|
|
528
|
+
end
|
|
529
|
+
|
|
530
|
+
# A row keeps each line to this many characters; one Array printed
|
|
531
|
+
# whole can fill tens of kilobytes.
|
|
532
|
+
MESSAGE_LINE = 300
|
|
533
|
+
|
|
534
|
+
def message_lines(head)
|
|
535
|
+
return [] if head.nil?
|
|
536
|
+
|
|
537
|
+
failure_lines(head).map { |line| cut_line(line) }
|
|
538
|
+
end
|
|
539
|
+
|
|
540
|
+
# The end of a line often holds what differs, after a path or a value
|
|
541
|
+
# both sides share, so a cut leaves out the middle.
|
|
542
|
+
def cut_line(line)
|
|
543
|
+
return line if line.length <= MESSAGE_LINE
|
|
544
|
+
|
|
545
|
+
half = MESSAGE_LINE / 2
|
|
546
|
+
left = line.length - MESSAGE_LINE
|
|
547
|
+
"#{line[0, half]}... (+#{left} #{left == 1 ? "character" : "characters"}; the whole message is in --json) ...#{line[-half..]}"
|
|
548
|
+
end
|
|
549
|
+
|
|
550
|
+
def failure_lines(head)
|
|
551
|
+
|
|
552
|
+
lines = head["message"].to_s.lines.map(&:strip).reject(&:empty?)
|
|
553
|
+
if lines[1]&.match?(/\AActual:/)
|
|
554
|
+
lines.first(2)
|
|
555
|
+
elsif lines[0]&.end_with?("exception expected, not") && lines[1]&.start_with?("Class: <")
|
|
556
|
+
# assert_raises names the error that came on the next two lines.
|
|
557
|
+
lines.first(3)
|
|
558
|
+
elsif lines[0]&.start_with?("expected: ") && lines[1]&.start_with?("got: ")
|
|
559
|
+
# RSpec's eq and eql print each side on its own line.
|
|
560
|
+
expected = lines[0].delete_prefix("expected: ")
|
|
561
|
+
actual = lines[1].delete_prefix("got: ")
|
|
562
|
+
[expected, actual].all? { |text| text.length <= LONG_LINE } ? lines.first(2) : changed_pair(expected, actual)
|
|
563
|
+
elsif lines[0]&.start_with?("expected ") && lines[1]&.start_with?("got ")
|
|
564
|
+
# RSpec's be and equal compare identity and print each side unlabeled.
|
|
565
|
+
lines.first(2)
|
|
566
|
+
elsif lines[1]&.start_with?("expected: ") && lines[2]&.start_with?("got: ")
|
|
567
|
+
# rspec-mocks names the call, then the arguments it wanted and got.
|
|
568
|
+
lines.first(3)
|
|
569
|
+
elsif lines[0] == "--- expected" && lines[1] == "+++ actual"
|
|
570
|
+
# Minitest prints a unified diff when either side spans lines or is
|
|
571
|
+
# long; its header says nothing, so show the first removed and added line.
|
|
572
|
+
# A context line starts with a space, which strip would take away.
|
|
573
|
+
changed = head["message"].to_s.lines.map(&:rstrip).reject(&:empty?).drop(2)
|
|
574
|
+
removed = changed.index { |line| line.start_with?("-") }
|
|
575
|
+
added = changed.index { |line| line.start_with?("+") }
|
|
576
|
+
if removed && added && removed < added && changed[removed...added].all? { |line| line.start_with?("-") }
|
|
577
|
+
changed_pair(changed[removed][1..], changed[added][1..])
|
|
578
|
+
else
|
|
579
|
+
# Lines that do not replace each other share no column.
|
|
580
|
+
[removed, added].compact.map { |index| "#{changed[index][0]}#{shortened(changed[index][1..])}" }
|
|
581
|
+
end
|
|
582
|
+
elsif lines[0] && lines[0].length > MESSAGE_LINE && (changed = rspec_diff_changes(head["message"].to_s))
|
|
583
|
+
[lines[0], *changed]
|
|
584
|
+
else
|
|
585
|
+
lines.first(1)
|
|
586
|
+
end
|
|
587
|
+
end
|
|
588
|
+
|
|
589
|
+
# The removed and added lines of the Diff RSpec prints under a matcher's
|
|
590
|
+
# message, such as have_attributes, whose one line shows both Hashes.
|
|
591
|
+
def rspec_diff_changes(message)
|
|
592
|
+
diff = message.lines.map(&:rstrip).drop_while { |line| line != "Diff:" }.drop(1)
|
|
593
|
+
changed = diff.select { |line| line.match?(/\A[-+](?![-+]{2} )/) }
|
|
594
|
+
changed.empty? ? nil : changed.first(2 * DIFF_KEYS)
|
|
595
|
+
end
|
|
596
|
+
|
|
597
|
+
# A long line, such as a whole Hash on one line, is hard to compare by
|
|
598
|
+
# eye. Show the Hash entries that differ, or else a window around the
|
|
599
|
+
# first column that differs. `message` in `--json` keeps the full text.
|
|
600
|
+
def changed_pair(expected, actual)
|
|
601
|
+
return ["-#{expected}", "+#{actual}"] if [expected, actual].all? { |text| text.length <= LONG_LINE }
|
|
602
|
+
|
|
603
|
+
hash_entry_lines(expected, actual) || array_element_lines(expected, actual) || column_lines(expected, actual)
|
|
604
|
+
end
|
|
605
|
+
|
|
606
|
+
def shortened(line)
|
|
607
|
+
line.length > LONG_LINE ? "#{line[0, (2 * WINDOW) + 1]}..." : line
|
|
608
|
+
end
|
|
609
|
+
|
|
610
|
+
# The elements one Array literal has and the other lacks, by index, from
|
|
611
|
+
# a longest common subsequence, so one inserted element is one line and
|
|
612
|
+
# not a shift of every element after it.
|
|
613
|
+
def array_element_lines(expected, actual)
|
|
614
|
+
old_items = array_elements(expected)
|
|
615
|
+
new_items = array_elements(actual)
|
|
616
|
+
return nil unless old_items && new_items && old_items.size * new_items.size <= MAX_ELEMENT_PAIRS
|
|
617
|
+
|
|
618
|
+
changes = element_changes(old_items, new_items)
|
|
619
|
+
return nil if changes.empty?
|
|
620
|
+
|
|
621
|
+
shown = changes.first(DIFF_KEYS)
|
|
622
|
+
more = changes.size > DIFF_KEYS ? " (+#{changes.size - DIFF_KEYS} more)" : ""
|
|
623
|
+
lines = ["array elements differ: #{shown.map { |index, _, _| "[#{index}]" }.join(", ")}#{more}"]
|
|
624
|
+
shown.each do |index, old_item, new_item|
|
|
625
|
+
old_entry = "[#{index}] #{old_item}" if old_item
|
|
626
|
+
new_entry = "[#{index}] #{new_item}" if new_item
|
|
627
|
+
lines.concat(old_entry && new_entry ? changed_pair(old_entry, new_entry) : [old_entry && "-#{old_entry}", new_entry && "+#{new_entry}"].compact)
|
|
628
|
+
end
|
|
629
|
+
lines
|
|
630
|
+
end
|
|
631
|
+
|
|
632
|
+
def array_elements(text)
|
|
633
|
+
result = Prism.parse(text)
|
|
634
|
+
body = result.success? ? result.value.statements.body : []
|
|
635
|
+
return nil unless body.size == 1 && body.first.is_a?(Prism::ArrayNode)
|
|
636
|
+
|
|
637
|
+
body.first.elements.map(&:slice)
|
|
638
|
+
end
|
|
639
|
+
|
|
640
|
+
# Each change as [index, old element, new element]: a removed element
|
|
641
|
+
# has no new one and an added one no old one. Within one run of changes,
|
|
642
|
+
# the removed and added elements pair up in order, as replacements.
|
|
643
|
+
def element_changes(old_items, new_items)
|
|
644
|
+
lengths = Array.new(old_items.size + 1) { Array.new(new_items.size + 1, 0) }
|
|
645
|
+
old_items.each_index.reverse_each do |i|
|
|
646
|
+
new_items.each_index.reverse_each do |j|
|
|
647
|
+
lengths[i][j] = old_items[i] == new_items[j] ? lengths[i + 1][j + 1] + 1 : [lengths[i + 1][j], lengths[i][j + 1]].max
|
|
648
|
+
end
|
|
649
|
+
end
|
|
650
|
+
runs = [[[], []]]
|
|
651
|
+
i = j = 0
|
|
652
|
+
while i < old_items.size || j < new_items.size
|
|
653
|
+
if i < old_items.size && j < new_items.size && old_items[i] == new_items[j]
|
|
654
|
+
runs << [[], []]
|
|
655
|
+
i += 1
|
|
656
|
+
j += 1
|
|
657
|
+
elsif j < new_items.size && (i == old_items.size || lengths[i][j + 1] > lengths[i + 1][j])
|
|
658
|
+
runs.last[1] << [j, new_items[j]]
|
|
659
|
+
j += 1
|
|
660
|
+
else
|
|
661
|
+
runs.last[0] << [i, old_items[i]]
|
|
662
|
+
i += 1
|
|
663
|
+
end
|
|
664
|
+
end
|
|
665
|
+
runs.flat_map do |removed, added|
|
|
666
|
+
Array.new([removed.size, added.size].max) do |k|
|
|
667
|
+
[(removed[k] || added[k]).first, removed[k]&.last, added[k]&.last]
|
|
668
|
+
end
|
|
669
|
+
end
|
|
670
|
+
end
|
|
671
|
+
|
|
672
|
+
def hash_entry_lines(expected, actual)
|
|
673
|
+
old_entries = hash_entries(expected)
|
|
674
|
+
new_entries = hash_entries(actual)
|
|
675
|
+
return nil unless old_entries && new_entries
|
|
676
|
+
|
|
677
|
+
keys = (old_entries.keys + new_entries.keys).uniq.reject { |key| old_entries[key] == new_entries[key] }
|
|
678
|
+
return nil if keys.empty?
|
|
679
|
+
|
|
680
|
+
lines = ["hash keys differ: #{keys.first(DIFF_KEYS).join(", ")}#{" (+#{keys.size - DIFF_KEYS} more)" if keys.size > DIFF_KEYS}"]
|
|
681
|
+
keys.first(DIFF_KEYS).each do |key|
|
|
682
|
+
old_entry = "#{key} => #{old_entries[key]}" if old_entries.key?(key)
|
|
683
|
+
new_entry = "#{key} => #{new_entries[key]}" if new_entries.key?(key)
|
|
684
|
+
lines.concat(old_entry && new_entry ? changed_pair(old_entry, new_entry) : [old_entry && "-#{old_entry}", new_entry && "+#{new_entry}"].compact)
|
|
685
|
+
end
|
|
686
|
+
lines
|
|
687
|
+
end
|
|
688
|
+
|
|
689
|
+
# The entries of a Hash literal as written by `inspect`, keyed by the
|
|
690
|
+
# key's text, or nil when the text is not one Hash literal.
|
|
691
|
+
def hash_entries(text)
|
|
692
|
+
result = Prism.parse(text)
|
|
693
|
+
body = result.success? ? result.value.statements.body : []
|
|
694
|
+
return nil unless body.size == 1 && body.first.is_a?(Prism::HashNode)
|
|
695
|
+
return nil unless body.first.elements.all?(Prism::AssocNode)
|
|
696
|
+
|
|
697
|
+
body.first.elements.to_h { |assoc| [assoc.key.slice.delete_suffix(":"), assoc.value.slice] }
|
|
698
|
+
end
|
|
699
|
+
|
|
700
|
+
def column_lines(expected, actual)
|
|
701
|
+
column = (0...[expected.length, actual.length].min).find { |index| expected[index] != actual[index] } || [expected.length, actual.length].min
|
|
702
|
+
start = [column - WINDOW, 0].max
|
|
703
|
+
lead = start.zero? ? "" : "..."
|
|
704
|
+
window = ->(text) { "#{lead}#{text[start, (2 * WINDOW) + 1]}#{"..." if text.length > start + (2 * WINDOW) + 1}" }
|
|
705
|
+
["-#{window.call(expected)}", "+#{window.call(actual)}", "#{" " * (1 + lead.length + column - start)}^ column #{column + 1}"]
|
|
706
|
+
end
|
|
707
|
+
|
|
708
|
+
def finish(lines)
|
|
709
|
+
"#{lines.join("\n")}\n"
|
|
710
|
+
end
|
|
711
|
+
end
|
|
712
|
+
end
|