constable-rails 0.1.0 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +159 -1
- data/README.md +129 -13
- data/lib/constable/case.rb +33 -0
- data/lib/constable/cli.rb +38 -5
- data/lib/constable/cold_case/rspec.rb +93 -1
- data/lib/constable/config.rb +53 -5
- data/lib/constable/identity.rb +19 -0
- data/lib/constable/importer/modernizer.rb +40 -0
- data/lib/constable/investigation.rb +10 -0
- data/lib/constable/jail.rb +34 -7
- data/lib/constable/matchers.rb +206 -2
- data/lib/constable/registry.rb +24 -0
- data/lib/constable/reporter.rb +230 -4
- data/lib/constable/runner.rb +131 -3
- data/lib/constable/selection.rb +23 -1
- data/lib/constable/storage/sqlite_adapter.rb +21 -1
- data/lib/constable/version.rb +1 -1
- data/lib/constable/warrants.rb +23 -5
- data/lib/constable/worker_databases.rb +83 -0
- data/lib/constable.rb +1 -0
- data/lib/generators/constable/install_generator.rb +79 -8
- data/lib/generators/constable/templates/case_helper.rb.tt +36 -15
- data/lib/generators/constable/templates/config.yml.tt +9 -0
- metadata +4 -3
data/lib/constable/reporter.rb
CHANGED
|
@@ -46,6 +46,11 @@ module Constable
|
|
|
46
46
|
|
|
47
47
|
DEFAULT_SLOWEST = 5
|
|
48
48
|
|
|
49
|
+
# Column the expanded stream right-aligns durations into. Descriptions are never
|
|
50
|
+
# truncated to fit it -- a clipped test name is not something you can grep for --
|
|
51
|
+
# so a long one simply pushes its own stamp out past the column.
|
|
52
|
+
EXPANDED_STAMP_COLUMN = 56
|
|
53
|
+
|
|
49
54
|
# Result::GLYPHS covers statuses. These four are summary vocabulary, not statuses:
|
|
50
55
|
# supervision and coverage are properties of a test, not outcomes of one.
|
|
51
56
|
GLYPHS = Result::GLYPHS.merge(
|
|
@@ -95,13 +100,14 @@ module Constable
|
|
|
95
100
|
0x1F900..0x1F9FF, 0x20000..0x3FFFD
|
|
96
101
|
].freeze
|
|
97
102
|
|
|
98
|
-
attr_reader :io, :config, :seed, :total
|
|
103
|
+
attr_reader :io, :config, :seed, :total, :mode
|
|
99
104
|
|
|
100
|
-
def initialize(io: $stdout, config: nil, color: nil, slowest: DEFAULT_SLOWEST)
|
|
105
|
+
def initialize(io: $stdout, config: nil, color: nil, slowest: DEFAULT_SLOWEST, mode: nil)
|
|
101
106
|
@io = io
|
|
102
107
|
@config = config || Constable.config
|
|
103
108
|
@color = resolve_color(color)
|
|
104
109
|
@slowest = slowest.to_i
|
|
110
|
+
@mode = resolve_mode(mode)
|
|
105
111
|
@io.set_encoding(Encoding::UTF_8) if @io.respond_to?(:set_encoding)
|
|
106
112
|
|
|
107
113
|
reset_stream!
|
|
@@ -141,6 +147,14 @@ module Constable
|
|
|
141
147
|
def flush!
|
|
142
148
|
return self unless streaming?
|
|
143
149
|
|
|
150
|
+
# The expanded stream never buffers -- every line was written as it happened, so
|
|
151
|
+
# there is nothing left to emit, only a blank line before the summary.
|
|
152
|
+
if expanded?
|
|
153
|
+
reset_stream!
|
|
154
|
+
writeln
|
|
155
|
+
return self
|
|
156
|
+
end
|
|
157
|
+
|
|
144
158
|
close_stream_line
|
|
145
159
|
pending_case_names.each { |name| open_stream_line(name) && close_stream_line }
|
|
146
160
|
writeln
|
|
@@ -169,6 +183,9 @@ module Constable
|
|
|
169
183
|
|
|
170
184
|
section_parole_violations(results)
|
|
171
185
|
section_failures(results)
|
|
186
|
+
section_warrants(results)
|
|
187
|
+
section_jailed(results)
|
|
188
|
+
section_parole(results)
|
|
172
189
|
section_warnings(warnings)
|
|
173
190
|
section_slowest(results)
|
|
174
191
|
rename_suggestions(suggestions)
|
|
@@ -193,6 +210,7 @@ module Constable
|
|
|
193
210
|
def failed? = !success?
|
|
194
211
|
def color? = @color
|
|
195
212
|
def finished? = @finished
|
|
213
|
+
def expanded? = @mode == :expanded
|
|
196
214
|
|
|
197
215
|
private
|
|
198
216
|
|
|
@@ -211,6 +229,8 @@ module Constable
|
|
|
211
229
|
end
|
|
212
230
|
|
|
213
231
|
def stream(result)
|
|
232
|
+
return stream_expanded(result) if expanded?
|
|
233
|
+
|
|
214
234
|
name = result.case_name.to_s
|
|
215
235
|
name = "(anonymous)" if name.empty?
|
|
216
236
|
glyph = paint(result.glyph, COLORS[result.status])
|
|
@@ -263,6 +283,77 @@ module Constable
|
|
|
263
283
|
# Glyphs may carry escape codes; count only the visible ones.
|
|
264
284
|
def count_glyphs(string) = strip_ansi(string).length
|
|
265
285
|
|
|
286
|
+
# --- expanded stream -----------------------------------------------------------
|
|
287
|
+
#
|
|
288
|
+
# One line per test instead of one glyph. The trade is deliberate: concise keeps a
|
|
289
|
+
# thousand-test suite on one screen, expanded tells you which test is hanging while
|
|
290
|
+
# it hangs, without waiting for the summary.
|
|
291
|
+
#
|
|
292
|
+
# Workers interleave, so a case can come back after another has spoken. It gets a
|
|
293
|
+
# second header rather than having its later tests silently appended under the
|
|
294
|
+
# wrong one -- the same honesty rule the concise stream follows.
|
|
295
|
+
def stream_expanded(result)
|
|
296
|
+
name = result.case_name.to_s
|
|
297
|
+
name = "(anonymous)" if name.empty?
|
|
298
|
+
|
|
299
|
+
if @stream_case != name
|
|
300
|
+
writeln if @stream_case
|
|
301
|
+
writeln(INDENT + paint(name, :bold))
|
|
302
|
+
@stream_case = name
|
|
303
|
+
@stream_open = true
|
|
304
|
+
end
|
|
305
|
+
|
|
306
|
+
writeln(expanded_line(result))
|
|
307
|
+
end
|
|
308
|
+
|
|
309
|
+
def expanded_line(result)
|
|
310
|
+
glyph = paint(result.glyph, COLORS[result.status])
|
|
311
|
+
line = "#{ENTRY_INDENT}#{glyph} #{expanded_description(result)}"
|
|
312
|
+
|
|
313
|
+
stamp = expanded_duration(result)
|
|
314
|
+
return line if stamp.nil?
|
|
315
|
+
|
|
316
|
+
# Pad to a column so the durations line up, but never truncate a description --
|
|
317
|
+
# a clipped test name is not something you can grep for.
|
|
318
|
+
visible = strip_ansi(line).length
|
|
319
|
+
gap = [EXPANDED_STAMP_COLUMN - visible, 1].max
|
|
320
|
+
"#{line}#{" " * gap}#{paint(stamp, :dim)}"
|
|
321
|
+
end
|
|
322
|
+
|
|
323
|
+
def expanded_description(result)
|
|
324
|
+
description = result.description.to_s
|
|
325
|
+
description = "(no description)" if description.empty?
|
|
326
|
+
# A jailed test never ran its body, so say why rather than implying it passed.
|
|
327
|
+
return "#{description} #{paint("— #{result.jail_reason}", :dim)}" if jail_reason_worth_showing?(result)
|
|
328
|
+
|
|
329
|
+
description
|
|
330
|
+
end
|
|
331
|
+
|
|
332
|
+
def jail_reason_worth_showing?(result)
|
|
333
|
+
result.status == :jailed && !result.jail_reason.to_s.strip.empty?
|
|
334
|
+
end
|
|
335
|
+
|
|
336
|
+
# Only real, measured time. A jailed test never ran, and "0ms" would be a claim
|
|
337
|
+
# about a body that was skipped.
|
|
338
|
+
def expanded_duration(result)
|
|
339
|
+
return nil if result.status == :jailed
|
|
340
|
+
return nil unless result.duration.to_f.positive?
|
|
341
|
+
|
|
342
|
+
format_test_duration(result.duration)
|
|
343
|
+
end
|
|
344
|
+
|
|
345
|
+
# The summary's durations are run-scale, where "12.4s" is the useful unit. One test
|
|
346
|
+
# is usually sub-second, and "0.0s" against every line says nothing at all -- so the
|
|
347
|
+
# expanded stream counts milliseconds until a test is slow enough for seconds to mean
|
|
348
|
+
# something.
|
|
349
|
+
def format_test_duration(seconds)
|
|
350
|
+
seconds = seconds.to_f
|
|
351
|
+
return format_duration(seconds) if seconds >= 1
|
|
352
|
+
|
|
353
|
+
milliseconds = (seconds * 1000).round
|
|
354
|
+
milliseconds.zero? ? "<1ms" : "#{milliseconds}ms"
|
|
355
|
+
end
|
|
356
|
+
|
|
266
357
|
# --- header and headline -------------------------------------------------------
|
|
267
358
|
|
|
268
359
|
def header_line(results, duration)
|
|
@@ -359,8 +450,12 @@ module Constable
|
|
|
359
450
|
each_entry(violations) do |result|
|
|
360
451
|
writeln(INDENT + paint("#{GLYPHS[:parole_violation]} #{result.case_name}", COLORS[:parole_violation]))
|
|
361
452
|
writeln(ENTRY_INDENT + paint(%("#{result.description}"), :dim))
|
|
453
|
+
writeln(ENTRY_INDENT + paint(result.location, :dim))
|
|
362
454
|
writeln(ENTRY_INDENT + parole_violation_sentence(result))
|
|
363
455
|
end
|
|
456
|
+
hint("Somebody trusted this test again and it let them down, so it is back on the " \
|
|
457
|
+
"docket. Fix it before the next constable jail parole — a second violation " \
|
|
458
|
+
"is the signal that the test, not the flake, is the problem.")
|
|
364
459
|
end
|
|
365
460
|
|
|
366
461
|
def parole_violation_sentence(result)
|
|
@@ -369,6 +464,95 @@ module Constable
|
|
|
369
464
|
sentence
|
|
370
465
|
end
|
|
371
466
|
|
|
467
|
+
# A section says what happened; a hint says what to do about it. One dim sentence,
|
|
468
|
+
# and only where there is a real next step -- a tip printed on every run stops being
|
|
469
|
+
# read on the second one.
|
|
470
|
+
def hint(text)
|
|
471
|
+
writeln
|
|
472
|
+
lines = wrap(text, width: RULE_WIDTH - INDENT.length - 2, indent: " ")
|
|
473
|
+
writeln(INDENT + paint("→ #{lines.first}", :dim))
|
|
474
|
+
lines.drop(1).each { |line| writeln(INDENT + paint(line, :dim)) }
|
|
475
|
+
end
|
|
476
|
+
|
|
477
|
+
# Every test currently under a warrant: it failed, then passed when rerun in
|
|
478
|
+
# isolation, so it is flaky rather than broken. Loud, but not build-blocking.
|
|
479
|
+
def section_warrants(results)
|
|
480
|
+
warranted = results.select(&:warranted?)
|
|
481
|
+
return if warranted.empty?
|
|
482
|
+
|
|
483
|
+
section("WARRANTS")
|
|
484
|
+
each_entry(warranted) do |result|
|
|
485
|
+
writeln(INDENT + paint("#{GLYPHS[:warrant]} #{result.case_name}", COLORS[:warranted]))
|
|
486
|
+
writeln(ENTRY_INDENT + paint(%("#{result.description}"), :dim))
|
|
487
|
+
writeln(ENTRY_INDENT + paint(result.location, :dim))
|
|
488
|
+
writeln(ENTRY_INDENT + warrant_sentence(result))
|
|
489
|
+
end
|
|
490
|
+
hint("A warrant is \"not reproducible\", not \"not a problem\" — it stops blocking the " \
|
|
491
|
+
"build and stays visible until someone deals with it. " \
|
|
492
|
+
"Fixed the flake? constable warrants release PATH:LINE")
|
|
493
|
+
end
|
|
494
|
+
|
|
495
|
+
def warrant_sentence(result)
|
|
496
|
+
statuses = Array(result.retries).map(&:to_sym)
|
|
497
|
+
return "Failed once, then passed on retry." if statuses.empty?
|
|
498
|
+
|
|
499
|
+
passed = statuses.count(:passed)
|
|
500
|
+
"Failed, then passed #{passed} of #{statuses.size} #{pluralize(statuses.size, "retry")} " \
|
|
501
|
+
"run in isolation."
|
|
502
|
+
end
|
|
503
|
+
|
|
504
|
+
# The docket. These never ran their bodies, so they are neither passing nor failing --
|
|
505
|
+
# which is exactly why they get their own category rather than being folded into
|
|
506
|
+
# either one.
|
|
507
|
+
def section_jailed(results)
|
|
508
|
+
jailed = results.select { |result| result.status == :jailed }
|
|
509
|
+
return if jailed.empty?
|
|
510
|
+
|
|
511
|
+
section("JAILED")
|
|
512
|
+
each_entry(jailed) do |result|
|
|
513
|
+
writeln(INDENT + paint("#{GLYPHS[:jailed]} #{result.case_name}", COLORS[:jailed]))
|
|
514
|
+
writeln(ENTRY_INDENT + paint(%("#{result.description}"), :dim))
|
|
515
|
+
writeln(ENTRY_INDENT + paint(result.location, :dim))
|
|
516
|
+
writeln(ENTRY_INDENT + jailed_sentence(result))
|
|
517
|
+
end
|
|
518
|
+
hint("Jailed means skipped and tracked, not passing. Think one is fixed? " \
|
|
519
|
+
"constable jail parole PATH:LINE runs it for real again — " \
|
|
520
|
+
"#{@config.parole_period} clean runs and it releases itself.")
|
|
521
|
+
end
|
|
522
|
+
|
|
523
|
+
def jailed_sentence(result)
|
|
524
|
+
reason = result.jail_reason.to_s.strip
|
|
525
|
+
sentence = reason.empty? ? "Body skipped; setup still ran." : "#{reason.capitalize}."
|
|
526
|
+
sentence += " Its #{ordinalize(result.times_jailed)} time in jail." if result.times_jailed.to_i > 1
|
|
527
|
+
sentence
|
|
528
|
+
end
|
|
529
|
+
|
|
530
|
+
# Out on parole and behaving. Worth naming every run, because the count only means
|
|
531
|
+
# something if you can see it moving.
|
|
532
|
+
def section_parole(results)
|
|
533
|
+
paroled = results.select { |result| result.parole_day && !result.parole_violation? }
|
|
534
|
+
return if paroled.empty?
|
|
535
|
+
|
|
536
|
+
section("ON PAROLE")
|
|
537
|
+
each_entry(paroled) do |result|
|
|
538
|
+
writeln(INDENT + paint("#{GLYPHS[:parole]} #{result.case_name}", COLORS[:parole]))
|
|
539
|
+
writeln(ENTRY_INDENT + paint(%("#{result.description}"), :dim))
|
|
540
|
+
writeln(ENTRY_INDENT + paint(result.location, :dim))
|
|
541
|
+
writeln(ENTRY_INDENT + parole_progress_sentence(result))
|
|
542
|
+
end
|
|
543
|
+
hint("A paroled test runs for real and is watched: one failure sends it straight " \
|
|
544
|
+
"back to jail. constable watchlist shows everything under supervision.")
|
|
545
|
+
end
|
|
546
|
+
|
|
547
|
+
def parole_progress_sentence(result)
|
|
548
|
+
day = result.parole_day.to_i
|
|
549
|
+
period = @config.parole_period
|
|
550
|
+
remaining = [period - day, 0].max
|
|
551
|
+
return "Day #{day} of #{period} — releases after this run." if remaining.zero?
|
|
552
|
+
|
|
553
|
+
"Day #{day} of #{period} — #{remaining} #{pluralize(remaining, "clean run")} to go."
|
|
554
|
+
end
|
|
555
|
+
|
|
372
556
|
def section_failures(results)
|
|
373
557
|
failures = results.select(&:failed?)
|
|
374
558
|
return if failures.empty?
|
|
@@ -432,11 +616,21 @@ module Constable
|
|
|
432
616
|
writeln(INDENT + paint("#{GLYPHS[:warning]} #{message}", COLORS[:warning]))
|
|
433
617
|
else
|
|
434
618
|
writeln(INDENT + paint("#{GLYPHS[:warning]} #{location}", COLORS[:warning]))
|
|
435
|
-
message.
|
|
619
|
+
warning_message_lines(message).each { |line| writeln(ENTRY_INDENT + line) }
|
|
436
620
|
end
|
|
437
621
|
end
|
|
438
622
|
end
|
|
439
623
|
|
|
624
|
+
# A warning carries the author's own words -- an unsafe block's reason, a cold case's
|
|
625
|
+
# count -- and those are easily ninety columns. Wrapped to the frame, but respecting
|
|
626
|
+
# any line breaks the message already chose.
|
|
627
|
+
def warning_message_lines(message)
|
|
628
|
+
message.to_s.lines.flat_map do |line|
|
|
629
|
+
text = line.chomp
|
|
630
|
+
text.empty? ? [""] : wrap(text, width: RULE_WIDTH - ENTRY_INDENT.length, indent: "")
|
|
631
|
+
end
|
|
632
|
+
end
|
|
633
|
+
|
|
440
634
|
def section_slowest(results)
|
|
441
635
|
# A jailed test never ran its body, so it has no honest duration. A parole
|
|
442
636
|
# violation did run -- and failing slowly is still worth seeing.
|
|
@@ -551,7 +745,28 @@ module Constable
|
|
|
551
745
|
end
|
|
552
746
|
|
|
553
747
|
def pluralize(count, word)
|
|
554
|
-
count.to_i == 1
|
|
748
|
+
return word if count.to_i == 1
|
|
749
|
+
# "retry" -> "retries". Only the consonant-y rule earns a special case; every other
|
|
750
|
+
# word this reporter pluralizes takes a plain "s".
|
|
751
|
+
return "#{word[0..-2]}ies" if word.end_with?("y") && !"aeiou".include?(word[-2].to_s)
|
|
752
|
+
|
|
753
|
+
"#{word}s"
|
|
754
|
+
end
|
|
755
|
+
|
|
756
|
+
# Hints are prose, and prose that runs past the frame reads as a mistake. Wrapped to
|
|
757
|
+
# the same 60 columns the rules use, with continuation lines aligned under the arrow.
|
|
758
|
+
def wrap(text, width:, indent:)
|
|
759
|
+
words = text.split
|
|
760
|
+
lines = [+""]
|
|
761
|
+
words.each do |word|
|
|
762
|
+
candidate = lines.last.empty? ? word : "#{lines.last} #{word}"
|
|
763
|
+
if candidate.length <= width || lines.last.empty?
|
|
764
|
+
lines[-1] = candidate
|
|
765
|
+
else
|
|
766
|
+
lines << +word
|
|
767
|
+
end
|
|
768
|
+
end
|
|
769
|
+
lines.each_with_index.map { |line, i| i.zero? ? line : indent + line }
|
|
555
770
|
end
|
|
556
771
|
|
|
557
772
|
def format_duration(seconds)
|
|
@@ -571,6 +786,17 @@ module Constable
|
|
|
571
786
|
|
|
572
787
|
# Colour is a nicety; correctness is not. NO_COLOR, a pipe, a dumb terminal or an
|
|
573
788
|
# explicit --no-color all fall back to plain text with identical layout.
|
|
789
|
+
# An explicit argument (the --expanded / --concise flags) beats the config file, which
|
|
790
|
+
# beats the default. An unrecognized value falls back rather than raising: a typo in
|
|
791
|
+
# config.yml should not stop a suite from running.
|
|
792
|
+
def resolve_mode(mode)
|
|
793
|
+
configured = @config.respond_to?(:output_mode) ? @config.output_mode : :concise
|
|
794
|
+
return configured if mode.nil?
|
|
795
|
+
|
|
796
|
+
mode = mode.to_s.strip.downcase.to_sym
|
|
797
|
+
Config::OUTPUT_MODES.include?(mode) ? mode : configured
|
|
798
|
+
end
|
|
799
|
+
|
|
574
800
|
def resolve_color(color)
|
|
575
801
|
return !!color unless color.nil?
|
|
576
802
|
return false if ENV["NO_COLOR"] && !ENV["NO_COLOR"].empty?
|
data/lib/constable/runner.rb
CHANGED
|
@@ -11,6 +11,14 @@ module Constable
|
|
|
11
11
|
# chose, since reordering someone's untouched legacy file is exactly the kind of surprise
|
|
12
12
|
# the cold-case story exists to avoid.
|
|
13
13
|
class Runner
|
|
14
|
+
# Thresholds for "this run is broken, not these tests" -- see #systemic_failure.
|
|
15
|
+
SYSTEMIC_MINIMUM = 5 # below this it is cheaper to believe the tests
|
|
16
|
+
SYSTEMIC_SHARE = 0.25 # of the whole run
|
|
17
|
+
SYSTEMIC_AGREEMENT = 0.8 # of the failures, failing identically
|
|
18
|
+
|
|
19
|
+
# `failed` rather than `count` or `tally`, both of which override an Enumerable method.
|
|
20
|
+
Systemic = Struct.new(:exception_class, :failed, :total, keyword_init: true)
|
|
21
|
+
|
|
14
22
|
# One unit of work. Native items are a single investigation; cold items are a whole
|
|
15
23
|
# file, because their engine owns the granularity inside it.
|
|
16
24
|
class Item
|
|
@@ -65,7 +73,11 @@ module Constable
|
|
|
65
73
|
Constable::Coverage.start!(config: @config, force: true) if coverage?
|
|
66
74
|
|
|
67
75
|
load_suite!
|
|
76
|
+
# Before anything is keyed on an identity -- selection, the docket, flake history --
|
|
77
|
+
# settle any two tests that happen to share a body.
|
|
78
|
+
Constable.registry.disambiguate_identities!
|
|
68
79
|
items = build_items
|
|
80
|
+
refuse_empty_selection!(items)
|
|
69
81
|
ordered = order(items)
|
|
70
82
|
|
|
71
83
|
run_id = @storage.start_run(seed: @seed, mode: mode_label, full: @selection.full?)
|
|
@@ -91,6 +103,10 @@ module Constable
|
|
|
91
103
|
@results = adjudicate(raw)
|
|
92
104
|
duration = monotonic - started
|
|
93
105
|
|
|
106
|
+
# Cold-case engines hold a live session -- for RSpec that is a configuration
|
|
107
|
+
# carrying `after(:suite)` hooks that have not fired yet. Tear it down before our
|
|
108
|
+
# own after_suite so the engine's cleanup runs inside the suite, not after it.
|
|
109
|
+
ColdCase.reset_engines!
|
|
94
110
|
Constable.configuration.run_after_suite!
|
|
95
111
|
@coverage_report = build_coverage_report if coverage?
|
|
96
112
|
|
|
@@ -172,6 +188,17 @@ module Constable
|
|
|
172
188
|
end
|
|
173
189
|
end
|
|
174
190
|
|
|
191
|
+
# A run that was *asked* for something specific and found nothing is a usage error, not
|
|
192
|
+
# a pass. `constable test test/cases/typo_case.rb` used to print "0 passed, 0 failed"
|
|
193
|
+
# and exit 0, so a mistyped path in a CI script produced a green build that ran no
|
|
194
|
+
# tests at all. A full run with an empty suite is a different thing and stays quiet.
|
|
195
|
+
def refuse_empty_selection!(items)
|
|
196
|
+
return unless items.empty?
|
|
197
|
+
return unless @selection.explicit?
|
|
198
|
+
|
|
199
|
+
raise Constable::Error, @selection.empty_selection_message
|
|
200
|
+
end
|
|
201
|
+
|
|
175
202
|
def build_items
|
|
176
203
|
native = native_items
|
|
177
204
|
cold = @selection.cold_targets_selected.map { |t| Item.new(path: t.path, kind: :cold) }
|
|
@@ -188,14 +215,29 @@ module Constable
|
|
|
188
215
|
|
|
189
216
|
# PATH:LINE means "the investigation at that line" -- but developers point at any line
|
|
190
217
|
# inside the block, so pick the investigation whose declaration is nearest above it.
|
|
218
|
+
#
|
|
219
|
+
# Bounded by the end of the file. Unbounded, `:999` on a twenty-line file quietly ran
|
|
220
|
+
# the last investigation in it: not the test the user asked for, not an error, and
|
|
221
|
+
# green either way. A line past the end is a typo, and no answer beats a wrong one.
|
|
191
222
|
def narrow_to_line(investigations, line)
|
|
192
223
|
exact = investigations.select { |inv| inv.line == line }
|
|
193
224
|
return exact if exact.any?
|
|
225
|
+
return [] unless line_within_file?(investigations.first, line)
|
|
194
226
|
|
|
195
227
|
nearest = investigations.select { |inv| inv.line <= line }.max_by(&:line)
|
|
196
228
|
nearest ? [nearest] : []
|
|
197
229
|
end
|
|
198
230
|
|
|
231
|
+
def line_within_file?(investigation, line)
|
|
232
|
+
path = investigation&.file
|
|
233
|
+
return false if path.nil?
|
|
234
|
+
|
|
235
|
+
path = File.join(@config.root, path) unless File.exist?(path)
|
|
236
|
+
return false unless File.exist?(path)
|
|
237
|
+
|
|
238
|
+
line <= File.foreach(path).count
|
|
239
|
+
end
|
|
240
|
+
|
|
199
241
|
def order(items)
|
|
200
242
|
native, cold = items.partition(&:native?)
|
|
201
243
|
[*native.shuffle(random: Random.new(@seed)), *cold]
|
|
@@ -205,13 +247,32 @@ module Constable
|
|
|
205
247
|
return [] if items.empty?
|
|
206
248
|
|
|
207
249
|
count = worker_count(items)
|
|
208
|
-
if count > 1 && forkable?
|
|
250
|
+
if count > 1 && forkable? && parallel_safe?
|
|
209
251
|
run_parallel(items, count)
|
|
210
252
|
else
|
|
211
253
|
run_serial(items)
|
|
212
254
|
end
|
|
213
255
|
end
|
|
214
256
|
|
|
257
|
+
# Forking is only safe once each worker has a database of its own. Without that,
|
|
258
|
+
# every worker opens the same one: on SQLite the run dissolves into "database is
|
|
259
|
+
# locked", and on a client/server database the tests quietly see each other's rows,
|
|
260
|
+
# which is worse. An app with no ActiveRecord has nothing to shard and is always safe.
|
|
261
|
+
#
|
|
262
|
+
# When we cannot shard, we run serially and say why. Slow is a trade-off; wrong is not.
|
|
263
|
+
def parallel_safe?
|
|
264
|
+
return true unless WorkerDatabases.active_record?
|
|
265
|
+
return true if WorkerDatabases.shardable?
|
|
266
|
+
|
|
267
|
+
Constable.warn!(
|
|
268
|
+
"parallel workers need one database per worker, and this app's ActiveRecord " \
|
|
269
|
+
"cannot provide them (active_record/test_databases did not load). Running " \
|
|
270
|
+
"serially instead -- pass --workers N once that is available.",
|
|
271
|
+
kind: :parallel
|
|
272
|
+
)
|
|
273
|
+
false
|
|
274
|
+
end
|
|
275
|
+
|
|
215
276
|
def worker_count(items)
|
|
216
277
|
requested = @workers || @config.parallel_workers
|
|
217
278
|
requested.to_i.clamp(1, items.size)
|
|
@@ -240,7 +301,11 @@ module Constable
|
|
|
240
301
|
# driver rightly complains about it.
|
|
241
302
|
@storage.close
|
|
242
303
|
|
|
243
|
-
|
|
304
|
+
# Same reasoning for the app's own connections: a child that inherits a live
|
|
305
|
+
# handle can corrupt it. Rails does exactly this before its own fork.
|
|
306
|
+
WorkerDatabases.before_fork!
|
|
307
|
+
|
|
308
|
+
buckets.each_with_index do |bucket, worker_index|
|
|
244
309
|
reader, writer = IO.pipe
|
|
245
310
|
# Marshal payloads are binary. Left in text mode, the first byte that isn't valid
|
|
246
311
|
# UTF-8 takes the worker down with an encoding error.
|
|
@@ -248,10 +313,21 @@ module Constable
|
|
|
248
313
|
writer.binmode
|
|
249
314
|
pid = fork do
|
|
250
315
|
reader.close
|
|
316
|
+
|
|
317
|
+
# Before a single test runs: build this worker's own database and point the
|
|
318
|
+
# process at it. Raises rather than falling back to the shared one, because a
|
|
319
|
+
# silent fallback is the bug we are here to prevent.
|
|
320
|
+
WorkerDatabases.after_fork!(worker_index)
|
|
321
|
+
|
|
251
322
|
bucket.each do |item|
|
|
252
323
|
run_item(item).each { |result| write_message(writer, :result, result.to_h) }
|
|
253
324
|
end
|
|
254
325
|
|
|
326
|
+
# A worker owns its own cold-case session, and it dies here. Fire the engine's
|
|
327
|
+
# after(:suite) hooks in the process that ran the before(:suite) half, before
|
|
328
|
+
# coverage is read -- the parent has no hooks to run on its behalf.
|
|
329
|
+
ColdCase.reset_engines!
|
|
330
|
+
|
|
255
331
|
# Ruby's Coverage counts lines in the process that executed them, so a worker's
|
|
256
332
|
# hits would die with it. They ride home on the same pipe as the results.
|
|
257
333
|
write_message(writer, :coverage, Constable::Coverage.peek_raw) if coverage?
|
|
@@ -513,6 +589,9 @@ module Constable
|
|
|
513
589
|
# Turns raw pass/fail into the verdict the build acts on: warrants decide whether a
|
|
514
590
|
# failure is even real, then jail decides whether it blocks.
|
|
515
591
|
def adjudicate(raw)
|
|
592
|
+
@systemic = systemic_failure(raw)
|
|
593
|
+
announce_systemic_failure(@systemic) if @systemic
|
|
594
|
+
|
|
516
595
|
raw.map do |result|
|
|
517
596
|
decided = warrants.adjudicate(
|
|
518
597
|
result,
|
|
@@ -520,10 +599,54 @@ module Constable
|
|
|
520
599
|
subject: investigation_for(result.identity)
|
|
521
600
|
) { |subject, _attempt| rerun_in_isolation(subject) }
|
|
522
601
|
|
|
523
|
-
|
|
602
|
+
next decided if jail_run?
|
|
603
|
+
|
|
604
|
+
jail.adjudicate(decided, jail_mode: jail_mode?, systemic: systemic?(decided))
|
|
524
605
|
end
|
|
525
606
|
end
|
|
526
607
|
|
|
608
|
+
# A run is "systemically broken" when a large share of it failed the same way: the
|
|
609
|
+
# database was down, a worker could not start, a shared fixture never loaded. Thirty
|
|
610
|
+
# tests did not each independently go bad in the same second.
|
|
611
|
+
#
|
|
612
|
+
# This matters because flake history reads "passed last run, failed this run" as
|
|
613
|
+
# evidence about a *test*, and jails it. One bad afternoon on CI could therefore
|
|
614
|
+
# quarantine a third of a healthy suite, and the docket -- which is supposed to be a
|
|
615
|
+
# record of tests worth distrusting -- fills up with tests that were never at fault.
|
|
616
|
+
def systemic_failure(results)
|
|
617
|
+
failures = results.select(&:failed?)
|
|
618
|
+
return nil if failures.size < SYSTEMIC_MINIMUM
|
|
619
|
+
return nil if failures.size < results.size * SYSTEMIC_SHARE
|
|
620
|
+
|
|
621
|
+
grouped = failures.group_by { |result| result.failure&.exception_class.to_s }
|
|
622
|
+
grouped.delete("")
|
|
623
|
+
return nil if grouped.empty?
|
|
624
|
+
|
|
625
|
+
exception_class, sharing = grouped.max_by { |_klass, group| group.size }
|
|
626
|
+
return nil if sharing.size < failures.size * SYSTEMIC_AGREEMENT
|
|
627
|
+
|
|
628
|
+
Systemic.new(exception_class: exception_class, failed: sharing.size, total: results.size)
|
|
629
|
+
end
|
|
630
|
+
|
|
631
|
+
# Only the failures that look like the outage are exempt. A genuine failure that
|
|
632
|
+
# happened to land in the same run is still a genuine failure.
|
|
633
|
+
def systemic?(result)
|
|
634
|
+
return false unless @systemic
|
|
635
|
+
return false unless result.failed?
|
|
636
|
+
|
|
637
|
+
result.failure&.exception_class.to_s == @systemic.exception_class
|
|
638
|
+
end
|
|
639
|
+
|
|
640
|
+
def announce_systemic_failure(systemic)
|
|
641
|
+
Constable.warn!(
|
|
642
|
+
"#{systemic.failed} of #{systemic.total} tests failed with the same error " \
|
|
643
|
+
"(#{systemic.exception_class}). That reads as one broken run rather than " \
|
|
644
|
+
"#{systemic.failed} newly flaky tests, so flake history and the jail docket were " \
|
|
645
|
+
"left alone. Fix the cause and run again.",
|
|
646
|
+
kind: :systemic
|
|
647
|
+
)
|
|
648
|
+
end
|
|
649
|
+
|
|
527
650
|
def investigation_for(identity)
|
|
528
651
|
@investigation_index ||= Constable.registry.investigations.to_h { |inv| [inv.identity, inv] }
|
|
529
652
|
@investigation_index[identity]
|
|
@@ -564,6 +687,11 @@ module Constable
|
|
|
564
687
|
|
|
565
688
|
def persist(run_id, results, coverage_report)
|
|
566
689
|
results.each do |result|
|
|
690
|
+
# The blotter is the evidence file. A result produced by an outage is not
|
|
691
|
+
# evidence about the test, so it is not filed -- otherwise the next run reads
|
|
692
|
+
# "failed, then passed" and draws a conclusion from a power cut.
|
|
693
|
+
next if systemic?(result)
|
|
694
|
+
|
|
567
695
|
@storage.record_result(run_id, result)
|
|
568
696
|
@storage.record_duration(result.identity, result.duration)
|
|
569
697
|
end
|
data/lib/constable/selection.rb
CHANGED
|
@@ -33,10 +33,13 @@ module Constable
|
|
|
33
33
|
@root = root.to_s
|
|
34
34
|
@full = full
|
|
35
35
|
@unsafe_only = unsafe_only
|
|
36
|
-
|
|
36
|
+
# Downcased: `--tier UNIT` used to match nothing at all and report a clean run.
|
|
37
|
+
@tier = tier.to_s.strip.downcase.to_sym unless tier.to_s.strip.empty?
|
|
37
38
|
@reason = nil
|
|
38
39
|
end
|
|
39
40
|
|
|
41
|
+
TIERS = %w[unit integration system].freeze
|
|
42
|
+
|
|
40
43
|
def full? = @full
|
|
41
44
|
def unsafe_only? = @unsafe_only
|
|
42
45
|
|
|
@@ -60,6 +63,25 @@ module Constable
|
|
|
60
63
|
end
|
|
61
64
|
end
|
|
62
65
|
|
|
66
|
+
# Did the user ask for something in particular? If so, finding nothing is an error
|
|
67
|
+
# rather than a clean run -- see Runner#refuse_empty_selection!.
|
|
68
|
+
def explicit? = @args.any? { |arg| !arg.to_s.strip.empty? } || !@tier.nil?
|
|
69
|
+
|
|
70
|
+
# Says which part of the request came up empty, because "0 tests" on its own does not
|
|
71
|
+
# tell you whether the path was wrong, the tier was, or both.
|
|
72
|
+
def empty_selection_message
|
|
73
|
+
if @tier && !TIERS.include?(@tier.to_s)
|
|
74
|
+
return "unknown tier #{@tier.inspect} -- expected one of #{TIERS.join(", ")}."
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
described = @args.reject { |arg| arg.to_s.strip.empty? }
|
|
78
|
+
subject = described.empty? ? "this run" : described.join(", ")
|
|
79
|
+
suffix = @tier ? " in the #{@tier} tier" : ""
|
|
80
|
+
|
|
81
|
+
"no tests matched #{subject}#{suffix}. Check the path, the line number, and " \
|
|
82
|
+
"whether the file is a case or a cold case."
|
|
83
|
+
end
|
|
84
|
+
|
|
63
85
|
def native_targets = targets.select(&:native?)
|
|
64
86
|
def cold_targets_selected = targets.select(&:cold?)
|
|
65
87
|
def empty? = targets.empty?
|
|
@@ -36,16 +36,36 @@ module Constable
|
|
|
36
36
|
|
|
37
37
|
def connect!
|
|
38
38
|
require_driver!
|
|
39
|
+
open_database!
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
# The blotter is the one file Constable owns outright, and it is disposable: it
|
|
43
|
+
# holds flake history, the docket and warrants, never a test. So when it cannot be
|
|
44
|
+
# opened, say that deleting it is a real option -- the raw
|
|
45
|
+
# `SQLite3::NotADatabaseException: file is not a database: PRAGMA journal_mode = WAL`
|
|
46
|
+
# tells a reader nothing about what to do next, and a blotter that got committed to
|
|
47
|
+
# git and then merged is exactly how it ends up unreadable.
|
|
48
|
+
def open_database!
|
|
39
49
|
FileUtils.mkdir_p(File.dirname(path))
|
|
40
50
|
@connection = SQLite3::Database.new(path)
|
|
41
51
|
@connection.results_as_hash = true
|
|
42
52
|
@connection.busy_timeout = BUSY_TIMEOUT_MS
|
|
43
|
-
# Readers never block the writer and the writer never blocks readers.
|
|
53
|
+
# Readers never block the writer and the writer never blocks readers. This is also
|
|
54
|
+
# the first statement to touch the file, so a corrupt blotter surfaces here.
|
|
44
55
|
@connection.execute("PRAGMA journal_mode = WAL")
|
|
45
56
|
# WAL + NORMAL is durable across process crashes, which is the only failure that
|
|
46
57
|
# matters here; a machine losing power mid-run costs us one run's bookkeeping.
|
|
47
58
|
@connection.execute("PRAGMA synchronous = NORMAL")
|
|
48
59
|
@connection
|
|
60
|
+
rescue SystemCallError => e
|
|
61
|
+
raise Constable::Error,
|
|
62
|
+
"Constable cannot open its blotter at #{path} (#{e.class}: #{e.message}). " \
|
|
63
|
+
"Point `storage.path` in .constable/config.yml somewhere writable."
|
|
64
|
+
rescue StandardError => e
|
|
65
|
+
raise Constable::Error,
|
|
66
|
+
"Constable's blotter at #{path} is not a readable database " \
|
|
67
|
+
"(#{e.class}). It holds flake history, the jail docket and warrants -- " \
|
|
68
|
+
"never your tests -- so deleting it is safe and starts that record fresh."
|
|
49
69
|
end
|
|
50
70
|
|
|
51
71
|
def require_driver!
|
data/lib/constable/version.rb
CHANGED
data/lib/constable/warrants.rb
CHANGED
|
@@ -222,17 +222,35 @@ module Constable
|
|
|
222
222
|
|
|
223
223
|
# The CLI speaks in file:line, the blotter is keyed by content hash. Warrant rows
|
|
224
224
|
# carry both; loaded investigations are the fallback.
|
|
225
|
-
|
|
225
|
+
# Every docket row a target could mean.
|
|
226
|
+
#
|
|
227
|
+
# The interesting case is a bare path. "test/cases/users_case.rb" with three tests
|
|
228
|
+
# on the docket is a question, not an instruction: picking one silently acts on a
|
|
229
|
+
# test the user never named -- and not even the first one, since the order is
|
|
230
|
+
# whatever storage returns. Callers ask for the candidates and refuse to guess.
|
|
231
|
+
def candidates(target)
|
|
226
232
|
text = target.to_s.strip
|
|
227
|
-
return
|
|
228
|
-
|
|
233
|
+
return [] if text.empty?
|
|
234
|
+
|
|
235
|
+
if text.match?(/\A[0-9a-f]{8,64}\z/) && (row = entry(text))
|
|
236
|
+
return [row]
|
|
237
|
+
end
|
|
229
238
|
|
|
230
239
|
file, line = Jail.split_target(text)
|
|
231
|
-
return
|
|
240
|
+
return [] if file.empty?
|
|
232
241
|
|
|
233
242
|
matches = entries.select { |e| Jail.same_path?(e.file, file) }
|
|
234
243
|
matches = matches.select { |e| e.line == line } if line
|
|
235
|
-
|
|
244
|
+
matches
|
|
245
|
+
end
|
|
246
|
+
|
|
247
|
+
def resolve(target)
|
|
248
|
+
matches = candidates(target)
|
|
249
|
+
return matches.first.identity if matches.size == 1
|
|
250
|
+
return nil unless matches.empty?
|
|
251
|
+
|
|
252
|
+
file, line = Jail.split_target(target.to_s.strip)
|
|
253
|
+
return nil if file.empty?
|
|
236
254
|
|
|
237
255
|
Jail.registry_identity(file, line)
|
|
238
256
|
end
|