constable-rails 0.1.0 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -46,6 +46,11 @@ module Constable
46
46
 
47
47
  DEFAULT_SLOWEST = 5
48
48
 
49
+ # Column the expanded stream right-aligns durations into. Descriptions are never
50
+ # truncated to fit it -- a clipped test name is not something you can grep for --
51
+ # so a long one simply pushes its own stamp out past the column.
52
+ EXPANDED_STAMP_COLUMN = 56
53
+
49
54
  # Result::GLYPHS covers statuses. These four are summary vocabulary, not statuses:
50
55
  # supervision and coverage are properties of a test, not outcomes of one.
51
56
  GLYPHS = Result::GLYPHS.merge(
@@ -95,13 +100,14 @@ module Constable
95
100
  0x1F900..0x1F9FF, 0x20000..0x3FFFD
96
101
  ].freeze
97
102
 
98
- attr_reader :io, :config, :seed, :total
103
+ attr_reader :io, :config, :seed, :total, :mode
99
104
 
100
- def initialize(io: $stdout, config: nil, color: nil, slowest: DEFAULT_SLOWEST)
105
+ def initialize(io: $stdout, config: nil, color: nil, slowest: DEFAULT_SLOWEST, mode: nil)
101
106
  @io = io
102
107
  @config = config || Constable.config
103
108
  @color = resolve_color(color)
104
109
  @slowest = slowest.to_i
110
+ @mode = resolve_mode(mode)
105
111
  @io.set_encoding(Encoding::UTF_8) if @io.respond_to?(:set_encoding)
106
112
 
107
113
  reset_stream!
@@ -141,6 +147,14 @@ module Constable
141
147
  def flush!
142
148
  return self unless streaming?
143
149
 
150
+ # The expanded stream never buffers -- every line was written as it happened, so
151
+ # there is nothing left to emit, only a blank line before the summary.
152
+ if expanded?
153
+ reset_stream!
154
+ writeln
155
+ return self
156
+ end
157
+
144
158
  close_stream_line
145
159
  pending_case_names.each { |name| open_stream_line(name) && close_stream_line }
146
160
  writeln
@@ -169,6 +183,9 @@ module Constable
169
183
 
170
184
  section_parole_violations(results)
171
185
  section_failures(results)
186
+ section_warrants(results)
187
+ section_jailed(results)
188
+ section_parole(results)
172
189
  section_warnings(warnings)
173
190
  section_slowest(results)
174
191
  rename_suggestions(suggestions)
@@ -193,6 +210,7 @@ module Constable
193
210
  def failed? = !success?
194
211
  def color? = @color
195
212
  def finished? = @finished
213
+ def expanded? = @mode == :expanded
196
214
 
197
215
  private
198
216
 
@@ -211,6 +229,8 @@ module Constable
211
229
  end
212
230
 
213
231
  def stream(result)
232
+ return stream_expanded(result) if expanded?
233
+
214
234
  name = result.case_name.to_s
215
235
  name = "(anonymous)" if name.empty?
216
236
  glyph = paint(result.glyph, COLORS[result.status])
@@ -263,6 +283,77 @@ module Constable
263
283
  # Glyphs may carry escape codes; count only the visible ones.
264
284
  def count_glyphs(string) = strip_ansi(string).length
265
285
 
286
+ # --- expanded stream -----------------------------------------------------------
287
+ #
288
+ # One line per test instead of one glyph. The trade is deliberate: concise keeps a
289
+ # thousand-test suite on one screen, expanded tells you which test is hanging while
290
+ # it hangs, without waiting for the summary.
291
+ #
292
+ # Workers interleave, so a case can come back after another has spoken. It gets a
293
+ # second header rather than having its later tests silently appended under the
294
+ # wrong one -- the same honesty rule the concise stream follows.
295
+ def stream_expanded(result)
296
+ name = result.case_name.to_s
297
+ name = "(anonymous)" if name.empty?
298
+
299
+ if @stream_case != name
300
+ writeln if @stream_case
301
+ writeln(INDENT + paint(name, :bold))
302
+ @stream_case = name
303
+ @stream_open = true
304
+ end
305
+
306
+ writeln(expanded_line(result))
307
+ end
308
+
309
+ def expanded_line(result)
310
+ glyph = paint(result.glyph, COLORS[result.status])
311
+ line = "#{ENTRY_INDENT}#{glyph} #{expanded_description(result)}"
312
+
313
+ stamp = expanded_duration(result)
314
+ return line if stamp.nil?
315
+
316
+ # Pad to a column so the durations line up, but never truncate a description --
317
+ # a clipped test name is not something you can grep for.
318
+ visible = strip_ansi(line).length
319
+ gap = [EXPANDED_STAMP_COLUMN - visible, 1].max
320
+ "#{line}#{" " * gap}#{paint(stamp, :dim)}"
321
+ end
322
+
323
+ def expanded_description(result)
324
+ description = result.description.to_s
325
+ description = "(no description)" if description.empty?
326
+ # A jailed test never ran its body, so say why rather than implying it passed.
327
+ return "#{description} #{paint("— #{result.jail_reason}", :dim)}" if jail_reason_worth_showing?(result)
328
+
329
+ description
330
+ end
331
+
332
+ def jail_reason_worth_showing?(result)
333
+ result.status == :jailed && !result.jail_reason.to_s.strip.empty?
334
+ end
335
+
336
+ # Only real, measured time. A jailed test never ran, and "0ms" would be a claim
337
+ # about a body that was skipped.
338
+ def expanded_duration(result)
339
+ return nil if result.status == :jailed
340
+ return nil unless result.duration.to_f.positive?
341
+
342
+ format_test_duration(result.duration)
343
+ end
344
+
345
+ # The summary's durations are run-scale, where "12.4s" is the useful unit. One test
346
+ # is usually sub-second, and "0.0s" against every line says nothing at all -- so the
347
+ # expanded stream counts milliseconds until a test is slow enough for seconds to mean
348
+ # something.
349
+ def format_test_duration(seconds)
350
+ seconds = seconds.to_f
351
+ return format_duration(seconds) if seconds >= 1
352
+
353
+ milliseconds = (seconds * 1000).round
354
+ milliseconds.zero? ? "<1ms" : "#{milliseconds}ms"
355
+ end
356
+
266
357
  # --- header and headline -------------------------------------------------------
267
358
 
268
359
  def header_line(results, duration)
@@ -359,8 +450,12 @@ module Constable
359
450
  each_entry(violations) do |result|
360
451
  writeln(INDENT + paint("#{GLYPHS[:parole_violation]} #{result.case_name}", COLORS[:parole_violation]))
361
452
  writeln(ENTRY_INDENT + paint(%("#{result.description}"), :dim))
453
+ writeln(ENTRY_INDENT + paint(result.location, :dim))
362
454
  writeln(ENTRY_INDENT + parole_violation_sentence(result))
363
455
  end
456
+ hint("Somebody trusted this test again and it let them down, so it is back on the " \
457
+ "docket. Fix it before the next constable jail parole — a second violation " \
458
+ "is the signal that the test, not the flake, is the problem.")
364
459
  end
365
460
 
366
461
  def parole_violation_sentence(result)
@@ -369,6 +464,95 @@ module Constable
369
464
  sentence
370
465
  end
371
466
 
467
+ # A section says what happened; a hint says what to do about it. One dim sentence,
468
+ # and only where there is a real next step -- a tip printed on every run stops being
469
+ # read on the second one.
470
+ def hint(text)
471
+ writeln
472
+ lines = wrap(text, width: RULE_WIDTH - INDENT.length - 2, indent: " ")
473
+ writeln(INDENT + paint("→ #{lines.first}", :dim))
474
+ lines.drop(1).each { |line| writeln(INDENT + paint(line, :dim)) }
475
+ end
476
+
477
+ # Every test currently under a warrant: it failed, then passed when rerun in
478
+ # isolation, so it is flaky rather than broken. Loud, but not build-blocking.
479
+ def section_warrants(results)
480
+ warranted = results.select(&:warranted?)
481
+ return if warranted.empty?
482
+
483
+ section("WARRANTS")
484
+ each_entry(warranted) do |result|
485
+ writeln(INDENT + paint("#{GLYPHS[:warrant]} #{result.case_name}", COLORS[:warranted]))
486
+ writeln(ENTRY_INDENT + paint(%("#{result.description}"), :dim))
487
+ writeln(ENTRY_INDENT + paint(result.location, :dim))
488
+ writeln(ENTRY_INDENT + warrant_sentence(result))
489
+ end
490
+ hint("A warrant is \"not reproducible\", not \"not a problem\" — it stops blocking the " \
491
+ "build and stays visible until someone deals with it. " \
492
+ "Fixed the flake? constable warrants release PATH:LINE")
493
+ end
494
+
495
+ def warrant_sentence(result)
496
+ statuses = Array(result.retries).map(&:to_sym)
497
+ return "Failed once, then passed on retry." if statuses.empty?
498
+
499
+ passed = statuses.count(:passed)
500
+ "Failed, then passed #{passed} of #{statuses.size} #{pluralize(statuses.size, "retry")} " \
501
+ "run in isolation."
502
+ end
503
+
504
+ # The docket. These never ran their bodies, so they are neither passing nor failing --
505
+ # which is exactly why they get their own category rather than being folded into
506
+ # either one.
507
+ def section_jailed(results)
508
+ jailed = results.select { |result| result.status == :jailed }
509
+ return if jailed.empty?
510
+
511
+ section("JAILED")
512
+ each_entry(jailed) do |result|
513
+ writeln(INDENT + paint("#{GLYPHS[:jailed]} #{result.case_name}", COLORS[:jailed]))
514
+ writeln(ENTRY_INDENT + paint(%("#{result.description}"), :dim))
515
+ writeln(ENTRY_INDENT + paint(result.location, :dim))
516
+ writeln(ENTRY_INDENT + jailed_sentence(result))
517
+ end
518
+ hint("Jailed means skipped and tracked, not passing. Think one is fixed? " \
519
+ "constable jail parole PATH:LINE runs it for real again — " \
520
+ "#{@config.parole_period} clean runs and it releases itself.")
521
+ end
522
+
523
+ def jailed_sentence(result)
524
+ reason = result.jail_reason.to_s.strip
525
+ sentence = reason.empty? ? "Body skipped; setup still ran." : "#{reason.capitalize}."
526
+ sentence += " Its #{ordinalize(result.times_jailed)} time in jail." if result.times_jailed.to_i > 1
527
+ sentence
528
+ end
529
+
530
+ # Out on parole and behaving. Worth naming every run, because the count only means
531
+ # something if you can see it moving.
532
+ def section_parole(results)
533
+ paroled = results.select { |result| result.parole_day && !result.parole_violation? }
534
+ return if paroled.empty?
535
+
536
+ section("ON PAROLE")
537
+ each_entry(paroled) do |result|
538
+ writeln(INDENT + paint("#{GLYPHS[:parole]} #{result.case_name}", COLORS[:parole]))
539
+ writeln(ENTRY_INDENT + paint(%("#{result.description}"), :dim))
540
+ writeln(ENTRY_INDENT + paint(result.location, :dim))
541
+ writeln(ENTRY_INDENT + parole_progress_sentence(result))
542
+ end
543
+ hint("A paroled test runs for real and is watched: one failure sends it straight " \
544
+ "back to jail. constable watchlist shows everything under supervision.")
545
+ end
546
+
547
+ def parole_progress_sentence(result)
548
+ day = result.parole_day.to_i
549
+ period = @config.parole_period
550
+ remaining = [period - day, 0].max
551
+ return "Day #{day} of #{period} — releases after this run." if remaining.zero?
552
+
553
+ "Day #{day} of #{period} — #{remaining} #{pluralize(remaining, "clean run")} to go."
554
+ end
555
+
372
556
  def section_failures(results)
373
557
  failures = results.select(&:failed?)
374
558
  return if failures.empty?
@@ -432,11 +616,21 @@ module Constable
432
616
  writeln(INDENT + paint("#{GLYPHS[:warning]} #{message}", COLORS[:warning]))
433
617
  else
434
618
  writeln(INDENT + paint("#{GLYPHS[:warning]} #{location}", COLORS[:warning]))
435
- message.each_line { |line| writeln(ENTRY_INDENT + line.chomp) }
619
+ warning_message_lines(message).each { |line| writeln(ENTRY_INDENT + line) }
436
620
  end
437
621
  end
438
622
  end
439
623
 
624
+ # A warning carries the author's own words -- an unsafe block's reason, a cold case's
625
+ # count -- and those are easily ninety columns. Wrapped to the frame, but respecting
626
+ # any line breaks the message already chose.
627
+ def warning_message_lines(message)
628
+ message.to_s.lines.flat_map do |line|
629
+ text = line.chomp
630
+ text.empty? ? [""] : wrap(text, width: RULE_WIDTH - ENTRY_INDENT.length, indent: "")
631
+ end
632
+ end
633
+
440
634
  def section_slowest(results)
441
635
  # A jailed test never ran its body, so it has no honest duration. A parole
442
636
  # violation did run -- and failing slowly is still worth seeing.
@@ -551,7 +745,28 @@ module Constable
551
745
  end
552
746
 
553
747
  def pluralize(count, word)
554
- count.to_i == 1 ? word : "#{word}s"
748
+ return word if count.to_i == 1
749
+ # "retry" -> "retries". Only the consonant-y rule earns a special case; every other
750
+ # word this reporter pluralizes takes a plain "s".
751
+ return "#{word[0..-2]}ies" if word.end_with?("y") && !"aeiou".include?(word[-2].to_s)
752
+
753
+ "#{word}s"
754
+ end
755
+
756
+ # Hints are prose, and prose that runs past the frame reads as a mistake. Wrapped to
757
+ # the same 60 columns the rules use, with continuation lines aligned under the arrow.
758
+ def wrap(text, width:, indent:)
759
+ words = text.split
760
+ lines = [+""]
761
+ words.each do |word|
762
+ candidate = lines.last.empty? ? word : "#{lines.last} #{word}"
763
+ if candidate.length <= width || lines.last.empty?
764
+ lines[-1] = candidate
765
+ else
766
+ lines << +word
767
+ end
768
+ end
769
+ lines.each_with_index.map { |line, i| i.zero? ? line : indent + line }
555
770
  end
556
771
 
557
772
  def format_duration(seconds)
@@ -571,6 +786,17 @@ module Constable
571
786
 
572
787
  # Colour is a nicety; correctness is not. NO_COLOR, a pipe, a dumb terminal or an
573
788
  # explicit --no-color all fall back to plain text with identical layout.
789
+ # An explicit argument (the --expanded / --concise flags) beats the config file, which
790
+ # beats the default. An unrecognized value falls back rather than raising: a typo in
791
+ # config.yml should not stop a suite from running.
792
+ def resolve_mode(mode)
793
+ configured = @config.respond_to?(:output_mode) ? @config.output_mode : :concise
794
+ return configured if mode.nil?
795
+
796
+ mode = mode.to_s.strip.downcase.to_sym
797
+ Config::OUTPUT_MODES.include?(mode) ? mode : configured
798
+ end
799
+
574
800
  def resolve_color(color)
575
801
  return !!color unless color.nil?
576
802
  return false if ENV["NO_COLOR"] && !ENV["NO_COLOR"].empty?
@@ -11,6 +11,14 @@ module Constable
11
11
  # chose, since reordering someone's untouched legacy file is exactly the kind of surprise
12
12
  # the cold-case story exists to avoid.
13
13
  class Runner
14
+ # Thresholds for "this run is broken, not these tests" -- see #systemic_failure.
15
+ SYSTEMIC_MINIMUM = 5 # below this it is cheaper to believe the tests
16
+ SYSTEMIC_SHARE = 0.25 # of the whole run
17
+ SYSTEMIC_AGREEMENT = 0.8 # of the failures, failing identically
18
+
19
+ # `failed` rather than `count` or `tally`, both of which override an Enumerable method.
20
+ Systemic = Struct.new(:exception_class, :failed, :total, keyword_init: true)
21
+
14
22
  # One unit of work. Native items are a single investigation; cold items are a whole
15
23
  # file, because their engine owns the granularity inside it.
16
24
  class Item
@@ -65,7 +73,11 @@ module Constable
65
73
  Constable::Coverage.start!(config: @config, force: true) if coverage?
66
74
 
67
75
  load_suite!
76
+ # Before anything is keyed on an identity -- selection, the docket, flake history --
77
+ # settle any two tests that happen to share a body.
78
+ Constable.registry.disambiguate_identities!
68
79
  items = build_items
80
+ refuse_empty_selection!(items)
69
81
  ordered = order(items)
70
82
 
71
83
  run_id = @storage.start_run(seed: @seed, mode: mode_label, full: @selection.full?)
@@ -91,6 +103,10 @@ module Constable
91
103
  @results = adjudicate(raw)
92
104
  duration = monotonic - started
93
105
 
106
+ # Cold-case engines hold a live session -- for RSpec that is a configuration
107
+ # carrying `after(:suite)` hooks that have not fired yet. Tear it down before our
108
+ # own after_suite so the engine's cleanup runs inside the suite, not after it.
109
+ ColdCase.reset_engines!
94
110
  Constable.configuration.run_after_suite!
95
111
  @coverage_report = build_coverage_report if coverage?
96
112
 
@@ -172,6 +188,17 @@ module Constable
172
188
  end
173
189
  end
174
190
 
191
+ # A run that was *asked* for something specific and found nothing is a usage error, not
192
+ # a pass. `constable test test/cases/typo_case.rb` used to print "0 passed, 0 failed"
193
+ # and exit 0, so a mistyped path in a CI script produced a green build that ran no
194
+ # tests at all. A full run with an empty suite is a different thing and stays quiet.
195
+ def refuse_empty_selection!(items)
196
+ return unless items.empty?
197
+ return unless @selection.explicit?
198
+
199
+ raise Constable::Error, @selection.empty_selection_message
200
+ end
201
+
175
202
  def build_items
176
203
  native = native_items
177
204
  cold = @selection.cold_targets_selected.map { |t| Item.new(path: t.path, kind: :cold) }
@@ -188,14 +215,29 @@ module Constable
188
215
 
189
216
  # PATH:LINE means "the investigation at that line" -- but developers point at any line
190
217
  # inside the block, so pick the investigation whose declaration is nearest above it.
218
+ #
219
+ # Bounded by the end of the file. Unbounded, `:999` on a twenty-line file quietly ran
220
+ # the last investigation in it: not the test the user asked for, not an error, and
221
+ # green either way. A line past the end is a typo, and no answer beats a wrong one.
191
222
  def narrow_to_line(investigations, line)
192
223
  exact = investigations.select { |inv| inv.line == line }
193
224
  return exact if exact.any?
225
+ return [] unless line_within_file?(investigations.first, line)
194
226
 
195
227
  nearest = investigations.select { |inv| inv.line <= line }.max_by(&:line)
196
228
  nearest ? [nearest] : []
197
229
  end
198
230
 
231
+ def line_within_file?(investigation, line)
232
+ path = investigation&.file
233
+ return false if path.nil?
234
+
235
+ path = File.join(@config.root, path) unless File.exist?(path)
236
+ return false unless File.exist?(path)
237
+
238
+ line <= File.foreach(path).count
239
+ end
240
+
199
241
  def order(items)
200
242
  native, cold = items.partition(&:native?)
201
243
  [*native.shuffle(random: Random.new(@seed)), *cold]
@@ -205,13 +247,32 @@ module Constable
205
247
  return [] if items.empty?
206
248
 
207
249
  count = worker_count(items)
208
- if count > 1 && forkable?
250
+ if count > 1 && forkable? && parallel_safe?
209
251
  run_parallel(items, count)
210
252
  else
211
253
  run_serial(items)
212
254
  end
213
255
  end
214
256
 
257
+ # Forking is only safe once each worker has a database of its own. Without that,
258
+ # every worker opens the same one: on SQLite the run dissolves into "database is
259
+ # locked", and on a client/server database the tests quietly see each other's rows,
260
+ # which is worse. An app with no ActiveRecord has nothing to shard and is always safe.
261
+ #
262
+ # When we cannot shard, we run serially and say why. Slow is a trade-off; wrong is not.
263
+ def parallel_safe?
264
+ return true unless WorkerDatabases.active_record?
265
+ return true if WorkerDatabases.shardable?
266
+
267
+ Constable.warn!(
268
+ "parallel workers need one database per worker, and this app's ActiveRecord " \
269
+ "cannot provide them (active_record/test_databases did not load). Running " \
270
+ "serially instead -- pass --workers N once that is available.",
271
+ kind: :parallel
272
+ )
273
+ false
274
+ end
275
+
215
276
  def worker_count(items)
216
277
  requested = @workers || @config.parallel_workers
217
278
  requested.to_i.clamp(1, items.size)
@@ -240,7 +301,11 @@ module Constable
240
301
  # driver rightly complains about it.
241
302
  @storage.close
242
303
 
243
- buckets.each do |bucket|
304
+ # Same reasoning for the app's own connections: a child that inherits a live
305
+ # handle can corrupt it. Rails does exactly this before its own fork.
306
+ WorkerDatabases.before_fork!
307
+
308
+ buckets.each_with_index do |bucket, worker_index|
244
309
  reader, writer = IO.pipe
245
310
  # Marshal payloads are binary. Left in text mode, the first byte that isn't valid
246
311
  # UTF-8 takes the worker down with an encoding error.
@@ -248,10 +313,21 @@ module Constable
248
313
  writer.binmode
249
314
  pid = fork do
250
315
  reader.close
316
+
317
+ # Before a single test runs: build this worker's own database and point the
318
+ # process at it. Raises rather than falling back to the shared one, because a
319
+ # silent fallback is the bug we are here to prevent.
320
+ WorkerDatabases.after_fork!(worker_index)
321
+
251
322
  bucket.each do |item|
252
323
  run_item(item).each { |result| write_message(writer, :result, result.to_h) }
253
324
  end
254
325
 
326
+ # A worker owns its own cold-case session, and it dies here. Fire the engine's
327
+ # after(:suite) hooks in the process that ran the before(:suite) half, before
328
+ # coverage is read -- the parent has no hooks to run on its behalf.
329
+ ColdCase.reset_engines!
330
+
255
331
  # Ruby's Coverage counts lines in the process that executed them, so a worker's
256
332
  # hits would die with it. They ride home on the same pipe as the results.
257
333
  write_message(writer, :coverage, Constable::Coverage.peek_raw) if coverage?
@@ -513,6 +589,9 @@ module Constable
513
589
  # Turns raw pass/fail into the verdict the build acts on: warrants decide whether a
514
590
  # failure is even real, then jail decides whether it blocks.
515
591
  def adjudicate(raw)
592
+ @systemic = systemic_failure(raw)
593
+ announce_systemic_failure(@systemic) if @systemic
594
+
516
595
  raw.map do |result|
517
596
  decided = warrants.adjudicate(
518
597
  result,
@@ -520,10 +599,54 @@ module Constable
520
599
  subject: investigation_for(result.identity)
521
600
  ) { |subject, _attempt| rerun_in_isolation(subject) }
522
601
 
523
- jail_run? ? decided : jail.adjudicate(decided, jail_mode: jail_mode?)
602
+ next decided if jail_run?
603
+
604
+ jail.adjudicate(decided, jail_mode: jail_mode?, systemic: systemic?(decided))
524
605
  end
525
606
  end
526
607
 
608
+ # A run is "systemically broken" when a large share of it failed the same way: the
609
+ # database was down, a worker could not start, a shared fixture never loaded. Thirty
610
+ # tests did not each independently go bad in the same second.
611
+ #
612
+ # This matters because flake history reads "passed last run, failed this run" as
613
+ # evidence about a *test*, and jails it. One bad afternoon on CI could therefore
614
+ # quarantine a third of a healthy suite, and the docket -- which is supposed to be a
615
+ # record of tests worth distrusting -- fills up with tests that were never at fault.
616
+ def systemic_failure(results)
617
+ failures = results.select(&:failed?)
618
+ return nil if failures.size < SYSTEMIC_MINIMUM
619
+ return nil if failures.size < results.size * SYSTEMIC_SHARE
620
+
621
+ grouped = failures.group_by { |result| result.failure&.exception_class.to_s }
622
+ grouped.delete("")
623
+ return nil if grouped.empty?
624
+
625
+ exception_class, sharing = grouped.max_by { |_klass, group| group.size }
626
+ return nil if sharing.size < failures.size * SYSTEMIC_AGREEMENT
627
+
628
+ Systemic.new(exception_class: exception_class, failed: sharing.size, total: results.size)
629
+ end
630
+
631
+ # Only the failures that look like the outage are exempt. A genuine failure that
632
+ # happened to land in the same run is still a genuine failure.
633
+ def systemic?(result)
634
+ return false unless @systemic
635
+ return false unless result.failed?
636
+
637
+ result.failure&.exception_class.to_s == @systemic.exception_class
638
+ end
639
+
640
+ def announce_systemic_failure(systemic)
641
+ Constable.warn!(
642
+ "#{systemic.failed} of #{systemic.total} tests failed with the same error " \
643
+ "(#{systemic.exception_class}). That reads as one broken run rather than " \
644
+ "#{systemic.failed} newly flaky tests, so flake history and the jail docket were " \
645
+ "left alone. Fix the cause and run again.",
646
+ kind: :systemic
647
+ )
648
+ end
649
+
527
650
  def investigation_for(identity)
528
651
  @investigation_index ||= Constable.registry.investigations.to_h { |inv| [inv.identity, inv] }
529
652
  @investigation_index[identity]
@@ -564,6 +687,11 @@ module Constable
564
687
 
565
688
  def persist(run_id, results, coverage_report)
566
689
  results.each do |result|
690
+ # The blotter is the evidence file. A result produced by an outage is not
691
+ # evidence about the test, so it is not filed -- otherwise the next run reads
692
+ # "failed, then passed" and draws a conclusion from a power cut.
693
+ next if systemic?(result)
694
+
567
695
  @storage.record_result(run_id, result)
568
696
  @storage.record_duration(result.identity, result.duration)
569
697
  end
@@ -33,10 +33,13 @@ module Constable
33
33
  @root = root.to_s
34
34
  @full = full
35
35
  @unsafe_only = unsafe_only
36
- @tier = tier&.to_sym
36
+ # Downcased: `--tier UNIT` used to match nothing at all and report a clean run.
37
+ @tier = tier.to_s.strip.downcase.to_sym unless tier.to_s.strip.empty?
37
38
  @reason = nil
38
39
  end
39
40
 
41
+ TIERS = %w[unit integration system].freeze
42
+
40
43
  def full? = @full
41
44
  def unsafe_only? = @unsafe_only
42
45
 
@@ -60,6 +63,25 @@ module Constable
60
63
  end
61
64
  end
62
65
 
66
+ # Did the user ask for something in particular? If so, finding nothing is an error
67
+ # rather than a clean run -- see Runner#refuse_empty_selection!.
68
+ def explicit? = @args.any? { |arg| !arg.to_s.strip.empty? } || !@tier.nil?
69
+
70
+ # Says which part of the request came up empty, because "0 tests" on its own does not
71
+ # tell you whether the path was wrong, the tier was, or both.
72
+ def empty_selection_message
73
+ if @tier && !TIERS.include?(@tier.to_s)
74
+ return "unknown tier #{@tier.inspect} -- expected one of #{TIERS.join(", ")}."
75
+ end
76
+
77
+ described = @args.reject { |arg| arg.to_s.strip.empty? }
78
+ subject = described.empty? ? "this run" : described.join(", ")
79
+ suffix = @tier ? " in the #{@tier} tier" : ""
80
+
81
+ "no tests matched #{subject}#{suffix}. Check the path, the line number, and " \
82
+ "whether the file is a case or a cold case."
83
+ end
84
+
63
85
  def native_targets = targets.select(&:native?)
64
86
  def cold_targets_selected = targets.select(&:cold?)
65
87
  def empty? = targets.empty?
@@ -36,16 +36,36 @@ module Constable
36
36
 
37
37
  def connect!
38
38
  require_driver!
39
+ open_database!
40
+ end
41
+
42
+ # The blotter is the one file Constable owns outright, and it is disposable: it
43
+ # holds flake history, the docket and warrants, never a test. So when it cannot be
44
+ # opened, say that deleting it is a real option -- the raw
45
+ # `SQLite3::NotADatabaseException: file is not a database: PRAGMA journal_mode = WAL`
46
+ # tells a reader nothing about what to do next, and a blotter that got committed to
47
+ # git and then merged is exactly how it ends up unreadable.
48
+ def open_database!
39
49
  FileUtils.mkdir_p(File.dirname(path))
40
50
  @connection = SQLite3::Database.new(path)
41
51
  @connection.results_as_hash = true
42
52
  @connection.busy_timeout = BUSY_TIMEOUT_MS
43
- # Readers never block the writer and the writer never blocks readers.
53
+ # Readers never block the writer and the writer never blocks readers. This is also
54
+ # the first statement to touch the file, so a corrupt blotter surfaces here.
44
55
  @connection.execute("PRAGMA journal_mode = WAL")
45
56
  # WAL + NORMAL is durable across process crashes, which is the only failure that
46
57
  # matters here; a machine losing power mid-run costs us one run's bookkeeping.
47
58
  @connection.execute("PRAGMA synchronous = NORMAL")
48
59
  @connection
60
+ rescue SystemCallError => e
61
+ raise Constable::Error,
62
+ "Constable cannot open its blotter at #{path} (#{e.class}: #{e.message}). " \
63
+ "Point `storage.path` in .constable/config.yml somewhere writable."
64
+ rescue StandardError => e
65
+ raise Constable::Error,
66
+ "Constable's blotter at #{path} is not a readable database " \
67
+ "(#{e.class}). It holds flake history, the jail docket and warrants -- " \
68
+ "never your tests -- so deleting it is safe and starts that record fresh."
49
69
  end
50
70
 
51
71
  def require_driver!
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Constable
4
- VERSION = "0.1.0"
4
+ VERSION = "1.0.0"
5
5
  end
@@ -222,17 +222,35 @@ module Constable
222
222
 
223
223
  # The CLI speaks in file:line, the blotter is keyed by content hash. Warrant rows
224
224
  # carry both; loaded investigations are the fallback.
225
- def resolve(target)
225
+ # Every docket row a target could mean.
226
+ #
227
+ # The interesting case is a bare path. "test/cases/users_case.rb" with three tests
228
+ # on the docket is a question, not an instruction: picking one silently acts on a
229
+ # test the user never named -- and not even the first one, since the order is
230
+ # whatever storage returns. Callers ask for the candidates and refuse to guess.
231
+ def candidates(target)
226
232
  text = target.to_s.strip
227
- return nil if text.empty?
228
- return text if text.match?(/\A[0-9a-f]{8,64}\z/) && entry(text)
233
+ return [] if text.empty?
234
+
235
+ if text.match?(/\A[0-9a-f]{8,64}\z/) && (row = entry(text))
236
+ return [row]
237
+ end
229
238
 
230
239
  file, line = Jail.split_target(text)
231
- return nil if file.empty?
240
+ return [] if file.empty?
232
241
 
233
242
  matches = entries.select { |e| Jail.same_path?(e.file, file) }
234
243
  matches = matches.select { |e| e.line == line } if line
235
- return matches.first.identity if matches.any?
244
+ matches
245
+ end
246
+
247
+ def resolve(target)
248
+ matches = candidates(target)
249
+ return matches.first.identity if matches.size == 1
250
+ return nil unless matches.empty?
251
+
252
+ file, line = Jail.split_target(target.to_s.strip)
253
+ return nil if file.empty?
236
254
 
237
255
  Jail.registry_identity(file, line)
238
256
  end