constable-rails 0.1.0 → 1.1.0

data/lib/constable/runner.rb

@@ -11,6 +11,14 @@ module Constable
   # chose, since reordering someone's untouched legacy file is exactly the kind of surprise
   # the cold-case story exists to avoid.
   class Runner
+    # Thresholds for "this run is broken, not these tests" -- see #systemic_failure.
+    SYSTEMIC_MINIMUM   = 5    # below this it is cheaper to believe the tests
+    SYSTEMIC_SHARE     = 0.25 # of the whole run
+    SYSTEMIC_AGREEMENT = 0.8  # of the failures, failing identically
+
+    # `failed` rather than `count` or `tally`, both of which override an Enumerable method.
+    Systemic = Struct.new(:exception_class, :failed, :total, keyword_init: true)
+
     # One unit of work. Native items are a single investigation; cold items are a whole
     # file, because their engine owns the granularity inside it.
     class Item
@@ -56,6 +64,19 @@ module Constable
     def jail_run?   = @jail_run
     def coverage?   = @coverage_requested
 
+    # Loads every case file and hands back the identities the suite actually defines,
+    # without running anything. `constable prune` needs this: which tests still exist is
+    # only knowable once the whole suite has been loaded.
+    def self.identities(config: Constable.config)
+      selection = Selection.new([], config: config, root: Constable.root, full: true)
+      runner = new(selection: selection, config: config,
+                   reporter: Reporter.new(io: StringIO.new, config: config, color: false),
+                   storage: Constable.storage, workers: 1)
+      runner.send(:load_suite!)
+      Constable.registry.disambiguate_identities!
+      Constable.registry.investigations.map(&:identity)
+    end
+
     # => Integer exit status (0 clean, 1 failures)
     def call
       # Before anything is loaded: Coverage only counts files required after it starts, so
@@ -65,7 +86,11 @@ module Constable
       Constable::Coverage.start!(config: @config, force: true) if coverage?
 
       load_suite!
+      # Before anything is keyed on an identity -- selection, the docket, flake history --
+      # settle any two tests that happen to share a body.
+      Constable.registry.disambiguate_identities!
       items = build_items
+      refuse_empty_selection!(items)
       ordered = order(items)
 
       run_id = @storage.start_run(seed: @seed, mode: mode_label, full: @selection.full?)
@@ -91,6 +116,10 @@ module Constable
       @results = adjudicate(raw)
       duration = monotonic - started
 
+      # Cold-case engines hold a live session -- for RSpec that is a configuration
+      # carrying `after(:suite)` hooks that have not fired yet. Tear it down before our
+      # own after_suite so the engine's cleanup runs inside the suite, not after it.
+      ColdCase.reset_engines!
       Constable.configuration.run_after_suite!
       @coverage_report = build_coverage_report if coverage?
 
@@ -127,6 +156,15 @@ module Constable
                                                           .find { |p| File.exist?(p) }
       require helper if helper
 
+      # Between requiring the helper and asking the selection anything.
+      #
+      # Ordering is the whole point. case_helper.rb is where Constable.configure runs, so
+      # its settings do not exist until the line above. But Selection memoizes its targets
+      # the first time it is asked for them, and `cold_cases` is one of the settings people
+      # will most want to set in Ruby -- ask first and the override arrives too late to
+      # matter, silently. Requiring the helper needs no selection, so this fits between.
+      @config.apply_overrides!(Constable.configuration.overrides)
+
       @selection.native_targets.each { |target| load_case_file(target.path) }
       helper
     end
@@ -172,6 +210,17 @@ module Constable
       end
     end
 
+    # A run that was *asked* for something specific and found nothing is a usage error, not
+    # a pass. `constable test test/cases/typo_case.rb` used to print "0 passed, 0 failed"
+    # and exit 0, so a mistyped path in a CI script produced a green build that ran no
+    # tests at all. A full run with an empty suite is a different thing and stays quiet.
+    def refuse_empty_selection!(items)
+      return unless items.empty?
+      return unless @selection.explicit?
+
+      raise Constable::Error, @selection.empty_selection_message
+    end
+
     def build_items
       native = native_items
       cold = @selection.cold_targets_selected.map { |t| Item.new(path: t.path, kind: :cold) }
@@ -188,14 +237,29 @@ module Constable
 
     # PATH:LINE means "the investigation at that line" -- but developers point at any line
     # inside the block, so pick the investigation whose declaration is nearest above it.
+    #
+    # Bounded by the end of the file. Unbounded, `:999` on a twenty-line file quietly ran
+    # the last investigation in it: not the test the user asked for, not an error, and
+    # green either way. A line past the end is a typo, and no answer beats a wrong one.
     def narrow_to_line(investigations, line)
       exact = investigations.select { |inv| inv.line == line }
       return exact if exact.any?
+      return [] unless line_within_file?(investigations.first, line)
 
       nearest = investigations.select { |inv| inv.line <= line }.max_by(&:line)
       nearest ? [nearest] : []
     end
 
+    def line_within_file?(investigation, line)
+      path = investigation&.file
+      return false if path.nil?
+
+      path = File.join(@config.root, path) unless File.exist?(path)
+      return false unless File.exist?(path)
+
+      line <= File.foreach(path).count
+    end
+
     def order(items)
       native, cold = items.partition(&:native?)
       [*native.shuffle(random: Random.new(@seed)), *cold]
@@ -205,13 +269,32 @@ module Constable
       return [] if items.empty?
 
       count = worker_count(items)
-      if count > 1 && forkable?
+      if count > 1 && forkable? && parallel_safe?
         run_parallel(items, count)
       else
         run_serial(items)
       end
     end
 
+    # Forking is only safe once each worker has a database of its own. Without that,
+    # every worker opens the same one: on SQLite the run dissolves into "database is
+    # locked", and on a client/server database the tests quietly see each other's rows,
+    # which is worse. An app with no ActiveRecord has nothing to shard and is always safe.
+    #
+    # When we cannot shard, we run serially and say why. Slow is a trade-off; wrong is not.
+    def parallel_safe?
+      return true unless WorkerDatabases.active_record?
+      return true if WorkerDatabases.shardable?
+
+      Constable.warn!(
+        "parallel workers need one database per worker, and this app's ActiveRecord " \
+        "cannot provide them (active_record/test_databases did not load). Running " \
+        "serially instead -- pass --workers N once that is available.",
+        kind: :parallel
+      )
+      false
+    end
+
     def worker_count(items)
       requested = @workers || @config.parallel_workers
       requested.to_i.clamp(1, items.size)
@@ -240,7 +323,11 @@ module Constable
       # driver rightly complains about it.
       @storage.close
 
-      buckets.each do |bucket|
+      # Same reasoning for the app's own connections: a child that inherits a live
+      # handle can corrupt it. Rails does exactly this before its own fork.
+      WorkerDatabases.before_fork!
+
+      buckets.each_with_index do |bucket, worker_index|
         reader, writer = IO.pipe
         # Marshal payloads are binary. Left in text mode, the first byte that isn't valid
         # UTF-8 takes the worker down with an encoding error.
@@ -248,10 +335,21 @@ module Constable
         writer.binmode
         pid = fork do
           reader.close
+
+          # Before a single test runs: build this worker's own database and point the
+          # process at it. Raises rather than falling back to the shared one, because a
+          # silent fallback is the bug we are here to prevent.
+          WorkerDatabases.after_fork!(worker_index)
+
           bucket.each do |item|
             run_item(item).each { |result| write_message(writer, :result, result.to_h) }
           end
 
+          # A worker owns its own cold-case session, and it dies here. Fire the engine's
+          # after(:suite) hooks in the process that ran the before(:suite) half, before
+          # coverage is read -- the parent has no hooks to run on its behalf.
+          ColdCase.reset_engines!
+
           # Ruby's Coverage counts lines in the process that executed them, so a worker's
           # hits would die with it. They ride home on the same pipe as the results.
           write_message(writer, :coverage, Constable::Coverage.peek_raw) if coverage?
@@ -513,6 +611,9 @@ module Constable
     # Turns raw pass/fail into the verdict the build acts on: warrants decide whether a
     # failure is even real, then jail decides whether it blocks.
     def adjudicate(raw)
+      @systemic = systemic_failure(raw)
+      announce_systemic_failure(@systemic) if @systemic
+
       raw.map do |result|
         decided = warrants.adjudicate(
           result,
@@ -520,10 +621,54 @@ module Constable
           subject: investigation_for(result.identity)
         ) { |subject, _attempt| rerun_in_isolation(subject) }
 
-        jail_run? ? decided : jail.adjudicate(decided, jail_mode: jail_mode?)
+        next decided if jail_run?
+
+        jail.adjudicate(decided, jail_mode: jail_mode?, systemic: systemic?(decided))
       end
     end
 
+    # A run is "systemically broken" when a large share of it failed the same way: the
+    # database was down, a worker could not start, a shared fixture never loaded. Thirty
+    # tests did not each independently go bad in the same second.
+    #
+    # This matters because flake history reads "passed last run, failed this run" as
+    # evidence about a *test*, and jails it. One bad afternoon on CI could therefore
+    # quarantine a third of a healthy suite, and the docket -- which is supposed to be a
+    # record of tests worth distrusting -- fills up with tests that were never at fault.
+    def systemic_failure(results)
+      failures = results.select(&:failed?)
+      return nil if failures.size < SYSTEMIC_MINIMUM
+      return nil if failures.size < results.size * SYSTEMIC_SHARE
+
+      grouped = failures.group_by { |result| result.failure&.exception_class.to_s }
+      grouped.delete("")
+      return nil if grouped.empty?
+
+      exception_class, sharing = grouped.max_by { |_klass, group| group.size }
+      return nil if sharing.size < failures.size * SYSTEMIC_AGREEMENT
+
+      Systemic.new(exception_class: exception_class, failed: sharing.size, total: results.size)
+    end
+
+    # Only the failures that look like the outage are exempt. A genuine failure that
+    # happened to land in the same run is still a genuine failure.
+    def systemic?(result)
+      return false unless @systemic
+      return false unless result.failed?
+
+      result.failure&.exception_class.to_s == @systemic.exception_class
+    end
+
+    def announce_systemic_failure(systemic)
+      Constable.warn!(
+        "#{systemic.failed} of #{systemic.total} tests failed with the same error " \
+        "(#{systemic.exception_class}). That reads as one broken run rather than " \
+        "#{systemic.failed} newly flaky tests, so flake history and the jail docket were " \
+        "left alone. Fix the cause and run again.",
+        kind: :systemic
+      )
+    end
+
     def investigation_for(identity)
       @investigation_index ||= Constable.registry.investigations.to_h { |inv| [inv.identity, inv] }
       @investigation_index[identity]
@@ -564,6 +709,11 @@ module Constable
 
     def persist(run_id, results, coverage_report)
       results.each do |result|
+        # The blotter is the evidence file. A result produced by an outage is not
+        # evidence about the test, so it is not filed -- otherwise the next run reads
+        # "failed, then passed" and draws a conclusion from a power cut.
+        next if systemic?(result)
+
         @storage.record_result(run_id, result)
         @storage.record_duration(result.identity, result.duration)
       end

data/lib/constable/selection.rb

@@ -33,10 +33,13 @@ module Constable
       @root        = root.to_s
       @full        = full
       @unsafe_only = unsafe_only
-      @tier        = tier&.to_sym
+      # Downcased: `--tier UNIT` used to match nothing at all and report a clean run.
+      @tier        = tier.to_s.strip.downcase.to_sym unless tier.to_s.strip.empty?
       @reason      = nil
     end
 
+    TIERS = %w[unit integration system].freeze
+
     def full?        = @full
     def unsafe_only? = @unsafe_only
 
@@ -60,6 +63,25 @@ module Constable
       end
     end
 
+    # Did the user ask for something in particular? If so, finding nothing is an error
+    # rather than a clean run -- see Runner#refuse_empty_selection!.
+    def explicit? = @args.any? { |arg| !arg.to_s.strip.empty? } || !@tier.nil?
+
+    # Says which part of the request came up empty, because "0 tests" on its own does not
+    # tell you whether the path was wrong, the tier was, or both.
+    def empty_selection_message
+      if @tier && !TIERS.include?(@tier.to_s)
+        return "unknown tier #{@tier.inspect} -- expected one of #{TIERS.join(", ")}."
+      end
+
+      described = @args.reject { |arg| arg.to_s.strip.empty? }
+      subject   = described.empty? ? "this run" : described.join(", ")
+      suffix    = @tier ? " in the #{@tier} tier" : ""
+
+      "no tests matched #{subject}#{suffix}. Check the path, the line number, and " \
+        "whether the file is a case or a cold case."
+    end
+
     def native_targets = targets.select(&:native?)
     def cold_targets_selected = targets.select(&:cold?)
     def empty? = targets.empty?

data/lib/constable/storage/sqlite_adapter.rb

@@ -36,16 +36,36 @@ module Constable
 
       def connect!
         require_driver!
+        open_database!
+      end
+
+      # The blotter is the one file Constable owns outright, and it is disposable: it
+      # holds flake history, the docket and warrants, never a test. So when it cannot be
+      # opened, say that deleting it is a real option -- the raw
+      # `SQLite3::NotADatabaseException: file is not a database: PRAGMA journal_mode = WAL`
+      # tells a reader nothing about what to do next, and a blotter that got committed to
+      # git and then merged is exactly how it ends up unreadable.
+      def open_database!
         FileUtils.mkdir_p(File.dirname(path))
         @connection = SQLite3::Database.new(path)
         @connection.results_as_hash = true
         @connection.busy_timeout = BUSY_TIMEOUT_MS
-        # Readers never block the writer and the writer never blocks readers.
+        # Readers never block the writer and the writer never blocks readers. This is also
+        # the first statement to touch the file, so a corrupt blotter surfaces here.
         @connection.execute("PRAGMA journal_mode = WAL")
         # WAL + NORMAL is durable across process crashes, which is the only failure that
         # matters here; a machine losing power mid-run costs us one run's bookkeeping.
         @connection.execute("PRAGMA synchronous = NORMAL")
         @connection
+      rescue SystemCallError => e
+        raise Constable::Error,
+              "Constable cannot open its blotter at #{path} (#{e.class}: #{e.message}). " \
+              "Point `storage.path` in .constable/config.yml somewhere writable."
+      rescue StandardError => e
+        raise Constable::Error,
+              "Constable's blotter at #{path} is not a readable database " \
+              "(#{e.class}). It holds flake history, the jail docket and warrants -- " \
+              "never your tests -- so deleting it is safe and starts that record fresh."
       end
 
       def require_driver!

data/lib/constable/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Constable
-  VERSION = "0.1.0"
+  VERSION = "1.1.0"
 end

data/lib/constable/warrants.rb

@@ -222,17 +222,68 @@ module Constable
 
     # The CLI speaks in file:line, the blotter is keyed by content hash. Warrant rows
     # carry both; loaded investigations are the fallback.
-    def resolve(target)
+    # Every docket row a target could mean.
+    #
+    # The interesting case is a bare path. "test/cases/users_case.rb" with three tests
+    # on the docket is a question, not an instruction: picking one silently acts on a
+    # test the user never named -- and not even the first one, since the order is
+    # whatever storage returns. Callers ask for the candidates and refuse to guess.
+    def candidates(target)
       text = target.to_s.strip
-      return nil if text.empty?
-      return text if text.match?(/\A[0-9a-f]{8,64}\z/) && entry(text)
+      return [] if text.empty?
+
+      if text.match?(/\A[0-9a-f]{8,64}\z/) && (row = entry(text))
+        return [row]
+      end
 
       file, line = Jail.split_target(text)
-      return nil if file.empty?
+      return [] if file.empty?
 
       matches = entries.select { |e| Jail.same_path?(e.file, file) }
       matches = matches.select { |e| e.line == line } if line
-      return matches.first.identity if matches.any?
+      matches
+    end
+
+    # Rows whose test no longer exists.
+    #
+    # A test's key is a content hash of its body, so editing a jailed test gives it a new
+    # identity and leaves the old row behind -- pointing at a file:line that may now hold
+    # something else entirely. That is content-hash identity working as designed, but over
+    # a few months the docket fills with tests nobody can find.
+    #
+    # `known` is every identity the loaded suite registered, so this is only meaningful
+    # after a full load. Two things are deliberately conservative about it:
+    #
+    #   * A known identity is never stale, even when the file recorded beside it is gone.
+    #     The path is a display label; the identity is the truth. A test that moved file
+    #     has an out-of-date label, not a missing test.
+    #   * A cold case is never pruned while its file exists. Cold-case tests cannot be
+    #     enumerated without running their own engine, so absence from `known` says
+    #     nothing about them.
+    def stale_entries(known, cold_case: nil)
+      known = Array(known)
+      entries.reject { |entry| known.include?(entry.identity) }
+             .select { |entry| gone?(entry, cold_case) }
+    end
+
+    def gone?(entry, cold_case)
+      relative = entry.file.to_s
+      return true if relative.empty?
+
+      path = File.absolute_path?(relative) ? relative : File.join(Constable.root, relative)
+      return true unless File.exist?(path)
+      return false if cold_case&.call(relative)
+
+      true
+    end
+
+    def resolve(target)
+      matches = candidates(target)
+      return matches.first.identity if matches.size == 1
+      return nil unless matches.empty?
+
+      file, line = Jail.split_target(target.to_s.strip)
+      return nil if file.empty?
 
       Jail.registry_identity(file, line)
     end

data/lib/constable/worker_databases.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+module Constable
+  # Per-worker databases for parallel runs.
+  #
+  # Forking N workers that all talk to one database is not a speed/safety trade, it is a
+  # correctness bug. On SQLite it shows up immediately and honestly -- every worker
+  # contends for the same file and the run dissolves into
+  # `SQLite3::BusyException: database is locked` -- and on a client/server database it
+  # shows up later and far worse, as tests seeing each other's rows.
+  #
+  # Rails already solved this for `rails test`: each worker gets its own database, named
+  # by appending the worker index, rebuilt from schema. This is the same thing, driven by
+  # Constable's runner rather than by ActiveSupport::Testing::Parallelization, because
+  # Constable does its own forking.
+  #
+  # Everything here is defensive. Constable runs in apps with no ActiveRecord at all --
+  # that is the whole point of the :unit tier -- so every entry point answers "no" rather
+  # than raising when the pieces are missing, and the runner falls back to a serial run.
+  module WorkerDatabases
+    module_function
+
+    # Is there an ActiveRecord in this process whose databases would be shared by forks?
+    def active_record?
+      defined?(::ActiveRecord::Base) ? true : false
+    end
+
+    # Can we actually give each worker its own database? Rails ships the machinery in
+    # active_record/test_databases, which is only loaded when someone asks for parallel
+    # tests -- so ask for it here rather than assuming.
+    def shardable?
+      return false unless active_record?
+
+      load_test_databases!
+      defined?(::ActiveRecord::TestDatabases) ? true : false
+    end
+
+    # Parent side, before the fork. A child inheriting a live connection is a corruption
+    # risk in exactly the way an inherited SQLite handle is.
+    def before_fork!
+      return false unless active_record?
+
+      ::ActiveRecord::Base.connection_handler.clear_all_connections!
+      true
+    rescue StandardError
+      false
+    end
+
+    # Child side, immediately after the fork and before any test runs. Builds
+    # `<database>_<index>` from schema and points this process at it.
+    #
+    # ENV["VERBOSE"] is silenced the way Rails silences it: schema loading is chatty, and
+    # stdout belongs to the reporter.
+    def after_fork!(index)
+      return false unless shardable?
+
+      ::ActiveRecord::TestDatabases.create_and_load_schema(index, env_name: env_name)
+      true
+    rescue StandardError => e
+      # A worker that cannot build its own database would otherwise silently fall back to
+      # sharing the parent's, which is the bug this module exists to prevent. Say so, and
+      # let the failure be a real one.
+      raise Constable::Error, "worker #{index} could not create its own test database: " \
+                              "#{e.class}: #{e.message}"
+    end
+
+    def env_name
+      if defined?(::ActiveRecord::ConnectionHandling::DEFAULT_ENV)
+        ::ActiveRecord::ConnectionHandling::DEFAULT_ENV.call
+      else
+        ENV["RAILS_ENV"] || "test"
+      end
+    end
+
+    def load_test_databases!
+      return if defined?(::ActiveRecord::TestDatabases)
+
+      require "active_record/test_databases"
+    rescue LoadError, StandardError
+      nil
+    end
+  end
+end

data/lib/constable.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "constable/version"
+require "constable/worker_databases"
 
 # Constable -- an opinionated, strict Rails testing framework.
 #
@@ -129,14 +130,60 @@ module Constable
   end
 
   # Code-level configuration set from test/case_helper.rb.
+  # The Ruby half of configuration, set from test/case_helper.rb.
+  #
+  # Anything that is *code* -- custom matchers, tier base classes, one-time global setup --
+  # can only live here. Everything that is merely a *setting* can live in either place, and
+  # this is the one that wins:
+  #
+  #   a CLI flag        for one run
+  #   Constable.configure   in case_helper.rb, because it is code and ran deliberately
+  #   .constable/config.yml the declared default for the project
+  #   Constable's defaults
+  #
+  # Until 1.1.0 the four accessors here were read by nothing at all: `c.parallel_workers = 4`
+  # was in the generated case_helper.rb, documented, and silently ignored.
   class Configuration
-    attr_accessor :parallel_workers, :seed, :coverage, :warrants
+    # Every setting .constable/config.yml understands, settable in Ruby as well. A nil is
+    # "not set here" rather than "set to nothing", so leaving one alone defers to the file.
+    SETTINGS = %i[
+      cold_cases warrants warrant_retries auto_relink parole_period
+      coverage coverage_threshold coverage_html fail_on_warnings parallel_workers
+      output tiers
+    ].freeze
+
+    # `storage` is the one setting that cannot live here, and the reason is ordering, not
+    # preference: the blotter handle is opened before test/case_helper.rb is loaded,
+    # because `constable jail`, `warrants`, `watchlist` and `status` all read the docket
+    # without booting the app at all. By the time this block runs, it is already open.
+    #
+    # Raising beats accepting the value and quietly using the old path -- silently
+    # ignoring a setting somebody wrote is the failure mode this whole class was fixed for.
+    SETTINGS_ONLY_IN_YAML = %i[storage].freeze
+
+    attr_accessor(*SETTINGS, :seed)
+
+    def storage=(_value)
+      raise ConfigurationError,
+            "storage must be set in .constable/config.yml, not Constable.configure. The " \
+            "blotter is opened before case_helper.rb loads, so that `constable jail` and " \
+            "`constable status` can read the docket without booting the app -- by the " \
+            "time this block runs the connection is already open."
+    end
 
     def initialize
       @before_suite_hooks = []
       @after_suite_hooks  = []
     end
 
+    # Only the settings actually assigned, ready to merge over the file's values.
+    def overrides
+      SETTINGS.each_with_object({}) do |name, out|
+        value = public_send(name)
+        out[name.to_s] = value unless value.nil?
+      end
+    end
+
     def before_suite(&block) = @before_suite_hooks << block
     def after_suite(&block)  = @after_suite_hooks << block