constable-rails 0.1.0

data/lib/constable/runner.rb

@@ -0,0 +1,697 @@
+# frozen_string_literal: true
+
+require "securerandom"
+
+module Constable
+  # Executes a selection and turns it into results.
+  #
+  # Ordering is random every run, because a suite that only passes in one order is a suite
+  # that will fail the first time anything is added to it. The seed is always printed and
+  # always replayable. Cold cases are exempt -- they keep whatever order their own engine
+  # chose, since reordering someone's untouched legacy file is exactly the kind of surprise
+  # the cold-case story exists to avoid.
+  class Runner
+    # One unit of work. Native items are a single investigation; cold items are a whole
+    # file, because their engine owns the granularity inside it.
+    class Item
+      attr_reader :investigation, :path, :kind
+
+      def initialize(investigation: nil, path: nil, kind: :native)
+        @investigation = investigation
+        @path = path
+        @kind = kind
+      end
+
+      def native? = @kind == :native
+      def cold?   = @kind == :cold
+      def identity = native? ? @investigation.identity : Identity.for_cold_case(@path, "file")
+      def label = native? ? @investigation.display_label : @path.to_s
+    end
+
+    attr_reader :config, :selection, :reporter, :storage, :seed, :results, :coverage_report
+
+    def initialize(selection:, config: Constable.config, reporter: nil, storage: nil,
+                   seed: nil, jail_mode: false, jail_run: false, warrants: nil, coverage: nil,
+                   workers: nil, verbose: false, io: $stdout)
+      @selection  = selection
+      @config     = config
+      @storage    = storage || Constable.storage
+      @seed       = (seed || ENV["CONSTABLE_SEED"] || SecureRandom.random_number(10_000)).to_i
+      @jail_mode  = jail_mode
+      @jail_run   = jail_run
+      @warrants_requested = warrants.nil? ? config.warrants? : warrants
+      @coverage_requested = coverage.nil? ? config.coverage? : coverage
+      @workers    = workers
+      @verbose    = verbose
+      @io         = io
+      @reporter   = reporter || Reporter.new(io: io, config: config)
+      @results    = []
+      @worker_coverage = {}
+      @warnings_before = Constable.warnings.size
+    end
+
+    def jail_mode?  = @jail_mode
+
+    # `constable jail run` exists precisely to run the bodies the docket normally skips.
+    def jail_run?   = @jail_run
+    def coverage?   = @coverage_requested
+
+    # => Integer exit status (0 clean, 1 failures)
+    def call
+      # Before anything is loaded: Coverage only counts files required after it starts, so
+      # starting it any later measures an empty application and reports a confident 100%.
+      # force:, because #coverage? has already folded --coverage together with the config
+      # setting, and asking the config again would throw the flag away.
+      Constable::Coverage.start!(config: @config, force: true) if coverage?
+
+      load_suite!
+      items = build_items
+      ordered = order(items)
+
+      run_id = @storage.start_run(seed: @seed, mode: mode_label, full: @selection.full?)
+      Constable.configuration.run_before_suite!
+
+      started = monotonic
+      # A cold-case file is one work item but an unknown number of tests until its own
+      # engine has run it, so claim a total only when every item is a native investigation.
+      announced = ordered.all?(&:native?) ? ordered.size : nil
+      @reporter.start(total: announced, seed: @seed)
+
+      # The "alone" half of the order audit has to happen before the suite has touched
+      # anything, so it runs here rather than alongside the results it will be compared to.
+      order_audit.record_isolated!(ordered.select(&:native?).map(&:investigation))
+
+      # Warm everything that reads the blotter while we are still single-process.
+      docket_snapshot
+      duration_index
+      known_before
+
+      raw = order_audit.audit(execute(ordered)) + load_error_results
+
+      @results = adjudicate(raw)
+      duration = monotonic - started
+
+      Constable.configuration.run_after_suite!
+      @coverage_report = build_coverage_report if coverage?
+
+      persist(run_id, @results, @coverage_report)
+      suggestions = rename_suggestions(@results)
+
+      @reporter.finish(
+        results: @results,
+        duration: duration,
+        seed: @seed,
+        coverage: @coverage_report,
+        suggestions: suggestions
+      )
+
+      exit_status(@results, @coverage_report)
+    end
+
+    private
+
+    def mode_label
+      return "jail_run" if jail_run?
+      return "jail" if jail_mode?
+      return "unsafe" if @selection.unsafe_only?
+
+      @selection.full? ? "full" : "diff"
+    end
+
+    # test/case_helper.rb is the app's own entry point -- it boots Rails, defines the tier
+    # base classes and loads support files. Everything else depends on it having run.
+    def load_suite!
+      add_suite_dirs_to_load_path!
+
+      helper = %w[test/case_helper.rb spec/case_helper.rb].map { |p| File.join(Constable.root, p) }
+                                                          .find { |p| File.exist?(p) }
+      require helper if helper
+
+      @selection.native_targets.each { |target| load_case_file(target.path) }
+      helper
+    end
+
+    # Case files open with `require "case_helper"`, the way an RSpec file opens with
+    # `require "rails_helper"`. That only resolves if the suite directory is on the load
+    # path, and nothing else puts it there.
+    def add_suite_dirs_to_load_path!
+      %w[test spec].each do |dir|
+        path = File.join(Constable.root, dir)
+        $LOAD_PATH.unshift(path) if File.directory?(path) && !$LOAD_PATH.include?(path)
+      end
+    end
+
+    def load_case_file(path)
+      require path
+    rescue StandardError, ScriptError => e
+      load_errors << [path, e]
+    end
+
+    def load_errors
+      @load_errors ||= []
+    end
+
+    # A case file that will not load is a failure, not a silence. Reporting it as a result
+    # puts it in the FAILURES section with its own error, instead of letting a whole file
+    # of tests quietly vanish from the run.
+    def load_error_results
+      load_errors.map do |path, error|
+        relative = path.to_s.delete_prefix("#{Constable.root}/")
+        result = Result.new(
+          identity: Identity.for_source("load-error:#{relative}"),
+          case_name: relative,
+          description: "could not be loaded",
+          file: relative,
+          line: 1,
+          kind: :native,
+          status: :errored,
+          failure: Failure.from_exception(error, context: "This file never ran. Nothing in it was tested.")
+        )
+        result.seed = @seed
+        result
+      end
+    end
+
+    def build_items
+      native = native_items
+      cold = @selection.cold_targets_selected.map { |t| Item.new(path: t.path, kind: :cold) }
+      native + cold
+    end
+
+    def native_items
+      @selection.native_targets.flat_map do |target|
+        investigations = Constable.registry.investigations_in(target.path)
+        investigations = narrow_to_line(investigations, target.line) if target.line
+        investigations.map { |inv| Item.new(investigation: inv) }
+      end
+    end
+
+    # PATH:LINE means "the investigation at that line" -- but developers point at any line
+    # inside the block, so pick the investigation whose declaration is nearest above it.
+    def narrow_to_line(investigations, line)
+      exact = investigations.select { |inv| inv.line == line }
+      return exact if exact.any?
+
+      nearest = investigations.select { |inv| inv.line <= line }.max_by(&:line)
+      nearest ? [nearest] : []
+    end
+
+    def order(items)
+      native, cold = items.partition(&:native?)
+      [*native.shuffle(random: Random.new(@seed)), *cold]
+    end
+
+    def execute(items)
+      return [] if items.empty?
+
+      count = worker_count(items)
+      if count > 1 && forkable?
+        run_parallel(items, count)
+      else
+        run_serial(items)
+      end
+    end
+
+    def worker_count(items)
+      requested = @workers || @config.parallel_workers
+      requested.to_i.clamp(1, items.size)
+    end
+
+    def forkable?
+      Process.respond_to?(:fork) && !@verbose
+    end
+
+    def run_serial(items)
+      items.flat_map do |item|
+        run_item(item).each { |result| @reporter.record(result) }
+      end
+    end
+
+    # Workers never write to the blotter -- they ship results back over a pipe and the
+    # parent is the sole writer. That keeps every storage adapter free of cross-process
+    # write contention without any locking of its own.
+    def run_parallel(items, count)
+      buckets = balance(items, count)
+      readers = []
+      pids = []
+
+      # Everything that reads the blotter has already been warmed, so the handle can go.
+      # A child inheriting a writable SQLite connection is a corruption risk, and the
+      # driver rightly complains about it.
+      @storage.close
+
+      buckets.each do |bucket|
+        reader, writer = IO.pipe
+        # Marshal payloads are binary. Left in text mode, the first byte that isn't valid
+        # UTF-8 takes the worker down with an encoding error.
+        reader.binmode
+        writer.binmode
+        pid = fork do
+          reader.close
+          bucket.each do |item|
+            run_item(item).each { |result| write_message(writer, :result, result.to_h) }
+          end
+
+          # Ruby's Coverage counts lines in the process that executed them, so a worker's
+          # hits would die with it. They ride home on the same pipe as the results.
+          write_message(writer, :coverage, Constable::Coverage.peek_raw) if coverage?
+
+          writer.close
+          exit!(0)
+        end
+        writer.close
+        readers << reader
+        pids << pid
+      end
+
+      collected = drain(readers)
+      pids.each { |pid| Process.waitpid(pid) rescue nil } # rubocop:disable Style/RescueModifier
+
+      # A warning raised inside a worker only ever reached that worker's memory, so the
+      # results carry them home. Nothing that bends the rules is allowed to go missing
+      # just because it happened in a subprocess.
+      collected.each { |result| Constable.warnings.concat(Array(result.warnings)) }
+      collected
+    end
+
+    # Every message on the pipe is tagged, because results are not the only thing a worker
+    # has to send home.
+    def write_message(writer, kind, body)
+      payload = Marshal.dump([kind, body])
+      writer.write([payload.bytesize].pack("N"))
+      writer.write(payload)
+      writer.flush
+    end
+
+    # Reads from every worker as results arrive, so the glyph stream stays live rather than
+    # arriving in one lump when the slowest worker finishes.
+    def drain(readers)
+      collected = []
+      buffers = Hash.new { |h, k| h[k] = String.new(encoding: Encoding::BINARY) }
+      open_readers = readers.dup
+
+      until open_readers.empty?
+        ready, = IO.select(open_readers, nil, nil, 1)
+        next unless ready
+
+        ready.each do |reader|
+          chunk = begin
+            reader.read_nonblock(65_536)
+          rescue IOError # EOFError is one of these -- the worker finished and closed its end.
+            nil
+          rescue IO::WaitReadable
+            next
+          end
+
+          if chunk.nil?
+            open_readers.delete(reader)
+            reader.close unless reader.closed?
+            next
+          end
+
+          buffers[reader] << chunk
+          extract(buffers[reader]).each do |kind, body|
+            case kind
+            when :result
+              result = Result.from_h(body)
+              collected << result
+              @reporter.record(result)
+            when :coverage
+              @worker_coverage = Constable::Coverage.merge_raw(@worker_coverage, body)
+            end
+          end
+        end
+      end
+
+      collected
+    end
+
+    def extract(buffer)
+      out = []
+      loop do
+        break if buffer.bytesize < 4
+
+        size = buffer.byteslice(0, 4).unpack1("N")
+        break if buffer.bytesize < 4 + size
+
+        payload = buffer.byteslice(4, size)
+        rest = buffer.byteslice(4 + size, buffer.bytesize - 4 - size)
+        buffer.replace(rest || String.new(encoding: Encoding::BINARY))
+        out << Marshal.load(payload) # rubocop:disable Security/MarshalLoad -- our own pipe
+      end
+      out
+    end
+
+    # Longest-processing-time-first: the slowest tests are handed out before the quick ones,
+    # so no worker is left holding a three-second test after everyone else has finished.
+    def balance(items, count)
+      index = duration_index
+      buckets = Array.new(count) { [] }
+      loads = Array.new(count, 0.0)
+
+      items.sort_by { |item| -index.fetch(item.identity, 0.0) }.each do |item|
+        slot = loads.index(loads.min)
+        buckets[slot] << item
+        loads[slot] += index.fetch(item.identity, 0.05)
+      end
+      buckets.reject(&:empty?)
+    end
+
+    def duration_index
+      @duration_index ||= begin
+        @storage.duration_index
+      rescue StandardError
+        {}
+      end
+    end
+
+    # --- running one item ------------------------------------------------------
+
+    def run_item(item)
+      item.cold? ? run_cold(item) : [run_native(item)]
+    end
+
+    def run_cold(item)
+      warnings_before = Constable.warnings.size
+      # The seed goes in, not just onto the results: Minitest randomizes its own method
+      # order, so passing it is what makes `constable test PATH --seed N` actually replay.
+      results = ColdCase.run_file(item.path, config: @config, seed: @seed)
+      raised = Constable.warnings[warnings_before..] || []
+
+      results.each { |r| r.seed = @seed }
+
+      # A cold case warns once per file, not once per test, so the warning rides home on
+      # the first result rather than being repeated on all of them.
+      results.first&.warnings&.concat(raised)
+      results
+    rescue StandardError => e
+      [Result.new(
+        identity: Identity.for_cold_case(item.path, "load"),
+        case_name: File.basename(item.path),
+        description: "failed to load",
+        file: item.path.to_s.delete_prefix("#{Constable.root}/"),
+        line: 0,
+        kind: :cold,
+        status: :errored,
+        failure: Failure.from_exception(e)
+      ).tap { |r| r.seed = @seed }]
+    end
+
+    def run_native(item)
+      investigation = item.investigation
+      jail_entry = docket_snapshot[investigation.identity]
+
+      return run_jailed_setup(investigation, jail_entry) if jail_entry&.jailed? && !jail_run?
+
+      result = execute_investigation(investigation)
+      result.seed = @seed
+      result
+    end
+
+    # A jailed test still runs its briefing and witnesses -- only the investigate body is
+    # skipped -- so setup rot surfaces on the next ordinary run rather than lying in wait
+    # until someone gets around to `constable jail run`.
+    def run_jailed_setup(investigation, entry)
+      started = monotonic
+      failure = nil
+
+      instance = Case.constable_instance_for(investigation)
+      begin
+        # Setup without a body, but with its teardown -- skipping the body is the point of
+        # the jail; skipping the cleanup would just leave the request session, the browser
+        # or whatever else a teardown releases open for whatever runs next.
+        Isolation.with_rollback(investigation.tier) do
+          raised = nil
+          begin
+            instance.run_setup(investigation)
+          rescue StandardError => e
+            raised = e
+          end
+          # On its own line, not behind an ||=: teardown has to run whether or not setup
+          # got that far, and the first failure is still the one worth reporting.
+          teardown_failure = instance.run_teardown
+          raised ||= teardown_failure
+          raise raised if raised
+        end
+      rescue StandardError => e
+        failure = Failure.from_exception(e, context: "setup for a jailed test still runs, and it failed")
+      ensure
+        teardown(instance)
+      end
+
+      result = Result.from_investigation(
+        investigation,
+        status: :jailed,
+        duration: monotonic - started,
+        failure: failure
+      )
+      result.jail_reason  = entry.reason
+      result.times_jailed = entry.times_jailed
+      result.seed = @seed
+      result
+    end
+
+    def execute_investigation(investigation)
+      started = monotonic
+      before = leak_check? ? Isolation.snapshot : nil
+      warnings_before = Constable.warnings.size
+      failure = nil
+      status = :passed
+
+      instance = Case.constable_instance_for(investigation)
+
+      begin
+        # run_investigation is the whole lifecycle: before_setup, briefings, body,
+        # teardowns, after_teardown. The teardown half runs inside the rollback whether or
+        # not the body raised, and a teardown that raises never masks the body's failure.
+        Isolation.with_rollback(investigation.tier) { instance.run_investigation(investigation) }
+      rescue AssertionFailed => e
+        status = :failed
+        failure = Failure.from_exception(e, context: e.context)
+      rescue StandardError => e
+        status = :errored
+        failure = Failure.from_exception(e)
+      ensure
+        # Time stays frozen and the network stays blocked only for the life of one
+        # investigation. Whatever the outcome, the next one starts from clean ground.
+        teardown(instance)
+      end
+
+      duration = monotonic - started
+      raised = Constable.warnings[warnings_before..] || []
+
+      if before
+        leaks = Isolation.diff(before, Isolation.snapshot)
+        if leaks.any?
+          Constable.warn!(
+            "state leaked out of this investigation: #{leaks.join("; ")}",
+            location: investigation.location,
+            kind: :leak
+          )
+        end
+      end
+
+      Result.from_investigation(
+        investigation,
+        status: status,
+        duration: duration,
+        failure: failure,
+        warnings: Constable.warnings[warnings_before..] || raised
+      )
+    end
+
+    # The leak check walks every user class's class variables, which is worth it per test
+    # but not worth it per test in a hot parallel loop on a huge suite.
+    def leak_check?
+      return @leak_check if defined?(@leak_check)
+
+      @leak_check = ENV["CONSTABLE_LEAK_CHECK"] != "0"
+    end
+
+    # --- adjudication ----------------------------------------------------------
+
+    # Turns raw pass/fail into the verdict the build acts on: warrants decide whether a
+    # failure is even real, then jail decides whether it blocks.
+    def adjudicate(raw)
+      raw.map do |result|
+        decided = warrants.adjudicate(
+          result,
+          requested: @warrants_requested,
+          subject: investigation_for(result.identity)
+        ) { |subject, _attempt| rerun_in_isolation(subject) }
+
+        jail_run? ? decided : jail.adjudicate(decided, jail_mode: jail_mode?)
+      end
+    end
+
+    def investigation_for(identity)
+      @investigation_index ||= Constable.registry.investigations.to_h { |inv| [inv.identity, inv] }
+      @investigation_index[identity]
+    end
+
+    # One test, run again from scratch, so a warrant's retries measure the test rather than
+    # whatever the rest of the suite left lying around.
+    def rerun_in_isolation(subject)
+      investigation = subject.is_a?(Investigation) ? subject : investigation_for(subject.to_s)
+      return nil unless investigation
+
+      execute_investigation(investigation)
+    end
+
+    def jail
+      @jail ||= Jail.new(config: @config, storage: @storage)
+    end
+
+    # The docket, read once in the parent and inherited by every worker through fork.
+    # A worker that queried it directly would be reaching into a database handle it does
+    # not own -- SQLite is explicit that a connection must not cross a fork -- and the
+    # answer cannot change mid-run anyway.
+    def docket_snapshot
+      @docket_snapshot ||= jail.entries.to_h { |entry| [entry.identity, entry] }
+    rescue StandardError
+      {}
+    end
+
+    def warrants
+      @warrants ||= Warrants.new(config: @config, storage: @storage)
+    end
+
+    def order_audit
+      @order_audit ||= OrderAudit.new(config: @config, storage: @storage)
+    end
+
+    # --- persistence and reporting --------------------------------------------
+
+    def persist(run_id, results, coverage_report)
+      results.each do |result|
+        @storage.record_result(run_id, result)
+        @storage.record_duration(result.identity, result.duration)
+      end
+
+      Constable::Coverage.record!(coverage_report, run_id, storage: @storage) if coverage_report
+
+      @storage.finish_run(run_id, totals: totals(results))
+    rescue StandardError => e
+      Constable.warn!("could not write to the blotter: #{e.message}", kind: :storage)
+    end
+
+    def totals(results)
+      {
+        total: results.size,
+        passed: results.count(&:passed?),
+        failed: results.count(&:failed?),
+        jailed: results.count(&:jailed?),
+        warranted: results.count(&:warranted?)
+      }
+    end
+
+    # The blotter's identities as they stood before this run wrote anything. Taken up
+    # front, because rename detection compares "what we used to know" against "what we
+    # just saw", and persisting first would make every test look familiar.
+    def known_before
+      @known_before ||= Array(@storage.known_identities).map(&:to_s)
+    rescue StandardError
+      []
+    end
+
+    # A test whose body changed gets a new identity, so an old one vanishing the same run
+    # a similar new one appears is usually a rename plus a tweak, not two separate edits.
+    # Confirming the relink also carries any jail or warrant entry across, which is what
+    # stops a fixed-but-still-jailed test from haunting the docket forever.
+    def rename_suggestions(results)
+      return [] unless @selection.full?
+
+      seen     = results.map(&:identity)
+      vanished = known_before - seen
+      fresh    = results.reject { |r| known_before.include?(r.identity) }
+      return [] if vanished.empty? || fresh.empty?
+
+      labels = vanished.to_h { |identity| [identity, label_for(identity)] }
+
+      fresh.filter_map do |result|
+        match = vanished.max_by { |old| similarity(labels[old], result.description) }
+        next if match.nil?
+
+        score = similarity(labels[match], result.description)
+        next if score < 0.5
+
+        suggestion = {
+          relinked: false, score: score,
+          old_identity: match, new_identity: result.identity,
+          old_label: labels[match], new_label: result.display_label
+        }
+
+        if @config.auto_relink? && score >= 0.85
+          @storage.relink(match, result.identity)
+          suggestion[:relinked] = true
+        end
+
+        suggestion
+      end
+    rescue StandardError
+      []
+    end
+
+    # The blotter keeps a display label alongside each identity purely so a vanished test
+    # can still be named in a suggestion.
+    def label_for(identity)
+      row = @storage.history_for(identity, limit: 1).first
+      return identity unless row
+
+      [row[:case_name], row[:description]].compact.join(" ").strip
+    rescue StandardError
+      identity
+    end
+
+    # Cheap token overlap -- enough to spot "creates a user" vs "creates a user with valid
+    # params" without pulling in a Levenshtein dependency for a hint that a human confirms.
+    def similarity(left, right)
+      a = left.to_s.downcase.scan(/\w+/)
+      b = right.to_s.downcase.scan(/\w+/)
+      return 0.0 if a.empty? || b.empty?
+
+      (a & b).size.to_f / [a.size, b.size].max
+    end
+
+    def exit_status(results, coverage_report)
+      return 1 if results.any?(&:failed?)
+      return 1 if @config.fail_on_warnings? && new_warnings.any?
+      return 1 if coverage_report && !coverage_report.meets_threshold?(@config)
+
+      0
+    end
+
+    def new_warnings
+      Constable.warnings[@warnings_before..] || []
+    end
+
+    # Combines what this process saw with everything the workers sent back. Cold cases
+    # count here too -- Coverage works at the process level, so it never knew which engine
+    # ran the code.
+    def build_coverage_report
+      merged = Constable::Coverage.merge_raw(Constable::Coverage.peek_raw, @worker_coverage)
+      report = Constable::Coverage.build_report(
+        merged,
+        config: @config,
+        root: Constable.root,
+        # Cold cases contribute their numbers but are never held to the diff gate, so a
+        # run carrying nothing else must not be gated at all.
+        gate: !@selection.unsafe_only?
+      )
+      Constable::Coverage.abort!
+      report
+    end
+
+    # The last thing to happen to an instance, after its own teardowns and after the
+    # transaction is gone: the runtime DSL's global state -- a frozen clock, a barred
+    # network, a half-open unsafe block -- belongs to the process, not to the test, so it
+    # is released once everything that might still depend on it has finished.
+    def teardown(instance)
+      instance._constable_dsl_teardown if instance.respond_to?(:_constable_dsl_teardown)
+    rescue StandardError => e
+      Constable.warn!("teardown after an investigation raised: #{e.message}", kind: :teardown)
+    end
+
+    def monotonic = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+  end
+end