snap_diff-capybara 2.0.0.beta3 → 2.0.0.beta4

data/lib/snap_diff/reporters/html.rb

@@ -8,7 +8,7 @@ require "json"
 # The auto-registration block at the bottom of this file registers with
 # SnapDiff::Reporting.
 require "snap_diff/reporting"
-require "capybara/screenshot/diff/config_legacy"
+require "snap_diff/config"
 
 module SnapDiff
   module Reporters
@@ -64,20 +64,36 @@ module SnapDiff
         @output_path ||= Pathname.new(@explicit_output_path || self.class.default_output_path)
       end
 
+      # Fork-parallel handoff (issue #258). A forked worker never reaches
+      # `Minitest.after_run`, so it hands its records to the parent as
+      # plain data instead. JSON on purpose -- boring, and a half-written
+      # file is discarded rather than half-read. Image paths were resolved
+      # against `output_path`, which is the same in both processes.
+      def dump_state
+        @mutex.synchronize { {"total" => @total, "failures" => @failures} }
+      end
+
+      def merge_state!(state)
+        @mutex.synchronize do
+          @total += state["total"]
+          @failures.concat(state["failures"].map { |entry| entry.transform_keys(&:to_sym) })
+        end
+      end
+
       def passed = total - failures.size
       def failed = failures.size
 
+      # Both customer personas named this path as the best output in the
+      # product, so it gets its own line -- but only when a report was
+      # actually written.
+      #
+      # The counts this used to carry moved to SnapDiff::Reporting (issue
+      # #269): they are printed for every user, and this file is not. See
+      # Reporting.counts_summary.
+      #
+      # @return [String, nil] nil when no report was written
       def summary
-        return if total.zero?
-
-        screenshots_label = (total == 1) ? "1 screenshot" : "#{total} screenshots"
-
-        if failures.empty?
-          "[snap_diff] #{screenshots_label} compared, no failures."
-        else
-          failures_label = (failures.size == 1) ? "1 failure" : "#{failures.size} failures"
-          "[snap_diff] #{screenshots_label} compared, #{failures_label}. Report: #{output_path}"
-        end
+        "[snap_diff] Report: #{output_path}" if @finalized
       end
 
       def render
@@ -89,8 +105,8 @@ module SnapDiff
       end
 
       def self.default_output_path
-        root = Capybara::Screenshot.root || Pathname.pwd
-        root / Capybara::Screenshot.save_path / "snap_diff_report.html"
+        root = SnapDiff.config.root || Pathname.pwd
+        root / SnapDiff.config.save_path / "snap_diff_report.html"
       end
 
       private

data/lib/snap_diff/reporting.rb

@@ -1,5 +1,9 @@
 # frozen_string_literal: true
 
+require "fileutils"
+require "json"
+require "tmpdir"
+
 module SnapDiff
   # Process-global reporter lifecycle: registration, per-test notification,
   # end-of-suite finalization. One list of reporters for the whole process,
@@ -13,10 +17,88 @@ module SnapDiff
   module Reporting
     @reporters = []
     @mutex = Mutex.new
+    @missing_baselines = Set.new
+    @rerecorded_baselines = Set.new
+    @matched_selectors = Set.new
+    @unmatched_selectors = Set.new
+    @verified = 0
+    @changed = 0
 
     class << self
       attr_reader :reporters, :mutex
 
+      # How many screenshots were compared to a committed baseline, and how
+      # many of those differed.
+      #
+      # These counters live HERE, not in a reporter (issue #269). Counting
+      # is core honesty; writing an HTML file is a feature. The summary
+      # exists to catch the failure modes no per-assertion rule can see -- a
+      # run where zero system tests executed, or where an inherited GIT_DIR
+      # redirected every baseline lookup -- and `0 verified` is the only
+      # signal for either. It shipped inside Reporters::HTML, the gem's one
+      # and only `register` call site, so the documented Rails setup (which
+      # requires just the Minitest integration) printed nothing at all.
+      attr_reader :verified, :changed
+
+      # Remembers a screenshot that had no COMMITTED baseline and was
+      # therefore never compared.
+      #
+      # @return [Boolean] true the first time this name is seen -- the
+      #   "warn once per screenshot" gate for ScreenshotMatcher, and the
+      #   tally behind {finalize!}'s summary line. Kept here rather than in
+      #   the matcher because the end-of-run summary is this module's job.
+      def record_missing_baseline(name)
+        @mutex.synchronize { !!@missing_baselines.add?(name) }
+      end
+
+      # How many screenshots were captured but never compared. The "new"
+      # count on the end-of-run summary line, and live state: it is the
+      # number of screenshots that actually went down the no-baseline path
+      # this run, not a number derived from what the run should have done.
+      def missing_baselines_count
+        @mutex.synchronize { @missing_baselines.size }
+      end
+
+      # Remembers whether a CSS selector (`skip_area`, `crop`) found
+      # anything the one time it was resolved. Fed at the point of use, so
+      # a selector that was configured but never reached cannot be named.
+      #
+      # Two sets rather than a counter: the question the summary answers is
+      # "did this selector match ANYWHERE in the run", and under
+      # fork-parallel the hits and the misses arrive from different
+      # processes. Subtracting at read time is the only shape that survives
+      # that merge (#266).
+      def record_selector_use(selector, matched:)
+        @mutex.synchronize do
+          (matched ? @matched_selectors : @unmatched_selectors) << selector
+        end
+      end
+
+      # @api private
+      # Per-test isolation for this gem's own suite: everything {finalize!}
+      # reports, cleared in one call. One surface rather than one reset per
+      # tally, so a tally added later cannot be forgotten at the call site.
+      def reset_run_totals!
+        @mutex.synchronize do
+          @missing_baselines.clear
+          @rerecorded_baselines.clear
+          @matched_selectors.clear
+          @unmatched_selectors.clear
+          @verified = 0
+          @changed = 0
+        end
+      end
+
+      # Remembers a screenshot re-recorded by `record: :all` -- captured as
+      # the new baseline with nothing compared against it. A separate tally
+      # from {record_missing_baseline} on purpose: "there was no baseline"
+      # and "there was one and we accepted the new rendering over it" are
+      # different facts, and the summary must not claim the first when the
+      # second happened.
+      def record_rerecorded_baseline(name)
+        @mutex.synchronize { !!@rerecorded_baselines.add?(name) }
+      end
+
       # Registers a reporter for the rest of the process. The canonical way
       # in: the append happens under the mutex, so concurrent registrations
       # cannot lose one (issue #217 item 2). `reporters` stays public and
@@ -34,6 +116,18 @@ module SnapDiff
       def notify(assertions)
         return if assertions.nil? || assertions.empty?
 
+        # Warned about and skipped, never raised: `notify` runs inside every
+        # test's teardown (SnapDiff.reset), and a raise here would abort the
+        # reset before it clears the registry -- leaking one test's
+        # assertions into the next. A tally must not be able to take a
+        # user's suite down. Same contract the reporter loop below applies,
+        # and just as loud: unconditional, not DEBUG-gated.
+        begin
+          count(assertions)
+        rescue => e
+          warn "[snap_diff] Could not tally the run (#{e.class}: #{e.message})"
+        end
+
         reporters_snapshot = @mutex.synchronize { @reporters.dup }
         return if reporters_snapshot.empty?
 
@@ -44,10 +138,70 @@ module SnapDiff
         end
       end
 
-      # End-of-suite hook: finalizes each reporter and prints its summary.
-      # A raising reporter is warned about and skipped; the rest are still
-      # finalized.
+      # Tallies a finished test's assertions. An assertion with no
+      # `compare` never reached a baseline, so it is neither verified nor
+      # changed -- it is counted, if at all, by {record_missing_baseline}.
+      def count(assertions)
+        verified = 0
+        changed = 0
+
+        assertions.each do |assertion|
+          compare = assertion.compare
+          next unless compare
+
+          verified += 1
+          changed += 1 if compare.difference&.different?
+        end
+
+        @mutex.synchronize do
+          @verified += verified
+          @changed += changed
+        end
+      end
+
+      # The last line of the run, and the only place it says what it
+      # actually did:
+      #
+      #   verified -- a committed baseline existed and was compared
+      #   changed  -- of those, the ones that differed
+      #   new      -- captured but NOT compared, for want of a committed
+      #               baseline: neither a pass nor a failure
+      #
+      # Printed on every run, passing or failing, reporter or no reporter,
+      # and never nil. "N screenshots compared" counted only what it
+      # compared, so it was silent about exactly the screenshots it did not
+      # -- and silent altogether when it compared nothing, which is the one
+      # case worth shouting about.
+      def counts_summary
+        verified, changed, new_count, rerecorded = @mutex.synchronize {
+          [@verified, @changed, @missing_baselines.size, @rerecorded_baselines.size]
+        }
+        line = "[snap_diff] #{verified} verified, #{changed} changed, #{new_count} new (not verified)."
+
+        # `record: :all` (#274) accepts the rendering as the new baseline
+        # without comparing, so those are neither verified nor changed --
+        # and not "new" either, which is a different fact. Only shown when
+        # it happened; the names are on their own line below.
+        line += " #{rerecorded} re-recorded (not verified)." if rerecorded.positive?
+
+        # The shout is for an UNEXPLAINED zero -- a suite that ran no system
+        # tests, a GIT_DIR pointed at the wrong repository. Re-recording
+        # explains it, and the user asked for it: shouting there is a false
+        # alarm, and false alarms are how the real one stops being read.
+        if verified.zero? && rerecorded.zero?
+          return "#{line} NOTHING WAS VERIFIED -- no screenshot was compared to a committed baseline."
+        end
+
+        line
+      end
+
+      # End-of-suite hook: prints the counts, then finalizes each reporter
+      # and prints its summary. A raising reporter is warned about and
+      # skipped; the rest are still finalized -- and the counts line is
+      # already out, so no reporter can take it down with it.
       def finalize!
+        $stdout.puts counts_summary
+
         @mutex.synchronize { @reporters.dup }.each do |reporter|
           reporter.finalize
           if (msg = reporter.summary)
@@ -56,6 +210,178 @@ module SnapDiff
         rescue => e
           warn "[snap_diff] Reporter #{reporter.class} failed (#{e.class}: #{e.message})"
         end
+
+        if (msg = missing_baselines_summary)
+          $stdout.puts msg
+        end
+
+        if (msg = rerecorded_baselines_summary)
+          $stdout.puts msg
+        end
+
+        if (msg = never_matched_selectors_summary)
+          $stdout.puts msg
+        end
+      end
+
+      # --- fork-parallel reports (issue #258) ---------------------------
+      #
+      # Under Rails' default `parallelize(workers: N)` the tests run in
+      # forked children, and Minitest skips `after_run` in a forked child
+      # (`allow_fork = false`, minitest.rb:64/79). So every worker holds
+      # records and never finalizes, while the parent finalizes and holds
+      # none: no report, no summary line. Pass/fail is unaffected -- the
+      # failures marshal back over DRb -- which is what makes it quiet.
+      #
+      # Rails runs `run_cleanup_hooks` INSIDE the worker just before it
+      # exits (parallelization/worker.rb:31), so each worker dumps its
+      # records there, and the parent merges the fragments in the
+      # `Minitest.after_run` it does reach.
+
+      # Registers the worker-side dump with Rails, once per process.
+      #
+      # Feature-detected twice over: the gem must load without Rails at
+      # all, and with `Bundler.require` it loads BEFORE
+      # ActiveSupport::TestCase exists, so the caller retries from
+      # `ActiveSupport.on_load`.
+      #
+      # @return [Boolean] true when the hook is registered (now or already)
+      def install_parallel_hooks!
+        return true if @parallel_owner_pid
+        return false unless defined?(::ActiveSupport::Testing::Parallelization)
+
+        @parallel_owner_pid = Process.pid
+        ::ActiveSupport::Testing::Parallelization.run_cleanup_hook { dump_parallel_fragment }
+        true
+      end
+
+      # Outside the repository on purpose: it holds run-scoped scratch, and
+      # the alternative -- somewhere under `save_path` -- is a directory
+      # users `git add`.
+      #
+      # Keyed by the pid recorded at install time, which happens in the
+      # parent before any fork: `Process.pid` here would give each worker a
+      # directory of its own that the parent never looks in.
+      def parallel_fragments_dir
+        File.join(Dir.tmpdir, "snap_diff-fragments-#{@parallel_owner_pid}")
+      end
+
+      # Worker side. Writes to a `.tmp` name and renames it into place, so
+      # a worker killed mid-write leaves nothing the merge will read.
+      def dump_parallel_fragment
+        payload = {
+          "missing_baselines" => @mutex.synchronize { @missing_baselines.to_a },
+          "rerecorded_baselines" => @mutex.synchronize { @rerecorded_baselines.to_a },
+          # Both halves, not the subtraction: `img` may match in this worker
+          # and miss in the next, and only the parent that merged every
+          # fragment can tell whether it matched anywhere in the run.
+          "matched_selectors" => @mutex.synchronize { @matched_selectors.to_a },
+          "unmatched_selectors" => @mutex.synchronize { @unmatched_selectors.to_a },
+          "verified" => @verified,
+          "changed" => @changed,
+          "reporters" => @mutex.synchronize { @reporters.dup }
+            .map { |reporter| reporter.dump_state if reporter.respond_to?(:dump_state) }
+        }
+
+        FileUtils.mkdir_p(parallel_fragments_dir)
+        tmp = File.join(parallel_fragments_dir, "#{Process.pid}.json.tmp")
+        File.write(tmp, JSON.generate(payload))
+        File.rename(tmp, File.join(parallel_fragments_dir, "#{Process.pid}.json"))
+      end
+
+      # Parent side, called just before {finalize!}. A no-op when nothing
+      # forked, which is what keeps serial and `with: :threads` -- both of
+      # which record in the process that finalizes -- exactly as they were.
+      #
+      # Reporters are matched by position: registration happens at require
+      # time, before any fork, so the list is identical in every process.
+      def merge_parallel_fragments!
+        return unless @parallel_owner_pid == Process.pid
+
+        Dir[File.join(parallel_fragments_dir, "*.json")].sort.each do |fragment|
+          payload = JSON.parse(File.read(fragment))
+
+          # Only "missing_baselines" is read without a default: it is the
+          # one key every version of this fragment has ever written. Every
+          # key added since is `fetch`ed with one, because the fragments
+          # directory is keyed by pid under the system temp dir -- a
+          # recycled pid can hand this merge a fragment left behind by an
+          # older version of the gem, and a partial payload must not take
+          # the run down.
+          @mutex.synchronize do
+            payload["missing_baselines"].each { |name| @missing_baselines << name }
+            payload.fetch("rerecorded_baselines", []).each { |name| @rerecorded_baselines << name }
+            payload.fetch("matched_selectors", []).each { |selector| @matched_selectors << selector }
+            payload.fetch("unmatched_selectors", []).each { |selector| @unmatched_selectors << selector }
+            @verified += payload.fetch("verified", 0)
+            @changed += payload.fetch("changed", 0)
+          end
+
+          reporters_snapshot = @mutex.synchronize { @reporters.dup }
+          payload["reporters"].each_with_index do |state, index|
+            reporter = reporters_snapshot[index]
+            reporter.merge_state!(state) if state && reporter.respond_to?(:merge_state!)
+          end
+        end
+
+        FileUtils.rm_rf(parallel_fragments_dir)
+      end
+
+      # The reporters' summary line carries the COUNT of screenshots that
+      # were captured without a committed baseline ("N new (not
+      # verified)"); this line names them, so the next thing the reader
+      # does is `git add` the right files.
+      #
+      # @return [String, nil] nil when every screenshot had a baseline
+      def missing_baselines_summary
+        names = @mutex.synchronize { @missing_baselines.to_a }
+        return if names.empty?
+
+        label = (names.size == 1) ? "1 screenshot" : "#{names.size} screenshots"
+        "[snap_diff] #{label} had no committed baseline and #{(names.size == 1) ? "was" : "were"} NOT compared: " \
+          "#{names.join(", ")}. Commit the captured file(s) to enable comparison."
+      end
+
+      # The other half of "nothing was compared", and the louder one:
+      # `record: :all` accepts whatever the page rendered as the new
+      # baseline. Names the screenshots that really went down that path this
+      # run, so `git add` lands on the right files -- and so nobody commits
+      # forty accepted regressions without being told they were accepted.
+      #
+      # @return [String, nil] nil when nothing was re-recorded
+      def rerecorded_baselines_summary
+        names = @mutex.synchronize { @rerecorded_baselines.to_a }
+        return if names.empty?
+
+        label = (names.size == 1) ? "1 screenshot" : "#{names.size} screenshots"
+        "[snap_diff] record: :all re-recorded #{label} WITHOUT comparing: #{names.join(", ")}. " \
+          "Review the result before committing -- an unintended change is accepted just as silently."
+      end
+
+      # The selectors that matched nothing in EVERY screenshot of the run.
+      #
+      # Since #272 a `skip_area` selector is resolved without waiting: it
+      # masks what is on the page at assertion time, and one that matches
+      # nothing produces an empty mask -- the unstable region is compared
+      # and the test flakes, silently. #275 declined to warn per screenshot
+      # because the gem cannot tell a typo from a legitimately image-less
+      # page, and the legitimate case would fire on every screenshot.
+      #
+      # A run-level tally has no such problem. A selector that matched
+      # SOMEWHERE is doing its job and is never mentioned; one that matched
+      # NOWHERE, all run, is a typo or a stale selector with high
+      # probability. Silent when the set is empty, on purpose: a line that
+      # prints on every run is a line users learn to skip.
+      #
+      # @return [String, nil] nil when every selector used matched somewhere
+      def never_matched_selectors_summary
+        names = @mutex.synchronize { (@unmatched_selectors - @matched_selectors).to_a }
+        return if names.empty?
+
+        label = (names.size == 1) ? "1 selector" : "#{names.size} selectors"
+        "[snap_diff] #{label} never matched anything in this run: " \
+          "#{names.map(&:inspect).join(", ")}. " \
+          "A selector that matches nothing masks nothing -- check for a typo or a stale selector."
       end
     end
   end

data/lib/snap_diff/screenshot_assertion.rb

@@ -33,7 +33,7 @@ module SnapDiff
   #
   # @return [String, nil] the pending message, or nil when there is nothing to report
   def self.pending_screenshots_message
-    return unless ::Capybara::Screenshot::Diff.pending_if_new && session.new_screenshots_present?
+    return unless SnapDiff.config.pending_if_new && session.new_screenshots_present?
 
     "No baseline for: #{session.new_screenshots.join(", ")}. Commit the captured screenshots to record them."
   end
@@ -96,7 +96,7 @@ module SnapDiff
     # @return [Array, nil] Returns an array of error messages if there are screenshot differences, otherwise nil.
     # @note This method is typically called at the end of a test to assert all screenshots are as expected.
     def self.verify_screenshots!(screenshots)
-      return unless ::Capybara::Screenshot.active? && ::Capybara::Screenshot::Diff.fail_on_difference
+      return unless SnapDiff.config.active? && SnapDiff.config.fail_on_difference
 
       test_screenshot_errors = screenshots.map do |assertion|
         assertion.validate
@@ -106,21 +106,6 @@ module SnapDiff
 
       test_screenshot_errors.empty? ? nil : test_screenshot_errors
     end
-
-    # Asserts that an image has not changed compared to its baseline.
-    #
-    # @param backtrace [Array(String)] The caller context, used for error reporting.
-    # @param name [String] The name of the screenshot being verified.
-    # @param comparison [Object] The comparison object containing the result and details of the comparison.
-    # @return [String, nil] Returns an error message if the screenshot differs from the baseline, otherwise nil.
-    # @note Legacy entry point; delegates to the instance verify flow
-    #   (pure question + explicit #archive_baseline! on pass).
-    def self.assert_image_not_changed(backtrace, name, comparison)
-      assertion = new(name)
-      assertion.caller = backtrace
-      assertion.compare = comparison
-      assertion.validate
-    end
   end
 
   class AssertionRegistry
@@ -129,9 +114,18 @@ module SnapDiff
     def initialize
       @assertions = []
       @new_screenshots = []
+      @checked_out_baselines = Set.new
       @screenshot_namer = SnapDiff::ScreenshotNamer.new
     end
 
+    # Called by Snap#checkout_base_screenshot when git really handed us a
+    # baseline. Same thread as the reading below -- a test's checkout and
+    # its comparison happen in one call stack -- so no synchronization is
+    # needed or wanted here.
+    def record_baseline_checkout(name)
+      @checked_out_baselines << name
+    end
+
     def add_assertion(assertion)
       return unless assertion&.compare
 
@@ -144,7 +138,21 @@ module SnapDiff
       !@assertions.empty?
     end
 
+    # "No baseline exists" reaches here for two very different reasons. One
+    # is legitimate and warned about: nothing was ever committed for this
+    # name. The other is impossible in a correct run -- git gave us a
+    # baseline moments ago and it is gone now -- and used to be recorded as
+    # if it were the first, leaving the test green having compared nothing.
+    # That is the silently-wrong case measured in #217: two concurrent
+    # tests asserting the SAME name, where one's archive_baseline! moves
+    # the baseline the other just checked out.
     def record_new_screenshot(name)
+      raise SnapDiff::Error.new(<<~ERROR.chomp, caller) if @checked_out_baselines.include?(name)
+        The baseline for '#{name}' was checked out and then disappeared before it could be compared -- nothing was verified.
+        Every artifact path derives from the screenshot name alone, so two tests asserting '#{name}' at the same time race on one set of files: the one that finishes first archives the baseline the other is still using.
+        Give those screenshots distinct names, or do not run them concurrently.
+      ERROR
+
       @new_screenshots.push(name)
     end
 
@@ -153,14 +161,10 @@ module SnapDiff
     end
 
     def verify(screenshots = assertions)
-      return unless ::Capybara::Screenshot.active? && ::Capybara::Screenshot::Diff.fail_on_difference
-
-      failed_screenshot = failed_assertions.first
       result = ScreenshotAssertion.verify_screenshots!(screenshots)
+      return unless result
 
-      if result
-        raise SnapDiff::ExpectationNotMet.new(result.join("\n\n"), failed_screenshot.caller)
-      end
+      raise SnapDiff::ExpectationNotMet.new(result.join("\n\n"), failed_assertions.first.caller)
     end
 
     def failed_assertions
@@ -170,6 +174,7 @@ module SnapDiff
     def reset
       @assertions.clear
       @new_screenshots.clear
+      @checked_out_baselines.clear
       @screenshot_namer = SnapDiff::ScreenshotNamer.new
     end
   end