simplecov 1.0.2 → 1.1.0

data/lib/simplecov/combine/coverage_accumulator.rb

@@ -0,0 +1,268 @@
+# frozen_string_literal: true
+
+require_relative "branches_combiner"
+require_relative "lines_combiner"
+require_relative "methods_combiner"
+
+module SimpleCov
+  module Combine
+    #
+    # Folds any number of resultsets' coverage into one, absorbing them one
+    # at a time.
+    #
+    # This replaces a pairwise `reduce` over the resultsets, which rebuilt
+    # the entire accumulated structure on every one of its N-1 steps: a
+    # fresh outer file hash, a fresh lines array for every file, and a fresh
+    # branch / method table for every file whose keys were re-interned from
+    # their tuples each time. On a 160-worker run over ~1,800 files that is
+    # ~290,000 whole-file rebuilds to produce ~1,800 files of output.
+    #
+    # The accumulated side is private to the fold, so it can be updated in
+    # place instead, and the interned tables only have to be turned back into
+    # tuple-keyed hashes once, at the end.
+    #
+    # Resultsets are absorbed one at a time rather than taken as a list, so
+    # `ResultMerger.merge_results` can keep reading and discarding them one
+    # file at a time — reading 100s of CI jobs' worth of coverage into memory
+    # at once is what that method is careful not to do.
+    #
+    class CoverageAccumulator
+      # A file some process actually loaded has at least one executed line;
+      # a simulated (never-loaded) file's lines are all `nil` or `0`. This is
+      # the signal the merge reconciles synthesized tuples on, and the one
+      # `ResultMerger` re-derives a merged result's not-loaded set from, so it
+      # lives here rather than being spelled out at each site.
+      #
+      # `Array()` plus the `Numeric` test rather than a bare
+      # `any?(&:positive?)` because this reads straight off a parsed
+      # resultset, which is external input: a file written by another
+      # SimpleCov version, or hand-edited, can carry anything under
+      # "lines" — a Hash coerces to pairs, a String to itself. Keeping a
+      # malformed entry to a wrong answer instead of a NoMethodError out
+      # of the middle of a merge is the point.
+      def self.executed?(lines)
+        counts = Array(lines) #: Array[untyped]
+        counts.any? { |count| count.is_a?(Numeric) && count.positive? }
+      end
+
+      #
+      # Folds `[command_names, coverage]` pairs into one merged coverage.
+      #
+      # `pairs` is only ever iterated, so a caller that reads resultsets off
+      # disk can hand in a lazy enumerable and never hold more than one in
+      # memory — which is what `ResultMerger.merge_results` is careful about.
+      #
+      # @return [Array] the concatenated command names and the merged coverage
+      #
+      def self.fold(pairs)
+        accumulator = new
+        command_names = [] #: Array[String]
+
+        pairs.each do |names, coverage|
+          command_names.concat(names)
+          accumulator.absorb(coverage)
+        end
+
+        [command_names, accumulator.result]
+      end
+
+      def initialize
+        @files = {} #: Hash[String, untyped]
+        @absorbed = false
+        # Whether a criterion is enabled can't change mid-fold, so read it once
+        # here rather than once per file. `branch_coverage?` reaches
+        # `Coverage.supported?`, and a large parallel run merges thousands of
+        # files.
+        @branch_coverage = SimpleCov.branch_coverage?
+        @method_coverage = SimpleCov.method_coverage?
+      end
+
+      #
+      # Folds one resultset's coverage (filename => per-file coverage) into
+      # the accumulator. A `nil` coverage is ignored — a resultset that
+      # carried nothing contributes nothing.
+      #
+      # @return [CoverageAccumulator] self, so absorbs can be chained
+      #
+      def absorb(coverage)
+        return self unless coverage
+
+        @absorbed = true
+        coverage.each do |filename, file_coverage|
+          @files[filename] = merge_file(@files[filename], file_coverage)
+        end
+
+        self
+      end
+
+      #
+      # The merged coverage, or `nil` when nothing was absorbed at all — the
+      # caller needs to tell "no results" apart from "results that cover
+      # nothing", and only the former means there is no report to build.
+      #
+      # A file only one resultset carried is returned exactly as it came in,
+      # untouched: with nothing to merge it into, copying it would only cost
+      # memory.
+      #
+      # @return [Hash, nil]
+      #
+      def result
+        return nil unless @absorbed
+
+        @files.transform_values do |entry|
+          entry.is_a?(MergedFile) ? entry.to_h : entry
+        end
+      end
+
+    private
+
+      # The first sighting of a file is kept as-is; the second promotes it to
+      # a `MergedFile` that owns its state, and every later one folds into
+      # that same object.
+      def merge_file(existing, file_coverage)
+        case existing
+        when nil then file_coverage
+        when MergedFile then existing.absorb(file_coverage)
+        else MergedFile.new(existing, branches: @branch_coverage, methods: @method_coverage).absorb(file_coverage)
+        end
+      end
+
+      #
+      # One file's accumulated coverage, against state this object owns
+      # outright rather than rebuilt for each merged pair.
+      #
+      class MergedFile
+        # A criterion is carried for this file when the data carries it, not
+        # when this process happens to measure it. A merge runs on behalf of
+        # the processes that produced the resultsets and does not necessarily
+        # share their configuration: `simplecov merge` never ran SimpleCov.start
+        # at all, and dropping a table it did not ask for would lose branch data
+        # the producers did measure. A nil table means nobody measured that
+        # criterion; an empty one means it was measured and the file has none.
+        def initialize(coverage, branches:, methods:)
+          @branch_coverage = branches
+          @method_coverage = methods
+          @lines = coverage["lines"]&.dup
+          # Branch coverage always reports a table, even an empty one, so
+          # `SourceFile` can tell "no branches in this file" from "branch
+          # coverage was off". Method coverage stays `nil` until some resultset
+          # actually carries methods.
+          @branches = BranchesCombiner.absorb({}, coverage["branches"]) if branches || coverage["branches"]
+          @methods = MethodsCombiner.absorb(nil, coverage["methods"]) if methods || coverage["methods"]
+        end
+
+        #
+        # Folds one more resultset's coverage for this file in.
+        #
+        # @return [MergedFile] self
+        #
+        def absorb(coverage)
+          reconcile_synthesized(coverage)
+          @lines = LinesCombiner.merge_into(@lines, coverage["lines"])
+          self
+        end
+
+        def to_h
+          merged = {"lines" => @lines} #: Hash[String, untyped]
+          merged["branches"] = BranchesCombiner.materialize(@branches) if @branches
+          merged["methods"] = MethodsCombiner.materialize(@methods) if @methods
+          merged
+        end
+
+      private
+
+        def new_table
+          {} #: Hash[untyped, untyped]
+        end
+
+        # When exactly one side of the merge was actually executed, its branch
+        # and method tuples are authoritative and the other side's are
+        # dropped. A simulated entry (SimulateCoverage backfills
+        # tracked-but-unloaded files) synthesizes those tuples statically, so
+        # a location that drifts from what Coverage emits would otherwise be
+        # unioned in by position and survive as a phantom, permanently-missed
+        # branch (see #1233). This contains any such drift to denominator
+        # inflation for files no process loaded, rather than a false miss on a
+        # covered file. Lines are never dropped: a simulated file's line shape
+        # is correct and carries the unloaded-file denominator (#1059).
+        #
+        # The accumulated side's executed-ness is re-read on every absorb
+        # because folding in an executed resultset can flip it, exactly as it
+        # would have flipped between two steps of the pairwise fold.
+        def reconcile_synthesized(coverage)
+          return absorb_tuples(coverage) unless judgeable?(@lines) && judgeable?(coverage["lines"])
+
+          accumulated_executed = executed?(@lines)
+          incoming_executed = executed?(coverage["lines"])
+
+          if accumulated_executed == incoming_executed
+            absorb_tuples(coverage)
+          elsif incoming_executed
+            replace_tuples(coverage)
+          else
+            drop_incoming_tuples(coverage)
+          end
+        end
+
+        # Both sides agree on whether they ran: union their tuples. A criterion
+        # only the incoming side carries comes into play here, because its
+        # tuples survive. It deliberately does not come into play on the path
+        # that drops them: promoting there would advertise an empty table
+        # sourced from data this merge just discarded, which claims the file
+        # has no branches when what is true is that nobody measured them.
+        def absorb_tuples(coverage)
+          @branches = BranchesCombiner.absorb(@branches || new_table, coverage["branches"]) if
+            @branches || coverage["branches"]
+          @methods = MethodsCombiner.absorb(@methods, coverage["methods"]) if @methods || coverage["methods"]
+        end
+
+        # Only the accumulated side ran, so the incoming tuples are synthesized
+        # and contribute nothing. When the discarded side carried a methods
+        # table, a measuring process still gets one — the empty table stands in
+        # for the blanked side, which is what a pair against a blank one used
+        # to produce for a file that has no methods yet. When no side has
+        # carried methods at all, the table stays nil (the rule `initialize`
+        # documents), the same answer the union path gives.
+        def drop_incoming_tuples(coverage)
+          @methods ||= {} if @method_coverage && coverage["methods"] #: Hash[untyped, untyped]
+        end
+
+        # Only the incoming side ran, so what's accumulated is synthesized:
+        # start its tuples over from the executed side's. The authoritative
+        # side decides which criteria the file carries, not just their tuples.
+        # Keeping a table in play that only the dropped side carried would make
+        # the answer depend on which side was seen first.
+        def replace_tuples(coverage)
+          @branches = authoritative_table(BranchesCombiner, coverage["branches"], @branch_coverage)
+          # Methods stay nil until some resultset carries them (see
+          # `initialize`), so a measuring process keeps an empty table only
+          # when one did.
+          @methods = authoritative_table(MethodsCombiner, coverage["methods"], @method_coverage && !@methods.nil?)
+        end
+
+        # The executed side's tuples when it has them. Otherwise `keep_empty`
+        # decides whether the criterion survives as an empty table, which is
+        # what keeps `SourceFile` able to tell "no branches here" from
+        # "branches were not measured".
+        def authoritative_table(combiner, table, keep_empty)
+          return combiner.absorb(new_table, table) if table
+
+          keep_empty ? new_table : nil
+        end
+
+        # Whether a side can be judged at all. Absent lines do not mean the side
+        # never ran: a branch-only or method-only run omits them even for the
+        # files it loaded, so treating that as synthesized would discard real
+        # tuples. Unknown means union, which is already what happens when
+        # neither side carries lines.
+        def judgeable?(lines)
+          !Array(lines).empty?
+        end
+
+        def executed?(lines)
+          CoverageAccumulator.executed?(lines)
+        end
+      end
+    end
+  end
+end

data/lib/simplecov/combine/identity_interner.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module SimpleCov
+  module Combine
+    #
+    # Builds the raw-key => interned-identity caches the tuple combiners keep.
+    #
+    # The cache maps a raw key to its interned identity on first sight and
+    # keeps it: the pairwise fold would otherwise re-derive the accumulator's
+    # identities on every one of its merges, always to the same answer.
+    #
+    # Identities exist only to be `combine`'s hash keys, and `Array#hash` is
+    # not memoized, so keying on the identity tuple itself would rehash its
+    # elements for every key of both sides of every merge. An Integer hashes
+    # as an immediate. Two keys share an id exactly when their identities are
+    # equal, so the string and array forms of one key still merge together.
+    #
+    module IdentityInterner
+    module_function
+
+      def build
+        ids = {} #: Hash[::Array[untyped], Integer]
+        Hash.new do |cache, key|
+          identity = yield(key)
+          cache[key] = ids[identity] ||= ids.size
+        end
+      end
+    end
+  end
+end

data/lib/simplecov/combine/interned_counts.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module SimpleCov
+  module Combine
+    # The inner merge loop the branches and methods combiners share:
+    # fold `source`'s tuple => count pairs into `target`, an interned
+    # table keyed by each tuple's merge identity and holding
+    # `[raw_key, count]` pairs. The first-seen raw key is kept for
+    # display; counts sum. `target` and its pairs are always built by
+    # this loop, so updating them in place can't reach data a caller
+    # still holds.
+    module InternedCounts
+    module_function
+
+      # @return [Hash] `target`
+      def absorb_counts(target, source, identities)
+        source.each do |key, count|
+          interned = identities[key]
+          entry = target[interned]
+          if entry
+            entry[1] += count
+          else
+            target[interned] = [key, count]
+          end
+        end
+        target
+      end
+    end
+  end
+end

data/lib/simplecov/combine/lines_combiner.rb

@@ -5,40 +5,68 @@ module SimpleCov
     #
     # Combine two different lines coverage results on same file
     #
-    # Should be called through `SimpleCov.combine`.
+    # Should be called through `CoverageAccumulator`.
     module LinesCombiner
     module_function
 
-      # Build a fresh array sized to the longer input. The previous
-      # implementation mutated whichever input was longer in place,
-      # which could surprise callers holding a reference to that array
-      # (e.g. the parsed `coverage` key of a resultset hash being
-      # passed into a second merge).
-      def combine(coverage_a, coverage_b)
-        size = [coverage_a.size, coverage_b.size].max
-        Array.new(size) { |i| merge_line_coverage(coverage_a[i], coverage_b[i]) }
-      end
-
+      # Folds `source` into `target` rather than building a third array.
+      # Only for a `target` the caller owns outright: a caller holding a
+      # reference to it (e.g. the parsed `coverage` key of a resultset
+      # hash being passed into a second merge) would see it change.
+      # `source` is never touched.
+      #
       # Two runs of the same source file should agree on which lines
       # are coverage-relevant (`nil` for comments / whitespace, `0`+
       # for executable). When they don't, treat "relevant on either
       # side" as relevant rather than masking a real `0` as `nil`,
       # which would silently drop an uncovered line from the
-      # denominator and inflate the percentage.
-      #
-      # Logic:
+      # denominator and inflate the percentage:
       #
       # => nil + nil = nil
       # => nil + int = int (preserves a relevant-but-uncovered 0)
       # => int + int = int (sum)
       #
-      # @param [Integer || nil] first_val
-      # @param [Integer || nil] second_val
-      # @return [Integer || nil]
-      def merge_line_coverage(first_val, second_val)
-        return nil if first_val.nil? && second_val.nil?
+      # The loop is written out rather than dispatching a block per
+      # element because this is the innermost loop of a merge: a
+      # 160-worker run over ~1,800 files folds tens of millions of line
+      # counts through it, and a call per count is a measurable share of
+      # that.
+      #
+      # @return [Array] the array to keep — `target` itself once there is one
+      def merge_into(target, source)
+        return target unless source
+        return source.dup unless target
+
+        size = source.size
+        target.concat(Array.new(size - target.size)) if target.size < size
+        sum_into(target, source, size)
+      end
+
+      # Split out only to keep `merge_into` short; it is the same loop.
+      #
+      # The `Integer` tests coerce malformed counts (a "3" written by a
+      # hand-edited or foreign resultset, a JSON float) with `to_i`, so
+      # external input merges to a wrong answer instead of raising out of
+      # the middle of a merge. An `is_a?` per count is far cheaper than
+      # the block dispatch this loop exists to avoid, and the well-formed
+      # fast path stays branch-for-branch what it was.
+      def sum_into(target, source, size)
+        index = 0
+        while index < size
+          if (value = source[index])
+            value = value.to_i unless value.is_a?(Integer)
+            existing = target[index]
+            target[index] = existing.is_a?(Integer) ? existing + value : coerce_add(existing, value)
+          end
+          index += 1
+        end
+        target
+      end
 
-        first_val.to_i + second_val.to_i
+      # The rare arm: `existing` is nil (line not yet relevant in the
+      # target) or malformed external input.
+      def coerce_add(existing, value)
+        existing.nil? ? value : existing.to_i + value
       end
     end
   end

data/lib/simplecov/combine/methods_combiner.rb

@@ -1,13 +1,15 @@
 # frozen_string_literal: true
 
 require_relative "../source_file/ruby_data_parser"
+require_relative "identity_interner"
+require_relative "interned_counts"
 
 module SimpleCov
   module Combine
     #
     # Combine different method coverage results on a single file.
     #
-    # Should be called through `SimpleCov.combine`.
+    # Should be called through `CoverageAccumulator`.
     module MethodsCombiner
     module_function
 
@@ -16,33 +18,56 @@ module SimpleCov
       #
       # Method coverage maps `[class, name, start_line, start_col, end_line,
       # end_col]` keys to hit counts. Keys are matched on their SOURCE
-      # identity — (name, location), ignoring the class element — because
-      # Ruby records one entry per receiver: the same `define_method` block
-      # defined onto different classes in different processes arrives with
-      # different (normalized) receivers for the same source method, and
-      # matching on the full key would keep both, letting a never-called
-      # receiver's 0 shadow a covered method after merge (issue #1234).
-      # Combining sums the hit counts for matching methods and preserves
-      # methods that only appear in one result.
+      # identity — the location, ignoring the class and name elements —
+      # because Ruby records one entry per defined method: the same
+      # `define_method` block defined onto different classes, or under
+      # different names, in different processes arrives with different
+      # receivers or names for the same source method, and matching on the
+      # full key would keep both, letting a never-called copy's 0 shadow a
+      # covered method after merge (issue #1234). Combining sums the hit
+      # counts for matching methods and preserves methods that only appear
+      # in one result.
       #
       # @return [Hash]
       #
       def combine(coverage_a, coverage_b)
-        merged = {} #: Hash[untyped, [untyped, Integer]]
-        [coverage_a, coverage_b].each_with_object(merged) do |coverage, memo|
-          coverage.each do |key, count|
-            method_key = source_identity(key)
-            retained_key, existing = memo[method_key] || [key, 0]
-            memo[method_key] = [retained_key, existing + count]
-          end
-        end
-
-        merged.values.to_h
+        merged = absorb({}, coverage_a) #: Hash[untyped, [untyped, Integer]]
+        materialize(absorb(merged, coverage_b))
+      end
+
+      # Folds `coverage` into `target`, an interned table keyed by method
+      # source identity. See `BranchesCombiner.absorb` for why a fold keeps
+      # its accumulator interned.
+      #
+      # A `nil` `target` stays `nil` until some resultset actually carries
+      # methods, so a merge can tell "no method data anywhere" apart from
+      # "method data that covers nothing".
+      #
+      # @return [Hash, nil] `target`, created on the first coverage seen
+      def absorb(target, coverage)
+        return target unless coverage
+
+        target ||= {} #: Hash[untyped, [untyped, Integer]]
+        InternedCounts.absorb_counts(target, coverage, identities)
+      end
+
+      # Turns an interned table back into the tuple-keyed hash the rest of
+      # SimpleCov reads. Done once, at the end of a fold.
+      #
+      # @return [Hash]
+      def materialize(target)
+        target.values.to_h
+      end
+
+      # Bounded by the project's method count, like `RubyDataParser`'s parse
+      # cache.
+      def identities
+        @identities ||= IdentityInterner.build { |key| source_identity(key) }
       end
 
       def source_identity(key)
-        _class_name, *identity = SourceFile::RubyDataParser.call(key)
-        identity
+        _class_name, _method_name, *location = SourceFile::RubyDataParser.call(key)
+        location.freeze
       end
     end
   end

data/lib/simplecov/combine/results_combiner.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative "coverage_accumulator"
+
 module SimpleCov
   module Combine
     # There might be reports from different kinds of tests,
@@ -12,50 +14,23 @@ module SimpleCov
 
       #
       # Combine process explanation
-      # => ResultCombiner: define all present files between results and start combine on file level.
-      # ==> FileCombiner: collect result of next combine levels lines, branches, and methods.
+      # => ResultsCombiner: hand every result to one accumulator.
+      # ==> CoverageAccumulator::MergedFile: hold one file's merged coverage.
       # ===> LinesCombiner: combine lines results.
       # ===> BranchesCombiner: combine branches results.
       # ===> MethodsCombiner: combine methods results.
       #
-      # @return [Hash]
-      #
-      def combine(*results)
-        initial = {} #: Hash[untyped, untyped]
-        results.reduce(initial) do |combined_results, next_result|
-          combine_result_sets(combined_results, next_result)
-        end
-      end
-
-      #
-      # Manage combining results on files level
-      #
-      # @param [Hash] combined_results
-      # @param [Hash] result
+      # Callers that read their results one at a time (`ResultMerger`, to
+      # keep a big CI run's resultsets out of memory all at once) should
+      # drive a `CoverageAccumulator` directly instead of collecting the
+      # results to pass here.
       #
       # @return [Hash]
       #
-      def combine_result_sets(combined_results, result)
-        results_files = combined_results.keys | result.keys
-
-        results_files.to_h do |file_name|
-          [file_name, combine_file_coverage(
-            combined_results[file_name],
-            result[file_name]
-          )]
-        end
-      end
-
-      #
-      # Combine two files coverage results
-      #
-      # @param [Hash] coverage_a
-      # @param [Hash] coverage_b
-      #
-      # @return [Hash]
-      #
-      def combine_file_coverage(coverage_a, coverage_b)
-        Combine.combine(Combine::FilesCombiner, coverage_a, coverage_b)
+      def combine(*results)
+        accumulator = CoverageAccumulator.new
+        results.each { |result| accumulator.absorb(result) }
+        accumulator.result || {}
       end
     end
   end

data/lib/simplecov/combine.rb

@@ -1,30 +1,12 @@
 # frozen_string_literal: true
 
 module SimpleCov
-  # Functionally for combining coverage results
+  # Namespace for the pieces that combine coverage results.
+  #
+  # `CoverageAccumulator` is the entry point: it folds any number of
+  # resultsets together, delegating one file's lines, branches and methods to
+  # the combiner modules alongside it.
   #
   module Combine
-  module_function
-
-    #
-    # Combine two coverage based on the given combiner_module.
-    #
-    # Combiners should always be called through this interface,
-    # as it takes care of short-circuiting of one of the coverages is nil.
-    #
-    # @return [Hash]
-    def combine(combiner_module, coverage_a, coverage_b)
-      return existing_coverage(coverage_a, coverage_b) if empty_coverage?(coverage_a, coverage_b)
-
-      combiner_module.combine(coverage_a, coverage_b)
-    end
-
-    def empty_coverage?(coverage_a, coverage_b)
-      !(coverage_a && coverage_b)
-    end
-
-    def existing_coverage(coverage_a, coverage_b)
-      coverage_a || coverage_b
-    end
   end
 end