simplecov 1.0.3 → 1.1.0

data/lib/simplecov/result_processing.rb

@@ -13,13 +13,27 @@ module SimpleCov
     # so all results in all files specified will be merged. Pass
     # `ignore_timeout: false` to honor it.
     #
-    def collate(result_filenames, profile = nil, ignore_timeout: true, &)
+    # `processes:` above 1 fans the merge out across that many forked worker
+    # processes, for a collate big enough that reading and parsing the
+    # resultsets dominates it. The report is identical either way, not merely
+    # equivalent — the workers visit the resultsets in the order the
+    # single-process merge visits them — and one process never forks at all.
+    # The count is deliberately not clamped to the machine's core count nor
+    # gated on some minimum number of resultsets: how many processes a collate
+    # job can afford is the caller's call, not SimpleCov's. Anything below 1 is
+    # taken as 1, so a count computed from arithmetic that can reach zero needs
+    # no guarding. Defaults to `SIMPLECOV_CONCURRENCY`, or 1 when that is
+    # unset. See `SimpleCov::ParallelResultMerger`.
+    #
+    def collate(result_filenames, profile = nil, processes: ENV.fetch("SIMPLECOV_CONCURRENCY", 1).to_i,
+                ignore_timeout: true, &)
       raise ArgumentError, "There are no reports to be merged" if result_filenames.empty?
 
       initial_setup(profile, &)
 
       # Use the ResultMerger to produce a single, merged result, ready to use.
-      @result = ResultMerger.merge_and_store(*result_filenames, ignore_timeout: ignore_timeout)
+      @result = ParallelResultMerger.merge_and_store(*result_filenames, processes: [1, processes].max,
+                                                                        ignore_timeout: ignore_timeout)
 
       @collating_result = true
       run_exit_tasks!
@@ -28,24 +42,22 @@ module SimpleCov
     end
 
     #
-    # Returns the result for the current coverage run, merging it across test suites
-    # from cache using SimpleCov::ResultMerger if use_merging is activated (default)
+    # Returns the result for the current coverage run. With merging enabled,
+    # every process stores its own slice, but only the finalization owner reads
+    # and caches the merged result.
     #
     def result
       return @result if result?
 
       use_merging = merging
 
-      # Collect our coverage result. When merging is off there is no merge
-      # step, so this per-process result is the final one and reports any
-      # dropped source files; otherwise the merged result does the reporting.
-      process_coverage_result(report: !use_merging) if defined?(Coverage) && Coverage.running?
+      collect_own_coverage(standalone: !use_merging)
 
       # If we're using merging of results, store the current result
       # first (if there is one), then merge the results and return those
       if use_merging
         SimpleCov::ResultMerger.store_result(@result) if result?
-        return @result unless finalize_merge?
+        return @result unless merge_finalization_owner?
 
         wait_for_other_processes
         @result = SimpleCov::ResultMerger.merged_result
@@ -54,6 +66,16 @@ module SimpleCov
       @result
     end
 
+    # Build this process's slice of the coverage result. `standalone` is true
+    # when no merge step follows, which makes this the final result: it is the
+    # one that reports dropped source files, and the one that injects unloaded
+    # files. When a merge does follow, both jobs belong to the merged result.
+    def collect_own_coverage(standalone:)
+      return unless defined?(Coverage) && Coverage.running?
+
+      process_coverage_result(report: standalone, inject_unloaded: standalone)
+    end
+
     # Returns nil if the result has not been computed, otherwise the result.
     def result?
       defined?(@result) && @result
@@ -79,7 +101,7 @@ module SimpleCov
     # construction). Files matched by no group fall into the implicit
     # "Ungrouped" bucket.
     def grouped(files, groups: SimpleCov.groups)
-      return {} if groups.empty?
+      return {} if GroupNames.validate!(groups.keys).empty?
 
       grouped = groups.transform_values do |filter|
         SimpleCov::FileList.new(files.select { |source_file| filter.matches?(source_file) })
@@ -87,7 +109,7 @@ module SimpleCov
 
       in_group  = grouped_file_set(grouped)
       ungrouped = files.reject { |source_file| in_group.include?(source_file) }
-      grouped["Ungrouped"] = SimpleCov::FileList.new(ungrouped) if ungrouped.any?
+      grouped[GroupNames::UNGROUPED] = SimpleCov::FileList.new(ungrouped) if ungrouped.any?
 
       grouped
     end
@@ -115,6 +137,29 @@ module SimpleCov
       coverage.floor(2)
     end
 
+    # Add simulated coverage for each of `candidate_paths` the result doesn't
+    # already carry, and return it with the set of paths added.
+    #
+    # @api private — the seam `SimpleCov::ResultMerger` injects through. Public
+    # only because the merge runs in another object, on behalf of processes
+    # whose configuration it may not share, so it supplies the paths itself.
+    def inject_unloaded_files(result, candidate_paths, synthesize: nil, lines: nil)
+      return [result, Set.new] if candidate_paths.empty?
+
+      # Synthesizing branch and method tuples means parsing every tracked file
+      # that wasn't loaded, which is about half the cost of simulating one.
+      # Nothing reads those tuples when neither criterion is enabled —
+      # `Combine::CoverageAccumulator` only combines them per criterion, and the
+      # statistics drop them the same way — so skip the parse. The same goes
+      # for line data, which a branch-only or method-only run neither reports
+      # nor receives from `Coverage` for the files it loaded. See #1250.
+      UnloadedFileInjector.call(
+        result, candidate_paths,
+        synthesize: synthesize.nil? ? branch_coverage? || method_coverage? : synthesize,
+        lines: lines.nil? ? line_coverage? : lines
+      )
+    end
+
   private
 
     def initial_setup(profile, &block)
@@ -126,14 +171,15 @@ module SimpleCov
       grouped.values.each_with_object(Set.new) { |file_list, set| set.merge(file_list) }
     end
 
-    # Finds files that were to be tracked but were not loaded, and
-    # initializes their line-by-line coverage to zero (or nil for
-    # comments / whitespace).
-    def add_not_loaded_files(result)
-      globs = unloaded_file_discovery_globs
-      return [result, Set.new] if globs.empty?
-
-      inject_unloaded_files(result.dup, discover_unloaded_paths(globs))
+    # Every path this process was told to track, whether or not it loaded them.
+    # Recorded on the result (and from there into the resultset) so that a merge
+    # in another process can inject the ones nobody loaded without needing this
+    # process's `cover` / `track_files` configuration. A standalone `collate`
+    # never ran `SimpleCov.start` and so has none of its own. See #1250.
+    def tracked_file_paths
+      UnloadedFileInjector.discover(
+        unloaded_file_discovery_globs, root: root, reject: filters.select(&:path_only?)
+      )
     end
 
     # Globs to expand on disk when injecting unloaded files into the
@@ -144,36 +190,29 @@ module SimpleCov
       [tracked_files, *cover_globs].compact
     end
 
-    # Expand the given globs relative to SimpleCov.root, not Dir.pwd —
-    # test runners that chdir (or CI scripts that invoke the suite
-    # from a subdir) would otherwise silently miss the unloaded-file
-    # injection and produce a different file set per environment. See
-    # issue #1106.
-    def discover_unloaded_paths(globs)
-      globs.flat_map { |glob| Dir.glob(glob, base: root) }.uniq
-    end
-
-    def inject_unloaded_files(result, candidate_paths)
-      not_loaded_files = candidate_paths.each_with_object(Set.new) do |file, set|
-        absolute_path = File.expand_path(file, root)
-        next if result.key?(absolute_path)
-
-        result[absolute_path] = SimulateCoverage.call(absolute_path)
-        set << absolute_path
-      end
-
-      [result, not_loaded_files]
-    end
-
     # Run all the steps that handle processing the raw coverage result.
     # `report:` is true only when this slice is the final result (merging
     # off); with merging on the merged result reports dropped source files,
     # so the per-process slice stays quiet to avoid one warning per worker.
-    def process_coverage_result(report:)
+    #
+    # `inject_unloaded:` is likewise false when a merge step follows. Only the
+    # union of every process's loaded files says what was really never loaded,
+    # so injecting here means each worker simulates nearly the whole project
+    # and all but one of those passes is merged away. See #1250.
+    def process_coverage_result(report:, inject_unloaded: true)
       raw = SimpleCov::UselessResultsRemover.call(Coverage.result)
       adapted = SimpleCov::ResultAdapter.call(raw)
-      result, not_loaded_files = add_not_loaded_files(adapted)
-      @result = SimpleCov::Result.new(result, not_loaded_files: not_loaded_files, report: report)
+      # What this process was told to track and did not load. Subtracting what
+      # it loaded matters because `Result#to_hash` serializes coverage after
+      # filtering: a file this process loaded and then filtered out would
+      # otherwise be recorded as tracked while absent from the stored coverage,
+      # and the merge would simulate it back in as never-loaded. See #1250.
+      tracked = tracked_file_paths - adapted.keys
+      result, not_loaded = inject_unloaded ? inject_unloaded_files(adapted, tracked) : [adapted, Set.new]
+      @result = SimpleCov::Result.new(
+        result, not_loaded_files: not_loaded, tracked_files: tracked,
+                run_id: SimpleCov.run_id, worker_id: SimpleCov.worker_id, report: report
+      )
     end
   end
 end

data/lib/simplecov/run_identity.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+require "securerandom"
+
+module SimpleCov
+  # Identifies one test invocation and each top-level parallel worker within
+  # it. Forked subprocesses inherit both values from their parent worker.
+  module RunIdentity
+  module_function
+
+    # Returns the run id together with its provenance: `true` when the id
+    # alone proves same-run membership (explicitly configured, derived from
+    # parallel_tests' per-invocation pid file, or freshly random), `false`
+    # when it was inferred from the parent pid, which consecutive runs
+    # launched by the same long-lived parent process share.
+    def generate
+      explicit = ENV.fetch("SIMPLECOV_RUN_ID", nil)
+      return [explicit, true] if explicit && !explicit.empty?
+
+      pid_file = ENV.fetch("PARALLEL_PID_FILE", nil)
+      return ["parallel-tests:#{File.basename(pid_file)}", true] if pid_file && !pid_file.empty?
+
+      return ["parallel-parent:#{Process.ppid}", false] if SimpleCov::ParallelAdapters.current
+
+      [SecureRandom.uuid, true]
+    end
+
+    def worker_id
+      explicit = ENV.fetch("SIMPLECOV_WORKER_ID", nil)
+      return explicit if explicit && !explicit.empty?
+
+      number = ENV.fetch("TEST_ENV_NUMBER", nil)
+      return number.empty? ? "1" : number if number
+
+      Process.pid.to_s
+    end
+
+    # Whether the current run id alone proves same-run membership. Decided
+    # where the id is generated, never re-inferred from the id's shape, so
+    # an explicit SIMPLECOV_RUN_ID that happens to look like an inferred
+    # one is still trusted.
+    def authoritative?
+      materialize_current
+      @authoritative
+    end
+
+    def current
+      materialize_current
+      @current
+    end
+
+    def materialize_current
+      @current, @authoritative = generate unless defined?(@current)
+    end
+
+    def current_worker_id
+      @current_worker_id ||= worker_id
+    end
+
+    def prepare
+      SimpleCov::ParallelAdapters.current
+      SimpleCov.run_id
+      SimpleCov.worker_id
+    end
+
+    # Mixed into SimpleCov after ParallelAdapters has loaded.
+    module Accessors
+      def run_id
+        SimpleCov::RunIdentity.current
+      end
+
+      def worker_id
+        SimpleCov::RunIdentity.current_worker_id
+      end
+    end
+  end
+end

data/lib/simplecov/simulate_coverage.rb

@@ -33,25 +33,46 @@ module SimpleCov
     # can't be parsed, fall back to the old empty hashes — old behavior,
     # old tradeoff.
     #
+    # Pass `synthesize: false` to skip the static analysis and return the
+    # empty hashes directly. Callers use it when neither branch nor method
+    # coverage is enabled, since nothing will read the tuples and the Prism
+    # parse is about half the cost of simulating a file. See #1250.
+    #
+    # Pass `lines: false` to omit the `"lines"` key entirely, mirroring what
+    # `Coverage.result` reports for a file loaded under a branch-only or
+    # method-only run. Emitting zeroed lines there would make a simulated
+    # file indistinguishable from one a sibling process actually loaded once
+    # the two are merged.
+    #
     # @return [Hash]
     #
-    def call(absolute_path)
+    def call(absolute_path, synthesize: true, lines: true)
       source_lines = read_lines(absolute_path)
-      lines = coverage_stub(absolute_path, source_lines) ||
-              LinesClassifier.new.classify(source_lines)
+      simulated = synthesized_tuples(source_lines, synthesize)
+      return simulated unless lines
+
+      classified = coverage_stub(absolute_path, source_lines) ||
+                   LinesClassifier.new.classify(source_lines)
+      {"lines" => classified}.merge(simulated)
+    end
+
+    # The branch and method tuples for a file, or empty hashes when the static
+    # analysis is skipped (nothing enabled reads them) or unavailable (no Prism,
+    # or the file doesn't parse).
+    def synthesized_tuples(source_lines, synthesize)
       empty = {"branches" => {}, "methods" => {}} #: Hash[String, Hash[untyped, untyped]]
-      synthesized = StaticCoverageExtractor.call(source_lines.join) || empty
+      synthesized = (StaticCoverageExtractor.call(source_lines.join) if synthesize) || empty
 
-      {
-        "lines" => lines,
-        "branches" => synthesized["branches"],
-        "methods" => synthesized["methods"]
-      }
+      {"branches" => synthesized["branches"], "methods" => synthesized["methods"]}
     end
 
+    # SystemCallError, not just ENOENT: a `track_files` glob can sweep
+    # up an unreadable file or a directory named like a Ruby file
+    # (EACCES, EISDIR), and simulation must degrade to "empty file"
+    # rather than crash the merge or report step.
     def read_lines(path)
       File.readlines(path)
-    rescue Errno::ENOENT
+    rescue SystemCallError
       []
     end
 
@@ -68,7 +89,10 @@ module SimpleCov
       classifier_output = LinesClassifier.new.classify(source_lines)
       stub.each_index { |idx| stub[idx] = nil if classifier_output[idx].nil? }
       stub
-    rescue Errno::ENOENT, SyntaxError
+    rescue SystemCallError, SyntaxError
+      # SystemCallError for the same reason as read_lines above:
+      # line_stub reads the file itself, so EACCES/EISDIR surface here
+      # too.
       nil
     end
   end

data/lib/simplecov/source_file/method.rb

@@ -18,10 +18,16 @@ module SimpleCov
         !skipped? && coverage.positive?
       end
 
+      # Criterion-level skips (nocov chunks, `# simplecov:disable` /
+      # `# simplecov:disable method` regions) arrive via `skipped!` from
+      # MethodBuilder. Deliberately NOT derived from the lines' skip
+      # state: a line-only directive around a def must not remove the
+      # method from method totals. Without line info there is nothing to
+      # report against, so such a method stays skipped.
       def skipped?
         return @skipped if defined?(@skipped)
 
-        @skipped = lines.all?(&:skipped?)
+        @skipped = lines.empty?
       end
 
       # Flag the method as skipped directly, without going through its lines.

data/lib/simplecov/source_file/ruby_data_parser.rb

@@ -97,9 +97,13 @@ module SimpleCov
       # Method coverage keys can contain inspect-format class references
       # like `#<Class:Foo>` or `#<Class:0x...>`, which aren't valid Ruby
       # syntax. Wrap them in quotes so Ripper can parse the surrounding
-      # array literal; downstream we treat them as opaque strings.
+      # array literal; downstream we treat them as opaque strings. The
+      # pattern recurses because singleton methods on instances nest one
+      # inspect segment inside another (`#<Class:#<Object:0x...>>`), and
+      # stopping at the first `>` leaves a dangling `>` that fails both
+      # Ripper passes and crashes the merge.
       def quote_inspected_class_segments(str)
-        str.gsub(/#<[^>]*>/) { |segment| %("#{segment.gsub('"', '\\"')}") }
+        str.gsub(/#<(?:[^<>]|\g<0>)*>/) { |segment| %("#{segment.gsub('"', '\\"')}") }
       end
     end
   end

data/lib/simplecov/source_file/skip_chunks.rb

@@ -22,19 +22,16 @@ module SimpleCov
         @src = src
       end
 
-      # `:method` ignores nocov chunks (Ruby's Coverage doesn't tie
-      # method entries to line ranges); `:line` / `:branch` honor both
-      # the nocov chunks and the per-criterion directive ranges.
+      # Every criterion honors the deprecated all-criteria `# :nocov:`
+      # chunks plus its own per-criterion directive ranges. Methods
+      # included: they carry a source range, and nocov has always meant
+      # "exclude everything here".
       def for(criterion)
-        if criterion == :method
-          directive_chunks.fetch(:method)
-        else
-          nocov_chunks + directive_chunks.fetch(criterion)
-        end
+        nocov_chunks + directive_chunks.fetch(criterion)
       end
 
-      # no_cov_chunks is zero indexed to work directly with the array
-      # holding the lines.
+      # Ranges of 1-based line numbers (see `with_index(1)` below);
+      # consumers subtract 1 to index into the zero-based lines array.
       def nocov_chunks
         @nocov_chunks ||= build_nocov_chunks
       end

data/lib/simplecov/source_file/source_loader.rb

@@ -17,17 +17,27 @@ module SimpleCov
         lines = [] #: Array[String]
         # The default encoding is UTF-8
         File.open(filename, "rb:UTF-8") do |file|
-          current_line = file.gets
+          current_line = scrub_invalid(file.gets)
 
           if current_line && shebang?(current_line)
             lines << current_line
-            current_line = file.gets
+            current_line = scrub_invalid(file.gets)
           end
 
           read_lines(file, lines, current_line)
         end
       end
 
+      # A line read as UTF-8 can still carry invalid bytes (a Latin-1
+      # source file without a magic comment, say). Replace them before
+      # any regex sees the line: the shebang and magic-comment checks
+      # would otherwise raise ArgumentError and take the report down.
+      def scrub_invalid(line)
+        return line if line.nil? || line.valid_encoding?
+
+        line.scrub
+      end
+
       def shebang?(line)
         SHEBANG_REGEX.match?(line)
       end
@@ -47,14 +57,20 @@ module SimpleCov
         end
       end
 
-      # invalid/undef replace are technically not really necessary but
-      # nice to have and work around a JRuby incompatibility. Setting
-      # these options on `file.set_encoding` doesn't seem to work
-      # properly, so it has to be done here.
+      # Guarantee every line leaves the loader as valid UTF-8, replacing
+      # what can't be represented: transcode non-UTF-8 lines (setting
+      # invalid/undef options on `file.set_encoding` doesn't work
+      # properly, and this also works around a JRuby incompatibility)
+      # and scrub UTF-8-tagged lines that carry invalid bytes, which
+      # would otherwise raise from every regex the classifier runs.
       def ensure_remove_undefs(file_lines)
         file_lines.each do |line|
           # simplecov:disable — defensive: only fires for non-UTF-8 source files
-          line.encode!("UTF-8", invalid: :replace, undef: :replace) unless line.encoding == Encoding::UTF_8
+          if line.encoding == Encoding::UTF_8
+            line.scrub! unless line.valid_encoding?
+          else
+            line.encode!("UTF-8", invalid: :replace, undef: :replace)
+          end
           # simplecov:enable
         end
       end

data/lib/simplecov/source_file/statistics.rb

@@ -22,35 +22,43 @@ module SimpleCov
     private
 
       def line_statistics
-        sf = @source_file
         {
-          line: CoverageStatistics.new(
-            total_strength: sf.lines.sum { |line| line.coverage.to_i },
-            covered: sf.covered_lines.size,
-            missed: sf.missed_lines.size,
-            omitted: sf.never_lines.size
+          line: coverage_statistics(
+            @source_file.covered_lines,
+            @source_file.missed_lines,
+            omitted: @source_file.never_lines.size
           )
         }
       end
 
+      # Files added via track_files but never loaded have no branch/method
+      # data. Report 0% instead of the empty-set default of 100% (see #902).
       def branch_statistics
         sf = @source_file
-        # Files added via track_files but never loaded/required have no
-        # branch data. Report 0% instead of misleading 100% (see #902).
-        if sf.not_loaded? && sf.covered_branches.empty? && sf.missed_branches.empty?
-          return {branch: CoverageStatistics.new(covered: 0, missed: 0, percent: 0.0)}
-        end
+        covered = sf.covered_branches
+        missed = sf.missed_branches
+        percent = 0.0 if sf.not_loaded? && covered.empty? && missed.empty?
 
-        {branch: CoverageStatistics.new(covered: sf.covered_branches.size, missed: sf.missed_branches.size)}
+        {branch: coverage_statistics(covered, missed, percent: percent)}
       end
 
       def method_statistics
         sf = @source_file
-        if sf.not_loaded? && sf.covered_methods.empty? && sf.missed_methods.empty?
-          return {method: CoverageStatistics.new(covered: 0, missed: 0, percent: 0.0)}
-        end
+        covered = sf.covered_methods
+        missed = sf.missed_methods
+        percent = 0.0 if sf.not_loaded? && covered.empty? && missed.empty?
+
+        {method: coverage_statistics(covered, missed, percent: percent)}
+      end
 
-        {method: CoverageStatistics.new(covered: sf.covered_methods.size, missed: sf.missed_methods.size)}
+      def coverage_statistics(covered, missed, omitted: 0, percent: nil)
+        CoverageStatistics.new(
+          total_strength: covered.sum { |item| item.coverage.to_i },
+          covered: covered.size,
+          missed: missed.size,
+          omitted: omitted,
+          percent: percent
+        )
       end
     end
   end