padded_cell 0.1.0 → 0.3.0

data/lib/padded_cell/chart/report.rb

@@ -0,0 +1,128 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'open3'
+require 'time'
+
+module PaddedCell
+  module Chart
+    # Writes the reports (natively), prints the summary, and reports the
+    # thresholds missed.
+    class Report
+      Stats = Data.define(:covered, :missed, :omitted, :hits, :percent, :strength) do
+        def total = covered + missed
+      end
+      CRITERIA = { lines: :line, branches: :branch, methods: :method }.freeze
+      WORST_FILES = 5
+
+      def initialize(document, dir:, io:)
+        @document = document
+        @settings = document.settings
+        @dir = dir
+        @io = io
+      end
+
+      # check: a whole, passing run: thresholds decide, and .last_run.json
+      # records it. False when a threshold is missed.
+      def call(check:)
+        result = PaddedCellNative.chart_report(@document.rows, @document.group_rows, options(check))
+        @total = stats(result[:total])
+        @files = result[:files].map { |name, file| [name, stats(file)] }
+        summary
+        warn_lost
+        return true unless check
+
+        result[:violations].each { report_violation(*it) }
+        result[:violations].empty?
+      end
+
+      private
+
+      def options(check)
+        now = Time.now
+        root = @document.root
+        {
+          dir: File.expand_path(@dir), formats: @settings[:formats], nocov_token: @settings[:nocov_token],
+          criteria: criteria.map(&:to_s), write_last_run: check, last_run: last_run,
+          minimum: limits(@settings[:minimum]), minimum_by_file: limits(@settings[:minimum_by_file]),
+          minimum_by_group: @settings[:minimum_by_group].map { |name, minimums| [name.to_s, limits(minimums)] },
+          maximum_drop: limits(@settings[:maximum_drop]),
+          root:, timestamp: now.iso8601(3), created_at: now.to_f, commit: commit(root),
+          project_name: File.basename(root).capitalize.tr('_', ' '), generator: "padded_cell #{VERSION}"
+        }
+      end
+
+      # [line, branch, method], nil where unset.
+      def limits(values) = %i[line branch method].map { values[it]&.to_f }
+
+      # The previous whole run's percents.
+      def last_run
+        previous = JSON.parse(File.read(File.join(@dir, '.last_run.json')), symbolize_names: true)[:result] || {}
+        limits({ line: previous[:line] || previous[:covered_percent], branch: previous[:branch],
+                 method: previous[:method] })
+      rescue Errno::ENOENT, JSON::ParserError
+        [nil, nil, nil]
+      end
+
+      def commit(root)
+        out, status = Open3.capture2e('git', '-C', root, 'rev-parse', 'HEAD')
+        out.strip if status.success?
+      rescue SystemCallError
+        nil
+      end
+
+      def stats(row) = row.transform_values { Stats.new(*it) }
+
+      # The criteria measured, as stats keys.
+      def criteria = @document.criteria.filter_map { CRITERIA[it.to_sym] }
+
+      def summary
+        @io.puts "\nCoverage report generated for padded_cell to #{@dir}"
+        criteria.each do |criterion|
+          stat = @total[criterion]
+          next if criterion != :line && stat.total.zero?
+
+          @io.puts format('%<label>s coverage: %<covered>d / %<total>d (%<percent>.2f%%)',
+                          label: criterion.capitalize, covered: stat.covered, total: stat.total,
+                          percent: stat.percent.floor(2))
+        end
+      end
+
+      def warn_lost
+        lost = @document.lost
+        unmeasured = @document.unmeasured
+        if lost[:workers].positive? || lost[:cells].positive?
+          @io.puts "Coverage is partial: #{lost[:workers]} workers and #{lost[:cells]} cells died before handing " \
+                   'over what they measured'
+        end
+        return if unmeasured.empty?
+
+        @io.puts "#{unmeasured.size} tracked files were loaded before coverage started, and are not reported: " \
+                 "#{unmeasured.first(3).join(', ')}#{', ...' if unmeasured.size > 3}"
+      end
+
+      def report_violation(scope, name, criterion, limit, actual)
+        label = criterion.capitalize
+        @io.puts case scope
+                 when :drop
+                   format('%<label>s coverage has dropped by %<actual>.2f%% since the last time ' \
+                          '(maximum allowed: %<limit>.2f%%).', label:, actual:, limit:)
+                 when :total then format('%<label>s coverage (%<actual>.2f%%) is below the expected minimum ' \
+                                         'coverage (%<limit>.2f%%).', label:, actual:, limit:)
+                 else format('%<label>s coverage by %<scope>s (%<actual>.2f%%) is below the expected minimum ' \
+                             'coverage (%<limit>.2f%%) in %<name>s.', label:, scope:, actual:, limit:, name:)
+                 end
+        worst_files(criterion) if scope == :total
+      end
+
+      def worst_files(criterion)
+        worst = @files.map { |name, file| [name, file[criterion].percent] }.select { it[1] < 100 }
+                      .min_by(WORST_FILES) { it[1] }
+        return if worst.empty?
+
+        @io.puts "  Lowest-coverage files (#{criterion}):"
+        worst.each { |name, percent| @io.puts format('    %<percent>6.2f%%  %<name>s', percent:, name:) }
+      end
+    end
+  end
+end

data/lib/padded_cell/chart/source_files.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+module PaddedCell
+  module Chart
+    # The files of a report: those measured and those tracked but never
+    # loaded, minus the filtered ones, in path order; and their groups.
+    class SourceFiles
+      # Tracked templates are not Ruby files Coverage could stub.
+      TEMPLATES = /\.(?:erb|haml|slim)\z/
+
+      # Tracked files loaded before measuring started (by a Rails rake task,
+      # Bundler.require...): their coverage is unknown, not 0%.
+      attr_reader :unmeasured
+
+      # files: Raw's {relative path => {lines:, branches:, methods:}}
+      def initialize(files, config:, root:)
+        @config = config
+        @root = root
+        loaded = loaded_features
+        never, @unmeasured = tracked(files).partition { !loaded.include?(File.join(root, it)) }
+        @rows = (files.map { |name, entry| row(name, entry) } + stubs(never))
+                .reject { |path, name| skipped?(path, name) }
+                .sort_by { it[1] }
+      end
+
+      # [[absolute path, name, lines, branches, methods, loaded], ...], for
+      # PaddedCellNative.chart_report.
+      attr_reader :rows
+
+      # [[name, [row index, ...]], ...]: the groups with files, then
+      # Ungrouped. None without groups.
+      def groups
+        return [] if @config.groups.empty?
+
+        files = @rows.map { |path, name| SourceFile.new(path, name) }
+        grouped = @config.groups.map { |name, filter| [name, files.each_index.select { filter.call(files[it]) }] }
+        ungrouped = files.each_index.to_a - grouped.flat_map(&:last)
+        (grouped + [['Ungrouped', ungrouped]]).reject { |_, members| members.empty? }
+      end
+
+      private
+
+      def row(name, entry) = [File.join(@root, name), name, entry[:lines], entry[:branches], entry[:methods], true]
+
+      # Files never loaded. Ruby compiles them, without running them: while
+      # Coverage runs, that gives their lines and branches as if loaded, none
+      # covered. Their methods only exist once run: the native side finds
+      # them in the source. A file that does not compile has every line but
+      # blank and comment ones missed.
+      def stubs(names)
+        paths = names.map { File.join(@root, it) }
+        compiled = paths.select { compile(it) }
+        measured = compiled.empty? ? {} : ::Coverage.peek_result
+        paths.zip(names).map do |path, name|
+          data = measured[path]
+          next [path, name, File.readlines(path).map { 0 }, nil, nil, false] unless data
+
+          entry = Raw.file(Delta.dense(data))
+          [path, name, entry[:lines], entry[:branches], nil, false]
+        end
+      end
+
+      def compile(path)
+        RubyVM::InstructionSequence.compile_file(path)
+      rescue SyntaxError
+        false
+      end
+
+      def tracked(files)
+        @config.tracked.flat_map { Dir.glob(it, base: @root) }.uniq.reject do |name|
+          files.key?(name) || TEMPLATES.match?(name) || !File.file?(File.join(@root, name))
+        end
+      end
+
+      def skipped?(path, name)
+        file = SourceFile.new(path, name)
+        @config.filters.any? { it.call(file) }
+      end
+
+      # Required files, also under the root's real path.
+      def loaded_features
+        pwd = "#{Dir.pwd}/"
+        $LOADED_FEATURES.each_with_object(Set.new) do |feature, set|
+          set << feature
+          set << File.join(@root, feature.delete_prefix(pwd)) if feature.start_with?(pwd)
+        end
+      end
+    end
+  end
+end

data/lib/padded_cell/chart.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require_relative 'chart/delta'
+require_relative 'chart/probe'
+require_relative 'chart/raw'
+require_relative 'chart/accumulator'
+require_relative 'chart/filter'
+require_relative 'chart/config'
+require_relative 'chart/source_files'
+require_relative 'chart/document'
+require_relative 'chart/merge'
+require_relative 'chart/report'
+
+module PaddedCell
+  # Built-in coverage: Ruby's Coverage measured in the
+  # runner, every worker and every cell, summed exactly in the runner. The
+  # chart, as the asylum keeps one per patient: inside, a name that is not
+  # Ruby's own Coverage; outside, it is all --coverage.
+  #
+  # Measuring starts before the suite loads, so what is measured comes from
+  # the command line (--coverage=lines,branches,methods,eval); the report is
+  # configured in the test helper, with PaddedCell.coverage (see Config).
+  module Chart
+    CRITERIA = %i[lines branches methods eval].freeze
+    DEFAULT_CRITERIA = %i[lines branches].freeze
+
+    class << self
+      # Every PaddedCell.coverage block, in one Config.
+      def config = @config ||= Config.new
+
+      def configure(profile = nil, &block)
+        config.profile(profile) if profile
+        config.instance_eval(&block) if block
+        config
+      end
+    end
+  end
+end

data/lib/padded_cell/cli.rb

@@ -2,6 +2,7 @@
 
 require 'optparse'
 require_relative 'runner'
+require_relative 'chart'
 
 module PaddedCell
   # The padded_cell command.
@@ -18,22 +19,33 @@ module PaddedCell
 
     module_function
 
-    # Exit status: 0 passed, 1 failed, 2 usage or load error, 130 interrupted.
+    # Exit status: 0 passed, 1 failed, 2 usage or load error, 3 coverage below
+    # a minimum, 130 interrupted.
     def start(argv, io: $stdout)
-      options = { io: }
+      options = { io:, coverage: coverage_criteria(ENV.fetch('PADDED_CELL_COVERAGE', nil)), **report_paths }
       parser(options).parse!(argv)
+      return merge(options) if options[:coverage_merge]
+
       options[:paths] = argv
       Runner.new(**options).run
-    rescue OptionParser::ParseError, ArgumentError, LoadFailure, Unsupported, SystemCallError => e
+    rescue OptionParser::ParseError, ArgumentError, RegexpError, LoadFailure, Unsupported, SystemCallError => e
       warn "padded_cell: #{e.message}"
       2
     end
 
+    # Report paths a CI runner can set without touching the command line;
+    # --json and --junit still win.
+    def report_paths
+      { json: ENV.fetch('PADDED_CELL_JSON', nil), junit: ENV.fetch('PADDED_CELL_JUNIT', nil) }
+        .reject { |_, path| path.nil? || path.empty? }
+    end
+
     def parser(options)
       OptionParser.new do |opts|
         opts.banner = BANNER
         opts.version = VERSION
         runner_options(opts, options)
+        coverage_options(opts, options)
         output_options(opts, options)
         opts.on('--version', 'Print the version') do
           options[:io].puts "padded_cell #{VERSION}"
@@ -50,7 +62,7 @@ module PaddedCell
       opts.on('-j', '--jobs N', Integer, "Workers (default: #{Etc.nprocessors})") do
         options[:jobs] = positive(it)
       end
-      opts.on('-s', '--seed SEED', Integer, 'Order seed (default: random, or $SEED)') { options[:seed] = it }
+      opts.on('-s', '--seed SEED', Integer, 'Order seed (default: random, or $SEED)') { options[:seed] = seed(it) }
       opts.on('-n', '-i', '--name', '--include PATTERN', 'Only tests matching (/regexp/ or exact name)') do
         options[:include] = it
       end
@@ -65,10 +77,41 @@ module PaddedCell
       opts.on('--timeout SECONDS', Float, 'Kill and fail a test running longer') do
         options[:timeout] = positive(it)
       end
+      opts.on('--budget SECONDS', Float, 'Stop dispatching once the run has taken SECONDS (and kill and ' \
+                                         'fail what runs past it); tests left out fail the run') do
+        options[:budget] = positive(it)
+      end
       opts.on('--fail-fast', 'Stop dispatching after the first failure') { options[:fail_fast] = true }
+      opts.on('--asylum[=N]', Integer, 'Retry failed tests up to N times (default 2) after the run, alone, ' \
+                                       'each in a fresh cell; report the flaky ones') do
+        options[:asylum] = it.nil? ? 2 : positive(it)
+      end
       opts.on('--list', 'List the tests that would run (static, loads nothing)') { options[:list] = true }
     end
 
+    def coverage_options(opts, options)
+      opts.on('--[no-]coverage[=CRITERIA]', 'Measure coverage: lines, branches, methods, eval ' \
+                                            "(default: #{Chart::DEFAULT_CRITERIA.join(',')}; " \
+                                            'or $PADDED_CELL_COVERAGE)') do
+        options[:coverage] = coverage_criteria(it.to_s)
+      end
+      opts.on('--coverage-dir DIR', 'Write coverage reports to DIR (default: coverage, or the test helper\'s)') do
+        options[:coverage_dir] = it
+      end
+      opts.on('--coverage-merge DIR,DIR', Array, 'Merge the coverage of shards (their --coverage-dir) ' \
+                                                 'into one report; runs no test') do
+        options[:coverage_merge] = it
+      end
+    end
+
+    # --coverage-merge: 0, or 3 when a threshold is missed.
+    def merge(options)
+      document = Chart::Merge.call(options[:coverage_merge])
+      dir = options[:coverage_dir] || 'coverage'
+      document.write(dir)
+      Chart::Report.new(document, dir:, io: options[:io]).call(check: document.complete?) ? 0 : 3
+    end
+
     def output_options(opts, options)
       opts.on('-v', '--verbose', 'One line per test, with timings') { options[:verbose] = true }
       opts.on('--show-skips', 'Show skipped tests at the end') { options[:show_skips] = true }
@@ -78,6 +121,13 @@ module PaddedCell
       opts.on('--no-timings', 'Neither read nor write timings') { options[:timings] = nil }
     end
 
+    # The planner takes an unsigned 64-bit seed.
+    def seed(value)
+      raise OptionParser::InvalidArgument, "#{value} (must be 0 to 2**64 - 1)" unless value.between?(0, (2**64) - 1)
+
+      value
+    end
+
     # OptionParser names the switch in the message.
     def positive(value)
       raise OptionParser::InvalidArgument, "#{value} (must be positive)" unless value.positive?
@@ -85,6 +135,22 @@ module PaddedCell
       value
     end
 
+    # "branches,methods" => [:lines, :branches, :methods] (lines always);
+    # "1" or "true" => the defaults; nil, "", "0" or "false" => off.
+    def coverage_criteria(value)
+      value = value.to_s.strip.downcase
+      return if ['', '0', 'false'].include?(value)
+      return Chart::DEFAULT_CRITERIA if %w[1 true].include?(value)
+
+      names = value.split(',').map { it.strip.to_sym }
+      unknown = names - Chart::CRITERIA
+      unless unknown.empty?
+        raise OptionParser::InvalidArgument, "coverage #{unknown.join(',')} (known: #{Chart::CRITERIA.join(', ')})"
+      end
+
+      [:lines] | names
+    end
+
     def shard(value)
       index, count = value.split('/', 2).map { Integer(it, 10, exception: false) }
       unless index && count&.positive? && index.between?(1, count)

data/lib/padded_cell/limits.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module PaddedCell
+  # What cuts a test short: its --timeout, and what is left of the run's
+  # --budget. A test's deadline never outlives the run's. Deadlines are
+  # PaddedCellNative::Deadline values in monotonic nanoseconds: one clock for
+  # the runner, its workers and their cells.
+  class Limits
+    NS = 1_000_000_000
+
+    def self.now = Process.clock_gettime(Process::CLOCK_MONOTONIC, :nanosecond)
+
+    # Seconds left before deadline (0.0 once passed), nil when never.
+    def self.remaining(deadline, now = self.now) = deadline.remaining_ns(now)&./(NS.to_f)
+
+    attr_reader :timeout, :budget
+
+    #   timeout: seconds per test, or nil
+    #   budget:  seconds for the whole run from started, or nil
+    def initialize(timeout: nil, budget: nil, started: Limits.now)
+      @timeout = timeout
+      @budget = budget
+      @run = budget ? PaddedCellNative::Deadline.after(started, nanoseconds(budget)) : PaddedCellNative::Deadline.never
+    end
+
+    # The deadline of a test starting now.
+    def for_test(now = Limits.now) = @run.derive(now, @timeout && nanoseconds(@timeout))
+
+    # True once the run's budget is gone: nothing more gets dispatched.
+    def spent?(now = Limits.now) = @run.expired?(now)
+
+    # Why a test that reached deadline was cut short.
+    def message(deadline)
+      if @budget && deadline.expires_at == @run.expires_at
+        "run budget of #{@budget}s spent"
+      else
+        "timed out after #{@timeout}s"
+      end
+    end
+
+    private
+
+    def nanoseconds(seconds) = (seconds * NS).round
+  end
+end

data/lib/padded_cell/loader.rb

@@ -53,11 +53,15 @@ module PaddedCell
     end
 
     def call(klass, name)
+      matches_name?(klass, name) && (!@selection.line_filtered? || line_selected?(klass, name))
+    end
+
+    def matches_name?(klass, name)
       id = "#{klass}##{name}"
       return false if @include && !matches?(@include, name, id)
       return false if @exclude && matches?(@exclude, name, id)
 
-      !@selection.line_filtered? || line_selected?(klass, name)
+      true
     end
 
     private