padded_cell 0.1.0 → 0.2.0

data/ext/padded_cell_native/ffi/src/chart.rs

@@ -0,0 +1,205 @@
+use std::path::PathBuf;
+
+use magnus::{Error, RArray, RHash, Ruby, Symbol, TryConvert, Value};
+use padded_cell::chart::{
+    self, Arm, CRITERIA, Condition, Criterion, FileCoverage, Format, Meta, Method, Options, Scope,
+    Span, Stats, Summary, Thresholds, Totals,
+};
+
+use crate::file_error;
+use padded_cell::FileError;
+
+/// Reads every source, writes the reports, checks the thresholds, and
+/// returns what it found:
+///
+/// ```text
+/// files    [[path, name, lines, branches or nil, methods or nil, loaded], ...]
+///            branches  [[type, l1, c1, l2, c2, [[type, l1, c1, l2, c2, hits], ...]], ...]
+///            methods   [[owner, name, l1, c1, l2, c2, hits], ...]
+/// groups   [[name, [file index, ...]], ...]
+/// options  {dir:, formats: ["lcov", "json"], nocov_token:, criteria: ["line", "branch"],
+///           minimum:, minimum_by_file:, maximum_drop:, last_run: [line, branch, method] (nil: none),
+///           minimum_by_group: [[name, [line, branch, method]], ...], write_last_run:,
+///           root:, timestamp:, created_at:, commit:, project_name:, generator:}
+///
+/// => {total: stats, groups: [[name, stats], ...], files: [[name, stats], ...],
+///     violations: [[scope, name or nil, criterion, limit, actual], ...]}
+///    stats  {line: [covered, missed, omitted, hits, percent, strength], branch: ..., method: ...}
+/// ```
+pub fn chart_report(
+    ruby: &Ruby,
+    files: RArray,
+    groups: Vec<(String, Vec<usize>)>,
+    options: RHash,
+) -> Result<RHash, Error> {
+    let option = |key: &str| ruby.to_symbol(key);
+    let formats = options
+        .fetch::<_, Vec<String>>(option("formats"))?
+        .iter()
+        .map(|name| name.parse::<Format>())
+        .collect::<Result<Vec<_>, _>>()
+        .map_err(|error| Error::new(ruby.exception_arg_error(), error.to_string()))?;
+    let measured: Vec<Criterion> = options
+        .fetch::<_, Vec<String>>(option("criteria"))?
+        .iter()
+        .filter_map(|name| CRITERIA.into_iter().find(|criterion| criterion.singular() == name))
+        .collect();
+    let thresholds = Thresholds {
+        minimum: options.fetch(option("minimum"))?,
+        minimum_by_file: options.fetch(option("minimum_by_file"))?,
+        minimum_by_group: options.fetch(option("minimum_by_group"))?,
+        maximum_drop: options.fetch(option("maximum_drop"))?,
+        last_run: options.fetch(option("last_run"))?,
+    };
+    let meta = Meta {
+        root: options.fetch(option("root"))?,
+        timestamp: options.fetch(option("timestamp"))?,
+        created_at: options.fetch(option("created_at"))?,
+        commit: options.fetch(option("commit"))?,
+        project_name: options.fetch(option("project_name"))?,
+        generator: options.fetch(option("generator"))?,
+    };
+    let dir: PathBuf = options.fetch(option("dir"))?;
+    let analysis = Options {
+        nocov_token: options.fetch(option("nocov_token"))?,
+        methods: measured.contains(&Criterion::Method),
+    };
+    let files = (0..files.len())
+        .map(|index| file(files.entry(index.cast_signed())?))
+        .collect::<Result<Vec<_>, Error>>()?;
+
+    let failed = |error: FileError| file_error(ruby, &error);
+    let reports = chart::analyze_all(&files, &analysis).map_err(failed)?;
+    let totals = Totals::of_files(&reports);
+    let groups: Vec<(String, Vec<usize>, Totals)> = groups
+        .into_iter()
+        .map(|(name, members)| {
+            let totals = Totals::of_files(members.iter().filter_map(|&index| reports.get(index)));
+            (name, members, totals)
+        })
+        .collect();
+    let named: Vec<(String, Totals)> =
+        groups.iter().map(|(name, _, totals)| (name.clone(), *totals)).collect();
+    let violations = chart::violations(&thresholds, &measured, &totals, &reports, &named);
+    let summary = Summary { files: &reports, measured, totals, groups, violations, meta };
+    for format in formats {
+        chart::write(&dir, format, &summary).map_err(failed)?;
+    }
+    if options.fetch(option("write_last_run"))? {
+        chart::write_last_run_file(&dir, &summary).map_err(failed)?;
+    }
+    result(ruby, &summary)
+}
+
+fn result(ruby: &Ruby, summary: &Summary<'_>) -> Result<RHash, Error> {
+    let result = ruby.hash_new();
+    result.aset(ruby.to_symbol("total"), totals(ruby, &summary.totals)?)?;
+    let groups = ruby.ary_new_capa(summary.groups.len());
+    for (name, _, group) in &summary.groups {
+        groups.push((name.as_str(), totals(ruby, group)?))?;
+    }
+    result.aset(ruby.to_symbol("groups"), groups)?;
+    let files = ruby.ary_new_capa(summary.files.len());
+    for report in summary.files {
+        let stats = criteria(ruby, |criterion| report.stats.with_percent(criterion))?;
+        files.push((report.file.name.as_str(), stats))?;
+    }
+    result.aset(ruby.to_symbol("files"), files)?;
+    let violations = ruby.ary_new_capa(summary.violations.len());
+    for violation in &summary.violations {
+        let (scope, name) = match &violation.scope {
+            Scope::Total => ("total", None),
+            Scope::File(name) => ("file", Some(name.as_str())),
+            Scope::Group(name) => ("group", Some(name.as_str())),
+            Scope::Drop => ("drop", None),
+        };
+        let criterion = violation.criterion.singular();
+        violations.push((
+            ruby.to_symbol(scope),
+            name,
+            ruby.to_symbol(criterion),
+            violation.limit,
+            violation.actual,
+        ))?;
+    }
+    result.aset(ruby.to_symbol("violations"), violations)?;
+    Ok(result)
+}
+
+fn totals(ruby: &Ruby, totals: &Totals) -> Result<RHash, Error> {
+    criteria(ruby, |criterion| {
+        let stats = totals.get(criterion);
+        (stats, stats.percent())
+    })
+}
+
+fn criteria(ruby: &Ruby, stats: impl Fn(Criterion) -> (Stats, f64)) -> Result<RHash, Error> {
+    let hash = ruby.hash_new();
+    for criterion in CRITERIA {
+        let (stats, percent) = stats(criterion);
+        let row =
+            (stats.covered, stats.missed, stats.omitted, stats.hits, percent, stats.strength());
+        hash.aset(ruby.to_symbol(criterion.singular()), row)?;
+    }
+    Ok(hash)
+}
+
+fn file(row: RArray) -> Result<FileCoverage, Error> {
+    Ok(FileCoverage {
+        path: row.entry(0)?,
+        name: row.entry(1)?,
+        lines: row.entry::<RArray>(2)?.to_vec()?,
+        branches: row.entry::<Option<RArray>>(3)?.map(conditions).transpose()?,
+        methods: row.entry::<Option<RArray>>(4)?.map(methods).transpose()?,
+        loaded: row.entry(5)?,
+    })
+}
+
+fn conditions(rows: RArray) -> Result<Vec<Condition>, Error> {
+    rows.into_iter()
+        .map(|row| {
+            let row = RArray::try_convert(row)?;
+            let arms = RArray::try_convert(row.entry(5)?)?
+                .into_iter()
+                .map(|arm| {
+                    let arm = RArray::try_convert(arm)?;
+                    Ok(Arm { kind: name(arm.entry(0)?)?, span: span(arm, 1)?, hits: arm.entry(5)? })
+                })
+                .collect::<Result<_, Error>>()?;
+            Ok(Condition { kind: name(row.entry(0)?)?, span: span(row, 1)?, arms })
+        })
+        .collect()
+}
+
+fn methods(rows: RArray) -> Result<Vec<Method>, Error> {
+    rows.into_iter()
+        .map(|row| {
+            let row = RArray::try_convert(row)?;
+            Ok(Method {
+                owner: row.entry(0)?,
+                name: name(row.entry(1)?)?,
+                span: span(row, 2)?,
+                hits: row.entry(6)?,
+            })
+        })
+        .collect()
+}
+
+/// Four integers from `at`: first line, first column, last line, last column.
+fn span(row: RArray, at: isize) -> Result<Span, Error> {
+    Ok(Span {
+        start_line: row.entry(at)?,
+        start_column: row.entry(at.strict_add(1))?,
+        end_line: row.entry(at.strict_add(2))?,
+        end_column: row.entry(at.strict_add(3))?,
+    })
+}
+
+/// A branch type or method name: a Symbol from Ruby's Coverage, a String
+/// once through JSON.
+fn name(value: Value) -> Result<String, Error> {
+    match Symbol::from_value(value) {
+        Some(symbol) => Ok(symbol.name()?.into_owned()),
+        None => String::try_convert(value),
+    }
+}

data/ext/padded_cell_native/ffi/src/lib.rs

@@ -18,6 +18,8 @@ use magnus::prelude::*;
 use magnus::{Error, RArray, RString, Ruby, function};
 use padded_cell::{Discovery, FileError};
 
+mod chart;
+
 /// `Errno::*` for the path, as Ruby's own file reading raises.
 fn file_error(ruby: &Ruby, error: &FileError) -> Error {
     let path = error.path.display().to_string();
@@ -116,5 +118,6 @@ fn init(ruby: &Ruby) -> Result<(), Error> {
     module.define_module_function("discover", function!(discover, 1))?;
     module.define_module_function("discover_files", function!(discover_files, 1))?;
     module.define_module_function("plan", function!(plan, 5))?;
+    module.define_module_function("chart_report", function!(chart::chart_report, 3))?;
     Ok(())
 }

data/lib/padded_cell/asylum.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+module PaddedCell
+  # --asylum: tests that failed get more attempts once the run is over, one
+  # at a time, each in a fresh cell. A test that passes on retry counts as
+  # passed but is reported as flaky, with every attempt and what changed:
+  #
+  #   attempt 1, same seed, run alone:  timing, contention, or state left by
+  #                                     the tests that ran around it
+  #   attempt 2+, another seed:         randomness or test order
+  #
+  # A test that fails every attempt keeps its first failure.
+  class Asylum
+    ALONE = 'timing, contention or state left by other tests'
+    RESEEDED = 'randomness or test order'
+
+    # The seed of retry attempt, derived from the run's seed so a report can
+    # be replayed. Attempt 1 keeps the run's seed.
+    def self.retry_seed(seed, attempt)
+      attempt == 1 ? seed : (seed + (attempt * 0x9e37)) % 0xffff
+    end
+
+    def initialize(tests, attempts:, seed:, timeout: nil)
+      @tests = tests
+      @attempts = attempts
+      @seed = seed
+      @timeout = timeout
+      @success = true
+    end
+
+    # False when a retry's worker failed (its cleanup hook).
+    def success? = @success
+
+    # failures: {test index => first failing result}. Yields (index, result)
+    # once per test, with its final result. Its metadata holds the attempts:
+    # :flaky when it passed on retry, :asylum when it never did. Both are
+    # plain data, for Marshal and the JUnit and JSON reports.
+    def call(failures)
+      history = failures.to_h { |index, result| [index, [attempt_entry(@seed, false, result)]] }
+      pending = failures.keys
+      1.upto(@attempts) do |attempt|
+        break if pending.empty?
+
+        pending -= retry_round(pending, history, attempt) do |index, result|
+          result.metadata[:flaky] = flaky(failures[index], history[index], attempt)
+          yield index, result
+        end
+      end
+      pending.each do |index|
+        failures[index].metadata[:asylum] = { attempts: history[index].size, history: history[index] }
+        yield index, failures[index]
+      end
+    end
+
+    private
+
+    # Runs the pending tests once more, alone; returns the ones that passed.
+    def retry_round(pending, history, attempt)
+      seed = self.class.retry_seed(@seed, attempt)
+      passed = []
+      pool = Pool.new(@tests, jobs: 1, isolation: :test, seed:, timeout: @timeout)
+      pool.run([pending.dup]) do |index, result|
+        recovered = result.passed? && !result.skipped?
+        history[index] << attempt_entry(seed, true, recovered ? nil : result)
+        next unless recovered
+
+        passed << index
+        yield index, result
+      end
+      @success &&= pool.success?
+      passed
+    end
+
+    # One attempt: its seed, whether it ran alone, and how it ended.
+    def attempt_entry(seed, alone, failed)
+      { seed:, alone:, outcome: failed ? summary(failed.failure)[:message] : 'passed' }
+    end
+
+    def flaky(first, history, attempt)
+      {
+        attempts: history.size, seed: history.last[:seed], cause: attempt == 1 ? ALONE : RESEEDED,
+        first_failure: summary(first.failure), history:
+      }
+    end
+
+    def summary(failure)
+      type = failure.is_a?(Minitest::UnexpectedError) ? failure.error.class.name : failure.class.name
+      { type:, message: failure.message.lines.first.to_s.strip }
+    end
+  end
+end

data/lib/padded_cell/cell.rb

@@ -12,15 +12,18 @@ module PaddedCell
     # result when the cell died or ran past timeout seconds.
     #
     #   close: descriptors the cell must not keep open (the worker's socket)
-    def run(klass, name, seed:, timeout: nil, close: [])
+    #   probe: the worker's Chart::Probe, which gets the cell's coverage
+    def run(klass, name, seed:, timeout: nil, close: [], probe: nil)
       # Binary: the result is Marshal data, and with Encoding.default_internal
       # set (Rails sets it) a text-mode pipe transcodes what is written to it.
       reader, writer = IO.pipe
       [reader, writer].each(&:binmode)
       started = Minitest.clock_time
-      pid = fork_cell(klass, name, seed, reader, writer, close)
+      pid = fork_cell(klass, name, seed, reader, writer, close, probe)
       writer.close
-      collect(pid, reader, klass, name, started, timeout)
+      delta, result = collect(pid, reader, klass, name, started, timeout)
+      delta ? probe.absorb(delta) : probe&.lost
+      result
     ensure
       reader&.close
       writer&.close unless writer&.closed?
@@ -34,7 +37,7 @@ module PaddedCell
       Results.crashed(klass, name, "exit(#{e.status}) called during the test")
     end
 
-    def fork_cell(klass, name, seed, reader, writer, close)
+    def fork_cell(klass, name, seed, reader, writer, close, probe)
       $stdout.flush
       $stderr.flush
       Process.fork do
@@ -43,7 +46,9 @@ module PaddedCell
           reader.close
           close.each(&:close)
           srand(seed)
-          writer.write(Results.dump(execute(klass, name)))
+          probe&.start
+          result = execute(klass, name)
+          writer.write(Results.envelope(probe&.capture, result))
           status = 0
         rescue Exception => e # anything: report it, then exit!
           warn "padded_cell: cell #{klass}##{name} failed to report: #{e.class}: #{e.message}"
@@ -54,19 +59,20 @@ module PaddedCell
       end
     end
 
+    # [coverage delta or nil, result]
     def collect(pid, reader, klass, name, started, timeout)
       data = read(reader, timeout && (started + timeout))
       elapsed = Minitest.clock_time - started
       if data.nil?
         Process.kill(:KILL, pid)
         Process.wait(pid)
-        return Results.crashed(klass, name, "timed out after #{timeout}s", time: elapsed)
+        return [nil, Results.crashed(klass, name, "timed out after #{timeout}s", time: elapsed)]
       end
 
       _, status = Process.wait2(pid)
-      return Results.load(data, klass, name) unless data.empty?
+      return Results.unwrap(data, klass, name) unless data.empty?
 
-      Results.crashed(klass, name, Results.describe(status, 'cell'), time: elapsed)
+      [nil, Results.crashed(klass, name, Results.describe(status, 'cell'), time: elapsed)]
     end
 
     # Everything until EOF, or nil when the deadline passes first.

data/lib/padded_cell/chart/accumulator.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+require 'coverage'
+require 'fileutils'
+require 'tmpdir'
+
+module PaddedCell
+  module Chart
+    # The runner's side. Starts Ruby's Coverage before the suite loads, hands
+    # each worker a Probe, then sums its own measure (loading the suite) with
+    # every frame the workers wrote.
+    class Accumulator
+      attr_reader :criteria
+
+      # criteria: what Ruby's Coverage measures, :lines always among them
+      def initialize(criteria)
+        @criteria = criteria
+        @root = File.realpath(Dir.pwd)
+        @roots = [Dir.pwd, @root].uniq.map { "#{it}/" }
+        @kept = {}
+        @lost = 0
+        @known = Set.new
+      end
+
+      # Nothing loaded before is measured.
+      def start
+        ::Coverage.setup(**@criteria.to_h { [it, true] })
+        ::Coverage.resume
+        @frames = Dir.mktmpdir('padded_cell-coverage')
+        self
+      end
+
+      # After the suite loaded, before the first fork: the files whose
+      # structure the runner will report itself. Forks inherit the set.
+      def snapshot
+        ::Coverage.peek_result.each_key { @known << it if keep?(it) }
+      end
+
+      # For a worker, right after its fork.
+      def probe
+        Probe.new(file: File.join(@frames, "#{Process.pid}.cov"), known: @known, keep: method(:keep?),
+                  methods: @criteria.include?(:methods), interval: Chart.config.interval)
+      end
+
+      # A worker died: what it measured since its last flush is gone.
+      def lost_worker = @lost += 1
+
+      # Sums everything (see Raw); nil for an interrupted run.
+      def finish(interrupted: false)
+        return if interrupted
+
+        total = Delta.capture(::Coverage.peek_result, Set.new, method(:keep?))
+        cells = 0
+        frames.each do |frame|
+          Delta.merge!(total, frame[:delta])
+          cells += frame[:lost]
+        end
+        Raw.build(normalize(total), root: @root, criteria: @criteria, lost: { workers: @lost, cells: })
+      ensure
+        FileUtils.rm_rf(@frames) if @frames
+      end
+
+      private
+
+      # Files under the project root, which exist (not eval'd code).
+      def keep?(path)
+        @kept.fetch(path) { @kept[path] = @roots.any? { path.start_with?(it) } && File.file?(path) }
+      end
+
+      # Every complete frame of every worker. A worker killed while writing
+      # leaves a truncated last frame, skipped.
+      def frames
+        Dir[File.join(@frames, '*.cov')].flat_map do |file|
+          data = File.binread(file)
+          offset = 0
+          frames = []
+          while offset + 4 <= data.bytesize
+            size = data.unpack1('N', offset:)
+            break if offset + 4 + size > data.bytesize
+
+            frames << Marshal.load(data.byteslice(offset + 4, size)) # written by our own workers
+            offset += 4 + size
+          end
+          frames
+        end
+      end
+
+      # Keys by real path relative to the root: a file required through a
+      # symlink and through its real path is one file. A file deleted during
+      # the run is not reported.
+      def normalize(total)
+        total.each_with_object({}) do |(path, entry), files|
+          relative = File.realpath(path).delete_prefix("#{@root}/")
+          next if relative.start_with?('/')
+
+          files[relative] ? Delta.merge!(files, { relative => entry }) : files[relative] = entry
+        rescue SystemCallError
+          next
+        end
+      end
+    end
+  end
+end

data/lib/padded_cell/chart/config.rb

@@ -0,0 +1,129 @@
+# frozen_string_literal: true
+
+module PaddedCell
+  module Chart
+    # A PaddedCell.coverage block, in the test helper:
+    #
+    #   PaddedCell.coverage(:rails) do
+    #     track "{app,lib}/**/*.rb"
+    #     skip "app/legacy/"
+    #     group "Operations", "app/concepts"
+    #     minimum line: 90, branch: 80
+    #     minimum_by_file line: 50
+    #     minimum_by_group "Models", line: 95
+    #     maximum_drop line: 1
+    #     formats :markdown, :lcov, :json
+    #   end
+    #
+    # Filters and groups match paths (see Filter). What is
+    # measured is decided before the suite loads, on the command line:
+    # --coverage=lines,branches,methods.
+    class Config
+      FORMATS = %i[markdown lcov json].freeze
+      # Left out unless cleared: bundled gems, hidden files, test files.
+      DEFAULT_FILTERS = ['/vendor/bundle/', /\A\..*/, %r{\A(test|features|spec|autotest)/}].freeze
+      RAILS_GROUPS = {
+        'Channels' => 'app/channels', 'Controllers' => 'app/controllers', 'Helpers' => 'app/helpers',
+        'Jobs' => %w[app/jobs app/workers], 'Libraries' => 'lib/', 'Mailers' => 'app/mailers',
+        'Models' => 'app/models', 'Views' => 'app/views'
+      }.freeze
+
+      attr_reader :filters, :groups, :tracked, :output_formats, :directory, :token, :interval, :minimums,
+                  :file_minimums, :group_minimums, :maximum_drops
+
+      def initialize
+        @filters = DEFAULT_FILTERS.map { Filter.build(it) }
+        @groups = {}
+        @tracked = []
+        @minimums = {}
+        @file_minimums = {}
+        @group_minimums = {}
+        @maximum_drops = {}
+        @output_formats = %i[markdown lcov json]
+        @directory = 'coverage'
+        @token = 'nocov'
+        @interval = Probe::INTERVAL
+      end
+
+      # :rails skips config/ and db/, groups the
+      # app's directories, tracks {app,lib}/**/*.rb.
+      def profile(name)
+        raise ArgumentError, "unknown coverage profile #{name.inspect} (known: :rails)" unless name.to_s == 'rails'
+
+        skip(%r{\Aconfig/}, %r{\Adb/})
+        RAILS_GROUPS.each { |group_name, filter| group(group_name, filter) }
+        track('{app,lib}/**/*.rb')
+      end
+
+      # Files to report even when the suite never loads them, at 0%.
+      def track(*globs) = @tracked.concat(globs)
+
+      # Files to leave out of the report.
+      def skip(*filters, &block)
+        filters.each { @filters << Filter.build(it) }
+        @filters << Filter.build(&block) if block
+      end
+      alias filter skip
+
+      # Removes every filter, the defaults included.
+      def clear_filters = @filters.clear
+
+      # A file can be in several groups; files in none are Ungrouped.
+      # Defining a group again replaces it.
+      def group(name, filter = nil, &) = @groups[name.to_s] = Filter.build(filter, &)
+
+      # Thresholds compare percents floored to 2 decimals; a whole, passing
+      # run that misses one exits 3.
+      def minimum(line: nil, branch: nil, method: nil) = @minimums.merge!({ line:, branch:, method: }.compact)
+
+      # For every file.
+      def minimum_by_file(line: nil, branch: nil, method: nil)
+        @file_minimums.merge!({ line:, branch:, method: }.compact)
+      end
+
+      def minimum_by_group(name, line: nil, branch: nil, method: nil)
+        (@group_minimums[name.to_s] ||= {}).merge!({ line:, branch:, method: }.compact)
+      end
+
+      # How many points coverage may lose since the previous whole run.
+      def maximum_drop(line: nil, branch: nil, method: nil)
+        @maximum_drops.merge!({ line:, branch:, method: }.compact)
+      end
+
+      def formats(*names)
+        unknown = names.map(&:to_sym) - FORMATS
+        unless unknown.empty?
+          raise ArgumentError, "unknown coverage formats: #{unknown.join(', ')} (known: #{FORMATS.join(', ')})"
+        end
+
+        @output_formats = names.map(&:to_sym)
+      end
+
+      # Where reports go (default: coverage). --coverage-dir wins.
+      def dir(path) = @directory = path
+
+      # `# :nocov:` opens and closes blocks of lines coverage ignores.
+      def nocov_token(token) = @token = token
+
+      # How often, at least, a worker hands over what it measured (seconds,
+      # default 10, stretched to keep it under 1% of the worker's time; 0:
+      # after every test). A worker that dies loses what it measured since.
+      def flush_interval(seconds) = @interval = seconds
+
+      # What a report needs once files are filtered and grouped, as stored
+      # in its Document.
+      def settings
+        {
+          formats: @output_formats.map(&:to_s), nocov_token: @token, minimum: @minimums,
+          minimum_by_file: @file_minimums, minimum_by_group: @group_minimums, maximum_drop: @maximum_drops
+        }
+      end
+
+      # Criteria come from the command line, before the suite (and this
+      # block) loads.
+      def enable_coverage(*)
+        raise ArgumentError, 'coverage criteria come from --coverage=lines,branches,methods, not the test helper'
+      end
+    end
+  end
+end