actionable-rb 1.9.0

data/lib/actionable/rspec/coverage/tracker.rb

@@ -0,0 +1,330 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+require 'oj'
+require_relative '../../../actionable'
+
+module Actionable
+  # Outcome coverage for a test suite (decision D22). While enabled, every real
+  # +.run+ of a named action class is recorded as +[class name, outcome, code]+,
+  # where outcome is +:success+, +:failure+ or +:skipped+. At the end of the
+  # suite those observations are compared with the outcomes each action can
+  # end with (its declarations, read at run time), and the outcomes no spec
+  # reached are reported.
+  #
+  # Loaded by +require 'actionable/rspec/coverage'+, which also installs the
+  # suite hook. Nothing here is loaded by the core.
+  module Coverage
+    # Prepended onto {Actionable::Action}'s singleton class, so every
+    # subclass's +.run+ (nested {Steps::Action} runs and {Action.run_each}
+    # included) passes through it. An RSpec stub of +.run+ defines the method
+    # on the subclass itself, ahead of this module, so a stubbed run is never
+    # recorded as the action being tested.
+    module Recorder
+      # Run the action and record the result's outcome.
+      #
+      # @return [Result]
+      def run(*, **)
+        result = super
+        Coverage.record(self, result)
+        result
+      end
+    end
+
+    @mutex = Mutex.new
+    @observed = Set.new
+    @stubbed = Set.new
+    @enabled = false
+
+    # The default {.new_gap_hint}.
+    DEFAULT_NEW_GAP_HINT = 'Write a spec that reaches each one, or add it to the baseline:'
+
+    class << self
+      # The class whose named descendants are reported. Defaults to
+      # {Actionable::Action}.
+      # @return [Class<Actionable::Action>]
+      attr_accessor :root
+
+      # Name filters for the report. A String matches as a name prefix, a
+      # Regexp by +match?+. With any include filter, only matching names are
+      # reported; an exclude filter drops matching names.
+      # @return [Array<String, Regexp>]
+      attr_accessor :include_prefixes, :exclude_prefixes
+
+      # Called before the actions are listed, so a host can load every action
+      # class first (+-> { Rails.application.eager_load! }+). Unloaded classes
+      # cannot be reported.
+      # @return [#call, nil]
+      attr_accessor :loader
+
+      # When true, report only actions with at least one real run, so a
+      # partial or targeted run doesn't flag every action it never touched.
+      # @return [Boolean]
+      attr_accessor :only_observed
+
+      # Path to the committed baseline of known gaps, a JSON or YAML file of
+      # +{"Action::Name" => [outcome keys]}+. With one set, the report fails
+      # on a gap the baseline doesn't list and on a stale entry; see
+      # {Report#baseline_problems}. +nil+ (the default) never fails.
+      # @return [String, nil]
+      attr_accessor :baseline
+
+      # The advice after "New untested outcomes." in {Report#baseline_message}.
+      # The default offers both fixes; a host whose baseline only shrinks can
+      # drop the second, e.g. +"Write a spec that reaches each one."+
+      # @return [String]
+      attr_accessor :new_gap_hint
+
+      # @return [Array<Symbol>] the gem failure codes counted as expected
+      #   outcomes; see {.gem_codes=}
+      attr_reader :gem_codes
+
+      # Count some of {Actionable::Action::GEM_FAILURE_CODES} as expected
+      # outcomes. +:invalid_input+ counts for an action whose input has a
+      # required field; +:invalid_output+ for one whose output has a required
+      # field. Defaults to none, so an adopter opts in.
+      #
+      # @param codes [Array<Symbol, String>]
+      # @raise [ArgumentError] for a code the gem never emits
+      # @return [void]
+      def gem_codes=(codes)
+        codes = Array(codes).map(&:to_sym)
+        unknown = codes - Actionable::Action::GEM_FAILURE_CODES
+        unless unknown.empty?
+          raise ArgumentError,
+            "not a gem failure code: #{unknown.map(&:inspect).join(", ")} " \
+            "(the gem emits #{Actionable::Action::GEM_FAILURE_CODES.map(&:inspect).join(", ")})"
+        end
+
+        @gem_codes = codes
+      end
+
+      # Restore every setting to its default.
+      #
+      # @return [void]
+      def reset_config!
+        @root = Actionable::Action
+        @include_prefixes = []
+        @exclude_prefixes = []
+        @loader = nil
+        @gem_codes = []
+        @only_observed = false
+        @baseline = nil
+        @new_gap_hint = DEFAULT_NEW_GAP_HINT
+      end
+
+      # The outcomes +action_class+ can end with, as outcome keys:
+      # +"success"+, +"failure:<code>"+ for each declared failure (D20) and
+      # +"skipped:<code>"+ for each declared skip (D21), plus any configured
+      # {.gem_codes} that apply. Read from the declarations at run time.
+      #
+      # @param action_class [Class<Actionable::Action>]
+      # @return [Array<String>]
+      def expected_outcomes(action_class)
+        keys = ['success']
+        keys.concat(action_class.declared_failures.keys.map { |code| "failure:#{code}" })
+        keys.concat(action_class.declared_skips.keys.map { |code| "skipped:#{code}" })
+        gem_codes.each { |code| keys << "failure:#{code}" if gem_code_applies?(action_class, code) }
+        keys.uniq
+      end
+
+      # The action classes to report: named descendants of {.root} that have
+      # steps, whose name still resolves to the class, filtered by
+      # {.include_prefixes} and {.exclude_prefixes}. Calls {.loader} first.
+      #
+      # @return [Array<Class<Actionable::Action>>] sorted by name
+      def actions
+        loader&.call
+        descendants(root).select { |klass| reportable?(klass) }.sort_by(&:name)
+      end
+
+      # The outcome key an observation counts toward: +"success"+, or
+      # +"<outcome>:<code>"+.
+      #
+      # @param outcome [Symbol, String]
+      # @param code [Symbol, String]
+      # @return [String]
+      def outcome_key(outcome, code)
+        outcome.to_s == 'success' ? 'success' : "#{outcome}:#{code}"
+      end
+
+      # Start recording runs. Installs {Recorder} the first time; safe to call
+      # again.
+      #
+      # @return [void]
+      def enable!
+        singleton = Actionable::Action.singleton_class
+        singleton.prepend(Recorder) unless singleton.ancestors.include?(Recorder)
+        @enabled = true
+      end
+
+      # Stop recording runs. {Recorder} stays installed but records nothing.
+      #
+      # @return [void]
+      def disable!
+        @enabled = false
+      end
+
+      # @return [Boolean] whether runs are being recorded
+      def enabled?
+        @enabled
+      end
+
+      # Record one real run. Ignores anonymous classes and anything that isn't
+      # a {Result}.
+      #
+      # @param action_class [Class<Actionable::Action>]
+      # @param result [Result]
+      # @return [void]
+      def record(action_class, result)
+        observation = observation_for(action_class, result)
+        @mutex.synchronize { @observed << observation } if observation
+      end
+
+      # Record one run answered by the gem's stub helpers
+      # (+allow_actionable_success+ and the rest). Kept apart from {.observed}:
+      # a stubbed outcome was never produced by the action, so it never counts
+      # as tested, but the report shows it.
+      #
+      # @param action_class [Class<Actionable::Action>]
+      # @param result [Result]
+      # @return [void]
+      def record_stub(action_class, result)
+        observation = observation_for(action_class, result)
+        @mutex.synchronize { @stubbed << observation } if observation
+      end
+
+      # The runs answered by a gem stub helper so far, deduplicated.
+      #
+      # @return [Set<Array(String, Symbol, Symbol)>] +[class name, outcome, code]+
+      def stubbed
+        @mutex.synchronize { @stubbed.dup }
+      end
+
+      # The real runs recorded so far, deduplicated.
+      #
+      # @return [Set<Array(String, Symbol, Symbol)>] +[class name, outcome, code]+
+      def observed
+        @mutex.synchronize { @observed.dup }
+      end
+
+      # Write the runs recorded so far to +path+ as JSON, for a sharded suite:
+      # each shard writes its own file and one {Report} merges them. Creates
+      # the directory.
+      #
+      # @param path [String]
+      # @return [String] +path+
+      def write(path)
+        FileUtils.mkdir_p(File.dirname(path))
+        data = {version: 1, observed: observed.to_a.sort, stubbed: stubbed.to_a.sort}
+        File.write(path, Oj.dump(data, mode: :compat))
+        path
+      end
+
+      # Read a file written by {.write}.
+      #
+      # @param path [String]
+      # @return [Hash{Symbol=>Set<Array(String, Symbol, Symbol)>}] +{observed:, stubbed:}+
+      def read(path)
+        data = Oj.load(File.read(path), mode: :compat)
+        %w[observed stubbed].to_h do |set|
+          [set.to_sym, Array(data[set]).to_set { |name, outcome, code| [name, outcome.to_sym, code.to_sym] }]
+        end
+      end
+
+      # Forget every recorded run, real and stubbed.
+      #
+      # @return [void]
+      def reset!
+        @mutex.synchronize do
+          @observed.clear
+          @stubbed.clear
+        end
+      end
+
+      private
+
+      # @return [Array<Class>] every subclass below +klass+, recursively
+      def descendants(klass)
+        klass.subclasses.flat_map { |subclass| [subclass, *descendants(subclass)] }
+      end
+
+      # Abstract actions are left out: an anonymous class has no name to
+      # report, and an action with no steps is a base class. A class whose
+      # name no longer resolves to it (a removed +stub_const+, a reloaded
+      # constant) is left out too.
+      def reportable?(klass)
+        name = klass.name
+        return false if name.nil? || klass.steps.empty?
+        return false unless resolves_to?(name, klass)
+        return false if include_prefixes.any? && include_prefixes.none? { |pattern| matches?(pattern, name) }
+
+        exclude_prefixes.none? { |pattern| matches?(pattern, name) }
+      end
+
+      def resolves_to?(name, klass)
+        Object.const_get(name).equal?(klass)
+      rescue NameError
+        false
+      end
+
+      def matches?(pattern, name)
+        pattern.is_a?(Regexp) ? pattern.match?(name) : name.start_with?(pattern.to_s)
+      end
+
+      def gem_code_applies?(action_class, code)
+        case code
+        when :invalid_input then input_can_be_invalid?(action_class)
+        when :invalid_output then output_schemas(action_class).any? { |schema| requires_field?(schema) }
+        end
+      end
+
+      # A fixed input with a required field, or a discriminated input with no
+      # default shape (an unmatched value fails) or a shape with one.
+      def input_can_be_invalid?(action_class)
+        return requires_field?(action_class.input_schema) if action_class.input_schema
+
+        dispatch = action_class.input_dispatch
+        return false unless dispatch
+
+        dispatch.default_schema.nil? || dispatch.schemas.any? { |schema| requires_field?(schema) }
+      end
+
+      # Every output schema a run can be held to: the success's, and each skip
+      # or failure code's (decision D23), any of which can end it as
+      # +invalid_output+.
+      def output_schemas(action_class)
+        skips = action_class.skip_declarations.keys.map { |code| action_class.skip_output_schema(code) }
+        failures = action_class.failure_declarations.keys.map { |code| action_class.failure_output_schema(code) }
+        [action_class.output_schema, *skips, *failures].compact
+      end
+
+      def requires_field?(schema)
+        return false unless schema
+
+        schema.metadata.to_h.values.any? { |field| field[:required] }
+      end
+
+      # @return [Array(String, Symbol, Symbol), nil]
+      def observation_for(action_class, result)
+        return unless enabled? && result.is_a?(Actionable::Result)
+
+        name = action_class.name
+        return if name.nil?
+
+        [name, outcome_of(result), result.code.to_sym]
+      end
+
+      # @param result [Result]
+      # @return [Symbol] +:success+, +:failure+ or +:skipped+
+      def outcome_of(result)
+        if result.failure? then :failure
+        elsif result.skipped? then :skipped
+        else :success
+        end
+      end
+    end
+
+    reset_config!
+  end
+end

data/lib/actionable/rspec/coverage.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+# Optional outcome coverage for an RSpec suite (decision D22). Load it with
+# `require 'actionable/rspec/coverage'`, typically from spec/support. Never
+# loaded by the core or by `actionable/rspec`.
+#
+# Requiring it starts recording every real action run and, when RSpec is
+# loaded, adds an end-of-suite hook: it writes the runs to the file named by
+# +ACTIONABLE_COVERAGE_FILE+ when that is set (one shard of a sharded suite),
+# and otherwise prints the report and holds it to {Actionable::Coverage.baseline}.
+require_relative 'coverage/tracker'
+require_relative 'coverage/report'
+
+module Actionable
+  module Coverage
+    class << self
+      # Start recording and, under RSpec, add the end-of-suite hook once.
+      #
+      # @return [void]
+      def install!
+        enable!
+        return if @installed || !defined?(::RSpec) || !::RSpec.respond_to?(:configure)
+
+        @installed = true
+        ::RSpec.configure { |config| config.after(:suite) { Actionable::Coverage.finish_suite! } }
+      end
+
+      # The end-of-suite step: write this process's runs to
+      # +ACTIONABLE_COVERAGE_FILE+ when it is set, since one shard can't judge
+      # the whole suite, and otherwise {.report!}.
+      #
+      # @param io [IO] where the text report goes
+      # @return [Report, String] the report, or the shard file's path
+      def finish_suite!(io: $stdout)
+        path = ENV['ACTIONABLE_COVERAGE_FILE'].to_s
+        return write(path) unless path.empty?
+
+        io.puts # the progress formatter leaves its line open
+        report!(io: io)
+      end
+    end
+  end
+end
+
+Actionable::Coverage.install!

data/lib/actionable/rspec/matchers.rb

@@ -0,0 +1,266 @@
+# frozen_string_literal: true
+
+module Actionable
+  module RSpec
+    # RSpec matcher methods mixed into example groups by +actionable/rspec+.
+    module Matchers
+      # Run an action and assert on its outcome.
+      #
+      #   expect(CreateInvoice).to perform_actionable(amount: 5).and_succeed
+      #   expect(CreateInvoice).to perform_actionable(bad).and_fail(:invalid_input)
+      #   expect(Boom).to          perform_actionable.and_raise(RuntimeError)
+      #
+      # @param args [Object] positional args forwarded to +.run+
+      # @param kwargs [Object] keyword args forwarded to +.run+
+      # @yieldparam result [Result, nil] the run's result (+nil+ when it raised)
+      # @yieldparam exception [StandardError, nil] what the run raised, if anything
+      # @yieldreturn [Object] truthy when the extra assertions hold
+      # @return [PerformActionable]
+      def perform_actionable(*args, **kwargs, &block)
+        PerformActionable.new(args, kwargs, block)
+      end
+    end
+
+    # The matcher object returned by {Matchers#perform_actionable}. Runs the
+    # action under test and checks the configured outcome (decision D12). Bare
+    # +perform_actionable+ expects a success; chain +and_succeed+ / +and_fail+ /
+    # +and_raise+ to be explicit. Code and message expectations match with +===+,
+    # so exact values, Regexps, and classes all work.
+    class PerformActionable
+      # Backtrace frames shown under +ACTIONABLE_SHORT_BACKTRACE+ when
+      # +ACTIONABLE_BACKTRACE_QTY+ is unset.
+      # @return [Integer]
+      DEFAULT_BACKTRACE_QTY = 10
+
+      # @param args [Array<Object>] positional args forwarded to +.run+
+      # @param kwargs [Hash{Symbol=>Object}] keyword args forwarded to +.run+
+      # @param block [Proc, nil] extra assertions, called with the result and exception
+      def initialize(args, kwargs, block)
+        @args = args
+        @kwargs = kwargs
+        @block = block
+        @outcome = :success
+      end
+
+      # Expect a successful result, optionally matching +message+.
+      # @param message [Object, nil] matched against the result's message with +===+
+      # @yieldparam result [Result, nil]
+      # @yieldparam exception [StandardError, nil]
+      # @yieldreturn [Object] truthy when the extra assertions hold
+      # @return [self]
+      def and_succeed(message = nil, &block)
+        configure(:success, message, &block)
+      end
+
+      # Expect a failed result, optionally matching +code+ and +message+. On an
+      # action that declares failures, a Symbol +code+ it doesn't declare raises
+      # +ArgumentError+ (decision D20).
+      # @param code [Object, nil] matched against the result's code with +==+
+      # @param message [Object, nil] matched against the result's message with +===+
+      # @yieldparam result [Result, nil]
+      # @yieldparam exception [StandardError, nil]
+      # @yieldreturn [Object] truthy when the extra assertions hold
+      # @return [self]
+      def and_fail(code = nil, message = nil, &block)
+        @expected_code = code
+        configure(:failure, message, &block)
+      end
+
+      # Expect a skipped result, optionally matching the reason +code+ and
+      # +message+ (decision D17). On an action that declares skips, a Symbol
+      # +code+ it doesn't declare raises +ArgumentError+ (decision D21).
+      # @param code [Object, nil] matched against the result's code with +==+
+      # @param message [Object, nil] matched against the result's message with +===+
+      # @yieldparam result [Result, nil]
+      # @yieldparam exception [StandardError, nil]
+      # @yieldreturn [Object] truthy when the extra assertions hold
+      # @return [self]
+      def and_skip(code = nil, message = nil, &block)
+        @expected_code = code
+        configure(:skip, message, &block)
+      end
+
+      # Expect the run to raise +error_class+, optionally matching +message+.
+      # @param error_class [Class<Exception>]
+      # @param message [Object, nil] matched against the exception's message with +===+
+      # @yieldparam result [Result, nil]
+      # @yieldparam exception [StandardError, nil]
+      # @yieldreturn [Object] truthy when the extra assertions hold
+      # @return [self]
+      def and_raise(error_class = StandardError, message = nil, &block)
+        @expected_error = error_class
+        configure(:raise, message, &block)
+      end
+
+      # @param action_class [Class<Actionable::Action>]
+      # @return [Boolean]
+      def matches?(action_class)
+        @action_class = action_class
+        ensure_code_declared!
+        perform
+        primary_match? && block_match?
+      end
+
+      # @param action_class [Class<Actionable::Action>]
+      # @return [Boolean]
+      def does_not_match?(action_class)
+        @action_class = action_class
+        ensure_code_declared!
+        perform
+        !primary_match?
+      end
+
+      # @return [String]
+      def failure_message
+        "expected #{@action_class} to #{expectation_phrase}, but #{actual_phrase}"
+      end
+
+      # @return [String]
+      def failure_message_when_negated
+        "expected #{@action_class} not to #{expectation_phrase}, but it did"
+      end
+
+      # @return [String]
+      def description
+        "perform actionable and #{@outcome}"
+      end
+
+      private
+
+      # @param outcome [Symbol] +:success+, +:failure+, +:skip+ or +:raise+
+      # @param message [Object, nil]
+      # @yieldparam result [Result, nil]
+      # @yieldparam exception [StandardError, nil]
+      # @yieldreturn [Object] truthy when the extra assertions hold
+      # @return [self]
+      def configure(outcome, message, &block)
+        @outcome = outcome
+        @expected_message = message
+        @block = block if block
+        self
+      end
+
+      # An +and_fail+ or +and_skip+ code the action could never end with is a
+      # broken spec, not a failed expectation, so it raises rather than not
+      # matching.
+      #
+      # @raise [ArgumentError] when the action declares the outcome's codes but not this one
+      # @return [void]
+      def ensure_code_declared!
+        return unless @expected_code.is_a?(Symbol)
+
+        check = {failure: :allows_failure_code?, skip: :allows_skip_code?}[@outcome]
+        return unless check && @action_class.respond_to?(check)
+        return if @action_class.public_send(check, @expected_code)
+
+        raise ArgumentError,
+          Actionable::RSpec.undeclared_code_message(@action_class, @expected_code, outcome: @outcome)
+      end
+
+      # Run the action, keeping its result or the error it raised.
+      # @return [void]
+      def perform
+        @result = nil
+        @exception = nil
+        @result = @action_class.run(*@args, **@kwargs)
+      rescue StandardError => e
+        @exception = e
+      end
+
+      # @return [Boolean, nil]
+      def primary_match?
+        case @outcome
+        when :success then succeeded?
+        when :failure then failed?
+        when :skip then skipped?
+        when :raise then raised?
+        end
+      end
+
+      # @return [Boolean, nil]
+      def succeeded?
+        @exception.nil? && @result&.success? && message_matches?(@result&.message)
+      end
+
+      # @return [Boolean, nil]
+      def failed?
+        @exception.nil? && @result&.failure? &&
+          (@expected_code.nil? || @expected_code == @result.code) &&
+          message_matches?(@result&.message)
+      end
+
+      # @return [Boolean, nil]
+      def skipped?
+        @exception.nil? && @result&.skipped? &&
+          (@expected_code.nil? || @expected_code == @result.code) &&
+          message_matches?(@result&.message)
+      end
+
+      # @return [Boolean]
+      def raised?
+        !@exception.nil? && @exception.is_a?(@expected_error) && message_matches?(@exception.message)
+      end
+
+      # @param actual [String, nil]
+      # @return [Boolean]
+      def message_matches?(actual)
+        @expected_message.nil? || @expected_message === actual # rubocop:disable Style/CaseEquality
+      end
+
+      # @return [Boolean]
+      def block_match?
+        return true unless @block
+
+        !!@block.call(@result, @exception)
+      end
+
+      # @return [String, nil]
+      def expectation_phrase
+        case @outcome
+        when :success then "succeed#{message_suffix}"
+        when :failure then "fail#{code_suffix}#{message_suffix}"
+        when :skip then "skip#{code_suffix}#{message_suffix}"
+        when :raise then "raise #{@expected_error}#{message_suffix}"
+        end
+      end
+
+      # @return [String]
+      def code_suffix
+        @expected_code ? " with code #{@expected_code.inspect}" : ''
+      end
+
+      # @return [String]
+      def message_suffix
+        @expected_message ? " with message #{@expected_message.inspect}" : ''
+      end
+
+      # @return [String]
+      def actual_phrase
+        if @exception
+          "it raised #{@exception.class}: #{@exception.message}\n#{formatted_backtrace}"
+        elsif @result
+          "got #{@result.inspect}"
+        else
+          'no result was produced'
+        end
+      end
+
+      # @return [String]
+      def formatted_backtrace
+        frames = @exception.backtrace || []
+        frames = frames.first(backtrace_qty) if short_backtrace?
+        frames.join("\n")
+      end
+
+      # @return [Boolean]
+      def short_backtrace?
+        ENV.key?('ACTIONABLE_SHORT_BACKTRACE')
+      end
+
+      # @return [Integer]
+      def backtrace_qty
+        (ENV['ACTIONABLE_BACKTRACE_QTY'] || DEFAULT_BACKTRACE_QTY).to_i
+      end
+    end
+  end
+end