yard_example_runner 0.1.0

data/lib/yard_example_runner/example/constant_sandbox.rb

@@ -0,0 +1,123 @@
+# frozen_string_literal: true
+
+module YardExampleRunner
+  class Example < ::Minitest::Spec
+    # Isolates constant definitions introduced during example evaluation
+    #
+    # Resolves the top-level class or module for a YARD definition path,
+    # snapshots the constants on both +Object+ and that scope, yields control
+    # to the caller, then removes any constants that were added during the
+    # block. This prevents one example's constant definitions from leaking
+    # into subsequent examples.
+    #
+    # @example
+    #   ConstantSandbox.new('MyClass#foo').isolate do |scope|
+    #     # evaluate code that may define constants…
+    #   end
+    #   # any constants defined inside the block are now removed
+    #
+    class ConstantSandbox
+      # Creates a sandbox for the given YARD definition path
+      #
+      # @param definition [String] a YARD path such as +"MyClass#method"+ or
+      #   +"MyClass.method"+
+      #
+      # @example
+      #   ConstantSandbox.new('MyClass#foo')
+      #
+      # @api private
+      #
+      def initialize(definition)
+        @scope = resolve_scope(definition)
+      end
+
+      # Snapshots constants, yields the resolved scope, then cleans up
+      #
+      # Any constants added to +Object+ or the resolved scope during the
+      # block are removed when the block returns (or raises), with one
+      # exception: constants added to +Object+ whose source file was newly
+      # loaded via +require+ during the block are preserved. This prevents
+      # one example's constant definitions from leaking into subsequent
+      # examples while still allowing +require+ calls to have their normal
+      # lasting effect (re-requiring a cached file would be a no-op, so
+      # stripping those constants would cause +NameError+ in later examples).
+      #
+      # The resolved scope is yielded so that callers can use it as the
+      # evaluation binding.
+      #
+      # @yield [scope] gives the resolved class/module (or +nil+) to the block
+      # @yieldparam scope [Class, Module, nil] the resolved scope constant
+      #
+      # @return [void]
+      #
+      # @example
+      #   sandbox.isolate { |scope| scope.class_eval(code) }
+      #
+      # @api private
+      #
+      def isolate
+        global_before = Object.constants
+        scope_before  = @scope.respond_to?(:constants) ? @scope.constants : nil
+        loaded_before = $LOADED_FEATURES.dup
+
+        yield @scope
+      ensure
+        loaded_during = $LOADED_FEATURES - loaded_before
+        clear_extra_constants(Object, global_before, skip_if_loaded_by: loaded_during)
+        clear_extra_constants(@scope, scope_before) if scope_before
+      end
+
+      private
+
+      # Resolves the top-level class or module constant for a YARD definition path
+      #
+      # Extracts the leading constant name from +definition+ (the portion before
+      # the first +#+ or +.+ separator), then returns the corresponding constant
+      # from +Object+ if it exists. Returns +nil+ if the definition does not start
+      # with a constant name or if the constant is not currently defined.
+      #
+      # @param definition [String] a YARD path such as +"MyClass#method"+
+      #
+      # @return [Class, Module, nil] the resolved constant, or +nil+
+      #
+      # @example
+      #   resolve_scope('MyClass#foo') # => MyClass
+      #
+      # @api private
+      #
+      def resolve_scope(definition)
+        name = definition.split(/#|\./).first
+        Object.const_get(name) if name&.match?(/\A[A-Z]/) && Object.const_defined?(name)
+      end
+
+      # Removes constants from +scope+ that were not present in +before+
+      #
+      # When +skip_if_loaded_by+ is non-empty, any constant on +scope+ whose
+      # source file (as reported by +Module#const_source_location+) appears in
+      # +skip_if_loaded_by+ is preserved rather than removed. This is used to
+      # retain constants introduced by +require+ calls during example evaluation.
+      #
+      # @param scope [Module] the scope to clean up
+      # @param before [Array<Symbol>] the constant names present before evaluation
+      # @param skip_if_loaded_by [Array<String>] absolute paths of files newly
+      #   loaded during evaluation; constants defined in these files are kept
+      #
+      # @return [void]
+      #
+      # @example
+      #   clear_extra_constants(Object, before_constants)
+      #
+      # @api private
+      #
+      def clear_extra_constants(scope, before, skip_if_loaded_by: [])
+        (scope.constants - before).each do |constant|
+          if skip_if_loaded_by.any?
+            source_file, = scope.const_source_location(constant.to_s)
+            next if source_file && skip_if_loaded_by.include?(source_file)
+          end
+          scope.__send__(:remove_const, constant)
+        end
+      end
+    end
+  end
+end

data/lib/yard_example_runner/example/evaluator.rb

@@ -0,0 +1,145 @@
+# frozen_string_literal: true
+
+module YardExampleRunner
+  class Example < ::Minitest::Spec
+    # Manages binding creation and code evaluation for example expressions
+    #
+    # Each {Evaluator} is constructed with a fallback binding (whose +self+ is
+    # the +Minitest::Spec+ instance, providing access to any methods included
+    # on {Example} such as +RSpec::Matchers+) and a snapshot of instance
+    # variables from the spec instance (set by +before+ hooks). These are used
+    # to build per-scope evaluation contexts that mirror the documented class's
+    # namespace.
+    #
+    # @see Example
+    #
+    class Evaluator
+      # Creates a new evaluator
+      #
+      # @param fallback_binding [Binding] a binding whose +self+ is the spec
+      #   instance, used when the expression is not scoped to a specific class
+      #
+      # @param instance_variables [Hash{Symbol => Object}] a snapshot of instance
+      #   variable names to values from the spec instance, transplanted into
+      #   class-scoped bindings so that hook-set state is accessible
+      #
+      # @example
+      #   Evaluator.new(fallback_binding: binding, instance_variables: {})
+      #
+      # @api private
+      #
+      def initialize(fallback_binding:, instance_variables:)
+        @fallback_binding = fallback_binding
+        @instance_variables = instance_variables
+      end
+
+      # Evaluates a Ruby code string in the given scope
+      #
+      # @param code [String] the Ruby expression to evaluate
+      # @param bind [Class, nil] the class scope to evaluate in, or +nil+ for
+      #   the default (fallback) binding
+      #
+      # @return [Object] the result of evaluating +code+
+      #
+      # @raise [StandardError] any error raised during evaluation propagates
+      #
+      # @example
+      #   evaluator.evaluate('1 + 1', nil) # => 2
+      #
+      # @api private
+      #
+      def evaluate(code, bind)
+        context(bind).eval(code)
+      end
+
+      # Evaluates a Ruby code string, capturing any +StandardError+ as a value
+      #
+      # If evaluation raises a +StandardError+, the error itself is returned
+      # instead of propagating. This allows callers to compare raised errors
+      # against expected error values.
+      #
+      # @param code [String] the Ruby expression to evaluate
+      # @param bind [Class, nil] the class scope to evaluate in
+      #
+      # @return [Object, StandardError] the result of evaluation, or the error
+      #
+      # @example
+      #   evaluator.evaluate_with_assertion('raise "oops"', nil) # => RuntimeError
+      #
+      # @api private
+      #
+      def evaluate_with_assertion(code, bind)
+        evaluate(code, bind)
+      rescue StandardError => e
+        e
+      end
+
+      private
+
+      # Returns or creates the cached evaluation context for the given scope
+      #
+      # @param bind [Class, nil] the class scope
+      #
+      # @return [Binding] a binding suitable for evaluating code in +bind+
+      #
+      # @example
+      #   context(MyClass) # => Binding
+      #
+      # @api private
+      #
+      def context(bind)
+        @contexts ||= {}.compare_by_identity
+        @contexts[bind] ||= build_context(bind)
+      end
+
+      # Builds an evaluation context for the given scope
+      #
+      # When +bind+ responds to +class_eval+, a new binding is opened inside
+      # that class and instance variables are transplanted into it. Otherwise
+      # the fallback binding (whose +self+ is the spec instance) is returned.
+      #
+      # @param bind [Class, nil] the class scope
+      #
+      # @return [Binding]
+      #
+      # @example
+      #   build_context(MyClass) # => Binding
+      #
+      # @api private
+      #
+      def build_context(bind)
+        if bind.respond_to?(:class_eval)
+          ctx = bind.class_eval('binding', __FILE__, __LINE__)
+          transplant_instance_variables(ctx)
+          ctx
+        else
+          @fallback_binding
+        end
+      end
+
+      # Copies instance variables into an evaluation binding
+      #
+      # Sets each instance variable from the snapshot as a local, then
+      # assigns it to the corresponding instance variable name via +eval+.
+      # This makes hook-set state (e.g. +@flag+) available in class-scoped
+      # bindings.
+      #
+      # @param ctx [Binding] the target binding
+      #
+      # @return [void]
+      #
+      # @example
+      #   transplant_instance_variables(ctx)
+      #
+      # @api private
+      #
+      def transplant_instance_variables(ctx)
+        @instance_variables.each do |ivar, value|
+          local = "__yard_example_runner__#{ivar.to_s.delete('@')}"
+          ctx.local_variable_set(local, value)
+          ctx.eval("#{ivar} = #{local}")
+        end
+      end
+    end
+  end
+end

data/lib/yard_example_runner/example.rb

@@ -0,0 +1,360 @@
+# frozen_string_literal: true
+
+require_relative 'example/constant_sandbox'
+require_relative 'example/evaluator'
+require_relative 'example/comparison'
+
+module YardExampleRunner
+  # Represents a YARD +@example+ tag as a runnable +Minitest::Spec+
+  #
+  # Each instance is populated from a single +@example+ tag by
+  # {YARD::CLI::RunExamples#build_spec} and holds everything needed to
+  # generate and execute a test:
+  #
+  # - {#definition} — the YARD path of the documented object
+  #   (e.g. +"MyClass#my_method"+), used as the spec description
+  #
+  # - {#filepath} — the source location of the documented object
+  #   (e.g. +"lib/my_class.rb:10"+), prepended to failure backtraces
+  #
+  # - {#expectations} — the list of {Expectation} objects parsed from the
+  #   example body, each pairing a Ruby expression to evaluate with an
+  #   optional expected return value
+  #
+  # Calling {#generate} dynamically defines and registers an anonymous
+  # +Minitest::Spec+ subclass that wraps the expectations in a single +it+
+  # block. The registered spec is then picked up by +Minitest.autorun+ when
+  # the process exits.
+  #
+  # Each expectation is evaluated inside a binding scoped to the owning
+  # object's class (if one can be resolved), so instance methods and
+  # constants are available without qualification, mirroring how the code
+  # appears in the source documentation.
+  #
+  # Evaluation is delegated to an {Evaluator} (binding management and
+  # +eval+), constant isolation is handled by {ConstantSandbox}, and
+  # assertion / matcher logic lives in the {Comparison} module.
+  #
+  # @see YARD::CLI::RunExamples
+  #
+  # @see Expectation
+  #
+  # @see Evaluator
+  #
+  # @see ConstantSandbox
+  #
+  # @see Comparison
+  #
+  # @api public
+  #
+  class Example < ::Minitest::Spec
+    include Comparison
+
+    # The YARD namespace path of the documented object (e.g. +Foo#bar+)
+    #
+    # @example
+    #   example.definition #=> 'Foo#bar'
+    #
+    # @return [String] namespace path of example
+    #
+    # @api public
+    attr_accessor :definition
+
+    # The source location of the documented object (e.g. +app/app.rb:10+)
+    #
+    # @example
+    #   example.filepath #=> 'app/app.rb:10'
+    #
+    # @return [String] filepath to definition
+    #
+    # @api public
+    attr_accessor :filepath
+
+    # The list of expectations parsed from the example body
+    #
+    # @example
+    #   example.expectations #=> []
+    #
+    # @return [Array<YardExampleRunner::Expectation>] expectations to be verified
+    #
+    # @api public
+    attr_accessor :expectations
+
+    # Dynamically defines and registers a +Minitest::Spec+ for this example
+    #
+    # Creates an anonymous subclass of this class and evaluates a +describe+/+it+
+    # block inside it. The steps are:
+    #
+    # 1. Calls +load_helpers+ to require any +example_runner_helper+ files found in
+    #    +.+, +support/+, +spec/+, or +test/+.
+    # 2. Skips silently if {YardExampleRunner.skips} contains a substring that
+    #    matches {#definition}.
+    # 3. Opens a +describe+ block keyed on {#definition}, which becomes the spec
+    #    group name reported by Minitest.
+    # 4. Registers any matching +before+/+after+ hooks via +register_hooks+. These
+    #    are registered by the user with {YardExampleRunner.before} and
+    #    {YardExampleRunner.after}.
+    # 5. Opens an +it+ block keyed on +name+ (the +@example+ tag title) that calls
+    #    +run_expectations+ to evaluate every {Expectation} in {#expectations}.
+    #
+    # The anonymous class and its specs are registered with Minitest's internal list
+    # by the +describe+ call. They will be executed when +Minitest.autorun+'s
+    # +at_exit+ hook fires.
+    #
+    # @example
+    #   example.generate
+    #
+    # @return [void]
+    #
+    def generate
+      self.class.send(:load_helpers)
+      return if skipped?
+
+      this = self
+      Class.new(this.class).class_eval do
+        describe this.definition do
+          register_hooks(example_name_for(this), YardExampleRunner.hooks, this)
+          it(this.name) { run_expectations(this) }
+        end
+      end
+    end
+
+    protected
+
+    # Returns +true+ if this example's {#definition} matches any skip pattern
+    #
+    # Iterates over {YardExampleRunner.skips} and returns +true+ as soon as a
+    # pattern is found that is a substring of {#definition}. Used by {#generate}
+    # to bail out before registering any +Minitest::Spec+ subclass.
+    #
+    # @example
+    #   example.skipped? #=> false
+    #
+    # @return [Boolean] +true+ if the example should be skipped, +false+ otherwise
+    #
+    # @api private
+    #
+    def skipped?
+      YardExampleRunner.skips.any? { |skip| definition.include?(skip) }
+    end
+
+    # Evaluates every {Expectation} in the given example
+    #
+    # Delegates constant isolation to {ConstantSandbox}, which snapshots
+    # the current constants on +Object+ and the resolved scope, yields the
+    # scope for evaluation, then removes any constants that were introduced
+    # during evaluation.
+    #
+    # @param example [Example] the example whose {Example#expectations} are to be run
+    #
+    # @example
+    #   run_expectations(example)
+    #
+    # @return [void]
+    #
+    # @api private
+    #
+    def run_expectations(example)
+      ConstantSandbox.new(example.definition).isolate do |scope|
+        example.expectations.each { |expectation| run_expectation(example, expectation, scope) }
+      end
+    end
+
+    # Evaluates a single {Expectation} within the given scope
+    #
+    # If the expectation has no expected value ({Expectation#expected} is +nil+),
+    # the actual expression is evaluated for its side-effects only via
+    # {#evaluate_actual}. Otherwise the actual and expected expressions are both
+    # evaluated and compared via {Comparison#verify_actual}.
+    #
+    # @param example [Example] the owning example, used for backtrace decoration
+    #
+    # @param expectation [Expectation] the expectation to evaluate
+    #
+    # @param scope [Class, nil] the class scope to evaluate expressions in, or
+    #   +nil+ to evaluate in the default binding
+    #
+    # @example
+    #   run_expectation(example, expectation, MyClass)
+    #
+    # @return [void]
+    #
+    # @api private
+    #
+    def run_expectation(example, expectation, scope)
+      if expectation.expected.nil?
+        evaluate_actual(example, expectation.actual, scope)
+      else
+        verify_actual(example, expectation.expected, expectation.actual, scope)
+      end
+    end
+
+    # Evaluates the actual expression for side-effects only
+    #
+    # Delegates to {Evaluator#evaluate} and re-raises any +StandardError+ after
+    # prepending the example's source location to the backtrace via
+    # {#add_filepath_to_backtrace}, so that failure output points to the
+    # documented source rather than this file.
+    #
+    # @param example [Example] the owning example, used to decorate error backtraces
+    #
+    # @param actual [String] the Ruby expression to evaluate
+    #
+    # @param bind [Class, nil] the class scope to evaluate the expression in, or
+    #   +nil+ to use the default binding
+    #
+    # @example
+    #   evaluate_actual(example, 'foo(1)', MyClass)
+    #
+    # @return [void]
+    #
+    # @raise [StandardError] re-raises any error raised during evaluation, with the
+    #   example's filepath prepended to the backtrace
+    #
+    # @api private
+    #
+    def evaluate_actual(example, actual, bind)
+      evaluator.evaluate(actual, bind)
+    rescue StandardError => e
+      add_filepath_to_backtrace(e, example.filepath)
+      raise e
+    end
+
+    # Returns the lazily-initialized {Evaluator} for this spec instance
+    #
+    # The evaluator is created with a fallback binding (whose +self+ is this
+    # spec instance, so that methods included on {Example} — such as
+    # +RSpec::Matchers+ — are accessible in evaluated code) and a snapshot of
+    # the spec instance's instance variables (set by +before+ hooks).
+    #
+    # @example
+    #   evaluator.evaluate('1 + 1', nil)
+    #
+    # @return [Evaluator]
+    #
+    # @api private
+    #
+    def evaluator
+      @evaluator ||= Evaluator.new(
+        fallback_binding: create_fallback_binding,
+        instance_variables: instance_variable_hash
+      )
+    end
+
+    # Prepends the example's filepath to an exception's backtrace
+    #
+    # @param exception [Exception] the exception to decorate
+    #
+    # @param filepath [String] the source location to prepend
+    #
+    # @example
+    #   add_filepath_to_backtrace(exception, 'app/app.rb:10')
+    #
+    # @return [void]
+    #
+    # @api private
+    #
+    def add_filepath_to_backtrace(exception, filepath)
+      exception.set_backtrace([filepath] + exception.backtrace)
+    end
+
+    private
+
+    # Returns a binding whose +self+ is this spec instance
+    #
+    # Because {Example} includes any modules the user adds (e.g.
+    # +RSpec::Matchers+), the returned binding automatically exposes those
+    # methods to code evaluated via the {Evaluator}'s fallback path.
+    #
+    # @example
+    #   create_fallback_binding
+    #
+    # @return [Binding]
+    #
+    # @api private
+    #
+    def create_fallback_binding
+      binding
+    end
+
+    # Snapshots instance variables as a +Hash+
+    #
+    # Returns a hash mapping instance variable names to their current values
+    # on this spec instance. The {Evaluator} uses this snapshot to transplant
+    # hook-set state into class-scoped bindings.
+    #
+    # @example
+    #   instance_variable_hash #=> { :@foo => 1 }
+    #
+    # @return [Hash{Symbol => Object}]
+    #
+    # @api private
+    #
+    def instance_variable_hash
+      instance_variables.to_h do |ivar|
+        [ivar, instance_variable_get(ivar)]
+      end
+    end
+
+    class << self
+      protected
+
+      # Requires any +example_runner_helper+ files found in known directories
+      #
+      # @example
+      #   load_helpers
+      #
+      # @return [void]
+      #
+      # @api private
+      #
+      def load_helpers
+        %w[. support spec test].each do |dir|
+          require "#{dir}/example_runner_helper" if File.exist?("#{dir}/example_runner_helper.rb")
+        end
+      end
+
+      # Returns the full example name including an optional title suffix
+      #
+      # @param example [Example] the example to build a name for
+      #
+      # @example
+      #   example_name_for(example) #=> 'Foo#bar'
+      #
+      # @return [String] the example name
+      #
+      # @api private
+      #
+      def example_name_for(example)
+        return example.definition if example.name.empty?
+
+        "#{example.definition}@#{example.name}"
+      end
+
+      # Registers matching before/after hooks on the current spec context
+      #
+      # @param example_name [String] the name of the example
+      #
+      # @param all_hooks [Hash{Symbol => Array<Hash>}] hooks grouped by type
+      #
+      # @param example [Example] the example being registered
+      #
+      # @example
+      #   register_hooks('Foo#bar', YardExampleRunner.hooks, example)
+      #
+      # @return [void]
+      #
+      # @api private
+      #
+      def register_hooks(example_name, all_hooks, example)
+        all_hooks.each do |type, hooks|
+          global_hooks = hooks.reject { |hook| hook[:test] }
+          test_hooks   = hooks.select { |hook| hook[:test] && example_name.include?(hook[:test]) }
+          __send__(type) do
+            (global_hooks + test_hooks).each { |hook| instance_exec(example, &hook[:block]) }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/yard_example_runner/expectation.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module YardExampleRunner
+  # @!parse
+  #   # Represents a single expected outcome parsed from a YARD +@example+ tag
+  #   #
+  #   # Each instance holds the Ruby expression to evaluate (+actual+) and the
+  #   # string representation of the value it should return (+expected+). When
+  #   # +expected+ is +nil+, the expression is evaluated for side-effects only and
+  #   # no assertion is made against its return value.
+  #   #
+  #   # @!attribute actual [r]
+  #   #   @return [String] the Ruby expression to evaluate
+  #   #
+  #   # @!attribute expected [r]
+  #   #   @return [String, nil] the expected return value, or +nil+ if no
+  #   #     assertion should be made
+  #   #
+  #   # @api public
+  #   #
+  #   class Expectation < Data
+  Expectation = Data.define(:actual, :expected)
+end