rspec-hermetic 0.1.0

data/lib/rspec/hermetic/evaluation.rb

@@ -0,0 +1,184 @@
+# frozen_string_literal: true
+
+require "json"
+require "fileutils"
+require "tmpdir"
+require "time"
+
+require_relative "../hermetic"
+
+module RSpec
+  module Hermetic
+    class Evaluation
+      Example = Struct.new(:metadata, :full_description, :location, :block, keyword_init: true) do
+        attr_reader :exception
+
+        def run
+          block.call
+        rescue StandardError => error
+          @exception = error
+          raise
+        end
+
+        def reporter
+          self
+        end
+
+        def message(_text); end
+      end
+
+      def initialize(output_path: "tmp/rspec_hermetic_evaluation.json")
+        @output_path = output_path
+      end
+
+      def run
+        report = {
+          "seeded_pollution" => seeded_pollution_results,
+          "generated_at" => Time.now.utc.iso8601
+        }
+        write(report)
+        report
+      end
+
+      private
+
+      def seeded_pollution_results
+        cases = [
+          seeded_case(:env) { ENV["RSPEC_HERMETIC_EVAL"] = "dirty" },
+          seeded_case(
+            :constants,
+            setup: method(:setup_constant_case),
+            cleanup: method(:cleanup_constant_case),
+            configure: proc { |config| config.constant_namespaces = [HermeticEvaluationState] }
+          ) do
+            HermeticEvaluationState::STORE << :dirty
+          end,
+          seeded_case(:globals, cleanup: method(:cleanup_global_case)) { $LOAD_PATH << "/tmp/rspec-hermetic-eval" },
+          seeded_case(:ruby_runtime, cleanup: method(:cleanup_ruby_runtime_case)) do
+            Thread.abort_on_exception = !Thread.abort_on_exception
+          end,
+          seeded_case(:rails, setup: method(:setup_rails_case), cleanup: method(:cleanup_rails_case)) do
+            I18n.locale = :ja
+          end,
+          seeded_case(:time, cleanup: method(:cleanup_time_case)) { setup_time_case },
+          seeded_case(:randomness, configure: proc { |config| config.randomness_seed_probe = true }) { Kernel.srand(12_345) },
+          seeded_case(:resources, cleanup: method(:cleanup_resource_case)) do
+            @evaluation_thread = Thread.new { sleep 5 }
+            sleep 0.05
+          end,
+          seeded_filesystem_case
+        ]
+
+        total = cases.length
+        detected = cases.count { |result| result["detected"] }
+        {
+          "total" => total,
+          "detected" => detected,
+          "recall" => total.zero? ? 0.0 : detected.fdiv(total),
+          "cases" => cases
+        }
+      end
+
+      def seeded_case(probe, setup: nil, cleanup: nil, configure: nil, &block)
+        setup&.call
+        config = Configuration.new
+        config.probes = [probe]
+        config.on_pollution = :report
+        configure&.call(config)
+        runner = Runner.new(config)
+        example = Example.new(
+          metadata: {},
+          full_description: "seeded #{probe} pollution",
+          location: "evaluation/#{probe}",
+          block: block
+        )
+        runner.call(example, nil)
+        result_for(probe, runner)
+      ensure
+        cleanup&.call
+        ENV.delete("RSPEC_HERMETIC_EVAL") if probe == :env
+      end
+
+      def seeded_filesystem_case
+        Dir.mktmpdir do |dir|
+          seeded_case(
+            :filesystem,
+            configure: proc do |config|
+              config.root_path = dir
+              config.filesystem_paths = ["."]
+            end
+          ) { File.write(File.join(dir, "dirty.txt"), "dirty") }
+        end
+      end
+
+      def result_for(probe, runner)
+        pollutions = runner.records.sum { |record| record[:verdict].pollutions.count }
+        warnings = runner.records.sum { |record| record[:verdict].warnings.count }
+        {
+          "probe" => probe.to_s,
+          "detected" => pollutions.positive? || warnings.positive?,
+          "pollutions" => pollutions,
+          "warnings" => warnings
+        }
+      end
+
+      def setup_constant_case
+        Object.const_set(:HermeticEvaluationState, Module.new)
+        HermeticEvaluationState.const_set(:STORE, [])
+      end
+
+      def cleanup_constant_case
+        Object.__send__(:remove_const, :HermeticEvaluationState) if Object.const_defined?(:HermeticEvaluationState)
+      end
+
+      def cleanup_global_case
+        $LOAD_PATH.delete("/tmp/rspec-hermetic-eval")
+      end
+
+      def cleanup_ruby_runtime_case
+        Thread.abort_on_exception = false
+      end
+
+      def setup_rails_case
+        @previous_i18n = Object.const_get(:I18n) if Object.const_defined?(:I18n)
+        Object.__send__(:remove_const, :I18n) if Object.const_defined?(:I18n)
+        i18n = Class.new do
+          class << self
+            attr_accessor :locale, :default_locale
+          end
+        end
+        i18n.locale = :en
+        i18n.default_locale = :en
+        Object.const_set(:I18n, i18n)
+      end
+
+      def cleanup_rails_case
+        Object.__send__(:remove_const, :I18n) if Object.const_defined?(:I18n)
+        Object.const_set(:I18n, @previous_i18n) if @previous_i18n
+        @previous_i18n = nil
+      end
+
+      def setup_time_case
+        @previous_time_now = Time.method(:now)
+        previous = @previous_time_now
+        Time.define_singleton_method(:now) { previous.call + 3600 }
+      end
+
+      def cleanup_time_case
+        Time.define_singleton_method(:now, @previous_time_now) if @previous_time_now
+        @previous_time_now = nil
+      end
+
+      def cleanup_resource_case
+        @evaluation_thread&.kill
+        @evaluation_thread&.join
+        @evaluation_thread = nil
+      end
+
+      def write(report)
+        FileUtils.mkdir_p(File.dirname(@output_path))
+        File.write(@output_path, "#{JSON.pretty_generate(report)}\n")
+      end
+    end
+  end
+end

data/lib/rspec/hermetic/evaluation_task.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+require "rake"
+
+require_relative "corpus_evaluation"
+require_relative "evaluation"
+
+module RSpec
+  module Hermetic
+    class EvaluationTask
+      extend ::Rake::DSL
+
+      DEFAULT_OUTPUT = "tmp/rspec_hermetic_evaluation.json"
+
+      def self.install(name = "hermetic:evaluate")
+        namespace_name, short_name = name.to_s.split(":", 2)
+
+        namespace namespace_name do
+          desc "Run rspec-hermetic seeded pollution evaluation and write a JSON recall report"
+          task short_name, [:output] do |_task, args|
+            output = args[:output] || ENV["HERMETIC_EVALUATION_OUTPUT"] || DEFAULT_OUTPUT
+            report = Evaluation.new(output_path: output).run
+            seeded = report.fetch("seeded_pollution")
+            puts "seeded recall #{seeded.fetch("detected")}/#{seeded.fetch("total")} (#{seeded.fetch("recall")})"
+            puts "wrote #{output}"
+          end
+        end
+      end
+
+      def self.install_corpus(name = "hermetic:evaluate_corpus")
+        namespace_name, short_name = name.to_s.split(":", 2)
+
+        namespace namespace_name do
+          desc "Run corpus evaluation using HERMETIC_COMMAND and optional HERMETIC_BASELINE_COMMAND"
+          task short_name, [:output] do |_task, args|
+            command = ENV.fetch("HERMETIC_COMMAND")
+            output = args[:output] || ENV["HERMETIC_CORPUS_OUTPUT"] || "tmp/rspec_hermetic_corpus.json"
+            report = CorpusEvaluation.new(
+              output_path: output,
+              project_path: ENV["HERMETIC_PROJECT"] || Dir.pwd,
+              baseline_command: ENV["HERMETIC_BASELINE_COMMAND"],
+              hermetic_command: command,
+              candidate_report_path: ENV["HERMETIC_CANDIDATES"] || "tmp/rspec_hermetic_candidates.json",
+              judgments_path: ENV["HERMETIC_JUDGMENTS"]
+            ).run
+            puts "corpus candidates #{report.fetch("candidate_count")}"
+            puts "wrote #{output}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rspec/hermetic/forensic.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require_relative "../hermetic"
+
+module RSpec
+  module Hermetic
+    module Forensic
+      def self.install!(rspec_config)
+        Hermetic.configure(rspec_config) do |configuration|
+          configuration.forensic = true
+          configuration.on_pollution = :report if configuration.on_pollution == :risky
+        end
+      end
+    end
+  end
+end
+
+if defined?(::RSpec) && ::RSpec.respond_to?(:configure)
+  ::RSpec.configure do |config|
+    ::RSpec::Hermetic::Forensic.install!(config)
+  end
+end

data/lib/rspec/hermetic/formatter.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+require_relative "change"
+
+module RSpec
+  module Hermetic
+    class Formatter
+      def self.pollution_message(example, result, change_set)
+        new.pollution_message(example, result, change_set)
+      end
+
+      def self.forensic_message(example, changes, last_writers)
+        new.forensic_message(example, changes, last_writers)
+      end
+
+      def self.probe_error_message(example, change_set)
+        new.probe_error_message(example, change_set)
+      end
+
+      def pollution_message(example, result, change_set)
+        lines = ["POLLUTER  #{example_location(example)}"]
+        result.pollutions.each { |change| lines << "  #{format_change(change)}" }
+        result.warnings.each { |change| lines << "  #{format_change(change)}  (append-only warning)" }
+        append_probe_errors(lines, change_set)
+        append_timings(lines, change_set)
+        lines << "  hint: this example left shared state changed for later examples"
+        lines.join("\n")
+      end
+
+      def forensic_message(example, changes, last_writers)
+        lines = ["VICTIM    #{example_location(example)} started with dirty shared state"]
+        changes.each do |change|
+          writer = last_writers[[change.probe, change.key]]
+          suffix = writer ? "  (last changed by #{writer})" : ""
+          lines << "  #{format_change(change)}#{suffix}"
+        end
+        lines.join("\n")
+      end
+
+      def probe_error_message(example, change_set)
+        lines = ["HERMETIC PROBE ERROR  #{example_location(example)}"]
+        append_probe_errors(lines, change_set)
+        append_timings(lines, change_set)
+        lines.join("\n")
+      end
+
+      private
+
+      def example_location(example)
+        location = example.respond_to?(:location) ? example.location : nil
+        description = example.respond_to?(:full_description) ? example.full_description : nil
+        [location, description].compact.join(" ")
+      end
+
+      def format_change(change)
+        "#{display_key(change)} #{display_value(change.before)} -> #{display_value(change.after)} (probe: #{change.probe})"
+      end
+
+      def display_key(change)
+        case change.probe
+        when :env
+          "ENV[#{change.key.inspect}]"
+        else
+          change.key.to_s
+        end.ljust(28)
+      end
+
+      def display_value(value)
+        return "<missing>" if value.equal?(Change::MISSING)
+
+        value.inspect
+      end
+
+      def append_probe_errors(lines, change_set)
+        change_set.before_errors.each do |probe, error|
+          lines << "  before probe error #{probe}: #{error}"
+        end
+        change_set.after_errors.each do |probe, error|
+          lines << "  after probe error #{probe}: #{error}"
+        end
+      end
+
+      def append_timings(lines, change_set)
+        return if change_set.timings.empty?
+
+        timing = change_set.timings.map do |probe, seconds|
+          "#{probe}=#{(seconds * 1000).round(2)}ms"
+        end.join(", ")
+        lines << "  probe timings: #{timing}"
+      end
+    end
+  end
+end

data/lib/rspec/hermetic/minitest.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+begin
+  gem "minitest"
+rescue Gem::LoadError
+  nil
+end
+
+require "minitest"
+
+require_relative "../hermetic"
+
+module RSpec
+  module Hermetic
+    module Minitest
+      class Example
+        attr_reader :metadata, :full_description, :location, :exception
+
+        def initialize(test, block)
+          @test = test
+          @block = block
+          @metadata = {}
+          @full_description = "#{test.class}##{test.name}"
+          @location = source_location(test)
+        end
+
+        def run
+          result = @block.call
+          @exception = result.failures.first if result.respond_to?(:failures) && result.failures.any?
+          result
+        rescue StandardError => error
+          @exception = error
+          raise
+        end
+
+        def reporter
+          nil
+        end
+
+        private
+
+        def source_location(test)
+          location = test.method(test.name).source_location
+          location ? location.join(":") : full_description
+        rescue NameError
+          full_description
+        end
+      end
+
+      module TestMethods
+        def run
+          result = nil
+          example = Example.new(self, proc { result = super() })
+          RSpec::Hermetic.minitest_runner.call(example, self)
+          result
+        end
+      end
+
+      def self.install!
+        return if @installed
+
+        ::Minitest::Test.prepend(TestMethods)
+        @installed = true
+      end
+    end
+
+    class << self
+      attr_reader :minitest_runner
+
+      def configure_minitest
+        yield configuration if block_given?
+        @minitest_runner = Runner.new(configuration)
+        Minitest.install!
+        configuration
+      end
+    end
+  end
+end
+
+RSpec::Hermetic.configure_minitest unless ENV["RSPEC_HERMETIC_MINITEST_AUTO_INSTALL"] == "false"

data/lib/rspec/hermetic/probe/base.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module RSpec
+  module Hermetic
+    module Probe
+      class Base
+        def initialize(configuration)
+          @configuration = configuration
+        end
+
+        def name
+          self.class.name.split("::").last.gsub(/([a-z])([A-Z])/, "\\1_\\2").downcase.to_sym
+        end
+      end
+    end
+  end
+end

data/lib/rspec/hermetic/probe/constants.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+require_relative "../stable_value"
+require_relative "base"
+
+module RSpec
+  module Hermetic
+    module Probe
+      class Constants < Base
+        def capture(_context)
+          roots = constant_roots
+          @seen = {}
+          @entry_count = 0
+          roots.each_with_object({}) do |(mod, prefix), values|
+            capture_module(values, mod, prefix, @configuration.constants_max_depth)
+          end
+        end
+
+        private
+
+        def constant_roots
+          namespaces = Array(@configuration.constant_namespaces)
+          return [[Object, nil]] if namespaces.empty?
+
+          namespaces.filter_map { |namespace| resolve_namespace(namespace) }
+        end
+
+        def resolve_namespace(namespace)
+          case namespace
+          when Module
+            [namespace, namespace.name]
+          else
+            mod = namespace.to_s.split("::").reject(&:empty?).inject(Object) do |parent, name|
+              parent.const_get(name, false)
+            end
+            [mod, namespace.to_s]
+          end
+        rescue NameError
+          nil
+        end
+
+        def capture_module(values, mod, prefix, depth)
+          return if @seen[mod.object_id]
+
+          @seen[mod.object_id] = true
+          mod.constants(false).each do |const_name|
+            full_name = [prefix, const_name].compact.join("::")
+            next if excluded?(full_name)
+
+            increment_entry_count!
+            value = mod.const_get(const_name, false)
+            values[full_name] = capture_value(value)
+            capture_module(values, value, full_name, depth - 1) if depth.positive? && value.is_a?(Module)
+          rescue NameError, StandardError
+            values[full_name] = :unreadable
+          end
+        end
+
+        def excluded?(full_name)
+          @configuration.constant_exclude_patterns.any? do |pattern|
+            pattern.is_a?(Regexp) ? pattern.match?(full_name) : File.fnmatch?(pattern.to_s, full_name)
+          end
+        end
+
+        def increment_entry_count!
+          @entry_count += 1
+          return if @entry_count <= @configuration.constants_max_entries
+
+          raise Error, "constants probe exceeded #{@configuration.constants_max_entries} entries"
+        end
+
+        def capture_value(value)
+          return StableValue.capture(value) unless @configuration.constants_graph
+
+          {
+            value: StableValue.capture(value),
+            reachable_graph: StableValue.reachable_graph_fingerprint(
+              value,
+              max_depth: @configuration.constants_graph_max_depth,
+              max_nodes: @configuration.constants_graph_max_nodes
+            )
+          }
+        end
+      end
+    end
+  end
+end

data/lib/rspec/hermetic/probe/env.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require_relative "base"
+
+module RSpec
+  module Hermetic
+    module Probe
+      class Env < Base
+        def capture(_context)
+          ENV.to_h
+        end
+      end
+    end
+  end
+end

data/lib/rspec/hermetic/probe/filesystem.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+require "digest"
+require "find"
+require "pathname"
+
+require_relative "base"
+
+module RSpec
+  module Hermetic
+    module Probe
+      class Filesystem < Base
+        def capture(_context)
+          roots.each_with_object({}) do |root, values|
+            scan_root(Pathname(root), values)
+          end
+        end
+
+        private
+
+        def roots
+          Array(@configuration.filesystem_paths).map do |path|
+            Pathname(path).absolute? ? path : File.join(@configuration.root_path, path)
+          end
+        end
+
+        def scan_root(root, values)
+          return unless root.exist?
+
+          count = 0
+          Find.find(root.to_s) do |path|
+            pathname = Pathname(path)
+            if excluded?(pathname)
+              Find.prune if pathname.directory?
+              next
+            end
+
+            rel = pathname.relative_path_from(Pathname(@configuration.root_path)).to_s
+            if too_deep?(rel)
+              Find.prune if pathname.directory?
+              next
+            end
+
+            values[rel] = file_entry(pathname)
+            count += 1
+            raise Error, "filesystem probe exceeded #{@configuration.filesystem_max_entries} entries" if count > @configuration.filesystem_max_entries
+          end
+        end
+
+        def excluded?(pathname)
+          rel = pathname.relative_path_from(Pathname(@configuration.root_path)).to_s
+          @configuration.filesystem_exclude_patterns.any? do |pattern|
+            File.fnmatch?(pattern.to_s, rel, File::FNM_PATHNAME | File::FNM_EXTGLOB)
+          end
+        rescue ArgumentError
+          false
+        end
+
+        def too_deep?(relative_path)
+          return false unless @configuration.filesystem_max_depth
+          return false if relative_path == "."
+
+          relative_path.split(File::SEPARATOR).length > @configuration.filesystem_max_depth
+        end
+
+        def file_entry(pathname)
+          stat = pathname.lstat
+          {
+            type: file_type(stat),
+            size: stat.file? ? stat.size : nil,
+            mtime_nsec: stat.mtime.nsec,
+            mtime: stat.mtime.to_i,
+            target: stat.symlink? ? pathname.readlink.to_s : nil,
+            sha256: content_hash(pathname, stat),
+            content: content_snapshot(pathname, stat)
+          }.compact
+        end
+
+        def file_type(stat)
+          return :file if stat.file?
+          return :directory if stat.directory?
+          return :symlink if stat.symlink?
+
+          :other
+        end
+
+        def content_hash(pathname, stat)
+          return nil unless stat.file?
+          return nil if @configuration.filesystem_content_hash_bytes.to_i <= 0
+          return nil if stat.size > @configuration.filesystem_content_hash_bytes
+
+          Digest::SHA256.file(pathname.to_s).hexdigest
+        rescue StandardError => error
+          "#{error.class}: #{error.message}"
+        end
+
+        def content_snapshot(pathname, stat)
+          return nil unless stat.file?
+          return nil if @configuration.filesystem_content_hash_bytes.to_i <= 0
+          return nil if stat.size > @configuration.filesystem_content_hash_bytes
+
+          File.binread(pathname.to_s)
+        rescue StandardError
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/rspec/hermetic/probe/globals.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require_relative "../stable_value"
+require_relative "base"
+
+module RSpec
+  module Hermetic
+    module Probe
+      class Globals < Base
+        WATCHED_GLOBALS = %i[
+          $LOAD_PATH
+          $LOADED_FEATURES
+          $PROGRAM_NAME
+          $VERBOSE
+          $DEBUG
+          $stdin
+          $stdout
+          $stderr
+        ].freeze
+
+        def capture(_context)
+          WATCHED_GLOBALS.each_with_object({}) do |name, values|
+            values[name.to_s] = StableValue.capture(eval(name.to_s, TOPLEVEL_BINDING))
+          rescue StandardError => error
+            values[name.to_s] = "#{error.class}: #{error.message}"
+          end
+        end
+      end
+    end
+  end
+end