rspec-hermetic 0.1.0

data/lib/rspec/hermetic/restorer.rb

@@ -0,0 +1,330 @@
+# frozen_string_literal: true
+
+require "fileutils"
+
+require_relative "change"
+
+module RSpec
+  module Hermetic
+    class Restorer
+      def initialize(configuration)
+        @configuration = configuration
+      end
+
+      def restore(change_set, before_snapshot, context: nil)
+        restore_env(before_snapshot) if reset?(:env)
+        restore_globals(before_snapshot) if reset?(:globals)
+        restore_ruby_runtime(before_snapshot) if reset?(:ruby_runtime)
+        restore_rails(before_snapshot) if reset?(:rails)
+        restore_time(context) if reset?(:time)
+        restore_randomness(before_snapshot) if reset?(:randomness)
+        restore_constants(change_set) if reset?(:constants)
+        restore_resources(change_set) if reset?(:resources)
+        restore_filesystem(change_set) if reset?(:filesystem)
+      end
+
+      private
+
+      def reset?(probe)
+        @configuration.auto_reset_probe?(probe)
+      end
+
+      def restore_env(before_snapshot)
+        before = before_snapshot.data.fetch(:env, {})
+        ENV.replace(before)
+      end
+
+      def restore_globals(before_snapshot)
+        before = before_snapshot.data.fetch(:globals, {})
+        $LOAD_PATH.replace(before["$LOAD_PATH"]) if before["$LOAD_PATH"].is_a?(Array)
+        $LOADED_FEATURES.replace(before["$LOADED_FEATURES"]) if before["$LOADED_FEATURES"].is_a?(Array)
+        $PROGRAM_NAME = before["$PROGRAM_NAME"] if before.key?("$PROGRAM_NAME")
+        $VERBOSE = before["$VERBOSE"] if before.key?("$VERBOSE")
+        $DEBUG = before["$DEBUG"] if before.key?("$DEBUG")
+      end
+
+      def restore_ruby_runtime(before_snapshot)
+        before = before_snapshot.data.fetch(:ruby_runtime, {})
+        $VERBOSE = before["$VERBOSE"] if before.key?("$VERBOSE")
+        $DEBUG = before["$DEBUG"] if before.key?("$DEBUG")
+        GC.stress = before["GC.stress"] if before.key?("GC.stress")
+        Thread.abort_on_exception = before["Thread.abort_on_exception"] if before.key?("Thread.abort_on_exception")
+        Thread.report_on_exception = before["Thread.report_on_exception"] if before.key?("Thread.report_on_exception")
+        restore_warnings(before)
+      end
+
+      def restore_warnings(before)
+        before.each do |key, value|
+          next unless key.start_with?("Warning[:")
+          next unless defined?(Warning)
+
+          category = key[/Warning\[:(.+)\]/, 1]&.to_sym
+          Warning[category] = value unless category.nil? || value == :unsupported
+        rescue ArgumentError
+          nil
+        end
+      end
+
+      def restore_rails(before_snapshot)
+        before = before_snapshot.data.fetch(:rails, {})
+        restore_i18n(before)
+        restore_time_zone(before)
+        restore_rails_config(before)
+        restore_action_mailer(before)
+      end
+
+      def restore_i18n(before)
+        return unless Object.const_defined?(:I18n)
+
+        ::I18n.locale = before["I18n.locale"] if before.key?("I18n.locale") && ::I18n.respond_to?(:locale=)
+        if before.key?("I18n.default_locale") && ::I18n.respond_to?(:default_locale=)
+          ::I18n.default_locale = before["I18n.default_locale"]
+        end
+      rescue StandardError
+        nil
+      end
+
+      def restore_time_zone(before)
+        return unless Object.const_defined?(:Time) && ::Time.respond_to?(:zone=)
+
+        ::Time.zone = before["Time.zone"] if before.key?("Time.zone")
+      rescue StandardError
+        nil
+      end
+
+      def restore_rails_config(before)
+        return unless Object.const_defined?(:Rails) &&
+                      ::Rails.respond_to?(:application) &&
+                      ::Rails.application.respond_to?(:config)
+
+        before.each do |key, value|
+          next unless key.start_with?("Rails.application.config.")
+
+          path = key.delete_prefix("Rails.application.config.")
+          write_path(::Rails.application.config, path, value)
+        rescue StandardError
+          nil
+        end
+      end
+
+      def restore_action_mailer(before)
+        return unless defined?(::ActionMailer::Base) && ::ActionMailer::Base.respond_to?(:deliveries)
+        return unless before.key?("ActionMailer::Base.deliveries.size")
+
+        ::ActionMailer::Base.deliveries.slice!(before["ActionMailer::Base.deliveries.size"]..)
+      rescue StandardError
+        nil
+      end
+
+      def restore_time(context)
+        context.travel_back if context.respond_to?(:travel_back)
+      rescue StandardError
+        nil
+      end
+
+      def restore_randomness(before_snapshot)
+        before = before_snapshot.data.fetch(:randomness, {})
+        Kernel.srand(before["Kernel.srand.seed"]) if before.key?("Kernel.srand.seed")
+      end
+
+      def restore_constants(change_set)
+        change_set.changes.select { |change| change.probe == :constants }.each do |change|
+          if change.addition?
+            remove_constant(change.key)
+          elsif restorable_value?(change.before)
+            set_constant(change.key, deep_dup(change.before))
+          end
+        end
+      end
+
+      def restore_resources(change_set)
+        change_set.changes.select { |change| change.probe == :resources }.each do |change|
+          restore_threads(change) if change.key == "threads"
+          restore_io(change) if change.key == "io"
+          restore_child_processes(change) if change.key == "child_processes"
+        end
+      end
+
+      def restore_threads(change)
+        before_ids = Array(change.before).map { |entry| entry[:object_id] || entry["object_id"] }
+        after_ids = Array(change.after).map { |entry| entry[:object_id] || entry["object_id"] }
+        leaked_ids = after_ids - before_ids
+        Thread.list.each do |thread|
+          next unless leaked_ids.include?(thread.object_id)
+          next if thread == Thread.current
+
+          thread.kill
+          thread.join(0.1)
+        end
+      end
+
+      def restore_io(change)
+        before_fds = Array(change.before).map { |entry| entry[:fileno] || entry["fileno"] }
+        after_fds = Array(change.after).map { |entry| entry[:fileno] || entry["fileno"] }
+        leaked_fds = after_fds - before_fds
+        ObjectSpace.each_object(IO) do |io|
+          next if io.closed?
+          next unless leaked_fds.include?(io.fileno)
+
+          io.close
+        rescue IOError
+          nil
+        end
+      end
+
+      def restore_filesystem(change_set)
+        changes = change_set.changes.select { |change| change.probe == :filesystem }
+        remove_added_paths(changes.select(&:addition?))
+        restore_removed_or_modified_paths(changes.reject(&:addition?))
+      end
+
+      def restore_child_processes(change)
+        before_pids = Array(change.before).map { |entry| entry[:pid] || entry["pid"] }
+        after_pids = Array(change.after).map { |entry| entry[:pid] || entry["pid"] }
+        leaked_pids = after_pids - before_pids
+        leaked_pids.each { |pid| terminate_child_process(pid) }
+      end
+
+      def terminate_child_process(pid)
+        Process.kill("TERM", pid)
+        10.times do
+          return if Process.waitpid(pid, Process::WNOHANG)
+
+          sleep 0.02
+        rescue Errno::ECHILD
+          return
+        end
+        Process.kill("KILL", pid)
+        Process.waitpid(pid)
+      rescue Errno::ESRCH, Errno::ECHILD, Errno::EPERM
+        nil
+      end
+
+      def remove_added_paths(changes)
+        changes.sort_by { |change| -change.key.to_s.count(File::SEPARATOR) }.each do |change|
+          path = safe_filesystem_path(change.key)
+          FileUtils.rm_rf(path) if path
+        end
+      end
+
+      def restore_removed_or_modified_paths(changes)
+        changes.sort_by { |change| change.key.to_s.count(File::SEPARATOR) }.each do |change|
+          path = safe_filesystem_path(change.key)
+          next unless path
+
+          if change.removal? || change.mutation?
+            restore_filesystem_entry(path, change.before)
+          end
+        end
+      end
+
+      def restore_filesystem_entry(path, entry)
+        return if entry.equal?(Change::MISSING)
+
+        case entry_value(entry, :type)
+        when :directory, "directory"
+          FileUtils.mkdir_p(path)
+        when :file, "file"
+          restore_file(path, entry)
+        when :symlink, "symlink"
+          restore_symlink(path, entry)
+        end
+        restore_mtime(path, entry)
+      end
+
+      def restore_file(path, entry)
+        FileUtils.mkdir_p(File.dirname(path))
+        content = entry_value(entry, :content)
+        return unless content
+
+        File.binwrite(path, content)
+      end
+
+      def restore_symlink(path, entry)
+        target = entry_value(entry, :target)
+        return unless target
+
+        FileUtils.mkdir_p(File.dirname(path))
+        FileUtils.rm_f(path)
+        File.symlink(target, path)
+      rescue NotImplementedError
+        nil
+      end
+
+      def restore_mtime(path, entry)
+        return unless File.exist?(path) || File.symlink?(path)
+
+        mtime = entry_value(entry, :mtime)
+        return unless mtime
+
+        File.utime(mtime, mtime, path)
+      rescue StandardError
+        nil
+      end
+
+      def safe_filesystem_path(key)
+        root = File.expand_path(@configuration.root_path)
+        path = File.expand_path(key, root)
+        return unless path == root || path.start_with?("#{root}#{File::SEPARATOR}")
+
+        path
+      end
+
+      def entry_value(entry, key)
+        entry[key] || entry[key.to_s]
+      end
+
+      def remove_constant(name)
+        parent, const_name = constant_parent(name)
+        parent.__send__(:remove_const, const_name) if parent.const_defined?(const_name, false)
+      rescue StandardError
+        nil
+      end
+
+      def set_constant(name, value)
+        parent, const_name = constant_parent(name)
+        parent.__send__(:remove_const, const_name) if parent.const_defined?(const_name, false)
+        parent.const_set(const_name, value)
+      rescue StandardError
+        nil
+      end
+
+      def constant_parent(name)
+        parts = name.to_s.split("::")
+        const_name = parts.pop
+        parent = parts.reject(&:empty?).reduce(Object) { |mod, part| mod.const_get(part, false) }
+        [parent, const_name.to_sym]
+      end
+
+      def restorable_value?(value)
+        value.equal?(Change::MISSING) || value.nil? || value == true || value == false ||
+          value.is_a?(Symbol) || value.is_a?(Numeric) || value.is_a?(String) ||
+          value.is_a?(Array) || value.is_a?(Hash)
+      end
+
+      def deep_dup(value)
+        Marshal.load(Marshal.dump(value))
+      rescue StandardError
+        value
+      end
+
+      def write_path(object, path, value)
+        segments = path.split(".")
+        target = segments[0...-1].reduce(object) { |current, segment| read_segment(current, segment) }
+        last = segments.last
+        if target.respond_to?(:"#{last}=")
+          target.public_send(:"#{last}=", value)
+        elsif target.respond_to?(:[]=)
+          target[last.to_sym] = value
+        end
+      end
+
+      def read_segment(object, segment)
+        return object.public_send(segment) if object.respond_to?(segment)
+        return object[segment.to_sym] if object.respond_to?(:[])
+
+        raise NoMethodError, "undefined segment #{segment.inspect}"
+      end
+    end
+  end
+end

data/lib/rspec/hermetic/runner.rb

@@ -0,0 +1,183 @@
+# frozen_string_literal: true
+
+require_relative "diff"
+require_relative "candidate_report"
+require_relative "formatter"
+require_relative "probe"
+require_relative "resource_tracker"
+require_relative "restorer"
+require_relative "snapshot"
+require_relative "verdict"
+
+module RSpec
+  module Hermetic
+    class Runner
+      attr_reader :records
+
+      def initialize(configuration)
+        @configuration = configuration
+        @records = []
+        @suite_baseline = nil
+        @last_writers = {}
+        @example_count = 0
+      end
+
+      def call(example, context)
+        @example_count += 1
+        probes = build_probes
+        before = Snapshot.capture(probes, context: context)
+        @suite_baseline ||= before
+
+        ResourceTracker.start! if @configuration.track_resource_origins
+        example.run
+      ensure
+        if before
+          after = Snapshot.capture(probes, context: context)
+          ResourceTracker.stop! if @configuration.track_resource_origins
+          change_set = Diff.between(before, after)
+          result = Verdict.new(@configuration).call(
+            change_set,
+            example_allowances: example_allowances(example)
+          )
+          record(example, change_set, result)
+          report_pollution(example, change_set, result)
+          report_probe_errors(example, change_set, result)
+          report_forensic_victim(example, before) if @configuration.forensic && failed?(example)
+          remember_writers(example, change_set)
+          restore_state(change_set, before, context)
+          fail_example!(example, result, change_set) if should_fail_for_pollution?(example, result)
+        end
+      end
+
+      private
+
+      def build_probes
+        Array(@configuration.probes).filter_map do |name|
+          next unless sampled?(name)
+
+          Probe.build(name, @configuration)
+        end
+      end
+
+      def sampled?(name)
+        return true if @example_count == 1
+
+        (@example_count % @configuration.probe_interval(name)).zero?
+      end
+
+      def example_allowances(example)
+        metadata = example.respond_to?(:metadata) ? example.metadata : {}
+        hermetic = metadata[:hermetic]
+        hermetic.is_a?(Hash) ? hermetic[:allow] : nil
+      end
+
+      def record(example, change_set, result)
+        return unless result.reportable? || change_set.after_errors.any?
+
+        @records << {
+          example: example,
+          changes: change_set,
+          verdict: result
+        }
+      end
+
+      def report_pollution(example, change_set, result)
+        return unless result.reportable?
+        return if @configuration.on_pollution == :fail
+
+        message = Formatter.pollution_message(example, result, change_set)
+        reporter_message(example, message)
+      end
+
+      def report_probe_errors(example, change_set, result)
+        return unless @configuration.report_probe_errors
+        return if result.reportable?
+        return unless change_set.errors?
+
+        reporter_message(example, Formatter.probe_error_message(example, change_set))
+      end
+
+      def report_forensic_victim(example, before_snapshot)
+        dirty = Diff.between(@suite_baseline, before_snapshot).changes
+        return if dirty.empty?
+
+        reporter_message(example, Formatter.forensic_message(example, dirty, @last_writers))
+        write_candidates(example, dirty)
+      end
+
+      def remember_writers(example, change_set)
+        change_set.changes.each do |change|
+          @last_writers[[change.probe, change.key]] = example_label(example)
+        end
+      end
+
+      def fail_example!(example, result, change_set)
+        message = Formatter.pollution_message(example, result, change_set)
+        raise Error, message
+      end
+
+      def should_fail_for_pollution?(example, result)
+        result.polluted? && @configuration.on_pollution == :fail && !failed?(example)
+      end
+
+      def restore_state(change_set, before_snapshot, context)
+        return unless @configuration.auto_reset
+
+        Restorer.new(@configuration).restore(change_set, before_snapshot, context: context)
+      end
+
+      def write_candidates(example, changes)
+        return unless @configuration.candidate_report_path
+
+        report = CandidateReport.new(File.join(@configuration.root_path, @configuration.candidate_report_path))
+        changes.each do |change|
+          polluter = @last_writers[[change.probe, change.key]]
+          next unless polluter
+
+          report.append(
+            "polluter" => polluter,
+            "victim" => example_label(example),
+            "probe" => change.probe,
+            "key" => change.key,
+            "before" => safe_json(change.before),
+            "after" => safe_json(change.after)
+          )
+        end
+      rescue StandardError => error
+        reporter_message(example, "HERMETIC CANDIDATE REPORT ERROR #{error.class}: #{error.message}")
+      end
+
+      def reporter_message(example, message)
+        reporter = if example.respond_to?(:reporter)
+                     example.reporter
+                   elsif defined?(::RSpec) && ::RSpec.respond_to?(:configuration)
+                     ::RSpec.configuration.reporter
+                   end
+
+        if reporter&.respond_to?(:message)
+          reporter.message(message)
+        else
+          warn(message)
+        end
+      end
+
+      def failed?(example)
+        example.respond_to?(:exception) && !example.exception.nil?
+      end
+
+      def example_label(example)
+        return example.location if example.respond_to?(:location) && example.location
+
+        example.respond_to?(:full_description) ? example.full_description : example.object_id.to_s
+      end
+
+      def safe_json(value)
+        return nil if value.equal?(Change::MISSING)
+
+        JSON.parse(JSON.generate(value))
+      rescue StandardError
+        value.inspect
+      end
+    end
+  end
+end

data/lib/rspec/hermetic/snapshot.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module RSpec
+  module Hermetic
+    class Snapshot
+      attr_reader :data, :errors, :timings
+
+      def initialize(data:, errors: {}, timings: {})
+        @data = data
+        @errors = errors
+        @timings = timings
+      end
+
+      def self.capture(probes, context: nil)
+        data = {}
+        errors = {}
+        timings = {}
+
+        probes.each do |probe|
+          started_at = monotonic_now
+          data[probe.name] = probe.capture(context)
+        rescue StandardError => error
+          errors[probe.name] = "#{error.class}: #{error.message}"
+          data[probe.name] = {}
+        ensure
+          timings[probe.name] = monotonic_now - started_at if started_at
+        end
+
+        new(data: data, errors: errors, timings: timings)
+      end
+
+      def self.monotonic_now
+        Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      end
+    end
+  end
+end

data/lib/rspec/hermetic/stable_value.rb

@@ -0,0 +1,140 @@
+# frozen_string_literal: true
+
+require "digest"
+
+begin
+  require "objspace"
+rescue LoadError
+  nil
+end
+
+module RSpec
+  module Hermetic
+    module StableValue
+      module_function
+
+      MAX_DEPTH = 3
+
+      def capture(value, depth: MAX_DEPTH, seen: {})
+        case value
+        when nil, true, false, Symbol, Numeric
+          value
+        when String
+          value.dup.freeze
+        when Array
+          return fingerprint(value) if depth <= 0
+
+          value.map { |item| capture(item, depth: depth - 1, seen: seen) }.freeze
+        when Hash
+          return fingerprint(value) if depth <= 0
+
+          value.keys.sort_by(&:to_s).to_h do |key|
+            [
+              capture(key, depth: depth - 1, seen: seen),
+              capture(value[key], depth: depth - 1, seen: seen)
+            ]
+          end.freeze
+        else
+          object_fingerprint(value, depth, seen)
+        end
+      end
+
+      def fingerprint(value)
+        dumped = Marshal.dump(value)
+        "#{value.class}:marshal:#{Digest::SHA256.hexdigest(dumped)}"
+      rescue StandardError
+        normalized = value.inspect.gsub(/0x[0-9a-f]+/i, "0x...")
+        "#{value.class}:inspect:#{Digest::SHA256.hexdigest(normalized)}"
+      end
+
+      def reachable_graph_fingerprint(value, max_depth:, max_nodes:)
+        return capture(value) unless ObjectSpace.respond_to?(:reachable_objects_from)
+
+        seen = {}
+        queue = [[value, 0]]
+        nodes = []
+
+        until queue.empty? || nodes.length >= max_nodes
+          object, depth = queue.shift
+          next if seen[object.object_id]
+
+          begin
+            seen[object.object_id] = true
+            nodes << graph_node(object, depth)
+            next if depth >= max_depth
+
+            ObjectSpace.reachable_objects_from(object).each do |child|
+              next if immediate?(child)
+
+              queue << [child, depth + 1]
+            end
+          rescue StandardError
+            next
+          end
+        end
+
+        {
+          graph: Digest::SHA256.hexdigest(Marshal.dump(nodes.sort_by(&:to_s))),
+          nodes: nodes.length,
+          truncated: !queue.empty?
+        }
+      end
+
+      def object_fingerprint(value, depth, seen)
+        return fingerprint(value) if depth <= 0
+        return "#{value.class}:cycle:#{seen[value.object_id]}" if seen.key?(value.object_id)
+
+        seen[value.object_id] = seen.length
+        dumped = Marshal.dump(value)
+        "#{value.class}:marshal:#{Digest::SHA256.hexdigest(dumped)}"
+      rescue StandardError
+        payload = {
+          class: value.class.name,
+          inspect: value.inspect.gsub(/0x[0-9a-f]+/i, "0x..."),
+          ivars: instance_variables_for(value, depth, seen),
+          class_vars: class_variables_for(value, depth, seen)
+        }
+        "#{value.class}:graph:#{Digest::SHA256.hexdigest(Marshal.dump(payload))}"
+      ensure
+        seen.delete(value.object_id)
+      end
+
+      def instance_variables_for(value, depth, seen)
+        value.instance_variables.sort_by(&:to_s).to_h do |ivar|
+          [ivar.to_s, capture(value.instance_variable_get(ivar), depth: depth - 1, seen: seen)]
+        rescue StandardError => error
+          [ivar.to_s, "#{error.class}: #{error.message}"]
+        end
+      end
+
+      def class_variables_for(value, depth, seen)
+        return {} unless value.is_a?(Module)
+
+        value.class_variables.sort_by(&:to_s).to_h do |ivar|
+          [ivar.to_s, capture(value.class_variable_get(ivar), depth: depth - 1, seen: seen)]
+        rescue StandardError => error
+          [ivar.to_s, "#{error.class}: #{error.message}"]
+        end
+      end
+
+      def graph_node(object, depth)
+        {
+          class: object.class.name,
+          depth: depth,
+          value: immediate?(object) ? object : capture(object, depth: 1)
+        }
+      rescue StandardError => error
+        {
+          class: object.class.name,
+          depth: depth,
+          error: "#{error.class}: #{error.message}"
+        }
+      end
+
+      def immediate?(object)
+        object.nil? || object == true || object == false ||
+          object.is_a?(Symbol) || object.is_a?(Numeric)
+      end
+    end
+  end
+end