rspec-hermetic 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: dabaae0436d65cd41bd97891a144263792b4898dc2e4788bd0bdcbea11fb8dda
+  data.tar.gz: d4cab7a93d453f008e32a5a16421b0ec22f1eb4570c27f5e82251a86ba474d6f
+SHA512:
+  metadata.gz: e71317f84c4f592a2246942f15f2623d770f0d805a2bfca6737dc959874cbb3636ac9dd9a5597016666c2f8ea6fc2f88a5bac638ac3b2002dfeb980fd7b9a0d4
+  data.tar.gz: 6a54c3c931a80d52c95ae8b37bb0491b8add9d4425384f9063da5df9365de203dddc612cae32928c92ec2c530c0e6bc3d7ca668aa38560fc4dfe6aada8577355

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2026 Yudai Takada
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,166 @@
+# rspec-hermetic
+
+`rspec-hermetic` detects examples that leave process-global state changed after
+they finish. It snapshots selected probes before and after each example, diffs
+the snapshots, applies allowlists, and reports the example as a polluter when
+state is still dirty.
+
+The implementation follows the PolDet idea from state-pollution testing:
+compare shared state around one test and show the before/after evidence.
+
+## Installation
+
+Install the gem and add it to the application's Gemfile:
+
+```bash
+bundle add rspec-hermetic
+```
+
+Or install from a local checkout:
+
+```ruby
+gem "rspec-hermetic", path: "../rspec-hermetic"
+```
+
+## Usage
+
+Configure it in `spec/spec_helper.rb`:
+
+```ruby
+require "rspec/hermetic"
+
+RSpec.configure do |config|
+  RSpec::Hermetic.configure(config) do |hermetic|
+    hermetic.probes = %i[env constants globals ruby_runtime rails time randomness resources]
+    hermetic.on_pollution = :risky # :risky, :fail, or :report
+    hermetic.constant_namespaces = [MyApp]
+    hermetic.filesystem_paths = ["app", "config"]
+    hermetic.randomness_seed_probe = true # opt in to Kernel.srand seed sampling
+    hermetic.resource_process_probe = true # opt in to child process scanning
+    hermetic.auto_reset = false # true or probe list, e.g. %i[env rails time]
+    hermetic.candidate_report_path = "tmp/rspec_hermetic_candidates.json"
+
+    hermetic.allow do |allow|
+      allow.env "CI", "RAILS_*"
+      allow.path "tmp/**", "log/test.log"
+      allow.constant(/\AFactoryBot::/)
+      allow.append_only "Warning.deduplicated"
+    end
+  end
+end
+```
+
+Use metadata for intentional pollution in one example:
+
+```ruby
+it "changes locale globally", hermetic: { allow: [:i18n_locale] } do
+  I18n.locale = :ja
+end
+```
+
+Available probes:
+
+- `env`: `ENV.to_h`
+- `constants`: configured namespaces, or top-level constants by default
+  using bounded ObjectSpace reachable-object fingerprints
+- `globals`: selected Ruby globals such as `$LOAD_PATH` and stdio
+- `ruby_runtime`: warning, encoding, GC, and thread runtime flags
+- `rails`: I18n, Time.zone, Rails.cache, mailer/job counts, CurrentAttributes
+- `time`: wall-clock/monotonic offset, ActiveSupport time helper state, and
+  Timecop travel state when present
+- `randomness`: Random/FactoryBot state that can be read safely, with optional
+  `Kernel.srand` seed sampling via `randomness_seed_probe`
+- `resources`: live threads, open IO objects, ActiveRecord pool state, and
+  optional child process scanning via `resource_process_probe`
+- `filesystem`: configured paths under `root_path` (`.` by default when enabled),
+  with metadata and small-file SHA256 content hashes
+
+`append_only` patterns only downgrade a change to a warning when the new value
+preserves the old value as a prefix/subset. Replacing existing values is still
+reported as pollution.
+
+The RSpec hook is installed outside existing `around(:each)` hooks on RSpec 3
+by using RSpec's hook collection when available. On unsupported RSpec versions
+it falls back to normal `around(:each)` registration.
+
+`auto_reset` is opt-in. Set it to `true` or a probe list to restore supported
+state after reporting: ENV, selected globals/runtime flags, Rails globals,
+time helpers, randomness seed, added constants, leaked threads/IO objects,
+leaked child processes, and filesystem additions/modifications/deletions when
+file contents were captured.
+
+Resource origin tracking is enabled by default. It annotates resources created
+while an example is running so leaked threads and IO objects can include the
+creation callsite in the report.
+
+For order-dependent failures, require forensic mode:
+
+```bash
+bundle exec rspec --seed 1234 --require rspec/hermetic/forensic
+```
+
+It compares a failing example's starting state with the suite baseline and
+reports the last example that changed each dirty key when known. Candidate
+pairs are written to `candidate_report_path` as JSON.
+
+To verify known polluter/victim candidates:
+
+```bash
+bundle exec rake hermetic:verify[tmp/rspec_hermetic_candidates.json]
+```
+
+The JSON file can be an array of objects or newline-delimited JSON:
+
+```json
+[
+  { "polluter": "spec/models/user_spec.rb:12", "victim": "spec/models/user_spec.rb:48" }
+]
+```
+
+Minitest can use the same detector:
+
+```ruby
+require "rspec/hermetic/minitest"
+
+RSpec::Hermetic.configure_minitest do |hermetic|
+  hermetic.probes = %i[env globals ruby_runtime resources filesystem]
+  hermetic.on_pollution = :fail
+end
+```
+
+Run the built-in seeded pollution evaluation:
+
+```bash
+bundle exec rake hermetic:evaluate[tmp/rspec_hermetic_evaluation.json]
+```
+
+The report includes probe-level seeded recall and is intended as the local
+baseline for the evaluation workflow described in the design document.
+
+Run an external corpus evaluation by providing commands through the environment:
+
+```bash
+HERMETIC_BASELINE_COMMAND="bundle exec rspec" \
+HERMETIC_COMMAND="bundle exec rspec --require rspec/hermetic/forensic" \
+HERMETIC_CANDIDATES="tmp/rspec_hermetic_candidates.json" \
+bundle exec rake hermetic:evaluate_corpus[tmp/rspec_hermetic_corpus.json]
+```
+
+If `HERMETIC_JUDGMENTS` points to a JSON file keyed by
+`polluter|victim|probe|key`, the corpus report includes false-alarm counts and
+rates alongside overhead and candidate counts.
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then run
+`rake spec` to run the tests. You can also run `bin/console` for an interactive
+prompt.
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at
+https://github.com/ydah/rspec-hermetic.
+
+## License
+
+The gem is available as open source under the terms of the MIT License.

data/lib/rspec/hermetic/allowlist.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+require "fileutils"
+
+module RSpec
+  module Hermetic
+    class Allowlist
+      attr_reader :env_patterns, :path_patterns, :constant_patterns, :append_only_patterns
+
+      def initialize
+        @env_patterns = []
+        @path_patterns = []
+        @constant_patterns = []
+        @append_only_patterns = []
+      end
+
+      def env(*patterns)
+        @env_patterns.concat(patterns)
+      end
+
+      def path(*patterns)
+        @path_patterns.concat(patterns)
+      end
+
+      def constant(*patterns)
+        @constant_patterns.concat(patterns)
+      end
+
+      def append_only(*patterns)
+        @append_only_patterns.concat(patterns)
+      end
+
+      def allowed?(change, example_allowances)
+        allowed_by_metadata?(change, example_allowances) ||
+          allowed_env?(change) ||
+          allowed_path?(change) ||
+          allowed_constant?(change)
+      end
+
+      def append_only?(change)
+        pattern_match?(append_only_patterns, change.key.to_s)
+      end
+
+      private
+
+      def allowed_by_metadata?(change, example_allowances)
+        allowances = Array(example_allowances).map(&:to_sym)
+        (change.token_candidates & allowances).any?
+      end
+
+      def allowed_env?(change)
+        change.probe == :env && pattern_match?(env_patterns, change.key.to_s)
+      end
+
+      def allowed_path?(change)
+        change.probe == :filesystem && pattern_match?(path_patterns, change.key.to_s)
+      end
+
+      def allowed_constant?(change)
+        change.probe == :constants && pattern_match?(constant_patterns, change.key.to_s)
+      end
+
+      def pattern_match?(patterns, value)
+        patterns.any? do |pattern|
+          case pattern
+          when Regexp
+            pattern.match?(value)
+          else
+            File.fnmatch?(pattern.to_s, value, File::FNM_PATHNAME | File::FNM_EXTGLOB)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rspec/hermetic/candidate_report.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require "json"
+require "fileutils"
+
+module RSpec
+  module Hermetic
+    class CandidateReport
+      def initialize(path)
+        @path = path
+      end
+
+      def append(candidate)
+        candidates = read
+        candidates << candidate
+        write(candidates)
+      end
+
+      def read
+        return [] unless @path && File.exist?(@path)
+
+        content = File.read(@path)
+        return [] if content.strip.empty?
+
+        parsed = JSON.parse(content)
+        parsed.is_a?(Array) ? parsed : [parsed]
+      rescue JSON::ParserError
+        read_json_lines
+      end
+
+      private
+
+      def read_json_lines
+        File.readlines(@path, chomp: true).filter_map do |line|
+          next if line.strip.empty?
+
+          JSON.parse(line)
+        end
+      end
+
+      def write(candidates)
+        FileUtils.mkdir_p(File.dirname(@path))
+        File.write(@path, "#{JSON.pretty_generate(candidates)}\n")
+      end
+    end
+  end
+end

data/lib/rspec/hermetic/change.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module RSpec
+  module Hermetic
+    MISSING_VALUE = Object.new.freeze
+
+    Change = Struct.new(:probe, :key, :before, :after, keyword_init: true) do
+      def addition?
+        before.equal?(self.class::MISSING) && !after.equal?(self.class::MISSING)
+      end
+
+      def removal?
+        after.equal?(self.class::MISSING) && !before.equal?(self.class::MISSING)
+      end
+
+      def mutation?
+        !addition? && !removal?
+      end
+
+      def token_candidates
+        [
+          probe.to_sym,
+          key.to_s.tr(".[]\"$/:-", "_").gsub(/__+/, "_").downcase.to_sym,
+          "#{probe}:#{key}".to_sym
+        ]
+      end
+
+      def append_only?
+        return true if addition?
+        return false if removal?
+
+        append_only_value?(before, after)
+      end
+
+      private
+
+      def append_only_value?(before_value, after_value)
+        if wrapped_value?(before_value) && wrapped_value?(after_value)
+          return append_only_value?(before_value[:value], after_value[:value])
+        end
+
+        if before_value.is_a?(String) && after_value.is_a?(String)
+          after_value.start_with?(before_value)
+        elsif before_value.is_a?(Array) && after_value.is_a?(Array)
+          after_value.length >= before_value.length && after_value.first(before_value.length) == before_value
+        elsif before_value.is_a?(Hash) && after_value.is_a?(Hash)
+          (before_value.keys - after_value.keys).empty? &&
+            before_value.all? { |key, value| after_value[key] == value }
+        else
+          false
+        end
+      end
+
+      def wrapped_value?(value)
+        value.is_a?(Hash) && value.key?(:value) && value.key?(:reachable_graph)
+      end
+    end
+
+    Change.const_set(:MISSING, MISSING_VALUE)
+  end
+end

data/lib/rspec/hermetic/configuration.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+require_relative "allowlist"
+
+module RSpec
+  module Hermetic
+    class Configuration
+      attr_accessor :probes,
+                    :on_pollution,
+                    :root_path,
+                    :constant_namespaces,
+                    :constant_exclude_patterns,
+                    :constants_max_depth,
+                    :constants_max_entries,
+                    :constants_graph,
+                    :constants_graph_max_depth,
+                    :constants_graph_max_nodes,
+                    :filesystem_paths,
+                    :filesystem_exclude_patterns,
+                    :filesystem_content_hash_bytes,
+                    :filesystem_max_entries,
+                    :filesystem_max_depth,
+                    :forensic,
+                    :randomness_seed_probe,
+                    :resource_process_probe,
+                    :rails_config_paths,
+                    :probe_sampling,
+                    :report_probe_errors,
+                    :candidate_report_path,
+                    :auto_reset,
+                    :track_resource_origins
+      attr_reader :allowlist
+
+      def initialize
+        @probes = %i[env constants globals ruby_runtime rails time randomness resources]
+        @on_pollution = :risky
+        @root_path = Dir.pwd
+        @constant_namespaces = []
+        @constant_exclude_patterns = []
+        @constants_max_depth = 0
+        @constants_max_entries = 1_000
+        @constants_graph = true
+        @constants_graph_max_depth = 3
+        @constants_graph_max_nodes = 1_000
+        @filesystem_paths = ["."]
+        @filesystem_exclude_patterns = %w[
+          .git
+          .git/**
+          .bundle
+          .bundle/**
+          vendor/bundle
+          vendor/bundle/**
+          node_modules
+          node_modules/**
+        ]
+        @filesystem_content_hash_bytes = 64 * 1024
+        @filesystem_max_entries = 1_000
+        @filesystem_max_depth = nil
+        @forensic = false
+        @randomness_seed_probe = false
+        @resource_process_probe = false
+        @rails_config_paths = %w[
+          cache_store
+          time_zone
+          active_job.queue_adapter
+          action_mailer.delivery_method
+          action_controller.allow_forgery_protection
+          i18n.default_locale
+        ]
+        @probe_sampling = {}
+        @report_probe_errors = true
+        @candidate_report_path = "tmp/rspec_hermetic_candidates.json"
+        @auto_reset = false
+        @track_resource_origins = true
+        @allowlist = Allowlist.new
+        @allowlist.path "tmp/**", "log/**", ".rspec_status", "coverage/**"
+      end
+
+      def allow
+        yield allowlist
+      end
+
+      def probe_interval(name)
+        interval = probe_sampling.fetch(name.to_sym, 1).to_i
+        interval.positive? ? interval : 1
+      end
+
+      def auto_reset_probe?(name)
+        auto_reset == true || Array(auto_reset).map(&:to_sym).include?(name.to_sym)
+      end
+    end
+  end
+end

data/lib/rspec/hermetic/corpus_evaluation.rb

@@ -0,0 +1,124 @@
+# frozen_string_literal: true
+
+require "json"
+require "fileutils"
+require "open3"
+require "time"
+
+require_relative "candidate_report"
+
+module RSpec
+  module Hermetic
+    class CorpusEvaluation
+      def initialize(
+        output_path:,
+        project_path: Dir.pwd,
+        baseline_command: nil,
+        hermetic_command:,
+        candidate_report_path: "tmp/rspec_hermetic_candidates.json",
+        judgments_path: nil
+      )
+        @output_path = output_path
+        @project_path = project_path
+        @baseline_command = baseline_command
+        @hermetic_command = hermetic_command
+        @candidate_report_path = candidate_report_path
+        @judgments_path = judgments_path
+      end
+
+      def run
+        baseline = run_command(@baseline_command) if @baseline_command
+        hermetic = run_command(@hermetic_command)
+        candidates = read_candidates
+        report = {
+          "project_path" => @project_path,
+          "generated_at" => Time.now.utc.iso8601,
+          "baseline" => baseline,
+          "hermetic" => hermetic,
+          "overhead" => overhead(baseline, hermetic),
+          "candidates" => candidates,
+          "candidate_count" => candidates.length,
+          "false_alarm" => false_alarm_summary(candidates)
+        }.compact
+        write(report)
+        report
+      end
+
+      private
+
+      def run_command(command)
+        started_at = monotonic_now
+        stdout, stderr, status = Open3.capture3(command, chdir: @project_path)
+        {
+          "command" => command,
+          "status" => status.exitstatus,
+          "success" => status.success?,
+          "duration_seconds" => monotonic_now - started_at,
+          "stdout_tail" => tail(stdout),
+          "stderr_tail" => tail(stderr)
+        }
+      end
+
+      def read_candidates
+        path = File.expand_path(@candidate_report_path, @project_path)
+        return [] unless File.exist?(path)
+
+        CandidateReport.new(path).read
+      end
+
+      def false_alarm_summary(candidates)
+        judgments = read_judgments
+        return { "judged" => 0, "false_alarms" => nil, "rate" => nil } if judgments.empty?
+
+        judged = candidates.filter_map { |candidate| judgment_for(judgments, candidate) }
+        false_alarms = judged.count { |judgment| judgment == "false_alarm" }
+        {
+          "judged" => judged.length,
+          "false_alarms" => false_alarms,
+          "rate" => judged.empty? ? nil : false_alarms.fdiv(judged.length)
+        }
+      end
+
+      def read_judgments
+        return {} unless @judgments_path && File.exist?(@judgments_path)
+
+        JSON.parse(File.read(@judgments_path))
+      rescue JSON::ParserError
+        {}
+      end
+
+      def judgment_for(judgments, candidate)
+        judgments[candidate_key(candidate)] ||
+          judgments[[candidate["polluter"], candidate["victim"], candidate["probe"], candidate["key"]].join("|")]
+      end
+
+      def candidate_key(candidate)
+        candidate.fetch("id") do
+          [candidate["polluter"], candidate["victim"], candidate["probe"], candidate["key"]].join("|")
+        end
+      end
+
+      def overhead(baseline, hermetic)
+        return nil unless baseline && hermetic && baseline["duration_seconds"].positive?
+
+        {
+          "seconds" => hermetic["duration_seconds"] - baseline["duration_seconds"],
+          "ratio" => hermetic["duration_seconds"].fdiv(baseline["duration_seconds"])
+        }
+      end
+
+      def write(report)
+        FileUtils.mkdir_p(File.dirname(@output_path))
+        File.write(@output_path, "#{JSON.pretty_generate(report)}\n")
+      end
+
+      def tail(text)
+        text.lines.last(50).join
+      end
+
+      def monotonic_now
+        Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      end
+    end
+  end
+end

data/lib/rspec/hermetic/diff.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+require_relative "change"
+
+module RSpec
+  module Hermetic
+    class ChangeSet
+      attr_reader :changes, :before_errors, :after_errors, :before_timings, :after_timings
+
+      def initialize(changes:, before_errors:, after_errors:, before_timings:, after_timings:)
+        @changes = changes
+        @before_errors = before_errors
+        @after_errors = after_errors
+        @before_timings = before_timings
+        @after_timings = after_timings
+      end
+
+      def empty?
+        changes.empty?
+      end
+
+      def errors?
+        before_errors.any? || after_errors.any?
+      end
+
+      def timings
+        probe_names = before_timings.keys | after_timings.keys
+        probe_names.to_h do |probe|
+          [probe, before_timings.fetch(probe, 0.0) + after_timings.fetch(probe, 0.0)]
+        end
+      end
+    end
+
+    class Diff
+      def self.between(before_snapshot, after_snapshot)
+        changes = []
+        probe_names = before_snapshot.data.keys | after_snapshot.data.keys
+
+        probe_names.each do |probe|
+          before_data = before_snapshot.data.fetch(probe, {})
+          after_data = after_snapshot.data.fetch(probe, {})
+          keys = before_data.keys | after_data.keys
+
+          keys.each do |key|
+            before_value = before_data.key?(key) ? before_data[key] : Change::MISSING
+            after_value = after_data.key?(key) ? after_data[key] : Change::MISSING
+            next if before_value == after_value
+
+            changes << Change.new(probe: probe, key: key, before: before_value, after: after_value)
+          end
+        end
+
+        ChangeSet.new(
+          changes: changes,
+          before_errors: before_snapshot.errors,
+          after_errors: after_snapshot.errors,
+          before_timings: before_snapshot.timings,
+          after_timings: after_snapshot.timings
+        )
+      end
+    end
+  end
+end