pinspec 0.1.0 → 0.3.0

data/lib/pinspec/emit/spec_writer.rb

@@ -18,7 +18,7 @@ module Pinspec
       Result = Data.define(:spec_path, :support_paths, :pinned_cases, :aspects)
 
       def initialize(app_root:, target:, plan:, corpus:, stability:, fk_map:, force: false,
-                     only_aspect: nil, spec_dir: nil, namer: nil)
+                     only_aspect: nil, spec_dir: nil)
         @app_root = app_root
         @target = target
         @plan = plan
@@ -28,7 +28,18 @@ module Pinspec
         @force = force
         @only_aspect = only_aspect
         @spec_dir = spec_dir || SPEC_DIR
-        @namer = namer
+      end
+
+      # The class half of a pin's filename, so a caller can find the pin for a class
+      # without knowing which method it froze.
+      def self.class_stem(class_name)
+        class_name.to_s
+                  .gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2')
+                  .gsub(/([a-z\d])([A-Z])/, '\1_\2')
+                  .downcase
+                  .tr("/", "_")
+                  .tr(":", "_")
+                  .squeeze("_")
       end
 
       def spec_path
@@ -101,7 +112,7 @@ module Pinspec
       end
 
       def file_stem
-        underscore(@target.class_name).tr("/", "_") + "_" + @target.method_name.to_s.gsub(/[^a-z0-9_]/i, "")
+        self.class.class_stem(@target.class_name) + "_" + @target.method_name.to_s.gsub(/[^a-z0-9_]/i, "")
       end
 
       def render
@@ -206,8 +217,13 @@ module Pinspec
         fingerprint = @plan.env_fingerprint
 
         <<~RUBY
+          # travel_to and travel_back live here. rspec-rails includes them only when a
+          # suite opts in, and a pin must not depend on that.
+          include ActiveSupport::Testing::TimeHelpers
+
           before do
           #{clock_guard(fingerprint)}  pinspec_clear_sinks
+            PinspecFactory.reset_sequences!
 
             travel_to Time.parse(#{PINSPEC_EPOCH.inspect})
             srand(#{PINSPEC_SEED})
@@ -254,11 +270,16 @@ module Pinspec
       end
 
       def create_call(payload)
+        attrs = (payload[:attrs] || {}).merge(assoc_arguments(payload))
+
         if payload[:factory]
-          return "PinspecFactory.create(:#{payload[:factory]})"
+          # The probe passes these to the factory too. Dropping them here built a
+          # different world in the emitted spec than the one that was captured.
+          return "PinspecFactory.create(:#{payload[:factory]})" if attrs.empty?
+
+          return "PinspecFactory.create(:#{payload[:factory]}, #{render_attrs(attrs)})"
         end
 
-        attrs = (payload[:attrs] || {}).merge(assoc_arguments(payload))
         "#{payload[:model]}.create!(#{render_attrs(attrs)})"
       end
 
@@ -321,9 +342,6 @@ module Pinspec
       end
 
       def description_for(input_case, observation)
-        named = descriptions[input_case.id]
-        return named if named
-
         outcome =
           case observation["status"]
           when "raised" then "raises #{observation.dig('error', 'class')}"
@@ -333,20 +351,6 @@ module Pinspec
         "#{@target.method_name} #{outcome} (#{input_case.id}, #{input_case.origin})"
       end
 
-      def descriptions
-        return @descriptions if defined?(@descriptions)
-
-        @descriptions =
-          if @namer&.enabled?
-            pairs = pinned.filter_map do |verdict|
-              input_case = @corpus.cases.find { |c| c.id == verdict.case_id }
-              [input_case, verdict.observation] if input_case
-            end
-            @namer.describe(pairs)
-          else
-            {}
-          end
-      end
 
       def value_shape(value)
         case value && value["t"]

data/lib/pinspec/errors.rb

@@ -75,8 +75,8 @@ module Pinspec
     self.exit_code = 10
   end
 
-  class EnvironmentRefused < Error
-    self.exit_code = 11
+  class ConfigInvalid < Error
+    self.exit_code = 13
   end
 
   class PinspecInternalError < Error

data/lib/pinspec/inputs/boundary.rb

@@ -19,11 +19,10 @@ module Pinspec
       DECIMAL_HINTS = %w[BigDecimal Decimal].freeze
 
       class << self
-        def values_for(param, column: nil, default_first: true)
+        def values_for(param, column: nil)
           declared = default_value(param)
           edges    = edges_for(param, column)
-
-          ordered = default_first ? [declared, *edges] : [*edges, declared]
+          ordered = [declared, *edges]
 
           present(ordered).map { |value| encode(value, param) }.uniq
         end

data/lib/pinspec/inputs/sample_runner.rb

@@ -48,7 +48,7 @@ module Pinspec
         unless status.success?
           raise ProbeFailure,
                 "the sampler exited #{status.exitstatus} in #{@app_root}.\n" \
-                "#{stderr.to_s.lines.last(12).join}"
+                "#{excerpt(stderr)}"
         end
 
         parse(stdout, stderr)
@@ -63,6 +63,20 @@ module Pinspec
         path
       end
 
+      # A boot failure says WHAT went wrong on its first line and where on the rest.
+      # Showing only the last N lines shows the bottom of a backtrace - gem_prelude,
+      # rubygems.rb - which is identical for every failure and identifies none of them.
+      # Diagnosing a real application's failure took three commands and a read of
+      # pinspec's own internals because of that.
+      def excerpt(text, head: 6, tail: 6)
+        lines = text.to_s.lines
+        return lines.join if lines.size <= head + tail
+
+        lines.first(head).join +
+          "  ... #{lines.size - head - tail} more line(s) ...\n" +
+          lines.last(tail).join
+      end
+
       def command
         return ["rails", "runner", SCRIPT_PATH] unless File.file?(File.join(@app_root, "Gemfile"))
 
@@ -70,9 +84,9 @@ module Pinspec
       end
 
       def environment
-        Runner::Sandbox::SCRUBBED_ENV
-          .merge("RAILS_ENV" => @rails_env, "DISABLE_SPRING" => "1", "TZ" => "UTC")
-          .merge(@env)
+        Runner::Runtime.for(@app_root).env
+                       .merge("RAILS_ENV" => @rails_env, "DISABLE_SPRING" => "1", "TZ" => "UTC")
+                       .merge(@env)
       end
 
       def parse(stdout, stderr)
@@ -80,8 +94,7 @@ module Pinspec
 
         if json.nil?
           raise ProbeFailure,
-                "the sampler produced no rows.\nstdout: #{stdout.to_s.lines.last(8).join}" \
-                "stderr: #{stderr.to_s.lines.last(8).join}"
+                "the sampler produced no rows.\nstdout: #{excerpt(stdout)}\nstderr: #{excerpt(stderr)}"
         end
 
         parsed = JSON.parse(json)

data/lib/pinspec/inputs/sampler.rb

@@ -7,41 +7,9 @@ module Pinspec
     class Sampler
       SCRIPT_PATH = "tmp/pinspec/sampler.rb"
 
-      ENV_PREFERENCE = %w[development test].freeze
-
-      PRODUCTION_HINTS = [
-        /(?<![a-z])prod(uction)?(?![a-z])/i,
-        /(?<![a-z])live(?![a-z])/i,
-        /(?<![a-z])master(?![a-z])/i
-      ].freeze
-
       MAX_DISTINCT_STATUSES = 6
 
       class << self
-        def choose_env(available:, counts: {}, override: nil)
-          return override if override
-
-          populated = ENV_PREFERENCE.find do |env|
-            available.include?(env) && counts.fetch(env, 0).positive?
-          end
-
-          populated || ENV_PREFERENCE.find { |env| available.include?(env) } || available.first
-        end
-
-        def production_like?(name)
-          PRODUCTION_HINTS.any? { |hint| name.to_s.match?(hint) }
-        end
-
-        def guard_production!(name, confirmed: false)
-          return unless production_like?(name)
-          return if confirmed
-
-          raise EnvironmentRefused,
-                "#{name.inspect} looks like a production database. pinspec only ever " \
-                "SELECTs, but it will not read production without being told to: " \
-                "re-run with --sample-db pointed somewhere else, or confirm explicitly."
-        end
-
         def script_for(requests)
           new(requests).script
         end

data/lib/pinspec/report/summary.rb

@@ -185,6 +185,7 @@ module Pinspec
 
       def coverage_caveats
         caveats = []
+        caveats << vacuous_caveat
         caveats << seq_caveat
         caveats << truncated_caveat
         caveats << quarantine_caveat
@@ -203,6 +204,35 @@ module Pinspec
         MD
       end
 
+      # The honest half of pin strength, and the free half. A case that returned an
+      # empty collection, wrote no rows, enqueued nothing and sent nothing did run -
+      # but almost any change to the target would still satisfy it. Measured on real
+      # code, these are exactly the pins that score weak or worthless.
+      def vacuous_caveat
+        return nil if @stability.nil?
+
+        count = @stability.stable.count { |verdict| vacuous?(verdict.observation) }
+        return nil if count.zero?
+
+        "#{count} pinned case(s) observed nothing happening: an empty or absent return " \
+          "value, no rows written, no jobs, no mail. The pin holds, but almost any " \
+          "change to the target would still satisfy it. Give the target a world with " \
+          "something in it - `--sample` reads real rows - before relying on these."
+      end
+
+      EMPTY_RETURNS = [[], {}, nil, false, ""].freeze
+
+      def vacuous?(observation)
+        return false unless observation["status"] == "returned"
+        return false unless observation["enqueued_jobs"].to_a.empty?
+        return false unless observation["mail_deliveries"].to_a.empty?
+        return false unless observation["db_delta"].to_h.values.all? { |n| n.to_i.zero? }
+
+        EMPTY_RETURNS.include?(Tags.decode(observation["return_value"]))
+      rescue StandardError
+        false
+      end
+
       def seq_caveat
         count = rendered_observations.scan('"seq"').size
         return nil if count.zero?

data/lib/pinspec/runner/capture.rb

@@ -17,7 +17,9 @@ module Pinspec
         @target_file = target
         @method = method
         @max_cases = max_cases
-        @boots = boots
+        # Two is the floor, not the default. One boot has nothing to compare against,
+        # so every case would be called stable by virtue of never being checked.
+        @boots = [boots.to_i, 2].max
         @compare_sql = compare_sql
         @sandbox_env = sandbox_env
         @output_dir = output_dir || File.join(app_root, Sandbox::PROBE_DIR)

data/lib/pinspec/runner/probe_generator.rb

@@ -574,6 +574,7 @@ module Pinspec
             begin
               pinspec_with_isolation do
                 begin
+                  PinspecFactory.reset_sequences!
                   pinspec_run_steps(records, refs)
                 rescue StandardError => setup_error
                   observation["status"] = "setup_error"
@@ -638,7 +639,9 @@ module Pinspec
             # Between cases: anything memoized in a cache or a CurrentAttributes slot
             # would otherwise leak into the next case.
             Rails.cache.clear if defined?(Rails) && Rails.respond_to?(:cache) && Rails.cache
-            ActiveSupport::CurrentAttributes.reset_all if defined?(ActiveSupport::CurrentAttributes)
+            if defined?(ActiveSupport::CurrentAttributes) && ActiveSupport::CurrentAttributes.respond_to?(:reset_all)
+              ActiveSupport::CurrentAttributes.reset_all
+            end
             pinspec_clear_sinks
             PinspecClock.travel_back
           end

data/lib/pinspec/runner/runtime.rb

@@ -0,0 +1,131 @@
+# frozen_string_literal: true
+
+module Pinspec
+  module Runner
+    class Runtime
+      MANAGERS = %i[rvm rbenv asdf mise chruby].freeze
+
+      Result = Data.define(:env, :ruby_version, :manager, :note) do
+        def detected?
+          !manager.nil?
+        end
+
+        def same_ruby?
+          manager.nil? && note.nil?
+        end
+      end
+
+      BUNDLER_VARS = %w[
+        BUNDLE_GEMFILE BUNDLE_PATH BUNDLE_BIN_PATH BUNDLE_APP_CONFIG
+        BUNDLER_VERSION BUNDLER_SETUP RUBYOPT RUBYLIB
+      ].freeze
+
+      def self.for(app_root, home: Dir.home, current: RUBY_VERSION)
+        new(app_root, home: home, current: current).resolve
+      end
+
+      def initialize(app_root, home: Dir.home, current: RUBY_VERSION)
+        @app_root = app_root
+        @home = home
+        @current = current
+      end
+
+      def resolve
+        wanted = declared_version
+        return Result.new(env: scrub, ruby_version: @current, manager: nil, note: nil) if wanted.nil?
+        return Result.new(env: scrub, ruby_version: @current, manager: nil, note: nil) if satisfied_by_current?(wanted)
+
+        found = MANAGERS.filter_map { |manager| send(:"#{manager}_env", wanted)&.then { |e| [manager, e] } }.first
+
+        if found.nil?
+          return Result.new(
+            env: scrub, ruby_version: wanted, manager: nil,
+            note: "#{@app_root} declares Ruby #{wanted} and this is #{@current}, but no " \
+                  "installation of #{wanted} was found under rvm, rbenv, asdf, mise or " \
+                  "chruby. Pass the app's runtime explicitly:\n" \
+                  "  --app-env PATH=/path/to/ruby-#{wanted}/bin:$PATH"
+          )
+        end
+
+        manager, env = found
+        Result.new(env: scrub.merge(env), ruby_version: wanted, manager: manager, note: nil)
+      end
+
+      def declared_version
+        from_file(".ruby-version") || from_tool_versions
+      end
+
+      private
+
+      # Only bundler's variables are removed. GEM_HOME and GEM_PATH are left alone:
+      # dropping them is what forced every invocation to hand the app's gemset back
+      # by hand, and they are only wrong when the app runs on a different Ruby - in
+      # which case the detected runtime overwrites them anyway.
+      def scrub
+        BUNDLER_VARS.to_h { |name| [name, nil] }
+      end
+
+      def from_file(name)
+        path = File.join(@app_root, name)
+        return nil unless File.file?(path)
+
+        Analyzer::Source.read(path).strip[/\d+\.\d+(\.\d+)?/]
+      end
+
+      def from_tool_versions
+        path = File.join(@app_root, ".tool-versions")
+        return nil unless File.file?(path)
+
+        Analyzer::Source.read(path)[/^\s*ruby\s+(\S+)/, 1]&.[](/\d+\.\d+(\.\d+)?/)
+      end
+
+      # A patch-level difference is not worth relocating the runtime for: 3.3.0 and
+      # 3.3.6 run the same code. A minor difference is.
+      def satisfied_by_current?(wanted)
+        @current == wanted || series(@current) == series(wanted)
+      end
+
+      def series(version)
+        version.to_s.split(".").first(2).join(".")
+      end
+
+      def rvm_env(wanted)
+        root = File.join(@home, ".rvm")
+        ruby = File.join(root, "rubies", "ruby-#{wanted}", "bin")
+        return nil unless File.directory?(ruby)
+
+        gems = File.join(root, "gems", "ruby-#{wanted}")
+        env = { "PATH" => prepend(ruby) }
+        if File.directory?(gems)
+          env["GEM_HOME"] = gems
+          env["GEM_PATH"] = "#{gems}:#{gems}@global"
+        end
+        env
+      end
+
+      def rbenv_env(wanted)
+        bin = File.join(@home, ".rbenv", "versions", wanted, "bin")
+        File.directory?(bin) ? { "PATH" => prepend(bin), "RBENV_VERSION" => wanted } : nil
+      end
+
+      def asdf_env(wanted)
+        bin = File.join(@home, ".asdf", "installs", "ruby", wanted, "bin")
+        File.directory?(bin) ? { "PATH" => prepend(bin) } : nil
+      end
+
+      def mise_env(wanted)
+        bin = File.join(@home, ".local", "share", "mise", "installs", "ruby", wanted, "bin")
+        File.directory?(bin) ? { "PATH" => prepend(bin) } : nil
+      end
+
+      def chruby_env(wanted)
+        bin = File.join(@home, ".rubies", "ruby-#{wanted}", "bin")
+        File.directory?(bin) ? { "PATH" => prepend(bin) } : nil
+      end
+
+      def prepend(bin)
+        [bin, ENV.fetch("PATH", "")].reject(&:empty?).join(File::PATH_SEPARATOR)
+      end
+    end
+  end
+end

data/lib/pinspec/runner/sandbox.rb

@@ -8,9 +8,17 @@ module Pinspec
   module Runner
     class Sandbox
       PROBE_DIR = "tmp/pinspec"
-      PROBE_FILE = "probe.rb"
 
-      DEFAULT_TIMEOUT = 600
+      # One probe file per run. A fixed path means two pinspec runs against the same
+      # application overwrite each other's probe between boots, and the result is not
+      # an error: the stability filter compares one target's observations against
+      # another's and reports the target as unstable, or crashes looking up a case id
+      # that belongs to a different corpus. Silent and wrong, which is worse than a
+      # lock. `pinspec pin app/services` is sequential, but a developer with two
+      # terminals or a CI that fans out is not.
+      def self.probe_filename(pid: Process.pid, object_id: nil)
+        "probe-#{pid}-#{object_id || rand(1 << 24)}.rb"
+      end
 
       FORCED_ENV = {
         "DISABLE_SPRING" => "1",
@@ -18,18 +26,7 @@ module Pinspec
         "RAILS_ENV" => "test"
       }.freeze
 
-      SCRUBBED_ENV = {
-        "BUNDLE_GEMFILE" => nil,
-        "BUNDLE_PATH" => nil,
-        "BUNDLE_BIN_PATH" => nil,
-        "BUNDLE_APP_CONFIG" => nil,
-        "BUNDLER_VERSION" => nil,
-        "BUNDLER_SETUP" => nil,
-        "RUBYOPT" => nil,
-        "RUBYLIB" => nil,
-        "GEM_HOME" => nil,
-        "GEM_PATH" => nil
-      }.freeze
+      SCRUBBED_ENV = Runtime::BUNDLER_VARS.to_h { |name| [name, nil] }.freeze
 
       Result = Data.define(:run, :observations, :env, :plan_id, :stderr) do
         def observation(case_id)
@@ -37,16 +34,14 @@ module Pinspec
         end
       end
 
-      def initialize(app_root:, probe_source:, timeout: DEFAULT_TIMEOUT, runner: nil, env: {})
+      def initialize(app_root:, probe_source:, env: {})
         @app_root = app_root
         @probe_source = probe_source
-        @timeout = timeout
-        @runner = runner
         @env = env
       end
 
       def probe_path
-        File.join(@app_root, PROBE_DIR, PROBE_FILE)
+        @probe_path ||= File.join(@app_root, PROBE_DIR, self.class.probe_filename(object_id: object_id))
       end
 
       def write_probe!
@@ -58,38 +53,63 @@ module Pinspec
       def capture(boots: 2)
         write_probe!
 
-        (1..boots).map do |run|
-          execute(run: run, seed: 40 + run)
-        end
+        results = (1..boots).map { |run| execute(run: run, seed: 40 + run) }
+
+        # Removed only on success. A probe left behind after a failure is the
+        # evidence; one left behind after every run just fills tmp/ with files a
+        # reader cannot tell apart.
+        FileUtils.rm_f(probe_path)
+        results
+      end
+
+      def runtime
+        @runtime ||= Runtime.for(@app_root)
       end
 
       private
 
       def execute(run:, seed:)
         command = runner_command
-        env = SCRUBBED_ENV
+        env = runtime.env
               .merge(FORCED_ENV)
               .merge("PINSPEC_SHUFFLE_SEED" => seed.to_s)
               .merge(@env)
 
         stdout, stderr, status = Open3.capture3(env, *command, chdir: @app_root)
+        stdout = stdout.to_s.dup.force_encoding(Encoding::UTF_8).scrub
+        stderr = stderr.to_s.dup.force_encoding(Encoding::UTF_8).scrub
 
         unless status.success?
           raise ProbeFailure,
                 "the probe exited #{status.exitstatus} in #{@app_root}.\n" \
-                "#{stderr.to_s.lines.last(15).join}"
+                "#{runtime.note ? "#{runtime.note}\n\n" : ''}" \
+                "#{excerpt(stderr)}"
         end
 
         parse(stdout, stderr, run)
       end
 
+      # A boot failure says WHAT went wrong on its first line and where on the rest.
+      # Showing only the last N lines shows the bottom of a backtrace - gem_prelude,
+      # rubygems.rb - which is identical for every failure and identifies none of them.
+      # Diagnosing a real application's failure took three commands and a read of
+      # pinspec's own internals because of that.
+      def excerpt(text, head: 6, tail: 6)
+        lines = text.to_s.lines
+        return lines.join if lines.size <= head + tail
+
+        lines.first(head).join +
+          "  ... #{lines.size - head - tail} more line(s) ...\n" +
+          lines.last(tail).join
+      end
+
       def parse(stdout, stderr, run)
         json = stdout.to_s.lines.reverse.find { |line| line.strip.start_with?("{") }
 
         if json.nil?
           raise ProbeFailure,
                 "the probe produced no observations.\n" \
-                "stdout: #{stdout.to_s.lines.last(10).join}\nstderr: #{stderr.to_s.lines.last(10).join}"
+                "stdout: #{excerpt(stdout)}\nstderr: #{excerpt(stderr)}"
         end
 
         parsed = JSON.parse(json)
@@ -106,9 +126,7 @@ module Pinspec
       end
 
       def runner_command
-        return @runner if @runner
-
-        relative = File.join(PROBE_DIR, PROBE_FILE)
+        relative = File.join(PROBE_DIR, File.basename(probe_path))
 
         if File.file?(File.join(@app_root, "Gemfile"))
           ["bundle", "exec", "rails", "runner", relative]

data/lib/pinspec/setup/context_builder.rb

@@ -5,8 +5,6 @@ require "digest"
 module Pinspec
   module Setup
     class ContextBuilder
-      MAX_GENERATIONS = 3
-
       class << self
         def build(target:, profile:, imports: [], generation: 1, tz: nil)
           new(target: target, profile: profile, imports: imports, generation: generation, tz: tz).build
@@ -33,8 +31,11 @@ module Pinspec
         refuse_attachment_targets!
 
         environment_steps
-        record_steps
+        # Imports first: create_record_for skips a table whose ref already exists, so
+        # building factory rows first meant every binding pointed at the factory
+        # record and the sampled row - the whole point of --sample - went unused.
         import_steps
+        record_steps
         context_steps
         subject_step
 
@@ -146,11 +147,13 @@ module Pinspec
         @resolver.table_for_type_hint(param.type_hint)
       end
 
+      # Only the hints that actually occur. Measured over 841 real service files, these
+      # ten are the only ones ever produced; the other thirty-one fired zero times and
+      # none of them has a Boundary value, so a hint that did somehow reach the list
+      # would be exempted from the refusal and then passed as nil - manufacturing the
+      # exact fiction the refusal exists to prevent.
       NON_MODEL_HINTS = %w[
-        String Symbol Integer Float Numeric Decimal BigDecimal Boolean TrueClass
-        FalseClass Hash Array Range Time Date DateTime Proc Class Module Object
-        NilClass Params Param Options Option Attributes Attribute Id Ids Name
-        Amount Quantity Count Total Price Rate Percentage Status State Kind Type
+        String Symbol Integer Float Boolean Hash Array Time Date Object
       ].freeze
 
       def refuse_unresolvable_model_param!(param)