pinspec 0.2.0 → 0.3.0

data/lib/pinspec/emit/spec_writer.rb

@@ -18,7 +18,7 @@ module Pinspec
       Result = Data.define(:spec_path, :support_paths, :pinned_cases, :aspects)
 
       def initialize(app_root:, target:, plan:, corpus:, stability:, fk_map:, force: false,
-                     only_aspect: nil, spec_dir: nil, namer: nil)
+                     only_aspect: nil, spec_dir: nil)
         @app_root = app_root
         @target = target
         @plan = plan
@@ -28,7 +28,18 @@ module Pinspec
         @force = force
         @only_aspect = only_aspect
         @spec_dir = spec_dir || SPEC_DIR
-        @namer = namer
+      end
+
+      # The class half of a pin's filename, so a caller can find the pin for a class
+      # without knowing which method it froze.
+      def self.class_stem(class_name)
+        class_name.to_s
+                  .gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2')
+                  .gsub(/([a-z\d])([A-Z])/, '\1_\2')
+                  .downcase
+                  .tr("/", "_")
+                  .tr(":", "_")
+                  .squeeze("_")
       end
 
       def spec_path
@@ -101,7 +112,7 @@ module Pinspec
       end
 
       def file_stem
-        underscore(@target.class_name).tr("/", "_") + "_" + @target.method_name.to_s.gsub(/[^a-z0-9_]/i, "")
+        self.class.class_stem(@target.class_name) + "_" + @target.method_name.to_s.gsub(/[^a-z0-9_]/i, "")
       end
 
       def render
@@ -206,8 +217,13 @@ module Pinspec
         fingerprint = @plan.env_fingerprint
 
         <<~RUBY
+          # travel_to and travel_back live here. rspec-rails includes them only when a
+          # suite opts in, and a pin must not depend on that.
+          include ActiveSupport::Testing::TimeHelpers
+
           before do
           #{clock_guard(fingerprint)}  pinspec_clear_sinks
+            PinspecFactory.reset_sequences!
 
             travel_to Time.parse(#{PINSPEC_EPOCH.inspect})
             srand(#{PINSPEC_SEED})
@@ -254,11 +270,16 @@ module Pinspec
       end
 
       def create_call(payload)
+        attrs = (payload[:attrs] || {}).merge(assoc_arguments(payload))
+
         if payload[:factory]
-          return "PinspecFactory.create(:#{payload[:factory]})"
+          # The probe passes these to the factory too. Dropping them here built a
+          # different world in the emitted spec than the one that was captured.
+          return "PinspecFactory.create(:#{payload[:factory]})" if attrs.empty?
+
+          return "PinspecFactory.create(:#{payload[:factory]}, #{render_attrs(attrs)})"
         end
 
-        attrs = (payload[:attrs] || {}).merge(assoc_arguments(payload))
         "#{payload[:model]}.create!(#{render_attrs(attrs)})"
       end
 
@@ -321,9 +342,6 @@ module Pinspec
       end
 
       def description_for(input_case, observation)
-        named = descriptions[input_case.id]
-        return named if named
-
         outcome =
           case observation["status"]
           when "raised" then "raises #{observation.dig('error', 'class')}"
@@ -333,20 +351,6 @@ module Pinspec
         "#{@target.method_name} #{outcome} (#{input_case.id}, #{input_case.origin})"
       end
 
-      def descriptions
-        return @descriptions if defined?(@descriptions)
-
-        @descriptions =
-          if @namer&.enabled?
-            pairs = pinned.filter_map do |verdict|
-              input_case = @corpus.cases.find { |c| c.id == verdict.case_id }
-              [input_case, verdict.observation] if input_case
-            end
-            @namer.describe(pairs)
-          else
-            {}
-          end
-      end
 
       def value_shape(value)
         case value && value["t"]

data/lib/pinspec/errors.rb

@@ -75,10 +75,6 @@ module Pinspec
     self.exit_code = 10
   end
 
-  class EnvironmentRefused < Error
-    self.exit_code = 11
-  end
-
   class ConfigInvalid < Error
     self.exit_code = 13
   end

data/lib/pinspec/inputs/boundary.rb

@@ -19,11 +19,10 @@ module Pinspec
       DECIMAL_HINTS = %w[BigDecimal Decimal].freeze
 
       class << self
-        def values_for(param, column: nil, default_first: true)
+        def values_for(param, column: nil)
           declared = default_value(param)
           edges    = edges_for(param, column)
-
-          ordered = default_first ? [declared, *edges] : [*edges, declared]
+          ordered = [declared, *edges]
 
           present(ordered).map { |value| encode(value, param) }.uniq
         end

data/lib/pinspec/inputs/sample_runner.rb

@@ -48,7 +48,7 @@ module Pinspec
         unless status.success?
           raise ProbeFailure,
                 "the sampler exited #{status.exitstatus} in #{@app_root}.\n" \
-                "#{stderr.to_s.lines.last(12).join}"
+                "#{excerpt(stderr)}"
         end
 
         parse(stdout, stderr)
@@ -63,6 +63,20 @@ module Pinspec
         path
       end
 
+      # A boot failure says WHAT went wrong on its first line and where on the rest.
+      # Showing only the last N lines shows the bottom of a backtrace - gem_prelude,
+      # rubygems.rb - which is identical for every failure and identifies none of them.
+      # Diagnosing a real application's failure took three commands and a read of
+      # pinspec's own internals because of that.
+      def excerpt(text, head: 6, tail: 6)
+        lines = text.to_s.lines
+        return lines.join if lines.size <= head + tail
+
+        lines.first(head).join +
+          "  ... #{lines.size - head - tail} more line(s) ...\n" +
+          lines.last(tail).join
+      end
+
       def command
         return ["rails", "runner", SCRIPT_PATH] unless File.file?(File.join(@app_root, "Gemfile"))
 
@@ -80,8 +94,7 @@ module Pinspec
 
         if json.nil?
           raise ProbeFailure,
-                "the sampler produced no rows.\nstdout: #{stdout.to_s.lines.last(8).join}" \
-                "stderr: #{stderr.to_s.lines.last(8).join}"
+                "the sampler produced no rows.\nstdout: #{excerpt(stdout)}\nstderr: #{excerpt(stderr)}"
         end
 
         parsed = JSON.parse(json)

data/lib/pinspec/inputs/sampler.rb

@@ -7,41 +7,9 @@ module Pinspec
     class Sampler
       SCRIPT_PATH = "tmp/pinspec/sampler.rb"
 
-      ENV_PREFERENCE = %w[development test].freeze
-
-      PRODUCTION_HINTS = [
-        /(?<![a-z])prod(uction)?(?![a-z])/i,
-        /(?<![a-z])live(?![a-z])/i,
-        /(?<![a-z])master(?![a-z])/i
-      ].freeze
-
       MAX_DISTINCT_STATUSES = 6
 
       class << self
-        def choose_env(available:, counts: {}, override: nil)
-          return override if override
-
-          populated = ENV_PREFERENCE.find do |env|
-            available.include?(env) && counts.fetch(env, 0).positive?
-          end
-
-          populated || ENV_PREFERENCE.find { |env| available.include?(env) } || available.first
-        end
-
-        def production_like?(name)
-          PRODUCTION_HINTS.any? { |hint| name.to_s.match?(hint) }
-        end
-
-        def guard_production!(name, confirmed: false)
-          return unless production_like?(name)
-          return if confirmed
-
-          raise EnvironmentRefused,
-                "#{name.inspect} looks like a production database. pinspec only ever " \
-                "SELECTs, but it will not read production without being told to: " \
-                "re-run with --sample-db pointed somewhere else, or confirm explicitly."
-        end
-
         def script_for(requests)
           new(requests).script
         end

data/lib/pinspec/report/summary.rb

@@ -185,6 +185,7 @@ module Pinspec
 
       def coverage_caveats
         caveats = []
+        caveats << vacuous_caveat
         caveats << seq_caveat
         caveats << truncated_caveat
         caveats << quarantine_caveat
@@ -203,6 +204,35 @@ module Pinspec
         MD
       end
 
+      # The honest half of pin strength, and the free half. A case that returned an
+      # empty collection, wrote no rows, enqueued nothing and sent nothing did run -
+      # but almost any change to the target would still satisfy it. Measured on real
+      # code, these are exactly the pins that score weak or worthless.
+      def vacuous_caveat
+        return nil if @stability.nil?
+
+        count = @stability.stable.count { |verdict| vacuous?(verdict.observation) }
+        return nil if count.zero?
+
+        "#{count} pinned case(s) observed nothing happening: an empty or absent return " \
+          "value, no rows written, no jobs, no mail. The pin holds, but almost any " \
+          "change to the target would still satisfy it. Give the target a world with " \
+          "something in it - `--sample` reads real rows - before relying on these."
+      end
+
+      EMPTY_RETURNS = [[], {}, nil, false, ""].freeze
+
+      def vacuous?(observation)
+        return false unless observation["status"] == "returned"
+        return false unless observation["enqueued_jobs"].to_a.empty?
+        return false unless observation["mail_deliveries"].to_a.empty?
+        return false unless observation["db_delta"].to_h.values.all? { |n| n.to_i.zero? }
+
+        EMPTY_RETURNS.include?(Tags.decode(observation["return_value"]))
+      rescue StandardError
+        false
+      end
+
       def seq_caveat
         count = rendered_observations.scan('"seq"').size
         return nil if count.zero?

data/lib/pinspec/runner/capture.rb

@@ -17,7 +17,9 @@ module Pinspec
         @target_file = target
         @method = method
         @max_cases = max_cases
-        @boots = boots
+        # Two is the floor, not the default. One boot has nothing to compare against,
+        # so every case would be called stable by virtue of never being checked.
+        @boots = [boots.to_i, 2].max
         @compare_sql = compare_sql
         @sandbox_env = sandbox_env
         @output_dir = output_dir || File.join(app_root, Sandbox::PROBE_DIR)

data/lib/pinspec/runner/probe_generator.rb

@@ -574,6 +574,7 @@ module Pinspec
             begin
               pinspec_with_isolation do
                 begin
+                  PinspecFactory.reset_sequences!
                   pinspec_run_steps(records, refs)
                 rescue StandardError => setup_error
                   observation["status"] = "setup_error"
@@ -638,7 +639,9 @@ module Pinspec
             # Between cases: anything memoized in a cache or a CurrentAttributes slot
             # would otherwise leak into the next case.
             Rails.cache.clear if defined?(Rails) && Rails.respond_to?(:cache) && Rails.cache
-            ActiveSupport::CurrentAttributes.reset_all if defined?(ActiveSupport::CurrentAttributes)
+            if defined?(ActiveSupport::CurrentAttributes) && ActiveSupport::CurrentAttributes.respond_to?(:reset_all)
+              ActiveSupport::CurrentAttributes.reset_all
+            end
             pinspec_clear_sinks
             PinspecClock.travel_back
           end

data/lib/pinspec/runner/sandbox.rb

@@ -20,8 +20,6 @@ module Pinspec
         "probe-#{pid}-#{object_id || rand(1 << 24)}.rb"
       end
 
-      DEFAULT_TIMEOUT = 600
-
       FORCED_ENV = {
         "DISABLE_SPRING" => "1",
         "TZ" => "UTC",
@@ -36,11 +34,9 @@ module Pinspec
         end
       end
 
-      def initialize(app_root:, probe_source:, timeout: DEFAULT_TIMEOUT, runner: nil, env: {})
+      def initialize(app_root:, probe_source:, env: {})
         @app_root = app_root
         @probe_source = probe_source
-        @timeout = timeout
-        @runner = runner
         @env = env
       end
 
@@ -80,24 +76,40 @@ module Pinspec
               .merge(@env)
 
         stdout, stderr, status = Open3.capture3(env, *command, chdir: @app_root)
+        stdout = stdout.to_s.dup.force_encoding(Encoding::UTF_8).scrub
+        stderr = stderr.to_s.dup.force_encoding(Encoding::UTF_8).scrub
 
         unless status.success?
           raise ProbeFailure,
                 "the probe exited #{status.exitstatus} in #{@app_root}.\n" \
                 "#{runtime.note ? "#{runtime.note}\n\n" : ''}" \
-                "#{stderr.to_s.lines.last(15).join}"
+                "#{excerpt(stderr)}"
         end
 
         parse(stdout, stderr, run)
       end
 
+      # A boot failure says WHAT went wrong on its first line and where on the rest.
+      # Showing only the last N lines shows the bottom of a backtrace - gem_prelude,
+      # rubygems.rb - which is identical for every failure and identifies none of them.
+      # Diagnosing a real application's failure took three commands and a read of
+      # pinspec's own internals because of that.
+      def excerpt(text, head: 6, tail: 6)
+        lines = text.to_s.lines
+        return lines.join if lines.size <= head + tail
+
+        lines.first(head).join +
+          "  ... #{lines.size - head - tail} more line(s) ...\n" +
+          lines.last(tail).join
+      end
+
       def parse(stdout, stderr, run)
         json = stdout.to_s.lines.reverse.find { |line| line.strip.start_with?("{") }
 
         if json.nil?
           raise ProbeFailure,
                 "the probe produced no observations.\n" \
-                "stdout: #{stdout.to_s.lines.last(10).join}\nstderr: #{stderr.to_s.lines.last(10).join}"
+                "stdout: #{excerpt(stdout)}\nstderr: #{excerpt(stderr)}"
         end
 
         parsed = JSON.parse(json)
@@ -114,8 +126,6 @@ module Pinspec
       end
 
       def runner_command
-        return @runner if @runner
-
         relative = File.join(PROBE_DIR, File.basename(probe_path))
 
         if File.file?(File.join(@app_root, "Gemfile"))

data/lib/pinspec/setup/context_builder.rb

@@ -5,8 +5,6 @@ require "digest"
 module Pinspec
   module Setup
     class ContextBuilder
-      MAX_GENERATIONS = 3
-
       class << self
         def build(target:, profile:, imports: [], generation: 1, tz: nil)
           new(target: target, profile: profile, imports: imports, generation: generation, tz: tz).build
@@ -33,8 +31,11 @@ module Pinspec
         refuse_attachment_targets!
 
         environment_steps
-        record_steps
+        # Imports first: create_record_for skips a table whose ref already exists, so
+        # building factory rows first meant every binding pointed at the factory
+        # record and the sampled row - the whole point of --sample - went unused.
         import_steps
+        record_steps
         context_steps
         subject_step
 
@@ -146,11 +147,13 @@ module Pinspec
         @resolver.table_for_type_hint(param.type_hint)
       end
 
+      # Only the hints that actually occur. Measured over 841 real service files, these
+      # ten are the only ones ever produced; the other thirty-one fired zero times and
+      # none of them has a Boundary value, so a hint that did somehow reach the list
+      # would be exempted from the refusal and then passed as nil - manufacturing the
+      # exact fiction the refusal exists to prevent.
       NON_MODEL_HINTS = %w[
-        String Symbol Integer Float Numeric Decimal BigDecimal Boolean TrueClass
-        FalseClass Hash Array Range Time Date DateTime Proc Class Module Object
-        NilClass Params Param Options Option Attributes Attribute Id Ids Name
-        Amount Quantity Count Total Price Rate Percentage Status State Kind Type
+        String Symbol Integer Float Boolean Hash Array Time Date Object
       ].freeze
 
       def refuse_unresolvable_model_param!(param)

data/lib/pinspec/types.rb

@@ -1,19 +1,10 @@
 # frozen_string_literal: true
 
 module Pinspec
-  PARAM_KINDS = %i[req opt rest keyreq key keyrest].freeze
 
   OPTIONAL_PARAM_KINDS   = %i[opt key rest keyrest].freeze
   POSITIONAL_PARAM_KINDS = %i[req opt rest].freeze
 
-  CONSTRUCTION_KINDS = %i[
-    new
-    class_method
-    interactor
-    dry_initializer
-    struct
-    model_instance
-  ].freeze
 
   Param = Data.define(:name, :kind, :default_source, :type_hint) do
     def positional?
@@ -49,7 +40,8 @@ module Pinspec
     :takes_block,
     :source_range,
     :referenced_constants,
-    :clock_sites
+    :clock_sites,
+    :construction_source
   ) do
     def singleton?
       construction_kind == :class_method
@@ -365,38 +357,6 @@ module Pinspec
     def dsl_module
       legacy_dsl ? "FactoryGirl" : "FactoryBot"
     end
-
-    def traits_for(name)
-      ancestry(name).flat_map(&:traits)
-    end
-
-    def ancestry(name)
-      chain = []
-      seen  = []
-      current = factory(name)
-
-      while current && !seen.include?(current.name)
-        seen << current.name
-        chain.unshift(current)
-        current = current.parent && factory(current.parent)
-      end
-
-      chain
-    end
-
-    def attributes_for(name, traits: [])
-      chain = ancestry(name)
-      merged = {}
-
-      chain.each { |f| f.attributes.each { |a| merged[a.name] = a } }
-
-      Array(traits).each do |trait_name|
-        found = chain.reverse.filter_map { |f| f.trait(trait_name) }.first
-        found&.attributes&.each { |a| merged[a.name] = a }
-      end
-
-      merged.values
-    end
   end
 
   Column = Data.define(
@@ -450,13 +410,7 @@ module Pinspec
   end
 
   SkippedStatement = Data.define(:kind, :table, :column, :references, :file, :line, :relevant) do
-    def relevant?
-      relevant == true
-    end
 
-    def tables_touched
-      ([table] + Array(references)).compact.uniq
-    end
 
     def to_s
       subject = column ? "#{table}.#{column}" : table
@@ -486,12 +440,5 @@ module Pinspec
       tables.flat_map { |t| t.columns.select(&:unknown_type?).map { |c| [t.name, c] } }
     end
 
-    def annotate_relevance(planned_tables)
-      wanted = Array(planned_tables).map(&:to_s)
-
-      with(skipped_statements: skipped_statements.map do |statement|
-        statement.with(relevant: statement.tables_touched.any? { |t| wanted.include?(t) })
-      end)
-    end
   end
 end

data/lib/pinspec/verify/verifier.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "fileutils"
+require "pathname"
 require "json"
 require "open3"
 
@@ -8,6 +10,8 @@ module Pinspec
     class Verifier
       CONFIGS = %i[isolated hostile neighbored].freeze
 
+      NEIGHBOUR_DIR = "tmp/pinspec"
+
       HOSTILE_TZ_CANDIDATES = ["Etc/GMT+8", "Etc/GMT-6"].freeze
 
       Outcome = Data.define(:config, :status, :diagnosis, :detail, :examples, :failures) do
@@ -52,6 +56,7 @@ module Pinspec
       def run(config)
         args, extra_env = arguments_for(config)
         stdout, stderr, _status = Open3.capture3(environment.merge(extra_env), *args, chdir: @app_root)
+        FileUtils.rm_f(@neighbour) if @neighbour
 
         summary = parse(stdout)
         return failure(config, stdout, stderr, summary) if summary.nil? || summary["failure_count"].to_i.positive?
@@ -72,7 +77,7 @@ module Pinspec
       end
 
       def arguments_for(config)
-        relative = @spec_path.sub("#{@app_root}/", "")
+        relative = relative_spec_path
         base = ["bundle", "exec", "rspec", "--format", "json"]
 
         case config
@@ -81,10 +86,40 @@ module Pinspec
         when :hostile
           [base + ["--seed", "7", relative], { "TZ" => hostile_tz, "LANG" => "C", "LC_ALL" => "C" }]
         when :neighbored
-          [base + [relative, relative], {}]
+          # A COPY alongside the original, because RSpec loads a given path once
+          # however many times it is named - so passing it twice re-ran nothing, and
+          # this configuration silently checked exactly what :isolated checks.
+          [base + [relative, neighbour_of(relative)], {}]
         end
       end
 
+      # Pathname does this properly: a string sub leaves an absolute path untouched
+      # when it does not start with the app root, which is exactly the case when
+      # someone runs `pinspec verify spec/foo_spec.rb` from inside their app.
+      def relative_spec_path
+        Pathname.new(File.expand_path(@spec_path))
+                .relative_path_from(Pathname.new(File.expand_path(@app_root)))
+                .to_s
+      rescue ArgumentError
+        @spec_path
+      end
+
+      # Written into the app's tmp, with its relative requires rewritten to point back
+      # at the support files the original sits beside. Removed again after the run -
+      # pinspec does not leave files in somebody's repository.
+      def neighbour_of(relative)
+        source = File.expand_path(relative, @app_root)
+        target = File.join(@app_root, NEIGHBOUR_DIR, "neighbour_#{File.basename(relative)}")
+        support = File.dirname(source)
+
+        FileUtils.mkdir_p(File.dirname(target))
+        File.write(target, Analyzer::Source.read(source)
+                                           .gsub('require_relative "support/', %(require_relative "#{support}/support/)))
+        @neighbour = target
+
+        File.join(NEIGHBOUR_DIR, File.basename(target))
+      end
+
       def hostile_tz
         HOSTILE_TZ_CANDIDATES.find { |zone| zone != @captured_tz } || HOSTILE_TZ_CANDIDATES.first
       end

data/lib/pinspec/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Pinspec
-  VERSION = "0.2.0"
+  VERSION = "0.3.0"
 
   PROBE_VERSION      = 3
   SERIALIZER_VERSION = 3

data/templates/factory_build.rb

@@ -4,12 +4,16 @@
 module PinspecFactory
   MAX_ATTEMPTS = 8
 
-  def self.attempts
-    @attempts ||= {}
-  end
-
-  def self.reset_attempts!
-    @attempts = {}
+  # factory_bot sequences are process-global and monotonic, so a pin whose world uses
+  # one is only reproducible while nothing else in the process built that factory
+  # first. Running the same pin twice in one process produced INV-1 then INV-2 and the
+  # second run failed against its own snapshot. Both hosts rewind before every case,
+  # so a case always sees the same sequence values.
+  def self.reset_sequences!
+    mod = defined?(FactoryBot) ? FactoryBot : (defined?(FactoryGirl) ? FactoryGirl : nil)
+    return unless mod.respond_to?(:rewind_sequences)
+
+    mod.rewind_sequences
   end
 
   def self.default_module
@@ -31,7 +35,6 @@ module PinspecFactory
                    factory_module.create(name.to_sym, attrs)
                  end
 
-        attempts[name.to_s] = attempt
         return record
       rescue StandardError => e
         raise unless retryable?(e)
@@ -40,7 +43,6 @@ module PinspecFactory
       end
     end
 
-    attempts[name.to_s] = MAX_ATTEMPTS
     raise last_error
   end
 
@@ -48,7 +50,4 @@ module PinspecFactory
     defined?(ActiveRecord::RecordInvalid) && error.is_a?(ActiveRecord::RecordInvalid)
   end
 
-  def self.fragile
-    attempts.select { |_name, count| count > 1 }
-  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pinspec
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Rehan Munir
@@ -75,6 +75,7 @@ files:
 - exe/pinspec
 - lib/pinspec.rb
 - lib/pinspec/analyzer/app_profile_reader.rb
+- lib/pinspec/analyzer/discovery.rb
 - lib/pinspec/analyzer/factory_registry.rb
 - lib/pinspec/analyzer/inflector.rb
 - lib/pinspec/analyzer/schema_reader.rb
@@ -83,7 +84,6 @@ files:
 - lib/pinspec/batch.rb
 - lib/pinspec/cli.rb
 - lib/pinspec/config.rb
-- lib/pinspec/emit/namer.rb
 - lib/pinspec/emit/spec_writer.rb
 - lib/pinspec/emit/stability_filter.rb
 - lib/pinspec/errors.rb