pinspec 0.1.0

data/lib/pinspec/validate/pin_scorer.rb

@@ -0,0 +1,164 @@
+# frozen_string_literal: true
+
+require "fileutils"
+
+module Pinspec
+  module Validate
+    class PinScorer
+      ASPECTS = %i[return error jobs mail].freeze
+      SCRATCH_DIR = "tmp/pinspec/aspects"
+
+      STRONG = 80.0
+      WEAK = 40.0
+
+      Score = Data.define(:aspect, :score, :verdict, :killed, :survived, :survivors, :note) do
+        def strong?
+          verdict == :strong
+        end
+      end
+
+      Report = Data.define(:subject, :scores, :skipped) do
+        def scored
+          scores.reject { |score| score.score.nil? }
+        end
+
+        def strong_ratio
+          return 0.0 if scored.empty?
+
+          (scored.count(&:strong?).to_f / scored.size * 100).round(1)
+        end
+
+        def surviving_all_aspects
+          sets = scored.map { |score| score.survivors.map { |s| mutant_key(s) } }
+          return [] if sets.empty? || sets.any?(&:empty?)
+
+          shared = sets.reduce(:&)
+          scored.first.survivors.select { |s| shared.include?(mutant_key(s)) }
+        end
+
+        def covered_by_another_aspect
+          all = scored.flat_map(&:survivors)
+          gaps = surviving_all_aspects.map { |s| mutant_key(s) }
+
+          all.reject { |s| gaps.include?(mutant_key(s)) }
+             .uniq { |s| mutant_key(s) }
+        end
+
+        def mutant_key(survivor)
+          [survivor["operator"], survivor["line"], survivor["token"]].join(":")
+        end
+      end
+
+      WRAPPER = "tmp/pinspec/run_suite.sh"
+
+      def initialize(app_root:, target:, plan:, corpus:, stability:, fk_map:, env: {}, test_command: nil)
+        @app_root = app_root
+        @target = target
+        @plan = plan
+        @corpus = corpus
+        @stability = stability
+        @fk_map = fk_map
+        @env = env
+        @test_command = test_command
+      end
+
+      def run
+        MutationAdapter.refuse_unless_available!
+        @command = @test_command || generate_wrapper
+
+        scores = present_aspects.map { |aspect| score_aspect(aspect) }
+
+        Report.new(subject: @target.qualified_name, scores: scores, skipped: excluded_aspects)
+      end
+
+      private
+
+      def present_aspects
+        found = []
+
+        @stability.stable.each do |verdict|
+          observation = verdict.observation
+          found << (observation["status"] == "raised" ? :error : :return)
+          found << :jobs unless observation["enqueued_jobs"].to_a.empty?
+          found << :mail unless observation["mail_deliveries"].to_a.empty?
+        end
+
+        found.uniq & ASPECTS
+      end
+
+      def excluded_aspects
+        ASPECTS - present_aspects
+      end
+
+      def score_aspect(aspect)
+        spec = write_variant(aspect)
+
+        outcome = MutationAdapter.new(
+          source_path: @target.file_path, subject: @target.qualified_name,
+          cwd: @app_root, test_command: @command
+        ).score(spec)
+
+        return Score.new(aspect: aspect, score: nil, verdict: :unscored, killed: nil,
+                         survived: nil, survivors: [], note: outcome.raw) unless outcome.scored?
+
+        Score.new(
+          aspect: aspect, score: outcome.score, verdict: verdict_for(aspect, outcome.score),
+          killed: outcome.killed, survived: outcome.survived, survivors: outcome.survivors,
+          note: caveat_for(aspect)
+        )
+      end
+
+      def verdict_for(aspect, score)
+        return :weak if score >= STRONG && understated?(aspect)
+        return :strong if score >= STRONG
+        return :weak if score >= WEAK
+
+        :worthless
+      end
+
+      def understated?(aspect)
+        @stability.stable.any? do |verdict|
+          rendered = verdict.observation.to_s
+
+          rendered.include?('"seq"') || rendered.include?('"truncated"')
+        end && %i[return error].include?(aspect)
+      end
+
+      def caveat_for(aspect)
+        return nil unless understated?(aspect)
+
+        "contains {\"t\":\"seq\"} or a truncated value, so it asserts less than a " \
+          "score alone suggests and is not counted as strong"
+      end
+
+      def generate_wrapper
+        return "bundle exec rspec %{files}" if @env.empty?
+
+        path = File.join(@app_root, WRAPPER)
+        FileUtils.mkdir_p(File.dirname(path))
+
+        exports = @env.map { |key, value| "export #{key}=#{value.to_s.inspect}" }.join("\n")
+        File.write(path, <<~SH)
+          #!/bin/bash
+          # Generated by pinspec. Establishes this application's own runtime, because
+          # the mutation backend runs in pinspec's Ruby and scrubs inherited pins.
+          #{exports}
+          exec bundle exec rspec "$@"
+        SH
+        FileUtils.chmod(0o755, path)
+
+        "./#{WRAPPER} %{files}"
+      end
+
+      def write_variant(aspect)
+        FileUtils.mkdir_p(File.join(@app_root, SCRATCH_DIR))
+
+        Emit::SpecWriter.new(
+          app_root: @app_root, target: @target, plan: @plan, corpus: @corpus,
+          stability: @stability, fk_map: @fk_map, force: true,
+          only_aspect: aspect, spec_dir: SCRATCH_DIR
+        ).write!.spec_path
+      end
+    end
+  end
+end

data/lib/pinspec/verify/verifier.rb

@@ -0,0 +1,149 @@
+# frozen_string_literal: true
+
+require "json"
+require "open3"
+
+module Pinspec
+  module Verify
+    class Verifier
+      CONFIGS = %i[isolated hostile neighbored].freeze
+
+      HOSTILE_TZ_CANDIDATES = ["Etc/GMT+8", "Etc/GMT-6"].freeze
+
+      Outcome = Data.define(:config, :status, :diagnosis, :detail, :examples, :failures) do
+        def green?
+          status == :green
+        end
+
+        def to_s
+          green? ? "#{config}: green (#{examples} examples)" : "#{config}: #{status} (#{diagnosis})"
+        end
+      end
+
+      DIAGNOSES = [
+        [/WebMock::NetConnectNotAllowedError|VCR::Errors/, :unpinnable_http],
+        [/captured with TZ=/, :tz_dependent],
+        [/uninitialized constant PinspecSerializer|PinspecSerializer.*NoMethodError/, :pinspec_internal],
+        [/RecordNotUnique|has already been taken/, :seed_collision],
+        [/DatabaseCleaner/, :db_cleaner_strategy],
+        [/cannot load such file -- (rails_helper|spec_helper)/, :spec_helper_missing],
+        [/LoadError|cannot load such file/, :rails_helper_variance]
+      ].freeze
+
+      def initialize(app_root:, spec_path:, env: {}, level: :isolated,
+                     captured_tz: Runner::Sandbox::FORCED_ENV.fetch("TZ"))
+        @app_root = app_root
+        @spec_path = spec_path
+        @env = env
+        @level = level
+        @captured_tz = captured_tz
+      end
+
+      def configs
+        @level == :full ? CONFIGS : [:isolated]
+      end
+
+      def verify
+        configs.map { |config| run(config) }
+      end
+
+      private
+
+      def run(config)
+        args, extra_env = arguments_for(config)
+        stdout, stderr, _status = Open3.capture3(environment.merge(extra_env), *args, chdir: @app_root)
+
+        summary = parse(stdout)
+        return failure(config, stdout, stderr, summary) if summary.nil? || summary["failure_count"].to_i.positive?
+
+        if summary["example_count"].to_i.zero?
+          return Outcome.new(config: config, status: :failed, diagnosis: :no_examples_ran,
+                             detail: excerpt(stdout, stderr), examples: 0, failures: 0)
+        end
+
+        if summary["errors_outside_of_examples_count"].to_i.positive?
+          return Outcome.new(config: config, status: :failed, diagnosis: :spec_load_error,
+                             detail: excerpt(stdout, stderr),
+                             examples: summary["example_count"], failures: 0)
+        end
+
+        Outcome.new(config: config, status: :green, diagnosis: nil, detail: nil,
+                    examples: summary["example_count"], failures: 0)
+      end
+
+      def arguments_for(config)
+        relative = @spec_path.sub("#{@app_root}/", "")
+        base = ["bundle", "exec", "rspec", "--format", "json"]
+
+        case config
+        when :isolated
+          [base + [relative], {}]
+        when :hostile
+          [base + ["--seed", "7", relative], { "TZ" => hostile_tz, "LANG" => "C", "LC_ALL" => "C" }]
+        when :neighbored
+          [base + [relative, relative], {}]
+        end
+      end
+
+      def hostile_tz
+        HOSTILE_TZ_CANDIDATES.find { |zone| zone != @captured_tz } || HOSTILE_TZ_CANDIDATES.first
+      end
+
+      def environment
+        Runner::Sandbox::SCRUBBED_ENV
+          .merge("RAILS_ENV" => "test", "DISABLE_SPRING" => "1", "TZ" => @captured_tz)
+          .merge(@env)
+      end
+
+      def parse(stdout)
+        json = stdout.to_s.lines.reverse.find { |line| line.strip.start_with?("{") }
+        return nil if json.nil?
+
+        parsed  = JSON.parse(json)
+        summary = parsed["summary"]
+        return nil if summary.nil?
+
+        summary.merge(
+          "errors_outside_of_examples_count" =>
+            summary["errors_outside_of_examples_count"] ||
+            parsed["errors_outside_of_examples_count"] || 0
+        )
+      rescue JSON::ParserError
+        nil
+      end
+
+      def failure(config, stdout, stderr, summary)
+        text = "#{stdout}\n#{stderr}"
+        found = DIAGNOSES.find { |pattern, _| text.match?(pattern) }
+
+        Outcome.new(
+          config: config,
+          status: :failed,
+          diagnosis: found ? found.last : :unknown,
+          detail: excerpt(stdout, stderr),
+          examples: summary && summary["example_count"],
+          failures: summary && summary["failure_count"]
+        )
+      end
+
+      def excerpt(stdout, stderr)
+        messages = failed_examples(stdout).map do |example|
+          "#{example['full_description']}\n#{example.dig('exception', 'message').to_s.lines.first(6).join}"
+        end
+
+        return messages.first(2).join("\n\n") unless messages.empty?
+
+        (stderr.to_s.empty? ? stdout.to_s : stderr.to_s).lines.last(20).join
+      end
+
+      def failed_examples(stdout)
+        json = stdout.to_s.lines.reverse.find { |line| line.strip.start_with?("{") }
+        return [] if json.nil?
+
+        JSON.parse(json)["examples"].to_a.reject { |example| example["status"] == "passed" }
+      rescue StandardError
+        []
+      end
+    end
+  end
+end

data/lib/pinspec/version.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module Pinspec
+  VERSION = "0.1.0"
+
+  PROBE_VERSION      = 3
+  SERIALIZER_VERSION = 3
+end

data/lib/pinspec.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require "zeitwerk"
+
+module Pinspec
+end
+
+require_relative "pinspec/version"
+require_relative "pinspec/errors"
+require_relative "pinspec/types"
+
+loader = Zeitwerk::Loader.for_gem
+loader.inflector.inflect("cli" => "CLI")
+loader.ignore("#{__dir__}/pinspec/version.rb")
+loader.ignore("#{__dir__}/pinspec/errors.rb")
+loader.ignore("#{__dir__}/pinspec/types.rb")
+loader.setup

data/templates/factory_build.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+# pinspec factory builder - generated. Do not hand-edit: re-run pinspec.
+module PinspecFactory
+  MAX_ATTEMPTS = 8
+
+  def self.attempts
+    @attempts ||= {}
+  end
+
+  def self.reset_attempts!
+    @attempts = {}
+  end
+
+  def self.default_module
+    return FactoryBot if defined?(FactoryBot)
+    return FactoryGirl if defined?(FactoryGirl)
+
+    raise "pinspec: the plan uses factories but neither FactoryBot nor FactoryGirl is loaded"
+  end
+
+  def self.create(name, attrs = {}, factory_module = nil)
+    factory_module ||= default_module
+    last_error = nil
+
+    1.upto(MAX_ATTEMPTS) do |attempt|
+      begin
+        record = if attrs.nil? || attrs.empty?
+                   factory_module.create(name.to_sym)
+                 else
+                   factory_module.create(name.to_sym, attrs)
+                 end
+
+        attempts[name.to_s] = attempt
+        return record
+      rescue StandardError => e
+        raise unless retryable?(e)
+
+        last_error = e
+      end
+    end
+
+    attempts[name.to_s] = MAX_ATTEMPTS
+    raise last_error
+  end
+
+  def self.retryable?(error)
+    defined?(ActiveRecord::RecordInvalid) && error.is_a?(ActiveRecord::RecordInvalid)
+  end
+
+  def self.fragile
+    attempts.select { |_name, count| count > 1 }
+  end
+end

data/templates/serializer.rb

@@ -0,0 +1,243 @@
+# frozen_string_literal: true
+
+module PinspecSerializer
+  MAX_DEPTH = 4
+  DEFAULT_MAX_COLLECTION = 50
+
+  VOLATILE = %w[created_at updated_at].freeze
+  DB_FUNCTION_DEFAULT = /\A(?:gen_random_uuid|now|uuid_generate_v4|CURRENT_|nextval)/i.freeze
+
+  class << self
+    def normalize(value, refs:, fk_map:, max_collection: DEFAULT_MAX_COLLECTION)
+      encode(value, refs: refs, fk_map: fk_map, max_collection: max_collection)
+    end
+
+    def encode(value, refs: {}, fk_map: {}, max_collection: DEFAULT_MAX_COLLECTION)
+      @refs = refs || {}
+      @fk_map = fk_map || {}
+      @max_collection = max_collection
+      @id_index = build_id_index(@refs)
+
+      visit(value, 0, [])
+    end
+
+    RETURNED_REF = "__returned__"
+
+    def build_id_index(refs)
+      index = {}
+
+      refs.each do |key, ref|
+        id = key.to_s.split(":").last
+        next unless id =~ /\A\d+\z/
+
+        numeric = id.to_i
+
+        if !index.key?(numeric)
+          index[numeric] = ref
+        elsif ref == RETURNED_REF || index[numeric] == RETURNED_REF
+          index[numeric] = RETURNED_REF
+        else
+          index[numeric] = :ambiguous
+        end
+      end
+
+      index
+    end
+
+    private
+
+    def visit(value, depth, seen)
+      return { "t" => "truncated" } if depth > MAX_DEPTH
+
+      case value
+      when nil then { "t" => "nil" }
+      when true then { "t" => "bool", "v" => true }
+      when false then { "t" => "bool", "v" => false }
+      when Integer then encode_integer(value)
+      when Float then encode_float(value)
+      when Symbol then { "t" => "sym", "v" => value.to_s }
+      when String then encode_string(value)
+      when Array then encode_array(value, depth, seen)
+      when Hash then encode_hash(value, depth, seen)
+      else encode_object(value, depth, seen)
+      end
+    end
+
+    def encode_integer(value)
+      ref = @id_index[value]
+      return { "t" => "ref", "v" => ref } if ref && ref != :ambiguous
+
+      { "t" => "int", "v" => value }
+    end
+
+    def encode_float(value)
+      return { "t" => "nan" } if value.nan?
+      return { "t" => "inf", "sign" => (value < 0 ? -1 : 1) } if value.infinite?
+
+      { "t" => "float", "v" => value.round(10) }
+    end
+
+    GLOBAL_ID = %r{\Agid://[^/]+/([A-Za-z0-9:_]+)/(.+)\z}.freeze
+
+    def encode_string(value)
+      if value.encoding == Encoding::ASCII_8BIT || !value.valid_encoding?
+        return { "t" => "bin", "v" => [value].pack("m0"), "enc" => value.encoding.to_s }
+      end
+
+      match = GLOBAL_ID.match(value)
+      return encode_global_id(match[1], match[2]) if match
+
+      { "t" => "str", "v" => value }
+    end
+
+    def encode_global_id(model, id)
+      out = { "t" => "gid", "model" => model }
+
+      ref = id =~ /\A\d+\z/ ? @id_index[id.to_i] : nil
+      if ref && ref != :ambiguous
+        out["ref"] = ref
+      else
+        out["id"] = "seq"
+      end
+
+      out
+    end
+
+    def encode_array(value, depth, seen)
+      if value.length > @max_collection
+        return {
+          "t" => "relation",
+          "total" => value.length,
+          "first" => value.first(@max_collection).map { |element| visit(element, depth + 1, seen) }
+        }
+      end
+
+      { "t" => "array", "v" => value.map { |element| visit(element, depth + 1, seen) } }
+    end
+
+    def encode_hash(value, depth, seen)
+      pairs = value.map do |key, element|
+        [visit(key, depth + 1, seen), visit(element, depth + 1, seen)]
+      end
+
+      { "t" => "hash", "v" => pairs }
+    end
+
+    def encode_object(value, depth, seen)
+      return { "t" => "cycle", "class" => value.class.name.to_s } if seen.include?(value.object_id)
+
+      nested = seen + [value.object_id]
+
+      return encode_record(value, depth, nested) if active_record?(value)
+      return encode_relation(value, depth, nested) if relation?(value)
+      return { "t" => "decimal", "v" => value.to_s("F") } if big_decimal?(value)
+      return encode_time(value) if value.is_a?(Time)
+      return encode_time(value.to_time) if defined?(DateTime) && value.is_a?(DateTime)
+      return { "t" => "date", "v" => value.strftime("%Y-%m-%d") } if defined?(Date) && value.is_a?(Date)
+      return { "t" => "exception", "class" => value.class.name.to_s, "message" => value.message.to_s } if value.is_a?(Exception)
+      return { "t" => "unpinnable", "class" => value.class.name.to_s } if unpinnable?(value)
+
+      encode_plain_object(value, depth, nested)
+    end
+
+    def encode_time(value)
+      { "t" => "time", "v" => value.getutc.strftime("%Y-%m-%dT%H:%M:%S.%6NZ") }
+    end
+
+    def encode_record(record, depth, seen)
+      table = record.class.table_name
+      out = {
+        "t" => "record",
+        "class" => record.class.name.to_s
+      }
+
+      ref = @refs[identity_key(table, record)]
+      out["ref"] = ref if ref
+
+      attributes = {}
+      record.attributes.each do |name, value|
+        next if name == record.class.primary_key
+        next if VOLATILE.include?(name)
+        next if db_function_default?(record.class, name)
+
+        attributes[name] = encode_attribute(table, name, value, depth, seen)
+      end
+
+      out["attributes"] = attributes
+      out
+    end
+
+    def encode_attribute(table, name, value, depth, seen)
+      target = @fk_map["#{table}.#{name}"]
+
+      if target && !value.nil?
+        ref = @refs["#{target}:#{value}"]
+        return { "t" => "ref", "v" => ref } if ref
+
+        return { "t" => "seq" }
+      end
+
+      return { "t" => "seq" } if unresolvable_id?(name, value)
+
+      visit(value, depth + 1, seen)
+    end
+
+    def unresolvable_id?(name, value)
+      return false if value.nil?
+      return true if name.to_s =~ /_id\z/ && value.is_a?(Integer)
+      return true if name.to_s =~ /_ids\z/ && value.is_a?(Array)
+
+      false
+    end
+
+    def identity_key(table, record)
+      "#{table}:#{record.id}"
+    end
+
+    def db_function_default?(klass, name)
+      column = klass.columns_hash[name]
+      return false if column.nil? || column.default_function.nil?
+
+      column.default_function.to_s =~ DB_FUNCTION_DEFAULT ? true : false
+    end
+
+    def encode_relation(relation, depth, seen)
+      total = relation.size
+      rows = relation.first(@max_collection)
+
+      {
+        "t" => "relation",
+        "total" => total,
+        "first" => rows.map { |row| visit(row, depth + 1, seen) }
+      }
+    end
+
+    def encode_plain_object(value, depth, seen)
+      state = {}
+
+      value.instance_variables.each do |name|
+        state[name.to_s] = visit(value.instance_variable_get(name), depth + 1, seen)
+      end
+
+      { "t" => "object", "class" => value.class.name.to_s, "ivars" => state }
+    end
+
+    def active_record?(value)
+      defined?(ActiveRecord::Base) && value.is_a?(ActiveRecord::Base)
+    end
+
+    def relation?(value)
+      defined?(ActiveRecord::Relation) && value.is_a?(ActiveRecord::Relation)
+    end
+
+    def big_decimal?(value)
+      defined?(BigDecimal) && value.is_a?(BigDecimal)
+    end
+
+    def unpinnable?(value)
+      value.is_a?(Proc) || value.is_a?(IO) ||
+        (defined?(Method) && value.is_a?(Method)) ||
+        (defined?(Thread) && value.is_a?(Thread))
+    end
+  end
+end