pinspec 0.1.0

data/lib/pinspec/runner/probe_generator.rb

@@ -0,0 +1,662 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Pinspec
+  module Runner
+    class ProbeGenerator
+      SERIALIZER_TEMPLATE = File.expand_path("../../../templates/serializer.rb", __dir__)
+      FACTORY_TEMPLATE = File.expand_path("../../../templates/factory_build.rb", __dir__)
+
+      class << self
+        def generate(plan:, corpus:, fk_map:, target:, max_collection: 50)
+          new(plan: plan, corpus: corpus, fk_map: fk_map, target: target,
+              max_collection: max_collection).generate
+        end
+
+        def serializer_source
+          File.read(SERIALIZER_TEMPLATE)
+        end
+
+        def factory_source
+          File.read(FACTORY_TEMPLATE)
+        end
+      end
+
+      def initialize(plan:, corpus:, fk_map:, target:, max_collection: 50)
+        @plan = plan
+        @corpus = corpus
+        @fk_map = fk_map
+        @target = target
+        @max_collection = max_collection
+      end
+
+      def payload
+        {
+          "pinspec_probe_version" => PROBE_VERSION,
+          "serializer" => SERIALIZER_VERSION,
+          "plan_id" => @plan.plan_id,
+          "isolation" => @plan.isolation.to_s,
+          "env_fingerprint" => stringify(@plan.env_fingerprint),
+          "fk_map" => @fk_map,
+          "max_collection" => @max_collection,
+          "target" => {
+            "class" => @target.class_name,
+            "method" => @target.method_name.to_s,
+            "construction" => @target.construction_kind.to_s,
+            "visibility" => @target.visibility.to_s
+          },
+          "setup_plan" => @plan.steps.map { |step| { "kind" => step.kind.to_s, "payload" => stringify(step.payload) } },
+          "cases" => @corpus.cases.map { |input_case| case_payload(input_case) }
+        }
+      end
+
+      def generate
+        <<~RUBY
+          # frozen_string_literal: true
+          #
+          # Generated by pinspec #{VERSION}. Do not edit: regenerate it.
+          #
+          # Runs inside the target application via `rails runner`. Requires only the
+          # standard library beyond what the app already loads, and is held to a Ruby
+          # 2.6 syntax floor because it runs in the app's Ruby, not pinspec's.
+          #
+          # Every case is wrapped in the plan's isolation regime and reversed
+          # afterwards. Nothing here writes outside a transaction unless the plan says
+          # the suite is untransacted, in which case the tables it touched are
+          # truncated instead.
+
+          require "json"
+
+          #{indent(self.class.serializer_source, 0)}
+
+          #{indent(self.class.factory_source, 0)}
+
+          PINSPEC = JSON.parse(<<'PINSPEC_PAYLOAD_JSON')
+          #{JSON.pretty_generate(payload)}
+          PINSPEC_PAYLOAD_JSON
+
+          #{indent(runtime, 0)}
+        RUBY
+      end
+
+      private
+
+      def case_payload(input_case)
+        {
+          "id" => input_case.id,
+          "origin" => input_case.origin.to_s,
+          "ctor_args" => input_case.ctor_args,
+          "ctor_kwargs" => input_case.ctor_kwargs,
+          "args" => input_case.args,
+          "kwargs" => input_case.kwargs
+        }
+      end
+
+      def stringify(value)
+        case value
+        when Hash then value.each_with_object({}) { |(key, inner), out| out[key.to_s] = stringify(inner) }
+        when Array then value.map { |inner| stringify(inner) }
+        when Symbol then value.to_s
+        else value
+        end
+      end
+
+      def indent(text, _spaces)
+        text.to_s
+      end
+
+      def runtime
+        [
+          runtime_environment,
+          runtime_isolation,
+          runtime_sinks,
+          runtime_sql,
+          runtime_steps,
+          runtime_invoke,
+          runtime_main
+        ].join("\n")
+      end
+
+      def runtime_environment
+        <<~'RUBY'
+          # --------------------------------------------------------------- guards --
+
+          unless defined?(ActiveRecord::Base)
+            abort("pinspec probe: ActiveRecord is not loaded; is this a Rails application?")
+          end
+
+          unless (defined?(Rails) && Rails.env.test?) || ENV["PINSPEC_UNSAFE"] == "1"
+            abort("pinspec probe: refusing to run outside RAILS_ENV=test (got " +
+                  (defined?(Rails) ? Rails.env.to_s : "no Rails") + ")")
+          end
+
+          require "active_support/testing/time_helpers"
+
+          module PinspecClock
+            extend ActiveSupport::Testing::TimeHelpers
+          end
+
+          # Forced once, for the whole process: a factory callback that enqueues during
+          # setup must land in a test sink rather than reaching a real queue.
+          ActiveJob::Base.queue_adapter = :test if defined?(ActiveJob::Base)
+          if defined?(ActionMailer::Base)
+            ActionMailer::Base.delivery_method = :test
+            ActionMailer::Base.perform_deliveries = true
+          end
+        RUBY
+      end
+
+      def runtime_isolation
+        <<~'RUBY'
+          # ------------------------------------------------------------ isolation --
+
+          # Raised to end a case early without it looking like a failure.
+          class PinspecCaseDone < StandardError; end
+          # The regime the PLAN chose, not the one this file would prefer. It has to
+          # be the plan's, because the emitted spec will run under the same one:
+          #
+          #   :transaction  wrap and roll back. after_commit callbacks never fire,
+          #                 in either host, and the divergence from production is
+          #                 documented rather than faked.
+          #   :truncation   do not wrap. after_commit callbacks DO fire - which is
+          #                 the whole reason this branch exists, because a suite that
+          #                 truncates would fire them in the emitted spec too, and a
+          #                 capture taken inside a transaction would have missed
+          #                 them. Reversed by truncating afterwards instead.
+          def pinspec_with_isolation
+            if PINSPEC["isolation"] == "truncation"
+              begin
+                yield
+              rescue PinspecCaseDone
+                nil
+              ensure
+                pinspec_truncate!
+              end
+            else
+              ActiveRecord::Base.transaction(requires_new: true) do
+                begin
+                  yield
+                rescue PinspecCaseDone
+                  nil
+                end
+
+                raise ActiveRecord::Rollback
+              end
+            end
+          end
+
+          # Everything the app owns, minus Rails' own bookkeeping. The safety claim
+          # is that a capture is reversible; under truncation that means leaving no
+          # rows behind rather than leaving no transaction behind.
+          def pinspec_truncate!
+            connection = ActiveRecord::Base.connection
+            tables = connection.tables - %w[schema_migrations ar_internal_metadata]
+            return if tables.empty?
+
+            if connection.respond_to?(:truncate_tables)
+              connection.truncate_tables(*tables)
+            else
+              tables.each { |table| connection.execute("DELETE FROM " + connection.quote_table_name(table)) }
+            end
+          rescue StandardError => e
+            warn("pinspec probe: could not truncate after a case: " + e.class.to_s + ": " + e.message.to_s)
+          end
+        RUBY
+      end
+
+      def runtime_sinks
+        <<~'RUBY'
+          # ---------------------------------------------------------------- sinks --
+
+          # Cleared after setup and before the target runs, so setup noise is never
+          # attributed to the target.
+          def pinspec_clear_sinks
+            if defined?(ActiveJob::Base) && ActiveJob::Base.queue_adapter.respond_to?(:enqueued_jobs)
+              ActiveJob::Base.queue_adapter.enqueued_jobs.clear
+              ActiveJob::Base.queue_adapter.performed_jobs.clear
+            end
+
+            ActionMailer::Base.deliveries.clear if defined?(ActionMailer::Base)
+          end
+
+          def pinspec_enqueued_jobs
+            return [] unless defined?(ActiveJob::Base)
+            adapter = ActiveJob::Base.queue_adapter
+            return [] unless adapter.respond_to?(:enqueued_jobs)
+
+            adapter.enqueued_jobs.map do |job|
+              {
+                "job" => job[:job].to_s,
+                "queue" => job[:queue].to_s,
+                "args" => PinspecSerializer.encode(job[:args], refs: $pinspec_refs,
+                                                              fk_map: PINSPEC["fk_map"],
+                                                              max_collection: PINSPEC["max_collection"])
+              }
+            end
+          end
+
+          def pinspec_mail_deliveries
+            return [] unless defined?(ActionMailer::Base)
+
+            ActionMailer::Base.deliveries.map do |mail|
+              { "to" => Array(mail.to).join(","), "subject" => mail.subject.to_s }
+            end
+          end
+        RUBY
+      end
+
+      def runtime_sql
+        <<~'RUBY'
+          # ------------------------------------------------------------------ sql --
+
+          PINSPEC_IGNORED_TABLES = %w[
+            versions ar_internal_metadata schema_migrations
+          ].freeze
+
+          # A breach is a statement that escapes the reversal pinspec relies on, and
+          # what counts depends on the regime. Under :transaction a COMMIT means the
+          # target broke out of the envelope. Under :truncation there IS no envelope
+          # - every write commits, and pinspec's own TRUNCATE follows the case - so
+          # only DDL and advisory locks are breaches there.
+          PINSPEC_BREACH =
+            if PINSPEC["isolation"] == "truncation"
+              /\A\s*(CREATE|ALTER|DROP)\b|pg_advisory_lock/i
+            else
+              /\A\s*(COMMIT|CREATE|ALTER|DROP|TRUNCATE)\b|pg_advisory_lock/i
+            end
+
+          # Rails logs its own column introspection as name "SCHEMA", once per model
+          # class per process - so the first case to touch each model would otherwise
+          # diverge from every later one and from the second boot. TRANSACTION covers
+          # BEGIN/COMMIT/SAVEPOINT, and payload[:cached] covers query-cache hits.
+          def pinspec_ignore_sql?(payload)
+            return true if %w[SCHEMA TRANSACTION].include?(payload[:name].to_s)
+            return true if payload[:cached]
+
+            sql = payload[:sql].to_s
+            return true if sql =~ /\A\s*(BEGIN|COMMIT|ROLLBACK|SAVEPOINT|RELEASE SAVEPOINT)/i
+            return true if PINSPEC_IGNORED_TABLES.any? { |table| sql.include?(table) }
+
+            false
+          end
+
+          def pinspec_fingerprint(sql)
+            text = sql.to_s.dup
+            text.gsub!(/'[^']*'/, "?")
+            text.gsub!(/\b\d+\b/, "?")
+            text.gsub!(/\$\d+/, "?")
+            text.gsub!(/\s+/, " ")
+            text.gsub!(/IN \((?:\?,\s*)*\?\)/i, "IN (?+)")
+            text.strip
+          end
+        RUBY
+      end
+
+      def runtime_steps
+        <<~'RUBY'
+          # ---------------------------------------------------------------- steps --
+
+          # Serializer-v3 tagged values arrive from cases.json; refs are resolved
+          # against the records this plan just built.
+          def pinspec_decode(tagged, records)
+            return tagged unless tagged.is_a?(Hash) && tagged.key?("t")
+
+            case tagged["t"]
+            when "nil" then nil
+            when "bool", "int", "float", "str" then tagged["v"]
+            when "sym" then tagged["v"].to_sym
+            when "decimal" then BigDecimal(tagged["v"])
+            when "date" then Date.parse(tagged["v"])
+            when "time" then Time.parse(tagged["v"])
+            when "nan" then Float::NAN
+            when "inf" then tagged["sign"].to_i < 0 ? -Float::INFINITY : Float::INFINITY
+            when "array" then tagged["v"].map { |element| pinspec_decode(element, records) }
+            when "hash"
+              out = {}
+              tagged["v"].each { |pair| out[pinspec_decode(pair[0], records)] = pinspec_decode(pair[1], records) }
+              out
+            when "ref"
+              records.fetch(tagged["v"]) do
+                raise "pinspec probe: case references unknown record " + tagged["v"].to_s
+              end
+            when "bin" then tagged["v"].unpack("m0").first
+            else tagged["v"]
+            end
+          end
+
+          def pinspec_run_steps(records, refs)
+            PINSPEC["setup_plan"].each do |step|
+              payload = step["payload"]
+
+              case step["kind"]
+              when "freeze_time"
+                PinspecClock.travel_to(Time.parse(payload["at"]))
+              when "seed_random"
+                srand(payload["seed"].to_i)
+              when "set_locale"
+                I18n.locale = payload["locale"] if defined?(I18n)
+              when "set_zone"
+                Time.zone = payload["zone"] if Time.respond_to?(:zone=)
+              when "set_flag"
+                if defined?(Flipper)
+                  payload["enabled"] ? Flipper.enable(payload["flag"].to_sym) : Flipper.disable(payload["flag"].to_sym)
+                end
+              when "create_record"
+                record = pinspec_create_record(payload, records)
+                pinspec_register(record, records, refs, payload["name"])
+              when "import_record"
+                record = pinspec_import_record(payload, records)
+                pinspec_register(record, records, refs, payload["name"]) if record
+              when "set_tenant"
+                if defined?(ActsAsTenant)
+                  ActsAsTenant.current_tenant = records[payload["record_ref"]]
+                end
+              when "stub_current"
+                pinspec_stub_current(payload, records)
+              when "set_whodunnit"
+                if defined?(PaperTrail)
+                  PaperTrail.request.whodunnit = records[payload["record_ref"]].try(:id).to_s
+                end
+              when "construct_subject"
+                nil # built per case, since its arguments are per case
+              end
+            end
+          end
+
+          # A real application bundles factory_bot with `require: false` and requires it
+          # from `rails_helper` - OFN does exactly this - and `rails runner` never
+          # reads rails_helper. So the probe loads it, rather than reporting that an
+          # app which plainly has 113 factories has none.
+          #
+          # `factory_bot_rails` is not enough on its own here: its railtie calls
+          # find_definitions during initialize!, which has already happened by the
+          # time this runs. So the definitions are found explicitly.
+          def pinspec_factory_module
+            return FactoryBot if defined?(FactoryBot)
+            return FactoryGirl if defined?(FactoryGirl)
+
+            begin
+              require "factory_bot"
+            rescue LoadError
+              begin
+                require "factory_girl"
+              rescue LoadError
+                return nil
+              end
+            end
+
+            mod = defined?(FactoryBot) ? FactoryBot : (defined?(FactoryGirl) ? FactoryGirl : nil)
+            return nil if mod.nil?
+
+            # Definitions are loaded once per process. A second call would raise
+            # DuplicateDefinitionError and take the whole probe down with it.
+            unless $pinspec_factories_found
+              $pinspec_factories_found = true
+              mod.find_definitions
+            end
+
+            mod
+          end
+
+          def pinspec_register(record, records, refs, name)
+            records[name] = record
+            refs[record.class.table_name.to_s + ":" + record.id.to_s] = name
+          end
+
+          def pinspec_create_record(payload, records)
+            attrs = {}
+            (payload["attrs"] || {}).each { |column, value| attrs[column] = value }
+            (payload["assoc_refs"] || {}).each do |column, ref|
+              attrs[column] = pinspec_decode(ref, records).id
+            end
+
+            if payload["factory"]
+              factory_module = pinspec_factory_module
+              raise "pinspec probe: plan wants factory :" + payload["factory"].to_s + " but factory_bot could not be loaded. " \
+                    "It is bundled with require: false, and this probe runs through rails runner, which does not read rails_helper" if factory_module.nil?
+
+              return PinspecFactory.create(payload["factory"], attrs.empty? ? {} : symbolize(attrs), factory_module)
+            end
+
+            payload["model"].constantize.create!(attrs)
+          end
+
+          # insert_all skips validations AND callbacks, which is the only honest way
+          # to recreate a legacy row: those rows routinely fail today's validations,
+          # and a before_save would rewrite the very attributes being imported.
+          def pinspec_import_record(payload, records)
+            model = payload["model"].constantize
+            attrs = {}
+
+            (payload["attrs"] || {}).each do |column, tagged|
+              value = pinspec_decode(tagged, records)
+              value = value.id if value.is_a?(ActiveRecord::Base)
+              attrs[column] = value
+            end
+
+            result = model.insert_all([attrs], returning: [model.primary_key])
+            key = result.rows.first && result.rows.first.first
+            return nil if key.nil?
+
+            found = model.find_by(model.primary_key => key)
+
+            # Refetch and diff: a fallback path that fires callbacks would silently
+            # rewrite what was imported, and that has to be visible.
+            (payload["attrs"] || {}).each do |column, tagged|
+              wanted = pinspec_decode(tagged, records)
+              next if wanted.is_a?(ActiveRecord::Base)
+              next if found[column].to_s == wanted.to_s
+
+              $pinspec_flags << "import_mutated"
+            end
+
+            found
+          end
+
+          def pinspec_stub_current(payload, records)
+            record = records[payload["record_ref"]]
+            return if record.nil?
+
+            if payload["kind"] == "current_attributes" && defined?(Current)
+              Current.user = record if Current.respond_to?(:user=)
+            end
+          end
+
+          def symbolize(hash)
+            out = {}
+            hash.each { |key, value| out[key.to_sym] = value }
+            out
+          end
+        RUBY
+      end
+
+      def runtime_invoke
+        <<~'RUBY'
+          # --------------------------------------------------------------- invoke --
+
+          def pinspec_build_subject(input_case, records)
+            construction = PINSPEC["target"]["construction"]
+            klass = PINSPEC["target"]["class"].constantize
+
+            return klass if construction == "class_method"
+
+            args = input_case["ctor_args"].map { |value| pinspec_decode(value, records) }
+            kwargs = {}
+            input_case["ctor_kwargs"].each { |name, value| kwargs[name.to_sym] = pinspec_decode(value, records) }
+
+            return records.values.last if construction == "model_instance"
+
+            pinspec_construct(klass, construction, args, kwargs)
+          end
+
+          def pinspec_construct(klass, construction, args, kwargs)
+            case construction
+            when "interactor" then klass
+            when "struct", "dry_initializer", "new"
+              if kwargs.empty?
+                args.empty? ? klass.new : klass.new(*args)
+              else
+                klass.new(*args, **kwargs)
+              end
+            else
+              klass.new
+            end
+          end
+
+          # One shim, because keyword forwarding of an EMPTY hash differs before Ruby
+          # 2.7 and open-coding it at each call site is how that bites.
+          def pinspec_invoke(subject, method_name, args, kwargs)
+            if kwargs.empty?
+              subject.send(method_name, *args)
+            else
+              subject.send(method_name, *args, **kwargs)
+            end
+          end
+        RUBY
+      end
+
+      def runtime_main
+        <<~'RUBY'
+          # ----------------------------------------------------------------- main --
+
+          $pinspec_refs = {}
+          $pinspec_flags = []
+
+          # Under truncation the plan's values are deterministic but the database is
+          # not: whatever ran before - the app's own suite, a previous pinspec run,
+          # seed data - is still there, and a leftover row collides with the plan on
+          # any unique index. Starting clean is not an extra liberty either: a suite
+          # in this regime has already declared its test database disposable.
+          pinspec_truncate! if PINSPEC["isolation"] == "truncation"
+
+          observations = []
+          order = (0...PINSPEC["cases"].length).to_a
+          order = order.shuffle(random: Random.new(ENV.fetch("PINSPEC_SHUFFLE_SEED", "42").to_i))
+
+          order.each do |index|
+            input_case = PINSPEC["cases"][index]
+
+            records = {}
+            refs = {}
+            $pinspec_refs = refs
+            $pinspec_flags = []
+            fingerprints = []
+            deltas = { "inserts" => 0, "updates" => 0, "deletes" => 0 }
+            started = Time.now.to_f
+
+            observation = {
+              "case_id" => input_case["id"],
+              "status" => "returned",
+              "return_value" => nil,
+              "error" => nil,
+              "setup_error" => nil,
+              "enqueued_jobs" => [],
+              "mail_deliveries" => [],
+              "sql_fingerprints" => [],
+              "db_delta" => deltas,
+              "flags" => [],
+              "duration_ms" => 0
+            }
+
+            subscriber = ActiveSupport::Notifications.subscribe("sql.active_record") do |*args|
+              payload = args.last
+              unless pinspec_ignore_sql?(payload)
+                sql = payload[:sql].to_s
+                $pinspec_flags << "escaped_transaction" if sql =~ PINSPEC_BREACH
+                fingerprints << pinspec_fingerprint(sql)
+                deltas["inserts"] += 1 if sql =~ /\A\s*INSERT/i
+                deltas["updates"] += 1 if sql =~ /\A\s*UPDATE/i
+                deltas["deletes"] += 1 if sql =~ /\A\s*DELETE/i
+              end
+            end
+
+            begin
+              pinspec_with_isolation do
+                begin
+                  pinspec_run_steps(records, refs)
+                rescue StandardError => setup_error
+                  observation["status"] = "setup_error"
+                  observation["setup_error"] = {
+                    "error" => { "class" => setup_error.class.name.to_s, "message" => setup_error.message.to_s }
+                  }
+                  raise PinspecCaseDone
+                end
+
+                # After setup, before the target: setup noise is not the target's.
+                pinspec_clear_sinks
+                fingerprints.clear
+                deltas["inserts"] = 0
+                deltas["updates"] = 0
+                deltas["deletes"] = 0
+
+                subject = pinspec_build_subject(input_case, records)
+                args = input_case["args"].map { |value| pinspec_decode(value, records) }
+                kwargs = {}
+                input_case["kwargs"].each { |name, value| kwargs[name.to_sym] = pinspec_decode(value, records) }
+
+                begin
+                  value = pinspec_invoke(subject, PINSPEC["target"]["method"], args, kwargs)
+                  observation["status"] = "returned"
+                  observation["return_value"] = PinspecSerializer.encode(
+                    value, refs: refs, fk_map: PINSPEC["fk_map"],
+                    max_collection: PINSPEC["max_collection"]
+                  )
+
+                  # Registered BEFORE the sinks are read: the commonest side effect
+                  # is perform_later(the_record_the_target_just_created.id), and
+                  # that id churns between runs unless it can name something.
+                  if value.is_a?(ActiveRecord::Base) && !value.id.nil?
+                    refs[value.class.table_name.to_s + ":" + value.id.to_s] = "__returned__"
+                  end
+                rescue StandardError, ScriptError => target_error
+                  observation["status"] = "raised"
+                  observation["error"] = {
+                    "class" => target_error.class.name.to_s,
+                    "message" => target_error.message.to_s
+                  }
+                end
+
+                observation["enqueued_jobs"] = pinspec_enqueued_jobs
+                observation["mail_deliveries"] = pinspec_mail_deliveries
+              end
+            rescue StandardError => outer
+              observation["status"] = "setup_error"
+              observation["setup_error"] = {
+                "error" => { "class" => outer.class.name.to_s, "message" => outer.message.to_s }
+              }
+            ensure
+              ActiveSupport::Notifications.unsubscribe(subscriber)
+            end
+
+            observation["sql_fingerprints"] = fingerprints
+            observation["flags"] = $pinspec_flags.uniq
+            observation["duration_ms"] = ((Time.now.to_f - started) * 1000).round
+
+            observations << observation
+
+            # Between cases: anything memoized in a cache or a CurrentAttributes slot
+            # would otherwise leak into the next case.
+            Rails.cache.clear if defined?(Rails) && Rails.respond_to?(:cache) && Rails.cache
+            ActiveSupport::CurrentAttributes.reset_all if defined?(ActiveSupport::CurrentAttributes)
+            pinspec_clear_sinks
+            PinspecClock.travel_back
+          end
+
+          puts JSON.generate(
+            "pinspec_probe_version" => PINSPEC["pinspec_probe_version"],
+            "serializer" => PINSPEC["serializer"],
+            "plan_id" => PINSPEC["plan_id"],
+            "env" => {
+              "ruby" => RUBY_VERSION,
+              "rails" => (defined?(Rails) ? Rails.version : nil),
+              "tz" => ENV["TZ"].to_s,
+              "shuffle_seed" => ENV.fetch("PINSPEC_SHUFFLE_SEED", "42")
+            },
+            "observations" => observations.sort_by { |observation| observation["case_id"] }
+          )
+        RUBY
+      end
+    end
+  end
+end