pinspec 0.1.0

data/lib/pinspec/errors.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+module Pinspec
+  class Error < StandardError
+    class << self
+      attr_writer :exit_code
+
+      def exit_code
+        @exit_code || 1
+      end
+    end
+
+    def exit_code
+      self.class.exit_code
+    end
+  end
+
+  class UnparsableSource < Error
+    self.exit_code = 1
+  end
+
+  class TargetNotFound < Error
+    self.exit_code = 2
+  end
+
+  class AmbiguousTarget < Error
+    self.exit_code = 3
+  end
+
+  class BlockRequired < Error
+    self.exit_code = 4
+  end
+
+  class UnresolvableSetup < Error
+    self.exit_code = 5
+
+    REASONS = %i[
+      association_cycle
+      attachment
+      opaque_constructor
+      unknown_column_type
+      apartment
+      unresolvable_parameter
+    ].freeze
+
+    attr_reader :reason
+
+    def initialize(reason, message = nil)
+      unless REASONS.include?(reason)
+        raise ArgumentError, "unknown UnresolvableSetup reason: #{reason.inspect}"
+      end
+
+      @reason = reason
+      super(message || reason.to_s.tr("_", " "))
+    end
+  end
+
+  class SchemaFormatUnsupported < Error
+    self.exit_code = 6
+  end
+
+  class ProbeFailure < Error
+    self.exit_code = 7
+  end
+
+  class NothingStableToPin < Error
+    self.exit_code = 8
+  end
+
+  class VerifyFailed < Error
+    self.exit_code = 9
+  end
+
+  class UnsupportedRailsVersion < Error
+    self.exit_code = 10
+  end
+
+  class EnvironmentRefused < Error
+    self.exit_code = 11
+  end
+
+  class PinspecInternalError < Error
+    self.exit_code = 12
+  end
+end

data/lib/pinspec/inputs/boundary.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+module Pinspec
+  module Inputs
+    module Boundary
+      BY_TYPE = {
+        "Integer" => [0, 1, -1],
+        "Float"   => [0.0, 1.0, -1.0],
+        "Boolean" => [true, false],
+        "String"  => ["", "pinspec"],
+        "Symbol"  => nil,
+        "Array"   => [[]],
+        "Hash"    => [{}],
+        "Proc"    => nil,
+        "Date"    => [PINSPEC_EPOCH[0, 10]],
+        "Time"    => [PINSPEC_EPOCH]
+      }.freeze
+
+      DECIMAL_HINTS = %w[BigDecimal Decimal].freeze
+
+      class << self
+        def values_for(param, column: nil, default_first: true)
+          declared = default_value(param)
+          edges    = edges_for(param, column)
+
+          ordered = default_first ? [declared, *edges] : [*edges, declared]
+
+          present(ordered).map { |value| encode(value, param) }.uniq
+        end
+
+        def base_value(param, column: nil)
+          declared = default_value(param)
+          return Tags.encode(declared) unless declared == :__none__
+
+          encode(Array(edges_for(param, column)).first, param)
+        end
+
+        def omittable?(param)
+          param.optional?
+        end
+
+        private
+
+        def encode(value, param)
+          return Tags.decimal(value) if decimal?(param) && !value.nil?
+
+          Tags.encode(value)
+        end
+
+        def decimal?(param)
+          DECIMAL_HINTS.include?(param.type_hint)
+        end
+
+        def present(values)
+          values.reject { |value| value == :__none__ }
+        end
+
+        def default_value(param)
+          return :__none__ if param.default_source.nil?
+
+          literal_from_source(param.default_source)
+        end
+
+        def literal_from_source(source)
+          text = source.to_s.strip
+
+          case text
+          when "nil"                then nil
+          when "true"               then true
+          when "false"              then false
+          when /\A:([A-Za-z_]\w*)\z/ then Regexp.last_match(1).to_sym
+          when /\A-?\d+\z/          then text.to_i
+          when /\A-?\d+\.\d+\z/     then text.to_f
+          when /\A"([^"]*)"\z/, /\A'([^']*)'\z/ then Regexp.last_match(1)
+          when "[]"                 then []
+          when "{}"                 then {}
+          else :__none__
+          end
+        end
+
+        def edges_for(param, column = nil)
+          hint = param.type_hint
+
+          return decimal_edges if DECIMAL_HINTS.include?(hint)
+          return BY_TYPE.fetch(hint) if BY_TYPE.key?(hint)
+
+          from_column = column && edges_for_column_type(column.type)
+          return from_column if from_column
+
+          [nil]
+        end
+
+        def edges_for_column_type(type)
+          case type
+          when :string, :text, :citext             then BY_TYPE.fetch("String")
+          when :integer, :bigint, :smallint        then BY_TYPE.fetch("Integer")
+          when :float                              then BY_TYPE.fetch("Float")
+          when :decimal, :numeric, :money          then decimal_edges
+          when :boolean                            then BY_TYPE.fetch("Boolean")
+          when :date                               then BY_TYPE.fetch("Date")
+          when :datetime, :timestamp, :timestamptz then BY_TYPE.fetch("Time")
+          when :json, :jsonb, :hstore              then BY_TYPE.fetch("Hash")
+          end
+        end
+
+        def decimal_edges
+          %w[0.0 1.0 -1.0]
+        end
+      end
+    end
+  end
+end

data/lib/pinspec/inputs/corpus.rb

@@ -0,0 +1,148 @@
+# frozen_string_literal: true
+
+module Pinspec
+  module Inputs
+    class Corpus
+      DEFAULT_MAX_CASES = 12
+
+      OMIT = :__pinspec_omit__
+
+      class << self
+        def build(target:, plan:, schema: nil, max_cases: DEFAULT_MAX_CASES, imports: [])
+          new(target: target, plan: plan, schema: schema, max_cases: max_cases, imports: imports).build
+        end
+      end
+
+      def initialize(target:, plan:, schema: nil, max_cases: DEFAULT_MAX_CASES, imports: [])
+        @target    = target
+        @plan      = plan
+        @schema    = schema
+        @max_cases = max_cases
+        @imports   = imports
+      end
+
+      def build
+        cases = [base_case]
+        cases.concat(boundary_cases(@max_cases - cases.size))
+
+        InputCorpus.new(cases: dedup(cases), setup_plan: @plan)
+      end
+
+      private
+
+      def base_case
+        build_case("c001", :defaults) { |param| base_value_for(param) }
+      end
+
+      def boundary_cases(budget)
+        return [] if budget <= 0
+
+        variations = round_robin(ctor_variations, method_variations).first(budget)
+
+        variations.each_with_index.map do |(param, value), index|
+          build_case(format("c%03d", index + 2), :boundary) do |candidate|
+            candidate.equal?(param) ? value : base_value_for(candidate)
+          end
+        end
+      end
+
+      def round_robin(first, second)
+        interleaved = []
+        [first.size, second.size].max.times do |index|
+          interleaved << first[index] if first[index]
+          interleaved << second[index] if second[index]
+        end
+
+        interleaved
+      end
+
+      def ctor_variations
+        variations_for(@target.initializer_params)
+      end
+
+      def method_variations
+        variations_for(@target.params)
+      end
+
+      def variations_for(params)
+        params.flat_map do |param|
+          next [] if bound?(param)
+          next [] if %i[rest keyrest].include?(param.kind)
+
+          base = base_value_for(param)
+          values = Boundary.values_for(param, column: column_for(param))
+                           .reject { |value| value == base }
+
+          values += [OMIT] if Boundary.omittable?(param)
+
+          values.map { |value| [param, value] }
+        end
+      end
+
+      def build_case(id, origin)
+        ctor_args, ctor_kwargs = split_params(@target.initializer_params) { |param| yield(param) }
+        args, kwargs           = split_params(@target.params) { |param| yield(param) }
+
+        InputCase.new(
+          id:          id,
+          ctor_args:   ctor_args,
+          ctor_kwargs: ctor_kwargs,
+          args:        args,
+          kwargs:      kwargs,
+          origin:      origin
+        )
+      end
+
+      def split_params(params)
+        positional = []
+        keyword    = {}
+
+        params.each do |param|
+          value = yield(param)
+          next if value == OMIT
+
+          if %i[keyreq key].include?(param.kind)
+            keyword[param.name.to_s] = value
+          elsif %i[req opt].include?(param.kind)
+            positional << value
+          end
+        end
+
+        [positional, keyword]
+      end
+
+      def base_value_for(param)
+        ref = @plan.binding_for(param.name)
+        return Tags.ref(ref) if ref
+
+        Boundary.base_value(param, column: column_for(param))
+      end
+
+      def column_for(param)
+        return nil if @schema.nil?
+
+        matches = @schema.tables.filter_map { |table| table.column(param.name) }
+        return nil if matches.empty?
+
+        types = matches.map(&:type).uniq
+        types.size == 1 ? matches.first : nil
+      end
+
+      def bound?(param)
+        !@plan.binding_for(param.name).nil?
+      end
+
+      def dedup(cases)
+        seen = {}
+
+        cases.each_with_object([]) do |input_case, kept|
+          signature = input_case.signature
+          next if seen[signature]
+
+          seen[signature] = true
+          kept << input_case
+        end
+      end
+    end
+  end
+end

data/lib/pinspec/inputs/hydrator.rb

@@ -0,0 +1,197 @@
+# frozen_string_literal: true
+
+require "digest"
+
+module Pinspec
+  module Inputs
+    class Hydrator
+      MAX_DEPTH        = 3
+      MAX_ROWS_PER_CLUSTER = 20
+
+      VOLATILE = %w[created_at updated_at].freeze
+      DB_FUNCTION_DEFAULT = /\A(?:->|gen_random_uuid|now|uuid_generate_v4|CURRENT_)/i.freeze
+
+      def initialize(schema:, redactor:, factories:, max_depth: MAX_DEPTH,
+                     max_rows: MAX_ROWS_PER_CLUSTER)
+        raise ArgumentError, "factories: is required - without it an imported row " \
+                            "gets a camelized table name, which is the wrong constant " \
+                            "on any app whose tables sit behind an engine prefix" if factories.nil?
+
+        @schema    = schema
+        @redactor  = redactor
+        @resolver  = Setup::DependencyResolver.new(schema, factories)
+        @max_depth = max_depth
+        @max_rows  = max_rows
+      end
+
+      def hydrate(table, rows, all_rows, target_source: "")
+        @notes = []
+        clusters = []
+        ordinal  = 0
+
+        @refs = {}
+        @counters = Hash.new(0)
+        @budget = @max_rows
+
+        Array(rows).each do |row|
+          ordinal += 1
+          clusters.concat(build_cluster(table, row, all_rows, ordinal, target_source))
+        end
+
+        [clusters, @notes]
+      end
+
+      private
+
+      def build_cluster(table, row, all_rows, ordinal, target_source)
+        emitted = []
+
+        walk = lambda do |current_table, current_row, depth|
+          return nil if current_row.nil?
+          return nil if depth > @max_depth
+          return nil if @budget <= 0
+
+          key = row_key(current_table, current_row)
+          return @refs[key] if @refs.key?(key)
+
+          @counters[current_table] += 1
+          ref = "imported_#{singular(current_table)}_#{@counters[current_table]}"
+          @refs[key] = ref
+          @budget -= 1
+
+          assoc_refs = {}
+
+          all_parents(current_table).each do |column, parent_table|
+            parent_id  = current_row[column] || current_row[column.to_sym]
+            parent_row = find_row(all_rows, parent_table, parent_id)
+
+            if parent_row.nil?
+              assoc_refs[column] = handle_missing_parent(current_table, column, parent_table)
+              next
+            end
+
+            parent_ref = walk.call(parent_table, parent_row, depth + 1)
+            assoc_refs[column] = parent_ref.nil? ? nil : Tags.ref(parent_ref)
+          end
+
+          emitted << build_import(current_table, current_row, ref, assoc_refs, ordinal, target_source)
+          ref
+        end
+
+        walk.call(table, row, 0)
+        emitted
+      end
+
+      def handle_missing_parent(table, column, parent_table)
+        nullable = column_of(table, column)&.nullable?
+
+        if nullable
+          note(:cross_cluster_nulled,
+               "#{table}.#{column} pointed outside the sampled rows; it is nullable, " \
+               "so the import leaves it null")
+          return Tags.encode(nil)
+        end
+
+        note(:row_discarded,
+             "#{table}.#{column} requires a #{parent_table} that was not sampled, so " \
+             "that row cannot be recreated and was dropped")
+        nil
+      end
+
+      def build_import(table, row, ref, assoc_refs, ordinal, target_source)
+        attrs, redacted = @redactor.redact(importable_attrs(table, row), ordinal: ordinal, table: table)
+
+        tagged = attrs.transform_values { |value| Tags.encode(value) }
+        tagged.merge!(assoc_refs.compact)
+
+        flags = []
+        reads = @redactor.reads_in(target_source, redacted)
+        flags << :redaction_read unless reads.empty?
+
+        reads.each do |read|
+          note(:redaction_read,
+               "the target reads #{read[:attribute]} at line #{read[:line]}, and that " \
+               "value was rewritten - the pin would freeze behaviour the app never has")
+        end
+
+        ImportCluster.new(
+          model:    model_for(table),
+          table:    table,
+          name:     ref,
+          attrs:    tagged,
+          source:   provenance(table, row),
+          redacted: redacted,
+          flags:    flags
+        )
+      end
+
+      def importable_attrs(table, row)
+        schema_table = @schema.table(table)
+        pk           = Array(schema_table&.primary_key)
+        fk_columns   = all_parents(table).map(&:first)
+
+        row.each_with_object({}) do |(column, value), out|
+          name = column.to_s
+          next if pk.include?(name)
+          next if VOLATILE.include?(name)
+          next if fk_columns.include?(name)
+          next if db_function_default?(schema_table, name)
+
+          out[name] = value
+        end
+      end
+
+      def db_function_default?(schema_table, column_name)
+        default = schema_table&.column(column_name)&.default
+        return false if default.nil?
+
+        default.to_s.match?(DB_FUNCTION_DEFAULT)
+      end
+
+      def provenance(table, row)
+        identifier = row["id"] || row[:id] || row.values.first
+
+        "sample:#{table}:#{Digest::SHA1.hexdigest("#{table}:#{identifier}")[0, 8]}"
+      end
+
+      def all_parents(table)
+        schema_table = @schema.table(table)
+        return [] if schema_table.nil?
+
+        schema_table.columns.filter_map do |column|
+          fk = @schema.fk_for(table, column.name)
+          fk ? [column.name, fk.to_table] : nil
+        end
+      end
+
+      def find_row(all_rows, table, id)
+        return nil if id.nil?
+
+        Array(all_rows[table] || all_rows[table.to_sym]).find do |row|
+          (row["id"] || row[:id]).to_s == id.to_s
+        end
+      end
+
+      def column_of(table, column_name)
+        @schema.table(table)&.column(column_name)
+      end
+
+      def row_key(table, row)
+        "#{table}:#{row['id'] || row[:id] || row.hash}"
+      end
+
+      def model_for(table)
+        @resolver.model_for(table)
+      end
+
+      def singular(table)
+        Analyzer::Inflector.singular_candidates(table).first
+      end
+
+      def note(kind, detail)
+        @notes << { kind: kind, detail: detail }
+        nil
+      end
+    end
+  end
+end

data/lib/pinspec/inputs/redactor.rb

@@ -0,0 +1,138 @@
+# frozen_string_literal: true
+
+require "digest"
+
+module Pinspec
+  module Inputs
+    class Redactor
+      BUILT_IN = %w[
+        email phone phone_number mobile
+        ssn social_security_number
+        dob date_of_birth birthdate
+        first_name last_name full_name middle_name maiden_name
+        address address1 address2 address_line1 address_line2 street city postcode zip zipcode
+        ip_address remote_ip
+        token api_key access_token refresh_token secret password_digest encrypted_password
+      ].freeze
+
+      def initialize(attributes: [], enabled: true)
+        @extra   = Array(attributes).map(&:to_s)
+        @enabled = enabled
+      end
+
+      def enabled?
+        @enabled
+      end
+
+      PERSON_TABLES = %w[
+        customers users people contacts employees members clients patients
+        subscribers accounts admins authors owners guests students staff
+      ].freeze
+
+      PERSON_COLUMNS = %w[name display_name nickname username handle].freeze
+
+      def redacts?(column_name, table: nil)
+        return false unless @enabled
+
+        name = column_name.to_s
+        return true if BUILT_IN.include?(name) || @extra.include?(name)
+
+        PERSON_COLUMNS.include?(name) && PERSON_TABLES.include?(table.to_s)
+      end
+
+      def redact(attrs, ordinal:, table: nil)
+        redacted = []
+
+        rewritten = attrs.each_with_object({}) do |(column, value), out|
+          if redacts?(column, table: table) && value.is_a?(String) && !value.empty?
+            out[column] = rewrite(column.to_s, value, ordinal)
+            redacted << column.to_s
+          else
+            out[column] = value
+          end
+        end
+
+        [rewritten, redacted]
+      end
+
+      def reads_in(source, columns)
+        return [] unless @enabled
+
+        Array(columns).uniq.filter_map do |column|
+          line = read_line(source, column.to_s)
+          next unless line
+
+          { attribute: column.to_s, line: line }
+        end
+      end
+
+      private
+
+      def read_line(source, column)
+        pattern = /(?:\.#{Regexp.escape(column)}\b|:#{Regexp.escape(column)}\b|["']#{Regexp.escape(column)}["'])/
+
+        source.to_s.lines.each_with_index do |text, index|
+          return index + 1 if text.match?(pattern)
+        end
+
+        nil
+      end
+
+      def rewrite(column, value, ordinal)
+        case column
+        when /email/                       then rewrite_email(value, ordinal)
+        when /phone|mobile/                then rewrite_digits(value, ordinal)
+        when /ssn|social_security/         then rewrite_digits(value, ordinal)
+        when /dob|birth/                   then rewrite_date(value)
+        when /token|key|secret|password/   then rewrite_hex(value, ordinal)
+        when /ip_address|remote_ip/        then rewrite_ip(value, ordinal)
+        else rewrite_filler(value, ordinal)
+        end
+      end
+
+      def rewrite_email(value, ordinal)
+        local, _, domain = value.rpartition("@")
+        return rewrite_filler(value, ordinal) if domain.empty? || local.empty?
+
+        "#{same_length_token(local, ordinal)}@#{domain}"
+      end
+
+      def rewrite_digits(value, ordinal)
+        counter = ordinal.abs
+        digits  = format("%0#{value.count('0-9')}d", counter)
+        index   = -1
+
+        value.gsub(/\d/) do
+          index += 1
+          digits[index] || "0"
+        end
+      end
+
+      def rewrite_date(value)
+        value.match?(/\A\d{4}-\d{2}-\d{2}/) ? "1970-01-01#{value[10..]}" : value
+      end
+
+      def rewrite_hex(value, ordinal)
+        digest = Digest::SHA256.hexdigest("pinspec:#{ordinal}")
+        (digest * ((value.length / digest.length) + 1))[0, value.length]
+      end
+
+      def rewrite_ip(value, ordinal)
+        return rewrite_digits(value, ordinal) if value.include?(":")
+
+        "192.0.2.#{(ordinal.abs % 254) + 1}"
+      end
+
+      def rewrite_filler(value, ordinal)
+        same_length_token(value, ordinal)
+      end
+
+      def same_length_token(value, ordinal)
+        seed = "p#{ordinal}"
+        return seed[0, value.length] if seed.length >= value.length
+
+        seed + ("x" * (value.length - seed.length))
+      end
+    end
+  end
+end