real_data_tests 0.1.0

data/lib/real_data_tests/pg_dump_generator.rb

@@ -0,0 +1,304 @@
+require 'csv'
+require 'tmpdir'
+require 'fileutils'
+require 'json'
+require 'set'
+
+module RealDataTests
+  class PgDumpGenerator
+    def initialize(records)
+      @records = records
+    end
+
+    def generate
+      sorted_records = sort_by_dependencies(@records)
+      insert_statements = collect_inserts(sorted_records)
+      insert_statements.join("\n")
+    end
+
+    private
+
+    def sort_by_dependencies(records)
+      # Group records by their model class
+      tables_with_records = records.group_by(&:class)
+
+      # Build dependency graph directly from the models we have
+      dependencies = build_dependency_graph(tables_with_records.keys)
+
+      # Sort models based on dependencies
+      sorted_models = topological_sort(dependencies)
+
+      # Map back to the actual records in dependency order
+      sorted_models.flat_map { |model| tables_with_records[model] || [] }
+    end
+
+    def build_dependency_graph(models)
+      models.each_with_object({}) do |model, deps|
+        # We only need to consider belongs_to associations since they represent
+        # the true foreign key dependencies that affect insert order
+        direct_dependencies = model.reflect_on_all_associations(:belongs_to)
+          .reject(&:polymorphic?) # Skip polymorphic associations
+          .map(&:klass)
+          .select { |klass| models.include?(klass) } # Only include models we actually have records for
+          .uniq
+
+        # For HABTM associations, we need to ensure the join tables are handled correctly
+        habtm_dependencies = model.reflect_on_all_associations(:has_and_belongs_to_many)
+          .map { |assoc| assoc.join_table_model }
+          .compact
+          .select { |join_model| models.include?(join_model) }
+          .uniq
+
+        deps[model] = (direct_dependencies + habtm_dependencies).uniq
+      end
+    end
+
+    def topological_sort(dependencies)
+      sorted = []
+      visited = Set.new
+      temporary = Set.new
+
+      dependencies.each_key do |model|
+        visit_model(model, dependencies, sorted, visited, temporary) unless visited.include?(model)
+      end
+
+      sorted
+    end
+
+    def visit_model(model, dependencies, sorted, visited, temporary)
+      return if visited.include?(model)
+
+      if temporary.include?(model)
+        # Provide more context in the error message
+        cycle = detect_cycle(model, dependencies, temporary)
+        raise "Circular dependency detected: #{cycle.map(&:name).join(' -> ')}"
+      end
+
+      temporary.add(model)
+
+      (dependencies[model] || []).each do |dependency|
+        visit_model(dependency, dependencies, sorted, visited, temporary) unless visited.include?(dependency)
+      end
+
+      temporary.delete(model)
+      visited.add(model)
+      sorted << model
+    end
+
+    def detect_cycle(start_model, dependencies, temporary)
+      cycle = [start_model]
+      current = dependencies[start_model]&.find { |dep| temporary.include?(dep) }
+
+      while current && current != start_model
+        cycle << current
+        current = dependencies[current]&.find { |dep| temporary.include?(dep) }
+      end
+
+      cycle << start_model if current == start_model
+      cycle
+    end
+
+    def collect_inserts(records)
+      records.map do |record|
+        table_name = record.class.table_name
+        columns = record.class.column_names
+
+        values = columns.map do |column|
+          if record.class.respond_to?(:defined_enums) && record.class.defined_enums.key?(column)
+            raw_value = record.read_attribute_before_type_cast(column)
+            raw_value.nil? ? 'NULL' : raw_value.to_s
+          else
+            quote_value(record[column], get_column_info(record.class, column))
+          end
+        end
+
+        <<~SQL.strip
+          INSERT INTO #{table_name}
+          (#{columns.join(', ')})
+          VALUES (#{values.join(', ')})
+          ON CONFLICT (id) DO NOTHING;
+        SQL
+      end
+    end
+
+    def get_column_info(model, column_name)
+      column = model.columns_hash[column_name]
+      {
+        type: column.type,
+        sql_type: column.sql_type,
+        array: column.array
+      }
+    end
+
+    def quote_value(value, column_info)
+      return 'NULL' if value.nil?
+
+      case column_info[:type]
+      when :integer, :decimal, :float
+        value.to_s
+      when :boolean
+        value.to_s
+      when :array, :json, :jsonb
+        parse_and_format_special_type(value, column_info)
+      else
+        if column_info[:array]
+          parse_and_format_array(value, column_info[:sql_type])
+        else
+          sanitize_string(value.to_s)
+        end
+      end
+    end
+
+    def parse_and_format_special_type(value, column_info)
+      if column_info[:array] || column_info[:type] == :array
+        parse_and_format_array(value, column_info[:sql_type])
+      else
+        # Handle JSON/JSONB
+        json_value = value.is_a?(String) ? value : value.to_json
+        sanitize_string(json_value)
+      end
+    end
+
+    def parse_and_format_array(value, sql_type)
+      # Always cast empty or string representations of empty arrays to proper type
+      if value.nil? || value == '[]' || value == '{}' || (value.is_a?(Array) && value.empty?)
+        base_type = extract_base_type(sql_type)
+        return "'{}'" + "::#{base_type}[]"
+      end
+
+      # Parse the array if it's a string
+      array_value = case value
+                   when String
+                     begin
+                       JSON.parse(value)
+                     rescue JSON::ParserError
+                       value.gsub(/[{}"]/, '').split(',')
+                     end
+                   when Array
+                     value
+                   else
+                     [value]
+                   end
+
+      # Format the array elements
+      elements = array_value.map do |element|
+        case element
+        when String
+          sanitize_string(element)
+        when Numeric
+          element.to_s
+        when nil
+          'NULL'
+        else
+          sanitize_string(element.to_s)
+        end
+      end
+
+      base_type = extract_base_type(sql_type)
+      "ARRAY[#{elements.join(',')}]::#{base_type}[]"
+    end
+
+    def extract_base_type(sql_type)
+      case sql_type
+      when /character varying\[\]/i, /varchar\[\]/i
+        'varchar'
+      when /text\[\]/i
+        'text'
+      when /integer\[\]/i
+        'integer'
+      when /bigint\[\]/i
+        'bigint'
+      when /jsonb\[\]/i
+        'jsonb'
+      when /json\[\]/i
+        'json'
+      else
+        sql_type.sub(/\[\]$/, '')
+      end
+    end
+
+    def format_array(value, column_info)
+      # Handle empty arrays
+      if value.nil? || value == '[]' || value == '{}' || (value.is_a?(Array) && value.empty?)
+        return "'{}'" + "::character varying[]" if column_info[:type] == :string
+        return "'{}'" + "::#{extract_base_type(column_info[:sql_type])}[]"
+      end
+
+      # Parse the array if it's a string
+      array_value = case value
+                   when String
+                     begin
+                       JSON.parse(value)
+                     rescue JSON::ParserError
+                       value.gsub(/[{}"]/, '').split(',')
+                     end
+                   when Array
+                     value
+                   else
+                     [value]
+                   end
+
+      # Format array elements
+      elements = array_value.map do |element|
+        case element
+        when String
+          sanitize_string(element)
+        when Numeric
+          element.to_s
+        when nil
+          'NULL'
+        else
+          sanitize_string(element.to_s)
+        end
+      end
+
+      # Use character varying[] for string arrays
+      array_type = if column_info[:type] == :string
+                    'character varying[]'
+                  else
+                    "#{extract_base_type(column_info[:sql_type])}[]"
+                  end
+
+      "ARRAY[#{elements.join(',')}]::#{array_type}"
+    end
+
+    def extract_base_type(sql_type)
+      case sql_type
+      when /character varying\[\]/i, /varchar\[\]/i
+        'character varying'
+      when /text\[\]/i
+        'text'
+      when /integer\[\]/i
+        'integer'
+      when /bigint\[\]/i
+        'bigint'
+      when /jsonb\[\]/i
+        'jsonb'
+      when /json\[\]/i
+        'json'
+      else
+        sql_type.sub(/\[\]$/, '')
+      end
+    end
+
+    def sanitize_string(str)
+      "'#{str.gsub("'", "''")}'"
+    end
+
+    def connection_options
+      config = if ActiveRecord::Base.respond_to?(:connection_db_config)
+        ActiveRecord::Base.connection_db_config.configuration_hash
+      else
+        ActiveRecord::Base.connection_config
+      end
+
+      options = []
+      options << "-h #{config[:host]}" if config[:host]
+      options << "-p #{config[:port]}" if config[:port]
+      options << "-U #{config[:username]}" if config[:username]
+      options << "-d #{config[:database]}"
+      options << "-q"  # Run quietly
+      options.join(" ")
+    end
+  end
+end

data/lib/real_data_tests/record_collector.rb

@@ -0,0 +1,117 @@
+module RealDataTests
+  class RecordCollector
+    def initialize(record)
+      @record = record
+      @collected_records = Set.new
+      @collection_stats = Hash.new { |h, k| h[k] = { count: 0, associations: Hash.new(0) } }
+      @processed_associations = Set.new
+      @association_path = []
+    end
+
+    def collect
+      puts "\nStarting record collection from: #{@record.class.name}##{@record.id}"
+      filter_mode = RealDataTests.configuration.association_filter_mode
+      filter_list = RealDataTests.configuration.association_filter_list
+      puts "Using #{filter_mode || 'no'} filter with #{filter_list.any? ? filter_list.join(', ') : 'no associations'}"
+      collect_record(@record)
+      print_collection_stats
+      @collected_records.to_a
+    end
+
+    private
+
+    def should_process_association?(record, association)
+      association_key = "#{record.class.name}##{record.id}:#{association.name}"
+      return false if @processed_associations.include?(association_key)
+      @processed_associations.add(association_key)
+
+      # Use the enhanced should_process_association? method
+      return false unless RealDataTests.configuration.should_process_association?(record, association.name)
+
+      # Check for prevented reciprocal loading
+      if RealDataTests.configuration.prevent_reciprocal?(record.class, association.name)
+        puts "  Skipping prevented reciprocal association: #{association.name} on #{record.class.name}"
+        return false
+      end
+
+      true
+    end
+
+    def collect_record(record)
+      return if @collected_records.include?(record)
+      return unless record # Guard against nil records
+
+      @collected_records.add(record)
+      @collection_stats[record.class.name][:count] += 1
+      collect_associations(record)
+    end
+
+    def collect_associations(record)
+      return unless record.class.respond_to?(:reflect_on_all_associations)
+
+      associations = record.class.reflect_on_all_associations
+      puts "\nProcessing associations for: #{record.class.name}##{record.id}"
+      puts "Found #{associations.length} associations"
+
+      associations.each do |association|
+        should_process = RealDataTests.configuration.should_process_association?(record, association.name)
+
+        unless should_process
+          puts "  Skipping #{RealDataTests.configuration.association_filter_mode == :whitelist ? 'non-whitelisted' : 'blacklisted'} association: #{association.name} for #{record.class.name}"
+          next
+        end
+
+        puts "  Processing #{association.macro} association: #{association.name}"
+        process_association(record, association)
+      end
+    end
+
+    def process_association(record, association)
+      begin
+        related_records = fetch_related_records(record, association)
+        count = related_records.length
+        puts "    Found #{count} related #{association.name} records"
+        @collection_stats[record.class.name][:associations][association.name] += count
+
+        related_records.each { |related_record| collect_record(related_record) }
+      rescue => e
+        puts "    Error processing association #{association.name}: #{e.message}"
+      end
+    end
+
+    def fetch_related_records(record, association)
+      case association.macro
+      when :belongs_to, :has_one
+        Array(record.public_send(association.name)).compact
+      when :has_many, :has_and_belongs_to_many
+        relation = record.public_send(association.name)
+
+        # Apply configured limit if it exists
+        if limit = RealDataTests.configuration.get_association_limit(record.class, association.name)
+          puts "  Applying configured limit of #{limit} records for #{record.class.name}.#{association.name}"
+          relation = relation.limit(limit)
+        end
+
+        relation.loaded? ? relation.to_a : relation.load.to_a
+      else
+        []
+      end
+    end
+
+    def print_collection_stats
+      puts "\n=== Collection Statistics ==="
+      @collection_stats.each do |model, stats|
+        puts "\n#{model}:"
+        puts "  Total records: #{stats[:count]}"
+        if stats[:associations].any?
+          puts "  Associations:"
+          stats[:associations].each do |assoc_name, count|
+            puts "    #{assoc_name}: #{count} records"
+          end
+        end
+      end
+      puts "\nTotal unique records collected: #{@collected_records.size}"
+      puts "==============================\n"
+    end
+  end
+end

data/lib/real_data_tests/rspec_helper.rb

@@ -0,0 +1,41 @@
+module RealDataTests
+  module RSpecHelper
+    def load_real_test_data(name)
+      dump_path = File.join(RealDataTests.configuration.dump_path, "#{name}.sql")
+      raise Error, "Test data file not found: #{dump_path}" unless File.exist?(dump_path)
+
+      ActiveRecord::Base.transaction do
+        # Disable foreign key checks
+        ActiveRecord::Base.connection.execute('SET session_replication_role = replica;')
+
+        begin
+          # Load the SQL dump quietly
+          result = system("psql #{connection_options} -q < #{dump_path}")
+          raise Error, "Failed to load test data: #{dump_path}" unless result
+
+        ensure
+          # Re-enable foreign key checks
+          ActiveRecord::Base.connection.execute('SET session_replication_role = DEFAULT;')
+        end
+      end
+    end
+
+    private
+
+    def connection_options
+      config = if ActiveRecord::Base.respond_to?(:connection_db_config)
+        ActiveRecord::Base.connection_db_config.configuration_hash
+      else
+        ActiveRecord::Base.connection_config
+      end
+
+      options = []
+      options << "-h #{config[:host]}" if config[:host]
+      options << "-p #{config[:port]}" if config[:port]
+      options << "-U #{config[:username]}" if config[:username]
+      options << "-d #{config[:database]}"
+      options << "-q"
+      options.join(" ")
+    end
+  end
+end

data/lib/real_data_tests/test_data_builder.rb

@@ -0,0 +1,34 @@
+module RealDataTests
+  class TestDataBuilder
+    def initialize(record, name: nil)
+      @record = record
+      @name = name || "#{record.class.name.underscore}_#{record.id}"
+    end
+
+    def create_dump_file
+      records = RealDataTests::RecordCollector.new(@record).collect
+
+      # Only anonymize if rules are configured
+      if RealDataTests.configuration.anonymization_rules.any?
+        puts "\nAnonymizing records..."
+        anonymizer = RealDataTests::DataAnonymizer.new(RealDataTests.configuration)
+        anonymizer.anonymize_records(records)
+      end
+
+      dump_content = RealDataTests::PgDumpGenerator.new(records).generate
+      dump_path = dump_file_path
+
+      FileUtils.mkdir_p(File.dirname(dump_path))
+      File.write(dump_path, dump_content)
+
+      puts "\nDump file created at: #{dump_path}"
+      dump_path
+    end
+
+    private
+
+    def dump_file_path
+      File.join(RealDataTests.configuration.dump_path, "#{@name}.sql")
+    end
+  end
+end

data/lib/real_data_tests/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module RealDataTests
+  VERSION = "0.1.0"
+end

data/lib/real_data_tests.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require 'real_data_tests/configuration'
+require 'real_data_tests/data_anonymizer'
+require 'real_data_tests/engine' if defined?(Rails)
+require 'real_data_tests/pg_dump_generator'
+require 'real_data_tests/record_collector'
+require 'real_data_tests/rspec_helper'
+require 'real_data_tests/test_data_builder'
+require 'real_data_tests/version'
+
+module RealDataTests
+  class Error < StandardError; end
+  class ConfigurationError < Error; end
+  class DumpFileError < Error; end
+
+  class << self
+    def configuration
+      @configuration ||= Configuration.new
+    end
+
+    def configure
+      yield(configuration) if block_given?
+      configuration
+    end
+
+    def reset_configuration!
+      @configuration = Configuration.new
+    end
+
+    def create_dump_file(record, name: nil)
+      raise ConfigurationError, "Configuration not initialized" unless @configuration
+
+      begin
+        TestDataBuilder.new(record, name: name).create_dump_file
+      rescue => e
+        raise DumpFileError, "Failed to create dump file: #{e.message}"
+      end
+    end
+
+    def root
+      File.expand_path('../..', __FILE__)
+    end
+
+    def env
+      @env ||= (ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development')
+    end
+  end
+end

data/sig/real_data_tests.rbs

@@ -0,0 +1,4 @@
+module RealDataTests
+  VERSION: String
+  # See the writing guide of rbs: https://github.com/ruby/rbs#guides
+end