real_data_tests 0.4.1 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: afa2db23f090493e45cd9becca7395e36e2a6d2720b0cf2687d170d2c88d4455
-  data.tar.gz: 267e784a01b732885aef2e00433044ab90ef6d2a9fb77516b55bebb7afdf1007
+  metadata.gz: 76313adb8becf99110aade7dae2741bb9fbb18426eab847ec46791dff93acec4
+  data.tar.gz: 2650573c1d3a7dc0d3e3866d0520c27696d4fe7d858b5d3b6380b3c88f640bd3
 SHA512:
-  metadata.gz: 1f042a6324814776aa3f513b0301850e021f8032a5d0740d125ffa11b0b8e778cbdac2cdfb24c6f2364b5d4cde0eda420039eca9a4e264b2c9da6af2a7b03ad3
-  data.tar.gz: ffbb471142b577bada70e151cb6cb06a6f460f6f6a4479f33278b98a459fbb3431ce5c55db5e8554682560a15dc355505d10fe3228c37bfd775bd0bcaa916d1d
+  metadata.gz: 6ef108ecbc7c30db555e8fb8792a08fce6af245763f524f0d6c1d8fe1bb59cff69d63e902eb46a785f8122fab502abb0dfd0803747348183f031cacdd6080ad0
+  data.tar.gz: 900f6e4f32dfbabae2a5492d7cc78bb8490f33b5b7610c9acd1b735e61e2837101ca0b7b8b74cd849d1cbc857805556935b821a87c56e7efa3005962a3179550

data/CHANGELOG.md

@@ -1,3 +1,17 @@
+## [Unreleased]
+### Added
+- **Load strategies**: SQL dump loading is now pluggable via `RealDataTests::LoadStrategies`
+  - `LoadStrategies::Native` — loads on the ActiveRecord connection (default only via `load_real_test_data_native`)
+  - `LoadStrategies::Psql` — the previous `psql` shell-out behavior, for dumps that require psql itself (meta-commands like `\set`, or dumps too large to read into memory)
+  - Inject via `load_real_test_data("dump", strategy: RealDataTests::LoadStrategies::Psql)`
+  - Custom strategies: subclass `LoadStrategies::Base` and implement `#call(dump_path)`
+
+### Changed
+- Reworked SQL dump loading around the new strategies
+  - `load_real_test_data` behavior is unchanged (psql shell-out, `LoadStrategies::Psql` default); pass `strategy: LoadStrategies::Native` to load transactionally on the ActiveRecord connection so data participates in the caller's transaction (e.g. DatabaseCleaner `:transaction` strategy) and rolls back with it
+  - Native loader: dumps without `COPY ... FROM stdin` blocks are executed as a single multi-statement `execute` (one server round-trip) instead of parsed and executed block-by-block; block parsing is kept only as the COPY fallback
+  - Native loader: `COPY ... FROM stdin` blocks are streamed through `raw_connection.copy_data` on the same libpq session, so COPY data is transactional too (previously broken — the whole COPY block went through `execute`)
+
 ## [0.4.1] - 2026-04-09
 ### Fixed
 - Fixed `datetime` microsecond precision loss in `PgDumpGenerator`

data/Gemfile.lock

@@ -1,8 +1,9 @@
 PATH
   remote: .
   specs:
-    real_data_tests (0.2.1)
+    real_data_tests (0.5.0)
       activerecord (>= 5.0)
+      csv
       faker (~> 3.0)
       pg (>= 1.1)
       rails (>= 5.0)
@@ -11,66 +12,68 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    actioncable (7.2.2.1)
-      actionpack (= 7.2.2.1)
-      activesupport (= 7.2.2.1)
+    actioncable (7.2.3.1)
+      actionpack (= 7.2.3.1)
+      activesupport (= 7.2.3.1)
       nio4r (~> 2.0)
       websocket-driver (>= 0.6.1)
       zeitwerk (~> 2.6)
-    actionmailbox (7.2.2.1)
-      actionpack (= 7.2.2.1)
-      activejob (= 7.2.2.1)
-      activerecord (= 7.2.2.1)
-      activestorage (= 7.2.2.1)
-      activesupport (= 7.2.2.1)
+    actionmailbox (7.2.3.1)
+      actionpack (= 7.2.3.1)
+      activejob (= 7.2.3.1)
+      activerecord (= 7.2.3.1)
+      activestorage (= 7.2.3.1)
+      activesupport (= 7.2.3.1)
       mail (>= 2.8.0)
-    actionmailer (7.2.2.1)
-      actionpack (= 7.2.2.1)
-      actionview (= 7.2.2.1)
-      activejob (= 7.2.2.1)
-      activesupport (= 7.2.2.1)
+    actionmailer (7.2.3.1)
+      actionpack (= 7.2.3.1)
+      actionview (= 7.2.3.1)
+      activejob (= 7.2.3.1)
+      activesupport (= 7.2.3.1)
       mail (>= 2.8.0)
       rails-dom-testing (~> 2.2)
-    actionpack (7.2.2.1)
-      actionview (= 7.2.2.1)
-      activesupport (= 7.2.2.1)
+    actionpack (7.2.3.1)
+      actionview (= 7.2.3.1)
+      activesupport (= 7.2.3.1)
+      cgi
       nokogiri (>= 1.8.5)
       racc
-      rack (>= 2.2.4, < 3.2)
+      rack (>= 2.2.4, < 3.3)
       rack-session (>= 1.0.1)
       rack-test (>= 0.6.3)
       rails-dom-testing (~> 2.2)
       rails-html-sanitizer (~> 1.6)
       useragent (~> 0.16)
-    actiontext (7.2.2.1)
-      actionpack (= 7.2.2.1)
-      activerecord (= 7.2.2.1)
-      activestorage (= 7.2.2.1)
-      activesupport (= 7.2.2.1)
+    actiontext (7.2.3.1)
+      actionpack (= 7.2.3.1)
+      activerecord (= 7.2.3.1)
+      activestorage (= 7.2.3.1)
+      activesupport (= 7.2.3.1)
       globalid (>= 0.6.0)
       nokogiri (>= 1.8.5)
-    actionview (7.2.2.1)
-      activesupport (= 7.2.2.1)
+    actionview (7.2.3.1)
+      activesupport (= 7.2.3.1)
       builder (~> 3.1)
+      cgi
       erubi (~> 1.11)
       rails-dom-testing (~> 2.2)
       rails-html-sanitizer (~> 1.6)
-    activejob (7.2.2.1)
-      activesupport (= 7.2.2.1)
+    activejob (7.2.3.1)
+      activesupport (= 7.2.3.1)
       globalid (>= 0.3.6)
-    activemodel (7.2.2.1)
-      activesupport (= 7.2.2.1)
-    activerecord (7.2.2.1)
-      activemodel (= 7.2.2.1)
-      activesupport (= 7.2.2.1)
+    activemodel (7.2.3.1)
+      activesupport (= 7.2.3.1)
+    activerecord (7.2.3.1)
+      activemodel (= 7.2.3.1)
+      activesupport (= 7.2.3.1)
       timeout (>= 0.4.0)
-    activestorage (7.2.2.1)
-      actionpack (= 7.2.2.1)
-      activejob (= 7.2.2.1)
-      activerecord (= 7.2.2.1)
-      activesupport (= 7.2.2.1)
+    activestorage (7.2.3.1)
+      actionpack (= 7.2.3.1)
+      activejob (= 7.2.3.1)
+      activerecord (= 7.2.3.1)
+      activesupport (= 7.2.3.1)
       marcel (~> 1.0)
-    activesupport (7.2.2.1)
+    activesupport (7.2.3.1)
       base64
       benchmark (>= 0.3)
       bigdecimal
@@ -79,135 +82,159 @@ GEM
       drb
       i18n (>= 1.6, < 2)
       logger (>= 1.4.2)
-      minitest (>= 5.1)
+      minitest (>= 5.1, < 6)
       securerandom (>= 0.3)
       tzinfo (~> 2.0, >= 2.0.5)
-    base64 (0.2.0)
-    benchmark (0.4.0)
-    bigdecimal (3.1.9)
+    base64 (0.3.0)
+    benchmark (0.5.0)
+    bigdecimal (4.1.2)
     builder (3.3.0)
+    cgi (0.5.2)
     concurrent-ruby (1.3.4)
-    connection_pool (2.5.0)
-    crass (1.0.6)
+    connection_pool (2.5.5)
+    crass (1.0.7)
+    csv (3.3.6)
     database_cleaner (2.1.0)
       database_cleaner-active_record (>= 2, < 3)
-    database_cleaner-active_record (2.2.0)
+    database_cleaner-active_record (2.2.2)
       activerecord (>= 5.a)
-      database_cleaner-core (~> 2.0.0)
+      database_cleaner-core (~> 2.0)
     database_cleaner-core (2.0.1)
-    date (3.4.1)
-    diff-lcs (1.5.1)
-    drb (2.2.1)
+    date (3.5.1)
+    diff-lcs (1.6.2)
+    drb (2.2.3)
+    erb (4.0.4.1)
+      cgi (>= 0.3.3)
     erubi (1.13.1)
-    faker (3.5.1)
+    faker (3.6.1)
       i18n (>= 1.8.11, < 2)
-    globalid (1.2.1)
+    globalid (1.4.0)
       activesupport (>= 6.1)
     i18n (1.14.6)
       concurrent-ruby (~> 1.0)
-    io-console (0.8.0)
-    irb (1.14.3)
+    io-console (0.8.2)
+    irb (1.18.0)
+      pp (>= 0.6.0)
+      prism (>= 1.3.0)
       rdoc (>= 4.0.0)
       reline (>= 0.4.2)
     logger (1.6.5)
-    loofah (2.24.0)
+    loofah (2.25.2)
       crass (~> 1.0.2)
       nokogiri (>= 1.12.0)
-    mail (2.8.1)
+    mail (2.9.1)
+      logger
       mini_mime (>= 0.1.1)
       net-imap
       net-pop
       net-smtp
-    marcel (1.0.4)
+    marcel (1.2.1)
     mini_mime (1.1.5)
-    minitest (5.25.4)
-    net-imap (0.5.5)
+    minitest (5.27.0)
+    net-imap (0.5.15)
       date
       net-protocol
     net-pop (0.1.2)
       net-protocol
     net-protocol (0.2.2)
       timeout
-    net-smtp (0.5.0)
+    net-smtp (0.5.1)
       net-protocol
-    nio4r (2.7.4)
-    nokogiri (1.18.1-x86_64-darwin)
+    nio4r (2.7.5)
+    nokogiri (1.18.10-arm64-darwin)
       racc (~> 1.4)
-    pg (1.5.9)
-    psych (5.2.2)
+    nokogiri (1.18.10-x86_64-darwin)
+      racc (~> 1.4)
+    nokogiri (1.18.10-x86_64-linux-gnu)
+      racc (~> 1.4)
+    pg (1.6.3-arm64-darwin)
+    pg (1.6.3-x86_64-darwin)
+    pg (1.6.3-x86_64-linux)
+    pp (0.6.4)
+      prettyprint
+    prettyprint (0.2.0)
+    prism (1.9.0)
+    psych (5.4.0)
       date
       stringio
     racc (1.8.1)
-    rack (3.1.8)
-    rack-session (2.1.0)
+    rack (3.2.6)
+    rack-session (2.1.2)
       base64 (>= 0.1.0)
       rack (>= 3.0.0)
     rack-test (2.2.0)
       rack (>= 1.3)
-    rackup (2.2.1)
+    rackup (2.3.1)
       rack (>= 3)
-    rails (7.2.2.1)
-      actioncable (= 7.2.2.1)
-      actionmailbox (= 7.2.2.1)
-      actionmailer (= 7.2.2.1)
-      actionpack (= 7.2.2.1)
-      actiontext (= 7.2.2.1)
-      actionview (= 7.2.2.1)
-      activejob (= 7.2.2.1)
-      activemodel (= 7.2.2.1)
-      activerecord (= 7.2.2.1)
-      activestorage (= 7.2.2.1)
-      activesupport (= 7.2.2.1)
+    rails (7.2.3.1)
+      actioncable (= 7.2.3.1)
+      actionmailbox (= 7.2.3.1)
+      actionmailer (= 7.2.3.1)
+      actionpack (= 7.2.3.1)
+      actiontext (= 7.2.3.1)
+      actionview (= 7.2.3.1)
+      activejob (= 7.2.3.1)
+      activemodel (= 7.2.3.1)
+      activerecord (= 7.2.3.1)
+      activestorage (= 7.2.3.1)
+      activesupport (= 7.2.3.1)
       bundler (>= 1.15.0)
-      railties (= 7.2.2.1)
-    rails-dom-testing (2.2.0)
+      railties (= 7.2.3.1)
+    rails-dom-testing (2.3.0)
       activesupport (>= 5.0.0)
       minitest
       nokogiri (>= 1.6)
-    rails-html-sanitizer (1.6.2)
-      loofah (~> 2.21)
+    rails-html-sanitizer (1.7.1)
+      loofah (~> 2.25, >= 2.25.2)
       nokogiri (>= 1.15.7, != 1.16.7, != 1.16.6, != 1.16.5, != 1.16.4, != 1.16.3, != 1.16.2, != 1.16.1, != 1.16.0.rc1, != 1.16.0)
-    railties (7.2.2.1)
-      actionpack (= 7.2.2.1)
-      activesupport (= 7.2.2.1)
+    railties (7.2.3.1)
+      actionpack (= 7.2.3.1)
+      activesupport (= 7.2.3.1)
+      cgi
       irb (~> 1.13)
       rackup (>= 1.0.0)
       rake (>= 12.2)
       thor (~> 1.0, >= 1.2.2)
+      tsort (>= 0.2)
       zeitwerk (~> 2.6)
-    rake (13.2.1)
-    rdoc (6.10.0)
+    rake (13.4.2)
+    rdoc (7.2.0)
+      erb
       psych (>= 4.0.0)
-    reline (0.6.0)
+      tsort
+    reline (0.6.3)
       io-console (~> 0.5)
-    rspec (3.13.0)
+    rspec (3.13.2)
       rspec-core (~> 3.13.0)
       rspec-expectations (~> 3.13.0)
       rspec-mocks (~> 3.13.0)
-    rspec-core (3.13.2)
+    rspec-core (3.13.6)
       rspec-support (~> 3.13.0)
-    rspec-expectations (3.13.3)
+    rspec-expectations (3.13.5)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
-    rspec-mocks (3.13.2)
+    rspec-mocks (3.13.8)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
     rspec-support (3.13.2)
     securerandom (0.4.1)
-    stringio (3.1.2)
-    thor (1.3.2)
-    timeout (0.4.3)
+    stringio (3.2.0)
+    thor (1.5.0)
+    timeout (0.6.1)
+    tsort (0.2.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
     useragent (0.16.11)
-    websocket-driver (0.7.7)
+    websocket-driver (0.8.2)
       base64
       websocket-extensions (>= 0.1.0)
     websocket-extensions (0.1.5)
     zeitwerk (2.6.18)
 
 PLATFORMS
+  arm64-darwin-24
   x86_64-darwin-21
+  x86_64-linux
 
 DEPENDENCIES
   database_cleaner (~> 2.0)

data/README.md

@@ -245,6 +245,36 @@ RSpec.describe "Blog" do
 end
 ```
 
+### 3. Load Strategies
+
+`load_real_test_data` accepts a strategy that controls how the dump is executed:
+
+```ruby
+# Default: shells out to psql. Data commits on a separate connection,
+# outside any test transaction. Supports psql meta-commands (\set, \connect).
+load_real_test_data("active_user_with_posts")
+
+# Native: executes on the ActiveRecord connection, so the data joins the
+# caller's transaction (e.g. DatabaseCleaner's :transaction strategy) and
+# rolls back with it.
+load_real_test_data("active_user_with_posts", strategy: RealDataTests::LoadStrategies::Native)
+
+# Shorthand for the Native strategy:
+load_real_test_data_native("active_user_with_posts")
+```
+
+Custom strategies subclass `RealDataTests::LoadStrategies::Base` and implement `#call(dump_path)`:
+
+```ruby
+class MyStrategy < RealDataTests::LoadStrategies::Base
+  def call(dump_path)
+    # execute the dump however you like
+  end
+end
+
+load_real_test_data("active_user_with_posts", strategy: MyStrategy)
+```
+
 ## Association Control
 
 Real Data Tests provides several ways to control how associations are collected and loaded.

data/lib/real_data_tests/load_strategies/base.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module RealDataTests
+  module LoadStrategies
+    # Base class for dump loading strategies. Subclasses implement #call,
+    # which receives the absolute path to the SQL dump file and executes it.
+    #
+    #   class MyStrategy < RealDataTests::LoadStrategies::Base
+    #     def call(dump_path)
+    #       # load the dump
+    #     end
+    #   end
+    #
+    #   load_real_test_data("dump", strategy: MyStrategy)
+    class Base
+      def self.call(dump_path)
+        new.call(dump_path)
+      end
+
+      def call(dump_path)
+        raise NotImplementedError, "#{self.class.name} must implement #call"
+      end
+    end
+  end
+end

data/lib/real_data_tests/load_strategies/native.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+module RealDataTests
+  module LoadStrategies
+    # Loads a SQL dump on the ActiveRecord connection, so the data
+    # participates in the caller's transaction (e.g. DatabaseCleaner's
+    # :transaction strategy) and rolls back with it.
+    #
+    # Dumps without COPY ... FROM stdin blocks are sent as a single
+    # multi-statement execute (one server round-trip). Dumps containing COPY
+    # blocks fall back to block-by-block execution, streaming COPY data
+    # through raw_connection.copy_data on the same libpq session.
+    class Native < Base
+      def call(dump_path)
+        sql_content = File.read(dump_path)
+
+        ActiveRecord::Base.transaction do
+          connection = ActiveRecord::Base.connection
+
+          # Disable foreign key checks
+          connection.execute('SET session_replication_role = replica;')
+
+          begin
+            if sql_content.match?(/^COPY .* FROM stdin/i)
+              blocks = SqlDumpParser.parse(sql_content)
+              blocks.each_with_index do |block, index|
+                execute_block(block, index + 1, blocks.length)
+              end
+            else
+              # No COPY blocks: send the whole dump in one round-trip. The
+              # server parses the multi-statement string, so semicolons inside
+              # string literals are handled correctly without client-side splitting.
+              connection.execute(sql_content)
+            end
+          ensure
+            connection.execute('SET session_replication_role = DEFAULT;')
+          end
+        end
+      end
+
+      private
+
+      def execute_block(block, index, total)
+        case block.type
+        when :insert
+          execute_insert_block(block, index, total)
+        when :copy
+          execute_copy_block(block, index, total)
+        else
+          execute_regular_block(block, index, total)
+        end
+      end
+
+      def execute_insert_block(block, index, total)
+        # Don't modify statements that already end with semicolon
+        statement = if block.content.strip.end_with?(';')
+          block.content
+        else
+          "#{block.content};"
+        end
+
+        begin
+          ActiveRecord::Base.connection.execute(statement)
+        rescue ActiveRecord::StatementInvalid => e
+          if e.message.include?('syntax error at or near "ON"')
+            # Try alternative formatting for ON CONFLICT
+            modified_statement = statement.gsub(/\)\s+ON\s+CONFLICT/, ') ON CONFLICT')
+            ActiveRecord::Base.connection.execute(modified_statement)
+          else
+            raise
+          end
+        end
+      end
+
+      def execute_copy_block(block, index, total)
+        lines = block.content.lines.map(&:chomp)
+        copy_statement = lines.shift
+        data_lines = lines.take_while { |line| line != '\\.' }
+
+        # raw_connection is the same libpq session as the AR connection, so the
+        # COPY participates in the surrounding transaction.
+        raw = ActiveRecord::Base.connection.raw_connection
+        raw.copy_data(copy_statement) do
+          data_lines.each { |line| raw.put_copy_data("#{line}\n") }
+        end
+      end
+
+      def execute_regular_block(block, index, total)
+        ActiveRecord::Base.connection.execute(block.content)
+      end
+    end
+  end
+end

data/lib/real_data_tests/load_strategies/psql.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module RealDataTests
+  module LoadStrategies
+    # Loads a SQL dump by shelling out to psql. The psql subprocess uses its
+    # own Postgres connection, so the loaded data commits immediately and does
+    # NOT participate in the caller's transaction (a transaction is
+    # per-connection; a child process can never join it).
+    #
+    # This is the default strategy of load_real_test_data. Prefer the Native
+    # strategy (load_real_test_data_native) unless the dump requires psql
+    # itself — e.g. psql meta-commands (\set, \connect) or dumps too large to
+    # read into memory.
+    class Psql < Base
+      def call(dump_path)
+        # Load the SQL dump quietly. Invoked without a shell (argv array +
+        # :in redirect), so paths and connection values need no escaping.
+        # Note: no transaction or session_replication_role handling here —
+        # psql runs on its own Postgres session, so nothing set on the
+        # ActiveRecord connection can affect the load.
+        result = system('psql', *connection_args, in: dump_path)
+        raise Error, "Failed to load test data: #{dump_path}" unless result
+      end
+
+      private
+
+      def connection_args
+        config = if ActiveRecord::Base.respond_to?(:connection_db_config)
+          ActiveRecord::Base.connection_db_config.configuration_hash
+        else
+          ActiveRecord::Base.connection_config
+        end
+        args = []
+        args += ['-h', config[:host].to_s] if config[:host]
+        args += ['-p', config[:port].to_s] if config[:port]
+        args += ['-U', config[:username].to_s] if config[:username]
+        args += ['-d', config[:database].to_s]
+        args << '-q'
+        args
+      end
+    end
+  end
+end

data/lib/real_data_tests/pg_dump_generator.rb

@@ -297,21 +297,5 @@ module RealDataTests
     def sanitize_string(str)
       "'#{str.gsub("'", "''")}'"
     end
-
-    def connection_options
-      config = if ActiveRecord::Base.respond_to?(:connection_db_config)
-        ActiveRecord::Base.connection_db_config.configuration_hash
-      else
-        ActiveRecord::Base.connection_config
-      end
-
-      options = []
-      options << "-h #{config[:host]}" if config[:host]
-      options << "-p #{config[:port]}" if config[:port]
-      options << "-U #{config[:username]}" if config[:username]
-      options << "-d #{config[:database]}"
-      options << "-q"  # Run quietly
-      options.join(" ")
-    end
   end
 end

data/lib/real_data_tests/rspec_helper.rb

@@ -1,376 +1,30 @@
 module RealDataTests
   module RSpecHelper
-    class SqlBlock
-      attr_reader :type, :content, :table_name
-
-      def initialize(content)
-        @content = content.strip
-        @type = determine_block_type
-        @table_name = extract_table_name if @type == :insert
-      end
-
-      private
-
-      def determine_block_type
-        case @content
-        when /\AINSERT INTO/i
-          :insert
-        when /\ACOPY.*FROM stdin/i
-          :copy
-        when /\AALTER TABLE/i
-          :alter
-        when /\ASET/i
-          :set
-        else
-          :other
-        end
-      end
-
-      def extract_table_name
-        if @content =~ /INSERT INTO\s+"?([^\s"(]+)"?\s/i
-          $1
-        end
-      end
-    end
-
-    def load_real_test_data(name)
+    # Loads a SQL dump using the given strategy (default: psql shell-out,
+    # matching pre-0.5 behavior — data commits outside the caller's
+    # transaction). Pass a different strategy to change how the dump is
+    # executed, e.g. transactional loading on the ActiveRecord connection:
+    #
+    #   load_real_test_data("dump", strategy: RealDataTests::LoadStrategies::Native)
+    def load_real_test_data(name, strategy: LoadStrategies::Psql)
       dump_path = File.join(RealDataTests.configuration.dump_path, "#{name}.sql")
       raise Error, "Test data file not found: #{dump_path}" unless File.exist?(dump_path)
-      ActiveRecord::Base.transaction do
-        # Disable foreign key checks
-        ActiveRecord::Base.connection.execute('SET session_replication_role = replica;')
-        begin
-          # Load the SQL dump quietly
-          result = system("psql #{connection_options} -q < #{dump_path}")
-          raise Error, "Failed to load test data: #{dump_path}" unless result
-        ensure
-          # Re-enable foreign key checks
-          ActiveRecord::Base.connection.execute('SET session_replication_role = DEFAULT;')
-        end
-      end
-    end
-
-    def load_real_test_data_native(name)
-      dump_path = File.join(RealDataTests.configuration.dump_path, "#{name}.sql")
-      raise Error, "Test data file not found: #{dump_path}" unless File.exist?(dump_path)
-
-      sql_content = File.read(dump_path)
-      blocks = parse_sql_blocks(sql_content)
-
-      ActiveRecord::Base.transaction do
-        connection = ActiveRecord::Base.connection
-
-        # Disable foreign key checks
-        connection.execute('SET session_replication_role = replica;')
-
-        begin
-          blocks.each_with_index do |block, index|
-            execute_block(block, index + 1, blocks.length)
-          end
-        ensure
-          connection.execute('SET session_replication_role = DEFAULT;')
-        end
-      end
-    end
-
-    private
-
-    def connection_options
-      config = if ActiveRecord::Base.respond_to?(:connection_db_config)
-        ActiveRecord::Base.connection_db_config.configuration_hash
-      else
-        ActiveRecord::Base.connection_config
-      end
-      options = []
-      options << "-h #{config[:host]}" if config[:host]
-      options << "-p #{config[:port]}" if config[:port]
-      options << "-U #{config[:username]}" if config[:username]
-      options << "-d #{config[:database]}"
-      options << "-q"
-      options.join(" ")
-    end
-
-    class SqlBlock
-      attr_reader :type, :content, :table_name
-
-      def initialize(content)
-        @content = content.strip
-        @type = determine_block_type
-        @table_name = extract_table_name if @type == :insert
-      end
-
-      private
-
-      def determine_block_type
-        if @content.match?(/\AINSERT INTO/i)
-          :insert
-        elsif @content.match?(/\ACOPY.*FROM stdin/i)
-          :copy
-        elsif @content.match?(/\AALTER TABLE/i)
-          :alter
-        elsif @content.match?(/\ASET/i)
-          :set
-        else
-          :other
-        end
-      end
-
-      def extract_table_name
-        if @content =~ /INSERT INTO\s+"?([^\s"(]+)"?\s/i
-          $1
-        end
-      end
-    end
-
-    def parse_sql_blocks(content)
-      blocks = []
-      current_block = []
-      in_copy_block = false
-
-      content.each_line do |line|
-        line = line.chomp
-
-        # Skip empty lines and comments unless in COPY block
-        next if !in_copy_block && (line.empty? || line.start_with?('--'))
-
-        # Handle start of COPY block
-        if !in_copy_block && line.upcase.match?(/\ACOPY.*FROM stdin/i)
-          current_block = [line]
-          in_copy_block = true
-          next
-        end
-
-        # Handle end of COPY block
-        if in_copy_block && line == '\\.'
-          current_block << line
-          blocks << SqlBlock.new(current_block.join("\n"))
-          current_block = []
-          in_copy_block = false
-          next
-        end
-
-        # Accumulate lines in COPY block
-        if in_copy_block
-          current_block << line
-          next
-        end
-
-        # Handle regular SQL statements
-        current_block << line
-        if line.end_with?(';')
-          blocks << SqlBlock.new(current_block.join("\n"))
-          current_block = []
-        end
-      end
-
-      # Handle any remaining block
-      blocks << SqlBlock.new(current_block.join("\n")) unless current_block.empty?
-      blocks
-    end
 
-    def execute_block(block, index, total)
-      case block.type
-      when :insert
-        execute_insert_block(block, index, total)
-      when :copy
-        execute_copy_block(block, index, total)
-      else
-        execute_regular_block(block, index, total)
-      end
+      strategy.call(dump_path)
     end
 
-    def execute_insert_block(block, index, total)
-      # puts "Executing INSERT block #{index}/#{total} for table: #{block.table_name}"
-      # Don't modify statements that already end with semicolon
-      statement = if block.content.strip.end_with?(';')
-        block.content
-      else
-        "#{block.content};"
-      end
-
-      begin
-        ActiveRecord::Base.connection.execute(statement)
-      rescue ActiveRecord::StatementInvalid => e
-        if e.message.include?('syntax error at or near "ON"')
-          # Try alternative formatting for ON CONFLICT
-          modified_statement = statement.gsub(/\)\s+ON\s+CONFLICT/, ') ON CONFLICT')
-          ActiveRecord::Base.connection.execute(modified_statement)
-        else
-          raise
-        end
-      end
-    end
-
-    def execute_copy_block(block, index, total)
-      # puts "Executing COPY block #{index}/#{total}"
-      ActiveRecord::Base.connection.execute(block.content)
-    end
-
-    def execute_regular_block(block, index, total)
-      # puts "Executing block #{index}/#{total} of type: #{block.type}"
-      ActiveRecord::Base.connection.execute(block.content)
-    end
-
-    def normalize_insert_statement(statement)
-      # First clean up any excess whitespace around parentheses
-      statement = statement.gsub(/\(\s+/, '(')
-                          .gsub(/\s+\)/, ')')
-                          .gsub(/\)\s+ON\s+CONFLICT/, ') ON CONFLICT')
-
-      # Ensure proper spacing around ON CONFLICT
-      if statement =~ /(.*?)\s*ON\s+CONFLICT\s+(.*?)\s*(?:DO\s+.*?)?\s*;\s*\z/i
-        base = $1.strip
-        conflict_part = $2.strip
-        action_part = $3&.strip || 'DO NOTHING'
-
-        # Rebuild the statement with consistent formatting
-        "#{base} ON CONFLICT #{conflict_part} #{action_part};"
-      else
-        # If no ON CONFLICT clause, just clean up the spacing
-        statement.strip.sub(/;?\s*$/, ';')
-      end
-    end
-
-    def split_sql_statements(sql)
-      statements = []
-      current_statement = ''
-      in_string = false
-      escaped = false
-
-      sql.each_char do |char|
-        if char == '\\'
-          escaped = !escaped
-        elsif char == "'" && !escaped
-          in_string = !in_string
-        elsif char == ';' && !in_string
-          # Add the completed statement
-          statements << current_statement.strip unless current_statement.strip.empty?
-          current_statement = ''
-          next
-        end
-        escaped = false
-        current_statement << char
-      end
-
-      # Add the last statement if it exists
-      statements << current_statement.strip unless current_statement.strip.empty?
-
-      # Normalize `ON CONFLICT` clauses
-      statements = statements.each_with_object([]) do |stmt, result|
-        if stmt.strip.upcase.start_with?('ON CONFLICT')
-          result[-1] = "#{result.last.strip} #{stmt.strip}"
-        else
-          result << stmt.strip
-        end
-      end
-
-      # Ensure semicolons and spacing
-      statements.map! do |stmt|
-        stmt = stmt.gsub(/\)\s*ON CONFLICT/, ') ON CONFLICT') # Normalize spacing
-        stmt.strip.end_with?(';') ? stmt.strip : "#{stmt.strip};"
-      end
-
-      statements
-    end
-
-    def extract_conflict_clause(statement)
-      # Use a more precise regex that handles multiple closing parentheses
-      if statement =~ /(.+?\))\s*(ON\s+CONFLICT\s+.*?)(?:;?\s*$)/i
-        [$1, $2.strip]
-      else
-        [statement.sub(/;?\s*$/, ''), nil]
-      end
-    end
-
-    def clean_sql_statement(statement)
-      # Match either INSERT INTO...VALUES or just VALUES
-      if statement =~ /(?:INSERT INTO.*)?VALUES\s*\(/i
-        # Split the statement into parts, being careful with the ending
-        if statement =~ /(.*?VALUES\s*\()(.*)(\)\s*(?:ON CONFLICT.*)?;?\s*$)/i
-          pre_values = $1
-          values_content = $2
-          post_values = $3
-
-          # Clean the values content while preserving complex JSON
-          cleaned_values = clean_complex_values(values_content)
-
-          # Reassemble the statement, ensuring exactly one semicolon at the end
-          statement = "#{pre_values}#{cleaned_values}#{post_values}"
-          statement = statement.gsub(/;*\s*$/, '')  # Remove any trailing semicolons and whitespace
-          statement += ";"
-        end
-      end
-      statement
-    end
-
-    def clean_complex_values(values_str)
-      current_value = ''
-      values = []
-      in_quotes = false
-      in_json = false
-      json_brace_count = 0
-      escaped = false
-
-      chars = values_str.chars
-      i = 0
-      while i < chars.length
-        char = chars[i]
-
-        case char
-        when '\\'
-          current_value << char
-          escaped = !escaped
-        when "'"
-          if !escaped
-            in_quotes = !in_quotes
-          end
-          escaped = false
-          current_value << char
-        when '{'
-          if !in_quotes
-            in_json = true
-            json_brace_count += 1
-          end
-          current_value << char
-        when '}'
-          if !in_quotes
-            json_brace_count -= 1
-            in_json = json_brace_count > 0
-          end
-          current_value << char
-        when ','
-          if !in_quotes && !in_json
-            values << clean_value(current_value.strip)
-            current_value = ''
-          else
-            current_value << char
-          end
-        else
-          escaped = false
-          current_value << char
-        end
-        i += 1
-      end
-
-      # Add the last value
-      values << clean_value(current_value.strip) unless current_value.strip.empty?
-
-      values.join(', ')
-    end
-
-    def clean_value(value)
-      return value if value.start_with?("'") # Already quoted
-      return value if value.start_with?("'{") # JSON object
-      return 'NULL' if value.upcase == 'NULL'
-      return value.downcase if ['true', 'false'].include?(value.downcase)
-      return value if value.match?(/^\d+$/) # Numbers
-
-      if value.match?(/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i)
-        "'#{value}'" # UUID
-      else
-        # Handle any other string value, including those with commas
-        "'#{value}'" # Other strings
-      end
+    # Loads a SQL dump on the ActiveRecord connection
+    # (LoadStrategies::Native), so the data participates in the caller's
+    # transaction (e.g. DatabaseCleaner's :transaction strategy) and rolls
+    # back with it. Shorthand for:
+    #
+    #   load_real_test_data(name, strategy: RealDataTests::LoadStrategies::Native)
+    #
+    # Unlike the default psql strategy, this never commits outside the test
+    # transaction, but it cannot process psql meta-commands (e.g. \set) and
+    # reads the whole dump into memory.
+    def load_real_test_data_native(name)
+      load_real_test_data(name, strategy: LoadStrategies::Native)
     end
   end
-end
+end

data/lib/real_data_tests/sql_dump_parser.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+module RealDataTests
+  # Splits a SQL dump into executable blocks. Regular statements become one
+  # block each (terminated by ";"); COPY ... FROM stdin blocks span from the
+  # COPY line through the "\." terminator, preserving the data lines between.
+  class SqlDumpParser
+    # Returns an array of SqlBlock in dump order.
+    def self.parse(content)
+      blocks = []
+      current_block = []
+      in_copy_block = false
+
+      content.each_line do |line|
+        line = line.chomp
+
+        # Skip empty lines and comments unless in COPY block
+        next if !in_copy_block && (line.empty? || line.start_with?('--'))
+
+        # Handle start of COPY block
+        if !in_copy_block && line.upcase.match?(/\ACOPY.*FROM stdin/i)
+          current_block = [line]
+          in_copy_block = true
+          next
+        end
+
+        # Handle end of COPY block
+        if in_copy_block && line == '\\.'
+          current_block << line
+          blocks << SqlBlock.new(current_block.join("\n"))
+          current_block = []
+          in_copy_block = false
+          next
+        end
+
+        # Accumulate lines in COPY block
+        if in_copy_block
+          current_block << line
+          next
+        end
+
+        # Handle regular SQL statements
+        current_block << line
+        if line.end_with?(';')
+          blocks << SqlBlock.new(current_block.join("\n"))
+          current_block = []
+        end
+      end
+
+      # Handle any remaining block
+      blocks << SqlBlock.new(current_block.join("\n")) unless current_block.empty?
+      blocks
+    end
+
+    # A single parsed block of the dump, tagged with its statement type.
+    class SqlBlock
+      attr_reader :type, :content, :table_name
+
+      def initialize(content)
+        @content = content.strip
+        @type = determine_block_type
+        @table_name = extract_table_name if @type == :insert
+      end
+
+      private
+
+      def determine_block_type
+        case @content
+        when /\AINSERT INTO/i
+          :insert
+        when /\ACOPY.*FROM stdin/i
+          :copy
+        when /\AALTER TABLE/i
+          :alter
+        when /\ASET/i
+          :set
+        else
+          :other
+        end
+      end
+
+      def extract_table_name
+        if @content =~ /INSERT INTO\s+"?([^\s"(]+)"?\s/i
+          $1
+        end
+      end
+    end
+  end
+end

data/lib/real_data_tests/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module RealDataTests
-  VERSION = "0.4.1"
+  VERSION = "0.5.0"
 end

data/lib/real_data_tests.rb

@@ -7,6 +7,10 @@ require_relative 'real_data_tests/data_anonymizer'
 require_relative 'real_data_tests/engine' if defined?(Rails)
 require_relative 'real_data_tests/pg_dump_generator'
 require_relative 'real_data_tests/record_collector'
+require_relative 'real_data_tests/sql_dump_parser'
+require_relative 'real_data_tests/load_strategies/base'
+require_relative 'real_data_tests/load_strategies/native'
+require_relative 'real_data_tests/load_strategies/psql'
 require_relative 'real_data_tests/rspec_helper'
 require_relative 'real_data_tests/test_data_builder'
 

data/real_data_tests.gemspec

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require_relative "lib/real_data_tests/version"
+
+Gem::Specification.new do |spec|
+  spec.name = "real_data_tests"
+  spec.version = RealDataTests::VERSION
+  spec.authors = ["Kevin Dias"]
+  spec.email = ["diasks2@gmail.com"]
+
+  spec.summary = "Create realistic test data from local db records"
+  spec.description = "A Ruby gem that helps create test data by analyzing and extracting real records and their associations from your database."
+  spec.homepage = "https://github.com/diasks2/real_data_tests"
+  spec.license = "MIT"
+  spec.required_ruby_version = ">= 2.6.0"
+
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = spec.homepage
+  spec.metadata["changelog_uri"] = spec.homepage
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files = Dir.chdir(__dir__) do
+    `git ls-files -z`.split("\x0").reject do |f|
+      (f == __FILE__) || f.match(%r{\A(?:(?:bin|test|spec|features)/|\.(?:git|travis|circleci)|appveyor)})
+    end
+  end
+  spec.bindir = "exe"
+  spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "rails", ">= 5.0"
+  spec.add_dependency "activerecord", ">= 5.0"
+  spec.add_dependency "thor", "~> 1.0"
+  spec.add_dependency "pg", ">= 1.1"
+  spec.add_dependency "csv"
+  spec.add_development_dependency "rake", "~> 13.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency "database_cleaner", "~> 2.0"
+  spec.add_development_dependency "database_cleaner-active_record"
+  spec.add_dependency "faker", "~> 3.0"
+end

metadata

@@ -1,14 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: real_data_tests
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.5.0
 platform: ruby
 authors:
 - Kevin Dias
-autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-04-09 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -66,6 +65,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '1.1'
+- !ruby/object:Gem::Dependency
+  name: csv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -156,11 +169,16 @@ files:
 - lib/real_data_tests/configuration.rb
 - lib/real_data_tests/data_anonymizer.rb
 - lib/real_data_tests/engine.rb
+- lib/real_data_tests/load_strategies/base.rb
+- lib/real_data_tests/load_strategies/native.rb
+- lib/real_data_tests/load_strategies/psql.rb
 - lib/real_data_tests/pg_dump_generator.rb
 - lib/real_data_tests/record_collector.rb
 - lib/real_data_tests/rspec_helper.rb
+- lib/real_data_tests/sql_dump_parser.rb
 - lib/real_data_tests/test_data_builder.rb
 - lib/real_data_tests/version.rb
+- real_data_tests.gemspec
 homepage: https://github.com/diasks2/real_data_tests
 licenses:
 - MIT
@@ -168,7 +186,6 @@ metadata:
   homepage_uri: https://github.com/diasks2/real_data_tests
   source_code_uri: https://github.com/diasks2/real_data_tests
   changelog_uri: https://github.com/diasks2/real_data_tests
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -183,8 +200,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.26
-signing_key:
+rubygems_version: 3.7.2
 specification_version: 4
 summary: Create realistic test data from local db records
 test_files: []