tapsoob 0.8.4 → 0.8.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 30cfe2b3b3f3c8e9daa9283eec5d3a3cdf6e6a689a60d21d8e4ad3306740a9aa
-  data.tar.gz: 3980451f8e14dd52c7fbf22de83cacf32ebc57de19c8c704a261575c5e92adb4
+  metadata.gz: dd37614b588e363ef7ae5fefdbc98188a194045f72906d496bef4ef67df04bf7
+  data.tar.gz: 9270a820fdd7600f943a6343c208d8752618f40fe29d6a81c3af32051f4475ee
 SHA512:
-  metadata.gz: baf8a2278be1d5499ad64a4ef89be355f7cc3e82f3efd759c51c254cc3e28567955fce8b6f5e4025bf451a08f8cf0fe7e236c09e536f6e09ffd71875eb276246
-  data.tar.gz: 20ae520ae8daa165509b91f3b448268b2a3927f2285e422a2540b5eab75d3b66839aad441b90c58c0b8e83edd43004c27921c41bdcf9c80088e8745f5892d523
+  metadata.gz: 1ac36335eca4bc18bff412ab892b05a6e13322f34debdf6a1decf40e95128066d86b9b598edc43e150e1414291aa883d5f93f8ba10695b1b1d7fa02480d620ad
+  data.tar.gz: 63bbb2832e9768287cc3e9189e7cbd919cf73b9707968ed8de4f9442ea2ab833216bcedcc043ebbd761f79a2f0ccab5fc60706efabfb9fa3f57abc8fb7422738

data/.gitlab-ci.yml

@@ -22,10 +22,11 @@ stages:
   rules:
     - if: $CI_COMMIT_TAG
       when: never
+    - if: $CI_COMMIT_MESSAGE =~ /Bumped version/
+      when: never
     - when: on_success
   variables:
     INTEGRATION_TESTS: "1"
-    SYSTEM_TESTS: "1"
   before_script:
     - apt-get update -qq && apt-get install -y -qq git
     - bundle install --quiet
@@ -49,10 +50,9 @@ test-sqlite:
     SRC_DATABASE_URL: "sqlite://tmp/tapsoob_src.db"
     DST_DATABASE_URL: "sqlite://tmp/tapsoob_dst.db"
     INTEGRATION_TESTS: "1"
-    SYSTEM_TESTS: "1"
   script:
     - mkdir -p tmp
-    - bundle exec rspec spec/unit spec/integration/sqlite_spec.rb spec/system/
+    - bundle exec rspec spec/unit spec/integration/sqlite_spec.rb
         --format progress
         --format RspecJunitFormatter --out rspec.xml
 
@@ -142,6 +142,8 @@ test-jruby:
   rules:
     - if: $CI_COMMIT_TAG
       when: never
+    - if: $CI_COMMIT_MESSAGE =~ /Bumped version/
+      when: never
     - when: on_success
   before_script:
     - apt-get update -qq && apt-get install -y -qq git libsqlite3-dev

data/lib/tapsoob/schema.rb

@@ -4,15 +4,16 @@ require 'sequel/extensions/schema_dumper'
 require 'sequel/extensions/migration'
 require 'erb'
 require 'json'
+require 'tapsoob/log'
 
 module Tapsoob
   module Schema
     extend self
 
     def dump(database_url, options = {})
-      db = Sequel.connect(database_url)
-      db.extension :schema_dumper
-      template = ERB.new <<-END_MIG
+      Sequel.connect(database_url) do |db|
+        db.extension :schema_dumper
+        template = ERB.new <<-END_MIG
 Class.new(Sequel::Migration) do
   def up
   <% db.send(:sort_dumped_tables, db.tables, {}).each do |table| %>
@@ -28,7 +29,8 @@ Class.new(Sequel::Migration) do
 end
 END_MIG
 
-      template.result(binding)
+        template.result(binding)
+      end
     end
 
     def dump_table(database_url_or_db, table, options)
@@ -67,15 +69,17 @@ END_MIG
     end
 
     def foreign_keys(database_url)
-      db = Sequel.connect(database_url)
-      db.extension :schema_dumper
-      db.dump_foreign_key_migration
+      Sequel.connect(database_url) do |db|
+        db.extension :schema_dumper
+        db.dump_foreign_key_migration
+      end
     end
 
     def indexes(database_url)
-      db = Sequel.connect(database_url)
-      db.extension :schema_dumper
-      db.dump_indexes_migration
+      Sequel.connect(database_url) do |db|
+        db.extension :schema_dumper
+        db.dump_indexes_migration
+      end
     end
 
     def indexes_individual(database_url)
@@ -104,6 +108,7 @@ END_MIG
     end
 
     def load(database_url_or_db, schema, options = { drop: false })
+      schema = rewrite_non_integer_primary_keys(schema)
       # Accept either a database URL or an existing connection object
       if database_url_or_db.is_a?(Sequel::Database)
         db = database_url_or_db
@@ -157,16 +162,38 @@ END_MIG
       end
     end
 
+    NON_INTEGER_PK_PATTERN = /^(\s*)primary_key\s+(:?\w+),\s*:type=>"([^"]+)"(.*)$/
+    INTEGER_DB_TYPES       = /\A(?:int(?:eger|\d+)?|bigint|smallint|serial|bigserial|smallserial)/i
+
+    # On PG 10+, Sequel's CreateTableGenerator injects `identity: true` into
+    # every primary_key call via serial_primary_key_options. PG rejects IDENTITY
+    # on non-integer types. Rewrite `primary_key :col, :type=>"varchar..."` to
+    # `column :col, "varchar...", primary_key: true, null: false` which bypasses
+    # that code path entirely.
+    def rewrite_non_integer_primary_keys(schema_str)
+      schema_str.gsub(NON_INTEGER_PK_PATTERN) do
+        indent, col, db_type, rest = $1, $2, $3, $4
+        if db_type =~ INTEGER_DB_TYPES
+          "#{indent}primary_key #{col}, :type=>\"#{db_type}\"#{rest}"
+        else
+          "#{indent}column #{col}, \"#{db_type}\", primary_key: true, null: false#{rest}"
+        end
+      end
+    end
+
     def reset_db_sequences(database_url)
-      db = Sequel.connect(database_url)
-      db.extension :schema_dumper
-      return unless db.respond_to?(:reset_primary_key_sequence)
-      db.tables.each do |table|
-        pk = db.primary_key(table)
-        next unless pk
-        pk_type = db.schema(table).find { |col, _| col.to_s == pk.to_s }&.last&.dig(:db_type)
-        next unless pk_type&.match?(/int|serial/i)
-        db.reset_primary_key_sequence(table)
+      Sequel.connect(database_url) do |db|
+        db.extension :schema_dumper
+        next unless db.respond_to?(:reset_primary_key_sequence)
+        db.tables.each do |table|
+          pk = db.primary_key(table)
+          next unless pk
+          pk_type = db.schema(table).find { |col, _| col.to_s == pk.to_s }&.last&.dig(:db_type)
+          next unless pk_type&.match?(/int|serial/i)
+          db.reset_primary_key_sequence(table)
+        rescue Sequel::DatabaseError => e
+          Tapsoob.log.warn "Could not reset sequence for table '#{table}': #{e.message.lines.first.chomp}"
+        end
       end
     end
   end

data/lib/tapsoob/version.rb

@@ -1,4 +1,4 @@
 # -*- encoding : utf-8 -*-
 module Tapsoob
-  VERSION = "0.8.4".freeze
+  VERSION = "0.8.6".freeze
 end

data/spec/integration/postgres_spec.rb

@@ -105,12 +105,35 @@ RSpec.describe 'PostgreSQL round-trip', :integration do
 
     after(:each) do
       @src_db.run("DROP TABLE IF EXISTS varchar_pk_table")
+      @dst_db.run("DROP TABLE IF EXISTS varchar_pk_table")
     end
 
-    it 'reset_db_sequences does not raise on a varchar primary key' do
-      expect {
-        Tapsoob::Schema.reset_db_sequences(@src_url)
-      }.not_to raise_error
+    it 'skips reset without logging a warning (no sequence attached to varchar PK)' do
+      expect(Tapsoob.log).not_to receive(:warn)
+      Tapsoob::Schema.reset_db_sequences(@src_url)
+    end
+
+    it 'round-trips the table without identity column errors' do
+      dump_dir = Dir.mktmpdir
+      begin
+        pull(src_url, dump_dir)
+        expect { push(dst_url, dump_dir) }.not_to raise_error
+        expect(dst_db[:varchar_pk_table].where(id: 'abc-123').count).to eq(1)
+      ensure
+        FileUtils.rm_rf(dump_dir)
+      end
+    end
+  end
+
+  context 'when reset_primary_key_sequence raises a DatabaseError' do
+    it 'logs a warning per failing table and does not re-raise' do
+      allow(Sequel).to receive(:connect).and_yield(@src_db)
+      allow(@src_db).to receive(:reset_primary_key_sequence).and_raise(
+        Sequel::DatabaseError, 'ERROR: identity column type must be smallint, integer, or bigint'
+      )
+
+      expect(Tapsoob.log).to receive(:warn).at_least(:once)
+      expect { Tapsoob::Schema.reset_db_sequences(@src_url) }.not_to raise_error
     end
   end
 end

data/spec/spec_helper.rb

@@ -42,12 +42,9 @@ RSpec.configure do |config|
   config.order = :random
   Kernel.srand config.seed
 
-  # Integration/system tests require a real DB — skip unless env vars are set.
+  # Integration tests require a real DB — skip unless env vars are set.
   config.filter_run_excluding :integration unless ENV['INTEGRATION_TESTS'] || ENV['SRC_DATABASE_URL']
-  config.filter_run_excluding :system      unless ENV['SYSTEM_TESTS']      || ENV['SRC_DATABASE_URL']
 
-  config.include DbHelpers,      :integration
-  config.include DbHelpers,      :system
+  config.include DbHelpers,       :integration
   config.include RoundTripHelper, :integration
-  config.include RoundTripHelper, :system
 end

data/spec/support/fixtures.rb

@@ -15,7 +15,7 @@ require 'sequel'
 #   null_heavy     – every nullable column is NULL for half the rows
 #
 module Fixtures
-  LARGE_TABLE_ROWS   = 150_000
+  LARGE_TABLE_ROWS   = 20_000
   DOCUMENT_ROWS      = 500
   ATTACHMENT_ROWS    = 200
   STANDARD_ROWS      = 1_000

data/spec/support/shared_examples/round_trip.rb

@@ -6,49 +6,53 @@
 # defined in DbHelpers (which delegate to the ivars set in before(:all)).
 
 RSpec.shared_examples 'a complete round-trip' do
+  # Pull once into a shared dir and reuse across all examples in this group.
+  before(:all) do
+    @shared_dump_dir = Dir.mktmpdir('tapsoob_shared_')
+    pull(src_url, @shared_dump_dir)
+  end
+
+  after(:all) do
+    FileUtils.rm_rf(@shared_dump_dir)
+  end
+
   it 'pulls without error' do
-    expect { pull(src_url, dump_dir) }.not_to raise_error
+    expect(File).to exist(File.join(@shared_dump_dir, 'schemas'))
   end
 
   it 'creates schema dump files for every table' do
-    pull(src_url, dump_dir)
     src_db.tables.each do |table|
-      expect(File).to exist(File.join(dump_dir, 'schemas', "#{table}.rb"))
+      expect(File).to exist(File.join(@shared_dump_dir, 'schemas', "#{table}.rb"))
     end
   end
 
   it 'creates data dump files for every seeded table' do
-    pull(src_url, dump_dir)
     %i[users orders products documents attachments events large_table null_heavy].each do |table|
-      expect(File).to exist(File.join(dump_dir, 'data', "#{table}.json"))
+      expect(File).to exist(File.join(@shared_dump_dir, 'data', "#{table}.json"))
     end
   end
 
   it 'pushes without error' do
-    pull(src_url, dump_dir)
-    expect { push(dst_url, dump_dir) }.not_to raise_error
+    expect { push(dst_url, @shared_dump_dir) }.not_to raise_error
   end
 
   it 'preserves row counts for all tables' do
-    round_trip(src_url, dst_url, dump_dir)
+    push(dst_url, @shared_dump_dir)
     expect_same_counts(src_db, dst_db)
   end
 
   it 'preserves NULL values in null_heavy' do
-    round_trip(src_url, dst_url, dump_dir)
-    null_rows = dst_db[:null_heavy].where(maybe_name: nil).count
-    expect(null_rows).to be > 0
+    push(dst_url, @shared_dump_dir)
+    expect(dst_db[:null_heavy].where(maybe_name: nil).count).to be > 0
   end
 
   it 'preserves string content in users.email' do
-    round_trip(src_url, dst_url, dump_dir)
-    src_emails = src_db[:users].select_map(:email).sort
-    dst_emails = dst_db[:users].select_map(:email).sort
-    expect(dst_emails).to eq(src_emails)
+    push(dst_url, @shared_dump_dir)
+    expect(dst_db[:users].select_map(:email).sort).to eq(src_db[:users].select_map(:email).sort)
   end
 
   it 'preserves BLOB payloads in attachments' do
-    round_trip(src_url, dst_url, dump_dir)
+    push(dst_url, @shared_dump_dir)
     src_db[:attachments].order(:id).each do |src_row|
       dst_row = dst_db[:attachments][id: src_row[:id]]
       expect(dst_row).not_to be_nil
@@ -57,7 +61,7 @@ RSpec.shared_examples 'a complete round-trip' do
   end
 
   it 'preserves large TEXT bodies in documents' do
-    round_trip(src_url, dst_url, dump_dir)
+    push(dst_url, @shared_dump_dir)
     src_db[:documents].order(:id).each do |src_row|
       dst_row = dst_db[:documents][id: src_row[:id]]
       expect(dst_row[:body]).to eq(src_row[:body])
@@ -65,19 +69,29 @@ RSpec.shared_examples 'a complete round-trip' do
   end
 
   it 'handles the no-PK events table' do
-    round_trip(src_url, dst_url, dump_dir)
+    push(dst_url, @shared_dump_dir)
     expect(dst_db[:events].count).to eq(src_db[:events].count)
   end
 end
 
 RSpec.shared_examples 'a parallel round-trip' do |workers:|
+  # Pull once, push with parallel workers.
+  before(:all) do
+    @parallel_dump_dir = Dir.mktmpdir('tapsoob_parallel_')
+    pull(src_url, @parallel_dump_dir, parallel: workers)
+  end
+
+  after(:all) do
+    FileUtils.rm_rf(@parallel_dump_dir)
+  end
+
   it "preserves row counts with #{workers} parallel workers" do
-    round_trip(src_url, dst_url, dump_dir, parallel: workers)
+    push(dst_url, @parallel_dump_dir, parallel: workers)
     expect_same_counts(src_db, dst_db)
   end
 
-  it "handles the large_table (>100K rows) with #{workers} workers" do
-    round_trip(src_url, dst_url, dump_dir, parallel: workers)
+  it "handles the large_table with #{workers} workers" do
+    push(dst_url, @parallel_dump_dir, parallel: workers)
     expect(dst_db[:large_table].count).to eq(Fixtures::LARGE_TABLE_ROWS)
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: tapsoob
 version: !ruby/object:Gem::Version
-  version: 0.8.4
+  version: 0.8.6
 platform: ruby
 authors:
 - Félix Bellanger
@@ -133,7 +133,6 @@ files:
 - spec/support/fixtures.rb
 - spec/support/round_trip_helper.rb
 - spec/support/shared_examples/round_trip.rb
-- spec/system/large_dataset_spec.rb
 - spec/unit/tapsoob/chunksize_spec.rb
 - spec/unit/tapsoob/data_stream_spec.rb
 - spec/unit/tapsoob/operation_base_spec.rb

data/spec/system/large_dataset_spec.rb

@@ -1,163 +0,0 @@
-require 'spec_helper'
-
-RSpec.describe 'Large dataset system tests', :system do
-  before(:all) do
-    @src_url = DbHelpers.adapt_url(ENV.fetch('SRC_DATABASE_URL', 'sqlite://tmp/tapsoob_system_src.db'))
-    @dst_url = DbHelpers.adapt_url(ENV.fetch('DST_DATABASE_URL', 'sqlite://tmp/tapsoob_system_dst.db'))
-
-    FileUtils.mkdir_p('tmp')
-    File.delete('tmp/tapsoob_system_src.db') rescue nil
-    File.delete('tmp/tapsoob_system_dst.db') rescue nil
-
-    @src_db = DbHelpers.connect(@src_url)
-    @dst_db = DbHelpers.connect(@dst_url)
-
-    Fixtures.create_tables(@src_db)
-    Fixtures.seed(@src_db)
-  end
-
-  before(:each) do
-    Fixtures.drop_tables(@dst_db)
-  end
-
-  after(:all) do
-    Fixtures.drop_tables(@src_db)
-    Fixtures.drop_tables(@dst_db)
-    DbHelpers.disconnect_all
-    File.delete('tmp/tapsoob_system_src.db') rescue nil
-    File.delete('tmp/tapsoob_system_dst.db') rescue nil
-  end
-
-  # ── large_table: intra-table parallelization threshold ───────────────────────
-
-  describe 'large_table (150K rows)' do
-    it 'transfers all rows in serial mode' do
-      round_trip(src_url, dst_url, dump_dir)
-      expect(dst_db[:large_table].count).to eq(Fixtures::LARGE_TABLE_ROWS)
-    end
-
-    it 'transfers all rows with parallel: 2' do
-      round_trip(src_url, dst_url, dump_dir, parallel: 2)
-      expect(dst_db[:large_table].count).to eq(Fixtures::LARGE_TABLE_ROWS)
-    end
-
-    it 'transfers all rows with parallel: 4' do
-      round_trip(src_url, dst_url, dump_dir, parallel: 4)
-      expect(dst_db[:large_table].count).to eq(Fixtures::LARGE_TABLE_ROWS)
-    end
-
-    it 'has no duplicate rows after parallel pull' do
-      round_trip(src_url, dst_url, dump_dir, parallel: 4)
-      total    = dst_db[:large_table].count
-      distinct = dst_db[:large_table].select(:id).distinct.count
-      expect(distinct).to eq(total)
-    end
-  end
-
-  # ── documents: large TEXT columns ────────────────────────────────────────────
-
-  describe 'documents table (large TEXT)' do
-    it 'preserves body content exactly' do
-      round_trip(src_url, dst_url, dump_dir)
-      src_db[:documents].order(:id).each do |src_row|
-        dst_row = dst_db[:documents][id: src_row[:id]]
-        expect(dst_row[:body]).to eq(src_row[:body]),
-          "body mismatch for document #{src_row[:id]}: " \
-          "src=#{src_row[:body]&.length} bytes dst=#{dst_row[:body]&.length} bytes"
-      end
-    end
-
-    it 'handles documents with nil body' do
-      round_trip(src_url, dst_url, dump_dir)
-      expect(dst_db[:documents].where(body: nil).count).to eq(src_db[:documents].where(body: nil).count)
-    end
-  end
-
-  # ── attachments: BLOB encoding/decoding ──────────────────────────────────────
-
-  describe 'attachments table (binary BLOBs up to 256 KB)' do
-    it 'preserves every byte of every payload' do
-      round_trip(src_url, dst_url, dump_dir)
-      mismatch_count = 0
-      src_db[:attachments].order(:id).each do |src_row|
-        dst_row = dst_db[:attachments][id: src_row[:id]]
-        mismatch_count += 1 unless dst_row[:payload].to_s.bytes == src_row[:payload].to_s.bytes
-      end
-      expect(mismatch_count).to eq(0)
-    end
-
-    it 'preserves size_bytes metadata' do
-      round_trip(src_url, dst_url, dump_dir)
-      src_db[:attachments].order(:id).each do |src_row|
-        dst_row = dst_db[:attachments][id: src_row[:id]]
-        expect(dst_row[:size_bytes]).to eq(src_row[:size_bytes])
-      end
-    end
-  end
-
-  # ── null_heavy: NULL preservation ────────────────────────────────────────────
-
-  describe 'null_heavy table' do
-    it 'preserves NULLs in every nullable column' do
-      round_trip(src_url, dst_url, dump_dir)
-      %i[maybe_name maybe_number maybe_score maybe_date maybe_text].each do |col|
-        src_nulls = src_db[:null_heavy].where(col => nil).count
-        dst_nulls = dst_db[:null_heavy].where(col => nil).count
-        expect(dst_nulls).to eq(src_nulls),
-          "NULL count mismatch for null_heavy.#{col}: src=#{src_nulls} dst=#{dst_nulls}"
-      end
-    end
-  end
-
-  # ── events: table without primary key ────────────────────────────────────────
-
-  describe 'events table (no primary key)' do
-    it 'uses the Base (non-keyed) stream' do
-      round_trip(src_url, dst_url, dump_dir)
-      expect(dst_db[:events].count).to eq(src_db[:events].count)
-    end
-  end
-
-  # ── adaptive chunksize: very small chunks ────────────────────────────────────
-
-  describe 'adaptive chunksize under load' do
-    it 'completes with chunksize=1 (extreme case)' do
-      small_src = DbHelpers.adapt_url('sqlite://tmp/tapsoob_small_src.db')
-      small_dst = DbHelpers.adapt_url('sqlite://tmp/tapsoob_small_dst.db')
-      small_dir = Dir.mktmpdir
-
-      begin
-        sdb = DbHelpers.connect(small_src)
-        sdb.create_table!(:small_test) { primary_key :id; String :v, size: 50 }
-        100.times { |i| sdb[:small_test].insert(v: "row_#{i}") }
-
-        round_trip(small_src, small_dst, small_dir, default_chunksize: 1)
-        expect(DbHelpers.connect(small_dst)[:small_test].count).to eq(100)
-      ensure
-        FileUtils.rm_rf(small_dir)
-        File.delete('tmp/tapsoob_small_src.db') rescue nil
-        File.delete('tmp/tapsoob_small_dst.db') rescue nil
-        # Reconnect suite DBs after disconnect_all clears the pool
-        DbHelpers.disconnect_all
-        @src_db = DbHelpers.connect(@src_url)
-        @dst_db = DbHelpers.connect(@dst_url)
-      end
-    end
-  end
-
-  # ── FK order: orders depends on users ────────────────────────────────────────
-
-  describe 'foreign key dependency ordering' do
-    it 'pushes users before orders (table_order.txt respected)' do
-      pull(src_url, dump_dir)
-      order_file = File.join(dump_dir, 'table_order.txt')
-      if File.exist?(order_file)
-        order = File.readlines(order_file).map(&:strip)
-        users_idx  = order.index('users')
-        orders_idx = order.index('orders')
-        expect(users_idx).to be < orders_idx if users_idx && orders_idx
-      end
-      expect { push(dst_url, dump_dir) }.not_to raise_error
-    end
-  end
-end