tapsoob 0.8.4 → 0.8.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 30cfe2b3b3f3c8e9daa9283eec5d3a3cdf6e6a689a60d21d8e4ad3306740a9aa
-  data.tar.gz: 3980451f8e14dd52c7fbf22de83cacf32ebc57de19c8c704a261575c5e92adb4
+  metadata.gz: 64dd145a81db1be94164524f8ec22cef232c76d7a3d455646c5289496c8e1ee2
+  data.tar.gz: 678fe127e48278964db8475c2d8001a085fb7a56546b4912b6fdf08a90ea2718
 SHA512:
-  metadata.gz: baf8a2278be1d5499ad64a4ef89be355f7cc3e82f3efd759c51c254cc3e28567955fce8b6f5e4025bf451a08f8cf0fe7e236c09e536f6e09ffd71875eb276246
-  data.tar.gz: 20ae520ae8daa165509b91f3b448268b2a3927f2285e422a2540b5eab75d3b66839aad441b90c58c0b8e83edd43004c27921c41bdcf9c80088e8745f5892d523
+  metadata.gz: 4263e7bdec44a5c31af5ebeba0fe1cc2b969e69f93a206a2c1b580a8141583eb94fd305626acc923d7499b20f38871294e41e1cdb9e14666fc2581e2fbbe8966
+  data.tar.gz: ebc49b6130cdcaa98ffe70405e8aac442012e451409a5d9e902c7c0518cf23dd7c3a158c850aac9392a9b349bae004b324efa42ad688d5dca10fd260f663fc2c

data/.gitlab-ci.yml

@@ -22,10 +22,11 @@ stages:
   rules:
     - if: $CI_COMMIT_TAG
       when: never
+    - if: $CI_COMMIT_MESSAGE =~ /Bumped version/
+      when: never
     - when: on_success
   variables:
     INTEGRATION_TESTS: "1"
-    SYSTEM_TESTS: "1"
   before_script:
     - apt-get update -qq && apt-get install -y -qq git
     - bundle install --quiet
@@ -49,10 +50,9 @@ test-sqlite:
     SRC_DATABASE_URL: "sqlite://tmp/tapsoob_src.db"
     DST_DATABASE_URL: "sqlite://tmp/tapsoob_dst.db"
     INTEGRATION_TESTS: "1"
-    SYSTEM_TESTS: "1"
   script:
     - mkdir -p tmp
-    - bundle exec rspec spec/unit spec/integration/sqlite_spec.rb spec/system/
+    - bundle exec rspec spec/unit spec/integration/sqlite_spec.rb
         --format progress
         --format RspecJunitFormatter --out rspec.xml
 
@@ -142,6 +142,8 @@ test-jruby:
   rules:
     - if: $CI_COMMIT_TAG
       when: never
+    - if: $CI_COMMIT_MESSAGE =~ /Bumped version/
+      when: never
     - when: on_success
   before_script:
     - apt-get update -qq && apt-get install -y -qq git libsqlite3-dev

data/lib/tapsoob/schema.rb

@@ -4,6 +4,7 @@ require 'sequel/extensions/schema_dumper'
 require 'sequel/extensions/migration'
 require 'erb'
 require 'json'
+require 'tapsoob/log'
 
 module Tapsoob
   module Schema
@@ -158,15 +159,18 @@ END_MIG
     end
 
     def reset_db_sequences(database_url)
-      db = Sequel.connect(database_url)
-      db.extension :schema_dumper
-      return unless db.respond_to?(:reset_primary_key_sequence)
-      db.tables.each do |table|
-        pk = db.primary_key(table)
-        next unless pk
-        pk_type = db.schema(table).find { |col, _| col.to_s == pk.to_s }&.last&.dig(:db_type)
-        next unless pk_type&.match?(/int|serial/i)
-        db.reset_primary_key_sequence(table)
+      Sequel.connect(database_url) do |db|
+        db.extension :schema_dumper
+        next unless db.respond_to?(:reset_primary_key_sequence)
+        db.tables.each do |table|
+          pk = db.primary_key(table)
+          next unless pk
+          pk_type = db.schema(table).find { |col, _| col.to_s == pk.to_s }&.last&.dig(:db_type)
+          next unless pk_type&.match?(/int|serial/i)
+          db.reset_primary_key_sequence(table)
+        rescue Sequel::DatabaseError => e
+          Tapsoob.log.warn "Could not reset sequence for table '#{table}': #{e.message.lines.first.chomp}"
+        end
       end
     end
   end

data/lib/tapsoob/version.rb

@@ -1,4 +1,4 @@
 # -*- encoding : utf-8 -*-
 module Tapsoob
-  VERSION = "0.8.4".freeze
+  VERSION = "0.8.5".freeze
 end

data/spec/integration/postgres_spec.rb

@@ -107,10 +107,21 @@ RSpec.describe 'PostgreSQL round-trip', :integration do
       @src_db.run("DROP TABLE IF EXISTS varchar_pk_table")
     end
 
-    it 'reset_db_sequences does not raise on a varchar primary key' do
-      expect {
-        Tapsoob::Schema.reset_db_sequences(@src_url)
-      }.not_to raise_error
+    it 'skips reset without logging a warning (no sequence attached to varchar PK)' do
+      expect(Tapsoob.log).not_to receive(:warn)
+      Tapsoob::Schema.reset_db_sequences(@src_url)
+    end
+  end
+
+  context 'when reset_primary_key_sequence raises a DatabaseError' do
+    it 'logs a warning per failing table and does not re-raise' do
+      allow(Sequel).to receive(:connect).and_yield(@src_db)
+      allow(@src_db).to receive(:reset_primary_key_sequence).and_raise(
+        Sequel::DatabaseError, 'ERROR: identity column type must be smallint, integer, or bigint'
+      )
+
+      expect(Tapsoob.log).to receive(:warn).at_least(:once)
+      expect { Tapsoob::Schema.reset_db_sequences(@src_url) }.not_to raise_error
     end
   end
 end

data/spec/spec_helper.rb

@@ -42,12 +42,9 @@ RSpec.configure do |config|
   config.order = :random
   Kernel.srand config.seed
 
-  # Integration/system tests require a real DB — skip unless env vars are set.
+  # Integration tests require a real DB — skip unless env vars are set.
   config.filter_run_excluding :integration unless ENV['INTEGRATION_TESTS'] || ENV['SRC_DATABASE_URL']
-  config.filter_run_excluding :system      unless ENV['SYSTEM_TESTS']      || ENV['SRC_DATABASE_URL']
 
-  config.include DbHelpers,      :integration
-  config.include DbHelpers,      :system
+  config.include DbHelpers,       :integration
   config.include RoundTripHelper, :integration
-  config.include RoundTripHelper, :system
 end

data/spec/support/fixtures.rb

@@ -15,7 +15,7 @@ require 'sequel'
 #   null_heavy     – every nullable column is NULL for half the rows
 #
 module Fixtures
-  LARGE_TABLE_ROWS   = 150_000
+  LARGE_TABLE_ROWS   = 20_000
   DOCUMENT_ROWS      = 500
   ATTACHMENT_ROWS    = 200
   STANDARD_ROWS      = 1_000

data/spec/support/shared_examples/round_trip.rb

@@ -6,49 +6,53 @@
 # defined in DbHelpers (which delegate to the ivars set in before(:all)).
 
 RSpec.shared_examples 'a complete round-trip' do
+  # Pull once into a shared dir and reuse across all examples in this group.
+  before(:all) do
+    @shared_dump_dir = Dir.mktmpdir('tapsoob_shared_')
+    pull(src_url, @shared_dump_dir)
+  end
+
+  after(:all) do
+    FileUtils.rm_rf(@shared_dump_dir)
+  end
+
   it 'pulls without error' do
-    expect { pull(src_url, dump_dir) }.not_to raise_error
+    expect(File).to exist(File.join(@shared_dump_dir, 'schemas'))
   end
 
   it 'creates schema dump files for every table' do
-    pull(src_url, dump_dir)
     src_db.tables.each do |table|
-      expect(File).to exist(File.join(dump_dir, 'schemas', "#{table}.rb"))
+      expect(File).to exist(File.join(@shared_dump_dir, 'schemas', "#{table}.rb"))
     end
   end
 
   it 'creates data dump files for every seeded table' do
-    pull(src_url, dump_dir)
     %i[users orders products documents attachments events large_table null_heavy].each do |table|
-      expect(File).to exist(File.join(dump_dir, 'data', "#{table}.json"))
+      expect(File).to exist(File.join(@shared_dump_dir, 'data', "#{table}.json"))
     end
   end
 
   it 'pushes without error' do
-    pull(src_url, dump_dir)
-    expect { push(dst_url, dump_dir) }.not_to raise_error
+    expect { push(dst_url, @shared_dump_dir) }.not_to raise_error
   end
 
   it 'preserves row counts for all tables' do
-    round_trip(src_url, dst_url, dump_dir)
+    push(dst_url, @shared_dump_dir)
     expect_same_counts(src_db, dst_db)
   end
 
   it 'preserves NULL values in null_heavy' do
-    round_trip(src_url, dst_url, dump_dir)
-    null_rows = dst_db[:null_heavy].where(maybe_name: nil).count
-    expect(null_rows).to be > 0
+    push(dst_url, @shared_dump_dir)
+    expect(dst_db[:null_heavy].where(maybe_name: nil).count).to be > 0
   end
 
   it 'preserves string content in users.email' do
-    round_trip(src_url, dst_url, dump_dir)
-    src_emails = src_db[:users].select_map(:email).sort
-    dst_emails = dst_db[:users].select_map(:email).sort
-    expect(dst_emails).to eq(src_emails)
+    push(dst_url, @shared_dump_dir)
+    expect(dst_db[:users].select_map(:email).sort).to eq(src_db[:users].select_map(:email).sort)
   end
 
   it 'preserves BLOB payloads in attachments' do
-    round_trip(src_url, dst_url, dump_dir)
+    push(dst_url, @shared_dump_dir)
     src_db[:attachments].order(:id).each do |src_row|
       dst_row = dst_db[:attachments][id: src_row[:id]]
       expect(dst_row).not_to be_nil
@@ -57,7 +61,7 @@ RSpec.shared_examples 'a complete round-trip' do
   end
 
   it 'preserves large TEXT bodies in documents' do
-    round_trip(src_url, dst_url, dump_dir)
+    push(dst_url, @shared_dump_dir)
     src_db[:documents].order(:id).each do |src_row|
       dst_row = dst_db[:documents][id: src_row[:id]]
       expect(dst_row[:body]).to eq(src_row[:body])
@@ -65,19 +69,29 @@ RSpec.shared_examples 'a complete round-trip' do
   end
 
   it 'handles the no-PK events table' do
-    round_trip(src_url, dst_url, dump_dir)
+    push(dst_url, @shared_dump_dir)
     expect(dst_db[:events].count).to eq(src_db[:events].count)
   end
 end
 
 RSpec.shared_examples 'a parallel round-trip' do |workers:|
+  # Pull once, push with parallel workers.
+  before(:all) do
+    @parallel_dump_dir = Dir.mktmpdir('tapsoob_parallel_')
+    pull(src_url, @parallel_dump_dir, parallel: workers)
+  end
+
+  after(:all) do
+    FileUtils.rm_rf(@parallel_dump_dir)
+  end
+
   it "preserves row counts with #{workers} parallel workers" do
-    round_trip(src_url, dst_url, dump_dir, parallel: workers)
+    push(dst_url, @parallel_dump_dir, parallel: workers)
     expect_same_counts(src_db, dst_db)
   end
 
-  it "handles the large_table (>100K rows) with #{workers} workers" do
-    round_trip(src_url, dst_url, dump_dir, parallel: workers)
+  it "handles the large_table with #{workers} workers" do
+    push(dst_url, @parallel_dump_dir, parallel: workers)
     expect(dst_db[:large_table].count).to eq(Fixtures::LARGE_TABLE_ROWS)
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: tapsoob
 version: !ruby/object:Gem::Version
-  version: 0.8.4
+  version: 0.8.5
 platform: ruby
 authors:
 - Félix Bellanger
@@ -133,7 +133,6 @@ files:
 - spec/support/fixtures.rb
 - spec/support/round_trip_helper.rb
 - spec/support/shared_examples/round_trip.rb
-- spec/system/large_dataset_spec.rb
 - spec/unit/tapsoob/chunksize_spec.rb
 - spec/unit/tapsoob/data_stream_spec.rb
 - spec/unit/tapsoob/operation_base_spec.rb

data/spec/system/large_dataset_spec.rb

@@ -1,163 +0,0 @@
-require 'spec_helper'
-
-RSpec.describe 'Large dataset system tests', :system do
-  before(:all) do
-    @src_url = DbHelpers.adapt_url(ENV.fetch('SRC_DATABASE_URL', 'sqlite://tmp/tapsoob_system_src.db'))
-    @dst_url = DbHelpers.adapt_url(ENV.fetch('DST_DATABASE_URL', 'sqlite://tmp/tapsoob_system_dst.db'))
-
-    FileUtils.mkdir_p('tmp')
-    File.delete('tmp/tapsoob_system_src.db') rescue nil
-    File.delete('tmp/tapsoob_system_dst.db') rescue nil
-
-    @src_db = DbHelpers.connect(@src_url)
-    @dst_db = DbHelpers.connect(@dst_url)
-
-    Fixtures.create_tables(@src_db)
-    Fixtures.seed(@src_db)
-  end
-
-  before(:each) do
-    Fixtures.drop_tables(@dst_db)
-  end
-
-  after(:all) do
-    Fixtures.drop_tables(@src_db)
-    Fixtures.drop_tables(@dst_db)
-    DbHelpers.disconnect_all
-    File.delete('tmp/tapsoob_system_src.db') rescue nil
-    File.delete('tmp/tapsoob_system_dst.db') rescue nil
-  end
-
-  # ── large_table: intra-table parallelization threshold ───────────────────────
-
-  describe 'large_table (150K rows)' do
-    it 'transfers all rows in serial mode' do
-      round_trip(src_url, dst_url, dump_dir)
-      expect(dst_db[:large_table].count).to eq(Fixtures::LARGE_TABLE_ROWS)
-    end
-
-    it 'transfers all rows with parallel: 2' do
-      round_trip(src_url, dst_url, dump_dir, parallel: 2)
-      expect(dst_db[:large_table].count).to eq(Fixtures::LARGE_TABLE_ROWS)
-    end
-
-    it 'transfers all rows with parallel: 4' do
-      round_trip(src_url, dst_url, dump_dir, parallel: 4)
-      expect(dst_db[:large_table].count).to eq(Fixtures::LARGE_TABLE_ROWS)
-    end
-
-    it 'has no duplicate rows after parallel pull' do
-      round_trip(src_url, dst_url, dump_dir, parallel: 4)
-      total    = dst_db[:large_table].count
-      distinct = dst_db[:large_table].select(:id).distinct.count
-      expect(distinct).to eq(total)
-    end
-  end
-
-  # ── documents: large TEXT columns ────────────────────────────────────────────
-
-  describe 'documents table (large TEXT)' do
-    it 'preserves body content exactly' do
-      round_trip(src_url, dst_url, dump_dir)
-      src_db[:documents].order(:id).each do |src_row|
-        dst_row = dst_db[:documents][id: src_row[:id]]
-        expect(dst_row[:body]).to eq(src_row[:body]),
-          "body mismatch for document #{src_row[:id]}: " \
-          "src=#{src_row[:body]&.length} bytes dst=#{dst_row[:body]&.length} bytes"
-      end
-    end
-
-    it 'handles documents with nil body' do
-      round_trip(src_url, dst_url, dump_dir)
-      expect(dst_db[:documents].where(body: nil).count).to eq(src_db[:documents].where(body: nil).count)
-    end
-  end
-
-  # ── attachments: BLOB encoding/decoding ──────────────────────────────────────
-
-  describe 'attachments table (binary BLOBs up to 256 KB)' do
-    it 'preserves every byte of every payload' do
-      round_trip(src_url, dst_url, dump_dir)
-      mismatch_count = 0
-      src_db[:attachments].order(:id).each do |src_row|
-        dst_row = dst_db[:attachments][id: src_row[:id]]
-        mismatch_count += 1 unless dst_row[:payload].to_s.bytes == src_row[:payload].to_s.bytes
-      end
-      expect(mismatch_count).to eq(0)
-    end
-
-    it 'preserves size_bytes metadata' do
-      round_trip(src_url, dst_url, dump_dir)
-      src_db[:attachments].order(:id).each do |src_row|
-        dst_row = dst_db[:attachments][id: src_row[:id]]
-        expect(dst_row[:size_bytes]).to eq(src_row[:size_bytes])
-      end
-    end
-  end
-
-  # ── null_heavy: NULL preservation ────────────────────────────────────────────
-
-  describe 'null_heavy table' do
-    it 'preserves NULLs in every nullable column' do
-      round_trip(src_url, dst_url, dump_dir)
-      %i[maybe_name maybe_number maybe_score maybe_date maybe_text].each do |col|
-        src_nulls = src_db[:null_heavy].where(col => nil).count
-        dst_nulls = dst_db[:null_heavy].where(col => nil).count
-        expect(dst_nulls).to eq(src_nulls),
-          "NULL count mismatch for null_heavy.#{col}: src=#{src_nulls} dst=#{dst_nulls}"
-      end
-    end
-  end
-
-  # ── events: table without primary key ────────────────────────────────────────
-
-  describe 'events table (no primary key)' do
-    it 'uses the Base (non-keyed) stream' do
-      round_trip(src_url, dst_url, dump_dir)
-      expect(dst_db[:events].count).to eq(src_db[:events].count)
-    end
-  end
-
-  # ── adaptive chunksize: very small chunks ────────────────────────────────────
-
-  describe 'adaptive chunksize under load' do
-    it 'completes with chunksize=1 (extreme case)' do
-      small_src = DbHelpers.adapt_url('sqlite://tmp/tapsoob_small_src.db')
-      small_dst = DbHelpers.adapt_url('sqlite://tmp/tapsoob_small_dst.db')
-      small_dir = Dir.mktmpdir
-
-      begin
-        sdb = DbHelpers.connect(small_src)
-        sdb.create_table!(:small_test) { primary_key :id; String :v, size: 50 }
-        100.times { |i| sdb[:small_test].insert(v: "row_#{i}") }
-
-        round_trip(small_src, small_dst, small_dir, default_chunksize: 1)
-        expect(DbHelpers.connect(small_dst)[:small_test].count).to eq(100)
-      ensure
-        FileUtils.rm_rf(small_dir)
-        File.delete('tmp/tapsoob_small_src.db') rescue nil
-        File.delete('tmp/tapsoob_small_dst.db') rescue nil
-        # Reconnect suite DBs after disconnect_all clears the pool
-        DbHelpers.disconnect_all
-        @src_db = DbHelpers.connect(@src_url)
-        @dst_db = DbHelpers.connect(@dst_url)
-      end
-    end
-  end
-
-  # ── FK order: orders depends on users ────────────────────────────────────────
-
-  describe 'foreign key dependency ordering' do
-    it 'pushes users before orders (table_order.txt respected)' do
-      pull(src_url, dump_dir)
-      order_file = File.join(dump_dir, 'table_order.txt')
-      if File.exist?(order_file)
-        order = File.readlines(order_file).map(&:strip)
-        users_idx  = order.index('users')
-        orders_idx = order.index('orders')
-        expect(users_idx).to be < orders_idx if users_idx && orders_idx
-      end
-      expect { push(dst_url, dump_dir) }.not_to raise_error
-    end
-  end
-end