tapsoob 0.8.4-java → 0.8.5-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c89e3fc3a94980f55559b6e4e32e7d2810beb829eb2b76d73e0464766c799c36
-  data.tar.gz: 24d8a8e5cca54038acf37c3a97aac5d9bc45743c91fad39cbabb17c332b32ec5
+  metadata.gz: bb9e75e4bea67137c1dff8803d7c49dbe96b6c46740eba03bb372cb8bf2d0c14
+  data.tar.gz: dda86ad6ec9d02734fce9f1ea9cedeafd745ebf7a0c316a0a329dfa461c61331
 SHA512:
-  metadata.gz: 01ef5f53d2977991b4e4f68829339e66fb196c2dc5757f6803f4621946004a40c484dcd3b2b1512c733bd07b77ef34cc57eb788180300085dd9d4f966c04727a
-  data.tar.gz: 79bd9ed407db74f20b6c0e5cefac70de4f4f4356aaac209ae04929c835db9fb4f49cfd33438f5039f1ce2dbf8351e89d7cbc4ee6b2e45a5b7384149bc2cf7b50
+  metadata.gz: '093b732d4cdc720f2e6dae958077581f31791236ea40ef57b84ef5a11f7d4282fe79ec8f6cbfffa1009d08c98f8e939a33440138cc901a61774f1f4d368f6ee6'
+  data.tar.gz: cd1fb4eaa2eab1aeb561a53ee4d1b1b6aef040ab461917c2577a75479060a692fb90ce9aedf601e04acd5fe8cdb8882759209b6fce9b8997c88c99336a20a6c3

data/.gitlab-ci.yml

@@ -22,10 +22,11 @@ stages:
   rules:
     - if: $CI_COMMIT_TAG
       when: never
+    - if: $CI_COMMIT_MESSAGE =~ /Bumped version/
+      when: never
     - when: on_success
   variables:
     INTEGRATION_TESTS: "1"
-    SYSTEM_TESTS: "1"
   before_script:
     - apt-get update -qq && apt-get install -y -qq git
     - bundle install --quiet
@@ -49,10 +50,9 @@ test-sqlite:
     SRC_DATABASE_URL: "sqlite://tmp/tapsoob_src.db"
     DST_DATABASE_URL: "sqlite://tmp/tapsoob_dst.db"
     INTEGRATION_TESTS: "1"
-    SYSTEM_TESTS: "1"
   script:
     - mkdir -p tmp
-    - bundle exec rspec spec/unit spec/integration/sqlite_spec.rb spec/system/
+    - bundle exec rspec spec/unit spec/integration/sqlite_spec.rb
         --format progress
         --format RspecJunitFormatter --out rspec.xml
 
@@ -142,6 +142,8 @@ test-jruby:
   rules:
     - if: $CI_COMMIT_TAG
       when: never
+    - if: $CI_COMMIT_MESSAGE =~ /Bumped version/
+      when: never
     - when: on_success
   before_script:
     - apt-get update -qq && apt-get install -y -qq git libsqlite3-dev

data/lib/tapsoob/schema.rb

@@ -4,6 +4,7 @@ require 'sequel/extensions/schema_dumper'
 require 'sequel/extensions/migration'
 require 'erb'
 require 'json'
+require 'tapsoob/log'
 
 module Tapsoob
   module Schema
@@ -158,15 +159,18 @@ END_MIG
     end
 
     def reset_db_sequences(database_url)
-      db = Sequel.connect(database_url)
-      db.extension :schema_dumper
-      return unless db.respond_to?(:reset_primary_key_sequence)
-      db.tables.each do |table|
-        pk = db.primary_key(table)
-        next unless pk
-        pk_type = db.schema(table).find { |col, _| col.to_s == pk.to_s }&.last&.dig(:db_type)
-        next unless pk_type&.match?(/int|serial/i)
-        db.reset_primary_key_sequence(table)
+      Sequel.connect(database_url) do |db|
+        db.extension :schema_dumper
+        next unless db.respond_to?(:reset_primary_key_sequence)
+        db.tables.each do |table|
+          pk = db.primary_key(table)
+          next unless pk
+          pk_type = db.schema(table).find { |col, _| col.to_s == pk.to_s }&.last&.dig(:db_type)
+          next unless pk_type&.match?(/int|serial/i)
+          db.reset_primary_key_sequence(table)
+        rescue Sequel::DatabaseError => e
+          Tapsoob.log.warn "Could not reset sequence for table '#{table}': #{e.message.lines.first.chomp}"
+        end
       end
     end
   end

data/lib/tapsoob/version.rb

@@ -1,4 +1,4 @@
 # -*- encoding : utf-8 -*-
 module Tapsoob
-  VERSION = "0.8.4".freeze
+  VERSION = "0.8.5".freeze
 end

data/spec/integration/postgres_spec.rb

@@ -107,10 +107,21 @@ RSpec.describe 'PostgreSQL round-trip', :integration do
       @src_db.run("DROP TABLE IF EXISTS varchar_pk_table")
     end
 
-    it 'reset_db_sequences does not raise on a varchar primary key' do
-      expect {
-        Tapsoob::Schema.reset_db_sequences(@src_url)
-      }.not_to raise_error
+    it 'skips reset without logging a warning (no sequence attached to varchar PK)' do
+      expect(Tapsoob.log).not_to receive(:warn)
+      Tapsoob::Schema.reset_db_sequences(@src_url)
+    end
+  end
+
+  context 'when reset_primary_key_sequence raises a DatabaseError' do
+    it 'logs a warning per failing table and does not re-raise' do
+      allow(Sequel).to receive(:connect).and_yield(@src_db)
+      allow(@src_db).to receive(:reset_primary_key_sequence).and_raise(
+        Sequel::DatabaseError, 'ERROR: identity column type must be smallint, integer, or bigint'
+      )
+
+      expect(Tapsoob.log).to receive(:warn).at_least(:once)
+      expect { Tapsoob::Schema.reset_db_sequences(@src_url) }.not_to raise_error
     end
   end
 end

data/spec/spec_helper.rb

@@ -42,12 +42,9 @@ RSpec.configure do |config|
   config.order = :random
   Kernel.srand config.seed
 
-  # Integration/system tests require a real DB — skip unless env vars are set.
+  # Integration tests require a real DB — skip unless env vars are set.
   config.filter_run_excluding :integration unless ENV['INTEGRATION_TESTS'] || ENV['SRC_DATABASE_URL']
-  config.filter_run_excluding :system      unless ENV['SYSTEM_TESTS']      || ENV['SRC_DATABASE_URL']
 
-  config.include DbHelpers,      :integration
-  config.include DbHelpers,      :system
+  config.include DbHelpers,       :integration
   config.include RoundTripHelper, :integration
-  config.include RoundTripHelper, :system
 end

data/spec/support/fixtures.rb

@@ -15,7 +15,7 @@ require 'sequel'
 #   null_heavy     – every nullable column is NULL for half the rows
 #
 module Fixtures
-  LARGE_TABLE_ROWS   = 150_000
+  LARGE_TABLE_ROWS   = 20_000
   DOCUMENT_ROWS      = 500
   ATTACHMENT_ROWS    = 200
   STANDARD_ROWS      = 1_000

data/spec/support/shared_examples/round_trip.rb

@@ -6,49 +6,53 @@
 # defined in DbHelpers (which delegate to the ivars set in before(:all)).
 
 RSpec.shared_examples 'a complete round-trip' do
+  # Pull once into a shared dir and reuse across all examples in this group.
+  before(:all) do
+    @shared_dump_dir = Dir.mktmpdir('tapsoob_shared_')
+    pull(src_url, @shared_dump_dir)
+  end
+
+  after(:all) do
+    FileUtils.rm_rf(@shared_dump_dir)
+  end
+
   it 'pulls without error' do
-    expect { pull(src_url, dump_dir) }.not_to raise_error
+    expect(File).to exist(File.join(@shared_dump_dir, 'schemas'))
   end
 
   it 'creates schema dump files for every table' do
-    pull(src_url, dump_dir)
     src_db.tables.each do |table|
-      expect(File).to exist(File.join(dump_dir, 'schemas', "#{table}.rb"))
+      expect(File).to exist(File.join(@shared_dump_dir, 'schemas', "#{table}.rb"))
     end
   end
 
   it 'creates data dump files for every seeded table' do
-    pull(src_url, dump_dir)
     %i[users orders products documents attachments events large_table null_heavy].each do |table|
-      expect(File).to exist(File.join(dump_dir, 'data', "#{table}.json"))
+      expect(File).to exist(File.join(@shared_dump_dir, 'data', "#{table}.json"))
     end
   end
 
   it 'pushes without error' do
-    pull(src_url, dump_dir)
-    expect { push(dst_url, dump_dir) }.not_to raise_error
+    expect { push(dst_url, @shared_dump_dir) }.not_to raise_error
   end
 
   it 'preserves row counts for all tables' do
-    round_trip(src_url, dst_url, dump_dir)
+    push(dst_url, @shared_dump_dir)
     expect_same_counts(src_db, dst_db)
   end
 
   it 'preserves NULL values in null_heavy' do
-    round_trip(src_url, dst_url, dump_dir)
-    null_rows = dst_db[:null_heavy].where(maybe_name: nil).count
-    expect(null_rows).to be > 0
+    push(dst_url, @shared_dump_dir)
+    expect(dst_db[:null_heavy].where(maybe_name: nil).count).to be > 0
   end
 
   it 'preserves string content in users.email' do
-    round_trip(src_url, dst_url, dump_dir)
-    src_emails = src_db[:users].select_map(:email).sort
-    dst_emails = dst_db[:users].select_map(:email).sort
-    expect(dst_emails).to eq(src_emails)
+    push(dst_url, @shared_dump_dir)
+    expect(dst_db[:users].select_map(:email).sort).to eq(src_db[:users].select_map(:email).sort)
   end
 
   it 'preserves BLOB payloads in attachments' do
-    round_trip(src_url, dst_url, dump_dir)
+    push(dst_url, @shared_dump_dir)
     src_db[:attachments].order(:id).each do |src_row|
       dst_row = dst_db[:attachments][id: src_row[:id]]
       expect(dst_row).not_to be_nil
@@ -57,7 +61,7 @@ RSpec.shared_examples 'a complete round-trip' do
   end
 
   it 'preserves large TEXT bodies in documents' do
-    round_trip(src_url, dst_url, dump_dir)
+    push(dst_url, @shared_dump_dir)
     src_db[:documents].order(:id).each do |src_row|
       dst_row = dst_db[:documents][id: src_row[:id]]
       expect(dst_row[:body]).to eq(src_row[:body])
@@ -65,19 +69,29 @@ RSpec.shared_examples 'a complete round-trip' do
   end
 
   it 'handles the no-PK events table' do
-    round_trip(src_url, dst_url, dump_dir)
+    push(dst_url, @shared_dump_dir)
     expect(dst_db[:events].count).to eq(src_db[:events].count)
   end
 end
 
 RSpec.shared_examples 'a parallel round-trip' do |workers:|
+  # Pull once, push with parallel workers.
+  before(:all) do
+    @parallel_dump_dir = Dir.mktmpdir('tapsoob_parallel_')
+    pull(src_url, @parallel_dump_dir, parallel: workers)
+  end
+
+  after(:all) do
+    FileUtils.rm_rf(@parallel_dump_dir)
+  end
+
   it "preserves row counts with #{workers} parallel workers" do
-    round_trip(src_url, dst_url, dump_dir, parallel: workers)
+    push(dst_url, @parallel_dump_dir, parallel: workers)
     expect_same_counts(src_db, dst_db)
   end
 
-  it "handles the large_table (>100K rows) with #{workers} workers" do
-    round_trip(src_url, dst_url, dump_dir, parallel: workers)
+  it "handles the large_table with #{workers} workers" do
+    push(dst_url, @parallel_dump_dir, parallel: workers)
     expect(dst_db[:large_table].count).to eq(Fixtures::LARGE_TABLE_ROWS)
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: tapsoob
 version: !ruby/object:Gem::Version
-  version: 0.8.4
+  version: 0.8.5
 platform: java
 authors:
 - Félix Bellanger
@@ -132,7 +132,6 @@ files:
 - spec/support/fixtures.rb
 - spec/support/round_trip_helper.rb
 - spec/support/shared_examples/round_trip.rb
-- spec/system/large_dataset_spec.rb
 - spec/unit/tapsoob/chunksize_spec.rb
 - spec/unit/tapsoob/data_stream_spec.rb
 - spec/unit/tapsoob/operation_base_spec.rb

data/spec/system/large_dataset_spec.rb

@@ -1,163 +0,0 @@
-require 'spec_helper'
-
-RSpec.describe 'Large dataset system tests', :system do
-  before(:all) do
-    @src_url = DbHelpers.adapt_url(ENV.fetch('SRC_DATABASE_URL', 'sqlite://tmp/tapsoob_system_src.db'))
-    @dst_url = DbHelpers.adapt_url(ENV.fetch('DST_DATABASE_URL', 'sqlite://tmp/tapsoob_system_dst.db'))
-
-    FileUtils.mkdir_p('tmp')
-    File.delete('tmp/tapsoob_system_src.db') rescue nil
-    File.delete('tmp/tapsoob_system_dst.db') rescue nil
-
-    @src_db = DbHelpers.connect(@src_url)
-    @dst_db = DbHelpers.connect(@dst_url)
-
-    Fixtures.create_tables(@src_db)
-    Fixtures.seed(@src_db)
-  end
-
-  before(:each) do
-    Fixtures.drop_tables(@dst_db)
-  end
-
-  after(:all) do
-    Fixtures.drop_tables(@src_db)
-    Fixtures.drop_tables(@dst_db)
-    DbHelpers.disconnect_all
-    File.delete('tmp/tapsoob_system_src.db') rescue nil
-    File.delete('tmp/tapsoob_system_dst.db') rescue nil
-  end
-
-  # ── large_table: intra-table parallelization threshold ───────────────────────
-
-  describe 'large_table (150K rows)' do
-    it 'transfers all rows in serial mode' do
-      round_trip(src_url, dst_url, dump_dir)
-      expect(dst_db[:large_table].count).to eq(Fixtures::LARGE_TABLE_ROWS)
-    end
-
-    it 'transfers all rows with parallel: 2' do
-      round_trip(src_url, dst_url, dump_dir, parallel: 2)
-      expect(dst_db[:large_table].count).to eq(Fixtures::LARGE_TABLE_ROWS)
-    end
-
-    it 'transfers all rows with parallel: 4' do
-      round_trip(src_url, dst_url, dump_dir, parallel: 4)
-      expect(dst_db[:large_table].count).to eq(Fixtures::LARGE_TABLE_ROWS)
-    end
-
-    it 'has no duplicate rows after parallel pull' do
-      round_trip(src_url, dst_url, dump_dir, parallel: 4)
-      total    = dst_db[:large_table].count
-      distinct = dst_db[:large_table].select(:id).distinct.count
-      expect(distinct).to eq(total)
-    end
-  end
-
-  # ── documents: large TEXT columns ────────────────────────────────────────────
-
-  describe 'documents table (large TEXT)' do
-    it 'preserves body content exactly' do
-      round_trip(src_url, dst_url, dump_dir)
-      src_db[:documents].order(:id).each do |src_row|
-        dst_row = dst_db[:documents][id: src_row[:id]]
-        expect(dst_row[:body]).to eq(src_row[:body]),
-          "body mismatch for document #{src_row[:id]}: " \
-          "src=#{src_row[:body]&.length} bytes dst=#{dst_row[:body]&.length} bytes"
-      end
-    end
-
-    it 'handles documents with nil body' do
-      round_trip(src_url, dst_url, dump_dir)
-      expect(dst_db[:documents].where(body: nil).count).to eq(src_db[:documents].where(body: nil).count)
-    end
-  end
-
-  # ── attachments: BLOB encoding/decoding ──────────────────────────────────────
-
-  describe 'attachments table (binary BLOBs up to 256 KB)' do
-    it 'preserves every byte of every payload' do
-      round_trip(src_url, dst_url, dump_dir)
-      mismatch_count = 0
-      src_db[:attachments].order(:id).each do |src_row|
-        dst_row = dst_db[:attachments][id: src_row[:id]]
-        mismatch_count += 1 unless dst_row[:payload].to_s.bytes == src_row[:payload].to_s.bytes
-      end
-      expect(mismatch_count).to eq(0)
-    end
-
-    it 'preserves size_bytes metadata' do
-      round_trip(src_url, dst_url, dump_dir)
-      src_db[:attachments].order(:id).each do |src_row|
-        dst_row = dst_db[:attachments][id: src_row[:id]]
-        expect(dst_row[:size_bytes]).to eq(src_row[:size_bytes])
-      end
-    end
-  end
-
-  # ── null_heavy: NULL preservation ────────────────────────────────────────────
-
-  describe 'null_heavy table' do
-    it 'preserves NULLs in every nullable column' do
-      round_trip(src_url, dst_url, dump_dir)
-      %i[maybe_name maybe_number maybe_score maybe_date maybe_text].each do |col|
-        src_nulls = src_db[:null_heavy].where(col => nil).count
-        dst_nulls = dst_db[:null_heavy].where(col => nil).count
-        expect(dst_nulls).to eq(src_nulls),
-          "NULL count mismatch for null_heavy.#{col}: src=#{src_nulls} dst=#{dst_nulls}"
-      end
-    end
-  end
-
-  # ── events: table without primary key ────────────────────────────────────────
-
-  describe 'events table (no primary key)' do
-    it 'uses the Base (non-keyed) stream' do
-      round_trip(src_url, dst_url, dump_dir)
-      expect(dst_db[:events].count).to eq(src_db[:events].count)
-    end
-  end
-
-  # ── adaptive chunksize: very small chunks ────────────────────────────────────
-
-  describe 'adaptive chunksize under load' do
-    it 'completes with chunksize=1 (extreme case)' do
-      small_src = DbHelpers.adapt_url('sqlite://tmp/tapsoob_small_src.db')
-      small_dst = DbHelpers.adapt_url('sqlite://tmp/tapsoob_small_dst.db')
-      small_dir = Dir.mktmpdir
-
-      begin
-        sdb = DbHelpers.connect(small_src)
-        sdb.create_table!(:small_test) { primary_key :id; String :v, size: 50 }
-        100.times { |i| sdb[:small_test].insert(v: "row_#{i}") }
-
-        round_trip(small_src, small_dst, small_dir, default_chunksize: 1)
-        expect(DbHelpers.connect(small_dst)[:small_test].count).to eq(100)
-      ensure
-        FileUtils.rm_rf(small_dir)
-        File.delete('tmp/tapsoob_small_src.db') rescue nil
-        File.delete('tmp/tapsoob_small_dst.db') rescue nil
-        # Reconnect suite DBs after disconnect_all clears the pool
-        DbHelpers.disconnect_all
-        @src_db = DbHelpers.connect(@src_url)
-        @dst_db = DbHelpers.connect(@dst_url)
-      end
-    end
-  end
-
-  # ── FK order: orders depends on users ────────────────────────────────────────
-
-  describe 'foreign key dependency ordering' do
-    it 'pushes users before orders (table_order.txt respected)' do
-      pull(src_url, dump_dir)
-      order_file = File.join(dump_dir, 'table_order.txt')
-      if File.exist?(order_file)
-        order = File.readlines(order_file).map(&:strip)
-        users_idx  = order.index('users')
-        orders_idx = order.index('orders')
-        expect(users_idx).to be < orders_idx if users_idx && orders_idx
-      end
-      expect { push(dst_url, dump_dir) }.not_to raise_error
-    end
-  end
-end