tapsoob 0.7.17 → 0.8.0

data/spec/support/shared_examples/round_trip.rb

@@ -0,0 +1,83 @@
+# Shared examples that any adapter-specific integration suite can include.
+# The including example group must define before(:all) that sets:
+#   @src_url, @dst_url  — Sequel connection URLs
+#   @src_db, @dst_db    — connected Sequel::Database objects
+# Individual examples access these via the src_url/dst_url/src_db/dst_db helpers
+# defined in DbHelpers (which delegate to the ivars set in before(:all)).
+
+RSpec.shared_examples 'a complete round-trip' do
+  it 'pulls without error' do
+    expect { pull(src_url, dump_dir) }.not_to raise_error
+  end
+
+  it 'creates schema dump files for every table' do
+    pull(src_url, dump_dir)
+    src_db.tables.each do |table|
+      expect(File).to exist(File.join(dump_dir, 'schemas', "#{table}.rb"))
+    end
+  end
+
+  it 'creates data dump files for every seeded table' do
+    pull(src_url, dump_dir)
+    %i[users orders products documents attachments events large_table null_heavy].each do |table|
+      expect(File).to exist(File.join(dump_dir, 'data', "#{table}.json"))
+    end
+  end
+
+  it 'pushes without error' do
+    pull(src_url, dump_dir)
+    expect { push(dst_url, dump_dir) }.not_to raise_error
+  end
+
+  it 'preserves row counts for all tables' do
+    round_trip(src_url, dst_url, dump_dir)
+    expect_same_counts(src_db, dst_db)
+  end
+
+  it 'preserves NULL values in null_heavy' do
+    round_trip(src_url, dst_url, dump_dir)
+    null_rows = dst_db[:null_heavy].where(maybe_name: nil).count
+    expect(null_rows).to be > 0
+  end
+
+  it 'preserves string content in users.email' do
+    round_trip(src_url, dst_url, dump_dir)
+    src_emails = src_db[:users].select_map(:email).sort
+    dst_emails = dst_db[:users].select_map(:email).sort
+    expect(dst_emails).to eq(src_emails)
+  end
+
+  it 'preserves BLOB payloads in attachments' do
+    round_trip(src_url, dst_url, dump_dir)
+    src_db[:attachments].order(:id).each do |src_row|
+      dst_row = dst_db[:attachments][id: src_row[:id]]
+      expect(dst_row).not_to be_nil
+      expect(dst_row[:payload].to_s.bytes).to eq(src_row[:payload].to_s.bytes)
+    end
+  end
+
+  it 'preserves large TEXT bodies in documents' do
+    round_trip(src_url, dst_url, dump_dir)
+    src_db[:documents].order(:id).each do |src_row|
+      dst_row = dst_db[:documents][id: src_row[:id]]
+      expect(dst_row[:body]).to eq(src_row[:body])
+    end
+  end
+
+  it 'handles the no-PK events table' do
+    round_trip(src_url, dst_url, dump_dir)
+    expect(dst_db[:events].count).to eq(src_db[:events].count)
+  end
+end
+
+RSpec.shared_examples 'a parallel round-trip' do |workers:|
+  it "preserves row counts with #{workers} parallel workers" do
+    round_trip(src_url, dst_url, dump_dir, parallel: workers)
+    expect_same_counts(src_db, dst_db)
+  end
+
+  it "handles the large_table (>100K rows) with #{workers} workers" do
+    round_trip(src_url, dst_url, dump_dir, parallel: workers)
+    expect(dst_db[:large_table].count).to eq(Fixtures::LARGE_TABLE_ROWS)
+  end
+end

data/spec/system/large_dataset_spec.rb

@@ -0,0 +1,163 @@
+require 'spec_helper'
+
+RSpec.describe 'Large dataset system tests', :system do
+  before(:all) do
+    @src_url = DbHelpers.adapt_url(ENV.fetch('SRC_DATABASE_URL', 'sqlite://tmp/tapsoob_system_src.db'))
+    @dst_url = DbHelpers.adapt_url(ENV.fetch('DST_DATABASE_URL', 'sqlite://tmp/tapsoob_system_dst.db'))
+
+    FileUtils.mkdir_p('tmp')
+    File.delete('tmp/tapsoob_system_src.db') rescue nil
+    File.delete('tmp/tapsoob_system_dst.db') rescue nil
+
+    @src_db = DbHelpers.connect(@src_url)
+    @dst_db = DbHelpers.connect(@dst_url)
+
+    Fixtures.create_tables(@src_db)
+    Fixtures.seed(@src_db)
+  end
+
+  before(:each) do
+    Fixtures.drop_tables(@dst_db)
+  end
+
+  after(:all) do
+    Fixtures.drop_tables(@src_db)
+    Fixtures.drop_tables(@dst_db)
+    DbHelpers.disconnect_all
+    File.delete('tmp/tapsoob_system_src.db') rescue nil
+    File.delete('tmp/tapsoob_system_dst.db') rescue nil
+  end
+
+  # ── large_table: intra-table parallelization threshold ───────────────────────
+
+  describe 'large_table (150K rows)' do
+    it 'transfers all rows in serial mode' do
+      round_trip(src_url, dst_url, dump_dir)
+      expect(dst_db[:large_table].count).to eq(Fixtures::LARGE_TABLE_ROWS)
+    end
+
+    it 'transfers all rows with parallel: 2' do
+      round_trip(src_url, dst_url, dump_dir, parallel: 2)
+      expect(dst_db[:large_table].count).to eq(Fixtures::LARGE_TABLE_ROWS)
+    end
+
+    it 'transfers all rows with parallel: 4' do
+      round_trip(src_url, dst_url, dump_dir, parallel: 4)
+      expect(dst_db[:large_table].count).to eq(Fixtures::LARGE_TABLE_ROWS)
+    end
+
+    it 'has no duplicate rows after parallel pull' do
+      round_trip(src_url, dst_url, dump_dir, parallel: 4)
+      total    = dst_db[:large_table].count
+      distinct = dst_db[:large_table].select(:id).distinct.count
+      expect(distinct).to eq(total)
+    end
+  end
+
+  # ── documents: large TEXT columns ────────────────────────────────────────────
+
+  describe 'documents table (large TEXT)' do
+    it 'preserves body content exactly' do
+      round_trip(src_url, dst_url, dump_dir)
+      src_db[:documents].order(:id).each do |src_row|
+        dst_row = dst_db[:documents][id: src_row[:id]]
+        expect(dst_row[:body]).to eq(src_row[:body]),
+          "body mismatch for document #{src_row[:id]}: " \
+          "src=#{src_row[:body]&.length} bytes dst=#{dst_row[:body]&.length} bytes"
+      end
+    end
+
+    it 'handles documents with nil body' do
+      round_trip(src_url, dst_url, dump_dir)
+      expect(dst_db[:documents].where(body: nil).count).to eq(src_db[:documents].where(body: nil).count)
+    end
+  end
+
+  # ── attachments: BLOB encoding/decoding ──────────────────────────────────────
+
+  describe 'attachments table (binary BLOBs up to 256 KB)' do
+    it 'preserves every byte of every payload' do
+      round_trip(src_url, dst_url, dump_dir)
+      mismatch_count = 0
+      src_db[:attachments].order(:id).each do |src_row|
+        dst_row = dst_db[:attachments][id: src_row[:id]]
+        mismatch_count += 1 unless dst_row[:payload].to_s.bytes == src_row[:payload].to_s.bytes
+      end
+      expect(mismatch_count).to eq(0)
+    end
+
+    it 'preserves size_bytes metadata' do
+      round_trip(src_url, dst_url, dump_dir)
+      src_db[:attachments].order(:id).each do |src_row|
+        dst_row = dst_db[:attachments][id: src_row[:id]]
+        expect(dst_row[:size_bytes]).to eq(src_row[:size_bytes])
+      end
+    end
+  end
+
+  # ── null_heavy: NULL preservation ────────────────────────────────────────────
+
+  describe 'null_heavy table' do
+    it 'preserves NULLs in every nullable column' do
+      round_trip(src_url, dst_url, dump_dir)
+      %i[maybe_name maybe_number maybe_score maybe_date maybe_text].each do |col|
+        src_nulls = src_db[:null_heavy].where(col => nil).count
+        dst_nulls = dst_db[:null_heavy].where(col => nil).count
+        expect(dst_nulls).to eq(src_nulls),
+          "NULL count mismatch for null_heavy.#{col}: src=#{src_nulls} dst=#{dst_nulls}"
+      end
+    end
+  end
+
+  # ── events: table without primary key ────────────────────────────────────────
+
+  describe 'events table (no primary key)' do
+    it 'uses the Base (non-keyed) stream' do
+      round_trip(src_url, dst_url, dump_dir)
+      expect(dst_db[:events].count).to eq(src_db[:events].count)
+    end
+  end
+
+  # ── adaptive chunksize: very small chunks ────────────────────────────────────
+
+  describe 'adaptive chunksize under load' do
+    it 'completes with chunksize=1 (extreme case)' do
+      small_src = DbHelpers.adapt_url('sqlite://tmp/tapsoob_small_src.db')
+      small_dst = DbHelpers.adapt_url('sqlite://tmp/tapsoob_small_dst.db')
+      small_dir = Dir.mktmpdir
+
+      begin
+        sdb = DbHelpers.connect(small_src)
+        sdb.create_table!(:small_test) { primary_key :id; String :v, size: 50 }
+        100.times { |i| sdb[:small_test].insert(v: "row_#{i}") }
+
+        round_trip(small_src, small_dst, small_dir, default_chunksize: 1)
+        expect(DbHelpers.connect(small_dst)[:small_test].count).to eq(100)
+      ensure
+        FileUtils.rm_rf(small_dir)
+        File.delete('tmp/tapsoob_small_src.db') rescue nil
+        File.delete('tmp/tapsoob_small_dst.db') rescue nil
+        # Reconnect suite DBs after disconnect_all clears the pool
+        DbHelpers.disconnect_all
+        @src_db = DbHelpers.connect(@src_url)
+        @dst_db = DbHelpers.connect(@dst_url)
+      end
+    end
+  end
+
+  # ── FK order: orders depends on users ────────────────────────────────────────
+
+  describe 'foreign key dependency ordering' do
+    it 'pushes users before orders (table_order.txt respected)' do
+      pull(src_url, dump_dir)
+      order_file = File.join(dump_dir, 'table_order.txt')
+      if File.exist?(order_file)
+        order = File.readlines(order_file).map(&:strip)
+        users_idx  = order.index('users')
+        orders_idx = order.index('orders')
+        expect(users_idx).to be < orders_idx if users_idx && orders_idx
+      end
+      expect { push(dst_url, dump_dir) }.not_to raise_error
+    end
+  end
+end

data/spec/unit/tapsoob/chunksize_spec.rb

@@ -0,0 +1,105 @@
+require 'spec_helper'
+require 'tapsoob/chunksize'
+
+RSpec.describe Tapsoob::Chunksize do
+  subject(:cs) { described_class.new(1000) }
+
+  describe '#initialize' do
+    it 'stores the initial chunksize' do
+      expect(cs.to_i).to eq(1000)
+    end
+
+    it 'starts with zero retries' do
+      expect(cs.retries).to eq(0)
+    end
+
+    it 'starts with zero idle_secs' do
+      expect(cs.idle_secs).to eq(0.0)
+    end
+  end
+
+  describe '#reset_chunksize' do
+    context 'with 0 retries (first failure)' do
+      it 'resets to 10' do
+        expect(cs.reset_chunksize).to eq(10)
+      end
+    end
+
+    context 'with 1 retry' do
+      it 'resets to 10' do
+        cs.retries = 1
+        expect(cs.reset_chunksize).to eq(10)
+      end
+    end
+
+    context 'with 2+ retries' do
+      it 'resets to 1' do
+        cs.retries = 2
+        expect(cs.reset_chunksize).to eq(1)
+      end
+    end
+  end
+
+  describe '#diff' do
+    before do
+      cs.start_time = 0.0
+      cs.end_time   = 10.0
+      cs.time_in_db = 3.0
+      cs.idle_secs  = 2.0
+    end
+
+    it 'returns end_time - start_time - time_in_db - idle_secs' do
+      expect(cs.diff).to eq(5.0)
+    end
+  end
+
+  describe '#calc_new_chunksize' do
+    def make(chunksize, diff_val)
+      c = described_class.new(chunksize)
+      # manufacture a diff by setting times such that diff == diff_val
+      c.start_time = 0.0
+      c.end_time   = diff_val
+      c.time_in_db = 0.0
+      c.idle_secs  = 0.0
+      c
+    end
+
+    it 'halves (roughly) when diff > 3.0' do
+      c = make(900, 3.5)
+      expect(c.calc_new_chunksize).to eq((900 / 3.0).ceil)
+    end
+
+    it 'decrements by 100 when diff is 1.1..3.0' do
+      c = make(900, 2.0)
+      expect(c.calc_new_chunksize).to eq(800)
+    end
+
+    it 'doubles when diff < 0.8' do
+      c = make(500, 0.5)
+      expect(c.calc_new_chunksize).to eq(1000)
+    end
+
+    it 'increments by 100 when diff is 0.8..1.1' do
+      c = make(500, 0.9)
+      expect(c.calc_new_chunksize).to eq(600)
+    end
+
+    it 'never returns less than 1' do
+      c = make(1, 5.0)
+      expect(c.calc_new_chunksize).to be >= 1
+    end
+
+    it 'holds chunksize unchanged when retries > 0' do
+      c = make(500, 0.5)
+      c.retries = 1
+      expect(c.calc_new_chunksize).to eq(500)
+    end
+  end
+
+  describe '#time_delta' do
+    it 'returns elapsed seconds for the block' do
+      delta = cs.time_delta { sleep 0.01 }
+      expect(delta).to be_between(0.005, 1.0)
+    end
+  end
+end

data/spec/unit/tapsoob/data_stream_spec.rb

@@ -0,0 +1,220 @@
+require 'spec_helper'
+require 'tapsoob/data_stream'
+
+# Shared low-level stream tests run against an in-process SQLite connection.
+# They exercise the fetch/encode/decode cycle, completion logic, and the
+# factory method – all without any filesystem dump.
+
+RSpec.describe Tapsoob::DataStream do
+  # ── shared DB setup ──────────────────────────────────────────────────────────
+
+  let(:db) do
+    d = connect_sqlite
+    d.extension :schema_dumper
+    d.create_table(:stream_test) do
+      primary_key :id
+      String  :label, size: 50
+      Integer :value
+    end
+    # Insert 50 rows
+    50.times { |i| d[:stream_test].insert(label: "row_#{i}", value: i) }
+    d
+  end
+
+  let(:db_nopk) do
+    d = connect_sqlite
+    d.extension :schema_dumper
+    d.create_table(:nopk_test) do
+      String  :key, size: 50
+      Integer :val
+    end
+    20.times { |i| d[:nopk_test].insert(key: "k#{i}", val: i) }
+    d
+  end
+
+  after { db.disconnect; db_nopk.disconnect }
+
+  # ── Base ─────────────────────────────────────────────────────────────────────
+
+  describe Tapsoob::DataStream::Base do
+    subject(:stream) do
+      described_class.new(db, { table_name: :stream_test, chunksize: 10 })
+    end
+
+    describe '#fetch' do
+      it 'returns [encoded_data, row_count, elapsed]' do
+        encoded, count, elapsed = stream.fetch
+        expect(encoded).to be_a(String)
+        expect(count).to eq(10)
+        expect(elapsed).to be_a(Float)
+      end
+
+      it 'advances offset on each call' do
+        stream.fetch
+        expect(stream.state[:offset]).to eq(10)
+        stream.fetch
+        expect(stream.state[:offset]).to eq(20)
+      end
+    end
+
+    describe '#complete?' do
+      it 'is false before all rows are fetched' do
+        stream.fetch
+        expect(stream.complete?).to be false
+      end
+
+      it 'is true after all rows are fetched' do
+        5.times { stream.fetch }
+        expect(stream.complete?).to be true
+      end
+    end
+
+    describe '#fetch_data_from_database' do
+      it 'yields a hash with :table_name, :header, :data, :types' do
+        encoded, _, _ = stream.fetch
+        data_params = {
+          state:        stream.to_hash,
+          checksum:     Tapsoob::Utils.checksum(encoded).to_s,
+          encoded_data: encoded
+        }
+        yielded = nil
+        stream.fetch_data_from_database(data_params) { |rows| yielded = rows }
+        expect(yielded).to include(:table_name, :header, :data)
+      end
+    end
+
+    describe '#parse_encoded_data' do
+      it 'raises CorruptedData on checksum mismatch' do
+        encoded, _, _ = stream.fetch
+        expect {
+          stream.parse_encoded_data(encoded, '0')
+        }.to raise_error(Tapsoob::CorruptedData)
+      end
+    end
+
+    describe '.factory' do
+      it 'returns Keyed stream for table with integer PK' do
+        result = described_class.factory(db, { table_name: :stream_test, chunksize: 10 }, {})
+        expect(result).to be_a(Tapsoob::DataStream::Keyed)
+      end
+
+      it 'returns Base stream for table without integer PK' do
+        result = described_class.factory(db_nopk, { table_name: :nopk_test, chunksize: 10 }, {})
+        expect(result).to be_a(Tapsoob::DataStream::Base)
+      end
+    end
+  end
+
+  # ── Keyed ────────────────────────────────────────────────────────────────────
+
+  describe Tapsoob::DataStream::Keyed do
+    subject(:stream) do
+      described_class.new(db, { table_name: :stream_test, chunksize: 10 })
+    end
+
+    it 'fetches all 50 rows without duplicates' do
+      all_ids = []
+      loop do
+        encoded, count, _ = stream.fetch
+        break if count == 0
+        data_params = {
+          state:        stream.to_hash,
+          checksum:     Tapsoob::Utils.checksum(encoded).to_s,
+          encoded_data: encoded
+        }
+        stream.fetch_data_from_database(data_params) do |rows|
+          id_idx = rows[:header].index(:id)
+          all_ids.concat(rows[:data].map { |r| r[id_idx] })
+        end
+        break if stream.complete?
+      end
+      expect(all_ids.uniq.size).to eq(50)
+      expect(all_ids.size).to eq(50)
+    end
+
+    describe '.calculate_pk_ranges' do
+      it 'returns the right number of ranges' do
+        ranges = described_class.calculate_pk_ranges(db, :stream_test, 4)
+        expect(ranges.size).to eq(4)
+      end
+
+      it 'covers the full PK range' do
+        min = db[:stream_test].min(:id)
+        max = db[:stream_test].max(:id)
+        ranges = described_class.calculate_pk_ranges(db, :stream_test, 4)
+        expect(ranges.first.first).to eq(min)
+        expect(ranges.last.last).to eq(max)
+      end
+    end
+  end
+
+  # ── KeyedPartition ───────────────────────────────────────────────────────────
+
+  describe Tapsoob::DataStream::KeyedPartition do
+    it 'fetches only rows within its assigned PK range' do
+      min = db[:stream_test].min(:id)
+      max = db[:stream_test].max(:id)
+      mid = (min + max) / 2
+
+      stream = described_class.new(db, {
+        table_name:      :stream_test,
+        chunksize:       100,
+        partition_range: [min, mid]
+      })
+
+      all_ids = []
+      until stream.complete?
+        encoded, count, _ = stream.fetch
+        break if count == 0
+        data_params = {
+          state:        stream.to_hash,
+          checksum:     Tapsoob::Utils.checksum(encoded).to_s,
+          encoded_data: encoded
+        }
+        stream.fetch_data_from_database(data_params) do |rows|
+          id_idx = rows[:header].index(:id)
+          all_ids.concat(rows[:data].map { |r| r[id_idx] })
+        end
+      end
+
+      expect(all_ids).to all(be_between(min, mid))
+    end
+  end
+
+  # ── Interleaved ──────────────────────────────────────────────────────────────
+
+  describe Tapsoob::DataStream::Interleaved do
+    it 'two workers together cover all rows without overlap' do
+      worker0 = described_class.new(db, {
+        table_name: :stream_test, chunksize: 10, worker_id: 0, num_workers: 2
+      })
+      worker1 = described_class.new(db, {
+        table_name: :stream_test, chunksize: 10, worker_id: 1, num_workers: 2
+      })
+
+      def drain(stream, db)
+        ids = []
+        until stream.complete?
+          encoded, count, _ = stream.fetch
+          break if count == 0
+          params = {
+            state:        stream.to_hash,
+            checksum:     Tapsoob::Utils.checksum(encoded).to_s,
+            encoded_data: encoded
+          }
+          stream.fetch_data_from_database(params) do |rows|
+            id_idx = rows[:header].index(:id)
+            ids.concat(rows[:data].map { |r| r[id_idx] })
+          end
+        end
+        ids
+      end
+
+      ids0 = drain(worker0, db)
+      ids1 = drain(worker1, db)
+
+      expect((ids0 + ids1).sort).to eq((ids0 + ids1).uniq.sort)
+      expect((ids0 + ids1).size).to eq(50)
+    end
+  end
+end