tapsoob 0.7.17 → 0.8.0

data/spec/unit/tapsoob/operation_base_spec.rb

@@ -0,0 +1,134 @@
+require 'spec_helper'
+require 'tapsoob/operation/base'
+require 'tapsoob/operation/pull'
+require 'tapsoob/operation/push'
+
+RSpec.describe Tapsoob::Operation::Base do
+  let(:url)     { sqlite_memory_url }
+  let(:dir)     { Dir.mktmpdir }
+  let(:base_opts) { { data: true, schema: true, progress: false, default_chunksize: 1000 } }
+
+  after { FileUtils.rm_rf(dir) }
+
+  subject(:op) { described_class.new(url, dir, base_opts) }
+
+  # ── table_filter / exclude_tables ────────────────────────────────────────────
+
+  describe '#apply_table_filter' do
+    context 'when no filter is set' do
+      it 'returns all tables unchanged' do
+        tables = ['users', 'orders', 'products']
+        expect(op.apply_table_filter(tables)).to eq(tables)
+      end
+    end
+
+    context 'with :tables filter' do
+      subject(:op) { described_class.new(url, dir, base_opts.merge(tables: ['users'])) }
+
+      it 'keeps only whitelisted tables' do
+        tables = ['users', 'orders']
+        expect(op.apply_table_filter(tables)).to eq(['users'])
+      end
+    end
+
+    context 'with :exclude_tables' do
+      subject(:op) { described_class.new(url, dir, base_opts.merge(exclude_tables: ['orders'])) }
+
+      it 'excludes the specified tables' do
+        tables = ['users', 'orders', 'products']
+        expect(op.apply_table_filter(tables)).to eq(['users', 'products'])
+      end
+    end
+
+    context 'with a Hash argument' do
+      subject(:op) { described_class.new(url, dir, base_opts.merge(tables: ['users'])) }
+
+      it 'filters the hash by whitelisted keys' do
+        tables = { 'users' => 10, 'orders' => 5 }
+        expect(op.apply_table_filter(tables)).to eq({ 'users' => 10 })
+      end
+    end
+  end
+
+  # ── parallelism ──────────────────────────────────────────────────────────────
+
+  describe '#parallel_workers' do
+    it 'defaults to 1 when no --parallel option given' do
+      expect(op.parallel_workers).to eq(1)
+    end
+
+    it 'respects the :parallel option' do
+      o = described_class.new(url, dir, base_opts.merge(parallel: 4))
+      expect(o.parallel_workers).to eq(4)
+    end
+
+    it 'is at least 1 for invalid values' do
+      o = described_class.new(url, dir, base_opts.merge(parallel: 0))
+      expect(o.parallel_workers).to be >= 1
+    end
+  end
+
+  describe '#table_parallel_workers' do
+    it 'returns 1 when dump_path is nil (piped mode)' do
+      o = described_class.new(url, nil, base_opts)
+      expect(o.table_parallel_workers(:big_table, 500_000)).to eq(1)
+    end
+
+    it 'returns 1 for tables under 100K rows' do
+      expect(op.table_parallel_workers(:small_table, 99_999)).to eq(1)
+    end
+
+    it 'returns >= 2 for tables at the 100K threshold' do
+      expect(op.table_parallel_workers(:big_table, 100_000)).to be >= 2
+    end
+
+    it 'returns 1 when --no-split is set' do
+      o = described_class.new(url, dir, base_opts.merge(no_split: true))
+      expect(o.table_parallel_workers(:big_table, 500_000)).to eq(1)
+    end
+  end
+
+  # ── completed_tables (thread safety) ─────────────────────────────────────────
+
+  describe '#add_completed_table' do
+    it 'adds the table name as a string' do
+      op.add_completed_table(:users)
+      expect(op.completed_tables).to include('users')
+    end
+
+    it 'is thread-safe under concurrent adds' do
+      threads = 20.times.map { |i| Thread.new { op.add_completed_table("table_#{i}") } }
+      threads.each(&:join)
+      expect(op.completed_tables.uniq.size).to eq(20)
+    end
+  end
+
+  # ── factory ──────────────────────────────────────────────────────────────────
+
+  describe '.factory' do
+    it 'returns a Pull operation for :pull' do
+      result = described_class.factory(:pull, url, dir, base_opts)
+      expect(result).to be_a(Tapsoob::Operation::Pull)
+    end
+
+    it 'returns a Push operation for :push' do
+      result = described_class.factory(:push, url, dir, base_opts)
+      expect(result).to be_a(Tapsoob::Operation::Push)
+    end
+
+    it 'raises on unknown type' do
+      expect {
+        described_class.factory(:unknown, url, dir, base_opts)
+      }.to raise_error(RuntimeError, /Unknown Operation Type/)
+    end
+  end
+
+  # ── session persistence ───────────────────────────────────────────────────────
+
+  describe '#to_hash' do
+    it 'includes klass, database_url, stream_state, completed_tables' do
+      h = op.to_hash
+      expect(h).to include(:klass, :database_url, :stream_state, :completed_tables)
+    end
+  end
+end

data/spec/unit/tapsoob/schema_spec.rb

@@ -0,0 +1,102 @@
+require 'spec_helper'
+require 'tapsoob/schema'
+
+RSpec.describe Tapsoob::Schema do
+  let(:db) do
+    d = connect_sqlite
+    d.extension :schema_dumper
+    d.create_table(:articles) do
+      primary_key :id
+      String  :title, null: false, size: 255
+      String  :body,  text: true
+      DateTime :published_at
+    end
+    d
+  end
+
+  after { db.disconnect }
+
+  # ── dump_table ───────────────────────────────────────────────────────────────
+
+  describe '.dump_table' do
+    it 'returns a Sequel migration string' do
+      result = described_class.dump_table(db, :articles, {})
+      expect(result).to include('Sequel::Migration')
+      expect(result).to include('articles')
+    end
+
+    it 'accepts a URL string' do
+      # sqlite::memory: opens a fresh empty DB on every connect, so dump_table
+      # (which opens its own connection) would see no tables. Use a temp file instead.
+      require 'tmpdir'
+      tmp_path = File.join(Dir.tmpdir, "tapsoob_schema_test_#{Process.pid}.db")
+      url = DbHelpers.adapt_url("sqlite://#{tmp_path}")
+      begin
+        tmp = DbHelpers.connect(url)
+        tmp.create_table(:t) { primary_key :id; String :v }
+        result = described_class.dump_table(url, :t, {})
+        expect(result).to include('Sequel::Migration')
+      ensure
+        DbHelpers.disconnect_all
+        File.delete(tmp_path) rescue nil
+      end
+    end
+  end
+
+  # ── load / round-trip ────────────────────────────────────────────────────────
+
+  describe '.load' do
+    it 'creates the table in the destination DB' do
+      schema_str = described_class.dump_table(db, :articles, {})
+
+      dest = connect_sqlite
+      dest.extension :schema_dumper
+      described_class.load(dest, schema_str)
+      expect(dest.table_exists?(:articles)).to be true
+      dest.disconnect
+    end
+
+    it 'drops then recreates table when drop: true' do
+      schema_str = described_class.dump_table(db, :articles, {})
+      dest = connect_sqlite
+      dest.extension :schema_dumper
+      described_class.load(dest, schema_str)
+      described_class.load(dest, schema_str, drop: true)
+      expect(dest.table_exists?(:articles)).to be true
+      dest.disconnect
+    end
+  end
+
+  # ── indexes_individual ───────────────────────────────────────────────────────
+
+  describe '.indexes_individual' do
+    let(:indexed_db) do
+      d = connect_sqlite
+      d.create_table(:idx_test) { primary_key :id; String :email, size: 100 }
+      d.add_index(:idx_test, :email)
+      d
+    end
+
+    after { indexed_db.disconnect }
+
+    it 'returns a JSON string' do
+      url = sqlite_memory_url
+      Sequel.connect(url) do |tmp|
+        tmp.create_table(:t) { primary_key :id; String :v }
+        tmp.add_index(:t, :v)
+        result = described_class.indexes_individual(url)
+        expect { JSON.parse(result) }.not_to raise_error
+      end
+    end
+  end
+
+  # ── reset_db_sequences ───────────────────────────────────────────────────────
+
+  describe '.reset_db_sequences' do
+    it 'runs without error on SQLite (which has no sequences)' do
+      expect {
+        described_class.reset_db_sequences(sqlite_memory_url)
+      }.not_to raise_error
+    end
+  end
+end

data/spec/unit/tapsoob/utils_spec.rb

@@ -0,0 +1,260 @@
+require 'spec_helper'
+require 'tapsoob/utils'
+
+RSpec.describe Tapsoob::Utils do
+  # ── checksum / valid_data ────────────────────────────────────────────────────
+
+  describe '.checksum' do
+    it 'returns a Zlib CRC32 integer' do
+      expect(described_class.checksum('hello')).to be_a(Integer)
+    end
+
+    it 'is deterministic for the same input' do
+      expect(described_class.checksum('test')).to eq(described_class.checksum('test'))
+    end
+
+    it 'differs for different inputs' do
+      expect(described_class.checksum('a')).not_to eq(described_class.checksum('b'))
+    end
+  end
+
+  describe '.valid_data?' do
+    let(:data) { 'some payload' }
+    let(:crc)  { described_class.checksum(data) }
+
+    it 'returns true when checksum matches' do
+      expect(described_class.valid_data?(data, crc)).to be true
+    end
+
+    it 'returns false when checksum does not match' do
+      expect(described_class.valid_data?(data, crc + 1)).to be false
+    end
+
+    it 'accepts crc as a string (mirrors production code path)' do
+      expect(described_class.valid_data?(data, crc.to_s)).to be true
+    end
+  end
+
+  # ── base64 round-trip ────────────────────────────────────────────────────────
+
+  describe '.base64encode / .base64decode' do
+    it 'round-trips plain text' do
+      text = 'Hello, Tapsoob!'
+      expect(described_class.base64decode(described_class.base64encode(text))).to eq(text)
+    end
+
+    it 'round-trips binary data faithfully' do
+      binary = Random.bytes(512)
+      decoded = described_class.base64decode(described_class.base64encode(binary))
+      expect(decoded.bytes).to eq(binary.bytes)
+    end
+
+    it 'round-trips empty string' do
+      expect(described_class.base64decode(described_class.base64encode(''))).to eq('')
+    end
+  end
+
+  # ── format_data ──────────────────────────────────────────────────────────────
+
+  describe '.format_data' do
+    let(:db) do
+      d = connect_sqlite
+      d.create_table(:fmt_test) do
+        primary_key :id
+        String  :name, size: 50
+        Integer :score
+        File    :payload
+      end
+      d
+    end
+
+    # Evaluate schema in the same let-chain so it always uses the same db instance.
+    let(:schema) { db.schema(:fmt_test) }
+
+    after { db.disconnect }
+
+    it 'returns {} for empty data' do
+      expect(described_class.format_data(db, [], schema: schema, table: :fmt_test)).to eq({})
+    end
+
+    it 'builds header, data, types for normal rows' do
+      data = [{ id: 1, name: 'Alice', score: 42, payload: nil }]
+      result = described_class.format_data(db, data, schema: schema, table: :fmt_test)
+      expect(result[:header]).to include(:id, :name, :score, :payload)
+      expect(result[:data].first).to include(1, 'Alice', 42)
+      expect(result[:types]).to include('integer', 'string')
+    end
+
+    it 'marks File columns as "blob" type' do
+      data = [{ id: 1, name: 'x', score: 0, payload: Sequel::SQL::Blob.new('bin') }]
+      result = described_class.format_data(db, data, schema: schema, table: :fmt_test)
+      payload_idx = result[:header].index(:payload)
+      expect(result[:types][payload_idx]).to eq('blob')
+    end
+
+    it 'converts Time objects to ISO-8601 strings' do
+      db2 = connect_sqlite
+      db2.extension :schema_dumper
+      db2.create_table(:ts_test) { primary_key :id; DateTime :ts }
+      t = Time.utc(2024, 6, 15, 12, 0, 0)
+      data = [{ id: 1, ts: t }]
+      result = described_class.format_data(db2, data,
+        schema: db2.schema(:ts_test), table: :ts_test)
+      ts_idx = result[:header].index(:ts)
+      expect(result[:data].first[ts_idx]).to eq('2024-06-15 12:00:00')
+      db2.disconnect
+    end
+  end
+
+  # ── encode_blobs ─────────────────────────────────────────────────────────────
+
+  describe '.encode_blobs' do
+    it 'base64-encodes Sequel::SQL::Blob values' do
+      blob = Sequel::SQL::Blob.new('binary data')
+      row  = { payload: blob }
+      described_class.encode_blobs(row, [:payload])
+      expect(row[:payload]).to eq(described_class.base64encode('binary data'))
+    end
+
+    it 'leaves non-blob string columns unchanged' do
+      row = { name: 'Alice' }
+      described_class.encode_blobs(row, [])
+      expect(row[:name]).to eq('Alice')
+    end
+
+    it 'encodes Blob values not in the columns list (auto-detect)' do
+      blob = Sequel::SQL::Blob.new('surprise')
+      row  = { payload: blob }
+      described_class.encode_blobs(row, [])
+      expect(row[:payload]).to eq(described_class.base64encode('surprise'))
+    end
+
+    it 'skips nil blob values' do
+      row = { payload: nil }
+      described_class.encode_blobs(row, [:payload])
+      expect(row[:payload]).to be_nil
+    end
+  end
+
+  # ── calculate_chunksize ──────────────────────────────────────────────────────
+
+  describe '.calculate_chunksize' do
+    it 'returns an Integer' do
+      result = described_class.calculate_chunksize(1000) { |_c| 0.5 }
+      expect(result).to be_a(Integer)
+    end
+
+    it 'increases chunksize when block returns fast time (< 0.8s)' do
+      result = described_class.calculate_chunksize(1000) { |_c| 0.1 }
+      expect(result).to be > 1000
+    end
+
+    it 'decreases chunksize when block returns slow time (> 3s)' do
+      # time_in_db must be near-zero so the real wall-clock diff drives the decision.
+      # We fake it by setting start/end times directly on the Chunksize object.
+      result = described_class.calculate_chunksize(900) do |c|
+        c.start_time = Time.now - 4.5  # pretend we started 4.5s ago
+        0.0                            # time_in_db ≈ 0 → diff = 4.5 > 3.0 → halve
+      end
+      expect(result).to be < 900
+    end
+
+    it 'retries up to 2 times on EPIPE and then re-raises' do
+      calls = 0
+      expect {
+        described_class.calculate_chunksize(1000) do |c|
+          calls += 1
+          raise Errno::EPIPE
+        end
+      }.to raise_error(Errno::EPIPE)
+      expect(calls).to eq(3) # initial + 2 retries
+    end
+  end
+
+  # ── primary_key / single_integer_primary_key / order_by ─────────────────────
+
+  describe 'primary key helpers' do
+    let(:db) do
+      d = connect_sqlite
+      d.create_table(:pk_test)  { primary_key :id; String :name }
+      d.create_table(:cpk_test) { String :a; String :b; primary_key [:a, :b] }
+      d.create_table(:nopk)     { String :x; String :y }
+      d
+    end
+
+    after { db.disconnect }
+
+    describe '.primary_key' do
+      it 'returns [:id] for a single-column integer PK' do
+        expect(described_class.primary_key(db, :pk_test)).to eq([:id])
+      end
+
+      it 'returns both columns for a composite PK' do
+        expect(described_class.primary_key(db, :cpk_test)).to match_array([:a, :b])
+      end
+    end
+
+    describe '.single_integer_primary_key' do
+      it 'is true for a single integer PK' do
+        expect(described_class.single_integer_primary_key(db, :pk_test)).to be true
+      end
+
+      it 'is false for composite PK' do
+        expect(described_class.single_integer_primary_key(db, :cpk_test)).to be false
+      end
+
+      it 'is false for a table with no PK' do
+        expect(described_class.single_integer_primary_key(db, :nopk)).to be false
+      end
+    end
+
+    describe '.order_by' do
+      it 'returns the PK column as an array for keyed tables' do
+        expect(described_class.order_by(db, :pk_test)).to eq([:id])
+      end
+
+      it 'returns all columns for tables without a single integer PK' do
+        result = described_class.order_by(db, :nopk)
+        expect(result).to match_array([:x, :y])
+      end
+    end
+  end
+
+  # ── export_rows / export_schema / export_indexes (filesystem) ────────────────
+
+  describe 'filesystem export helpers' do
+    let(:dir) { Dir.mktmpdir }
+    after { FileUtils.rm_rf(dir) }
+
+    describe '.export_rows' do
+      it 'appends NDJSON lines to the data file' do
+        FileUtils.mkdir_p(File.join(dir, 'data'))
+        rows = { table_name: :users, header: [:id, :name], data: [[1, 'Alice']] }
+        described_class.export_rows(dir, :users, rows)
+        described_class.export_rows(dir, :users, rows)
+        lines = File.readlines(File.join(dir, 'data', 'users.json'))
+        expect(lines.size).to eq(2)
+        expect(JSON.parse(lines.first)).to have_key('table_name')
+      end
+    end
+
+    describe '.export_schema' do
+      it 'writes the schema string to schemas/<table>.rb' do
+        FileUtils.mkdir_p(File.join(dir, 'schemas'))
+        described_class.export_schema(dir, :users, 'Class.new(Sequel::Migration) { def up; end }')
+        content = File.read(File.join(dir, 'schemas', 'users.rb'))
+        expect(content).to include('Sequel::Migration')
+      end
+    end
+
+    describe '.export_indexes' do
+      it 'appends index migration strings as NDJSON' do
+        FileUtils.mkdir_p(File.join(dir, 'indexes'))
+        described_class.export_indexes(dir, :users, 'add_index :users, :email')
+        described_class.export_indexes(dir, :users, 'add_index :users, :name')
+        lines = File.readlines(File.join(dir, 'indexes', 'users.json'))
+        expect(lines.size).to eq(2)
+      end
+    end
+  end
+end

data/spec/unit/tapsoob/version_spec.rb

@@ -0,0 +1,8 @@
+require 'spec_helper'
+require 'tapsoob/version'
+
+RSpec.describe Tapsoob do
+  it 'has a version string' do
+    expect(Tapsoob::VERSION).to match(/\A\d+\.\d+\.\d+\z/)
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: tapsoob
 version: !ruby/object:Gem::Version
-  version: 0.7.17
+  version: 0.8.0
 platform: ruby
 authors:
 - Félix Bellanger
@@ -125,9 +125,21 @@ files:
 - lib/tapsoob/utils.rb
 - lib/tapsoob/version.rb
 - lib/tasks/tapsoob.rake
-- spec/lib/tapsoob/chunksize_spec.rb
-- spec/lib/tapsoob/version_spec.rb
+- spec/integration/mysql_spec.rb
+- spec/integration/postgres_spec.rb
+- spec/integration/sqlite_spec.rb
 - spec/spec_helper.rb
+- spec/support/db_helpers.rb
+- spec/support/fixtures.rb
+- spec/support/round_trip_helper.rb
+- spec/support/shared_examples/round_trip.rb
+- spec/system/large_dataset_spec.rb
+- spec/unit/tapsoob/chunksize_spec.rb
+- spec/unit/tapsoob/data_stream_spec.rb
+- spec/unit/tapsoob/operation_base_spec.rb
+- spec/unit/tapsoob/schema_spec.rb
+- spec/unit/tapsoob/utils_spec.rb
+- spec/unit/tapsoob/version_spec.rb
 - tapsoob.gemspec
 homepage: https://github.com/Keeguon/tapsoob
 licenses:

data/spec/lib/tapsoob/chunksize_spec.rb

@@ -1,92 +0,0 @@
-require 'spec_helper'
-require 'tapsoob/chunksize'
-
-describe Tapsoob::Chunksize do
-  subject(:tapsoob) { Tapsoob::Chunksize.new(1) }
-  let(:chunksize) { double('chunksize') }
-  chunksize = Tapsoob::Chunksize.new(chunksize)
-  describe '#new' do
-    it 'works' do
-      result = Tapsoob::Chunksize.new(chunksize)
-      expect(result).not_to be_nil
-    end
-
-    describe '#initialize' do
-      it { should respond_to :chunksize }
-      it { should respond_to :idle_secs }
-      it { should respond_to :retries }
-    end
-  end
-
-  describe '#to_i' do
-
-    it { expect(tapsoob.to_i).to eq(1) }
-    it { expect(tapsoob.to_i).to be_a(Integer) }
-    it 'works' do
-      chunksize = Tapsoob::Chunksize.new(chunksize)
-      result = chunksize.to_i
-      expect(result).not_to be_nil
-    end
-
-    context 'converts to type integer' do
-      it { expect(tapsoob.to_i).to eq(1) }
-      it { expect(tapsoob.to_i).to be_an(Integer) }
-    end
-  end
-
-  describe '#reset_chunksize' do
-
-    context 'retries <= 1' do
-      it { expect(tapsoob.retries).to eq(0) }
-      it { expect(tapsoob.reset_chunksize).to eq(10) }
-    end
-
-    it 'works' do
-      chunksize = Tapsoob::Chunksize.new(chunksize)
-      result = chunksize.reset_chunksize
-      expect(result).not_to be_nil
-    end
-  end
-
-
-  describe '#diff' do
-    it 'works' do
-      chunksize = Tapsoob::Chunksize.new(chunksize)
-      chunksize.start_time = 1
-      chunksize.end_time = 10
-      chunksize.time_in_db = 2
-      chunksize.idle_secs = 3
-      result = chunksize.diff
-      expect(result).not_to be_nil
-    end
-  end
-
-  describe '#time_in_db=' do
-    it 'works' do
-      chunksize = Tapsoob::Chunksize.new(chunksize)
-      result = chunksize.time_in_db = (1)
-      expect(result).not_to be_nil
-    end
-  end
-
-  describe '#time_delta' do
-    it 'works' do
-      chunksize = double('chunksize')
-      chunksize = Tapsoob::Chunksize.new(chunksize)
-      result = chunksize.time_delta
-      expect(result).not_to be_nil
-    end
-  end
-
-  describe '#calc_new_chunksize' do
-    it 'works' do
-      chunksize = Tapsoob::Chunksize.new(1)
-      chunksize.start_time = 1
-      chunksize.end_time = 10
-      chunksize.time_in_db = 2
-      chunksize.idle_secs = 3
-      result = chunksize.calc_new_chunksize
-      expect(result).not_to be_nil
-    end
-  end
-end

data/spec/lib/tapsoob/version_spec.rb

@@ -1,7 +0,0 @@
-require 'spec_helper'
-
-describe Tapsoob::Chunksize do
-  it 'has a version number' do
-    expect(Tapsoob::Chunksize).not_to be nil
-  end
-end