tapsoob 0.8.5 → 0.8.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 64dd145a81db1be94164524f8ec22cef232c76d7a3d455646c5289496c8e1ee2
-  data.tar.gz: 678fe127e48278964db8475c2d8001a085fb7a56546b4912b6fdf08a90ea2718
+  metadata.gz: 612a3ebed6eb1091017de69eafd6907418e3d23685d0dfd0e88c1ec4fc1589ab
+  data.tar.gz: 4de16885893a76191c7e372f36884ae34af6c4b87f629c18d568f9371a79e6ac
 SHA512:
-  metadata.gz: 4263e7bdec44a5c31af5ebeba0fe1cc2b969e69f93a206a2c1b580a8141583eb94fd305626acc923d7499b20f38871294e41e1cdb9e14666fc2581e2fbbe8966
-  data.tar.gz: ebc49b6130cdcaa98ffe70405e8aac442012e451409a5d9e902c7c0518cf23dd7c3a158c850aac9392a9b349bae004b324efa42ad688d5dca10fd260f663fc2c
+  metadata.gz: 474583bb8a802d100f6603f4de125e60ed9583addf961252a3bc2496e00174d9aed54012c317a3df92077db5d806b3a4f700541d17bd7a5b134bd7c3162e86c5
+  data.tar.gz: b65389a1279e6b86bf132e0f2878913b868a91e49405eeebfaf4c998f941f0dadbf80b8f74d5c388aa90c7dcd45473577f1619d64e5ccf6a988c46b2b694d8f0

data/lib/tapsoob/cli/schema.rb

@@ -29,7 +29,7 @@ module Tapsoob
 
       desc "dump_table DATABASE_URL TABLE", "Dump a table from a database using a database URL"
       def dump_table(database_url, table)
-        puts Tapsoob::Schema.dump_table(database_url, table)
+        puts Tapsoob::Schema.dump_table(database_url, table, {})
       end
 
       desc "foreign_keys DATABASE_URL", "Dump foreign_keys from a database using a database URL"

data/lib/tapsoob/schema.rb

@@ -11,9 +11,9 @@ module Tapsoob
     extend self
 
     def dump(database_url, options = {})
-      db = Sequel.connect(database_url)
-      db.extension :schema_dumper
-      template = ERB.new <<-END_MIG
+      Sequel.connect(database_url) do |db|
+        db.extension :schema_dumper
+        template = ERB.new <<-END_MIG
 Class.new(Sequel::Migration) do
   def up
   <% db.send(:sort_dumped_tables, db.tables, {}).each do |table| %>
@@ -29,7 +29,8 @@ Class.new(Sequel::Migration) do
 end
 END_MIG
 
-      template.result(binding)
+        template.result(binding)
+      end
     end
 
     def dump_table(database_url_or_db, table, options)
@@ -68,15 +69,17 @@ END_MIG
     end
 
     def foreign_keys(database_url)
-      db = Sequel.connect(database_url)
-      db.extension :schema_dumper
-      db.dump_foreign_key_migration
+      Sequel.connect(database_url) do |db|
+        db.extension :schema_dumper
+        db.dump_foreign_key_migration
+      end
     end
 
     def indexes(database_url)
-      db = Sequel.connect(database_url)
-      db.extension :schema_dumper
-      db.dump_indexes_migration
+      Sequel.connect(database_url) do |db|
+        db.extension :schema_dumper
+        db.dump_indexes_migration
+      end
     end
 
     def indexes_individual(database_url)
@@ -105,6 +108,7 @@ END_MIG
     end
 
     def load(database_url_or_db, schema, options = { drop: false })
+      schema = rewrite_non_integer_primary_keys(schema)
       # Accept either a database URL or an existing connection object
       if database_url_or_db.is_a?(Sequel::Database)
         db = database_url_or_db
@@ -158,6 +162,25 @@ END_MIG
       end
     end
 
+    NON_INTEGER_PK_PATTERN = /^(\s*)primary_key\s+(:?\w+),\s*:type=>"([^"]+)"(.*)$/
+    INTEGER_DB_TYPES       = /\A(?:int(?:eger|\d+)?|bigint|smallint|serial|bigserial|smallserial)/i
+
+    # On PG 10+, Sequel's CreateTableGenerator injects `identity: true` into
+    # every primary_key call via serial_primary_key_options. PG rejects IDENTITY
+    # on non-integer types. Rewrite `primary_key :col, :type=>"varchar..."` to
+    # `column :col, "varchar...", primary_key: true, null: false` which bypasses
+    # that code path entirely.
+    def rewrite_non_integer_primary_keys(schema_str)
+      schema_str.gsub(NON_INTEGER_PK_PATTERN) do
+        indent, col, db_type, rest = $1, $2, $3, $4
+        if db_type =~ INTEGER_DB_TYPES
+          "#{indent}primary_key #{col}, :type=>\"#{db_type}\"#{rest}"
+        else
+          "#{indent}column #{col}, \"#{db_type}\", primary_key: true, null: false#{rest}"
+        end
+      end
+    end
+
     def reset_db_sequences(database_url)
       Sequel.connect(database_url) do |db|
         db.extension :schema_dumper

data/lib/tapsoob/version.rb

@@ -1,4 +1,4 @@
 # -*- encoding : utf-8 -*-
 module Tapsoob
-  VERSION = "0.8.5".freeze
+  VERSION = "0.8.7".freeze
 end

data/spec/integration/postgres_spec.rb

@@ -105,12 +105,24 @@ RSpec.describe 'PostgreSQL round-trip', :integration do
 
     after(:each) do
       @src_db.run("DROP TABLE IF EXISTS varchar_pk_table")
+      @dst_db.run("DROP TABLE IF EXISTS varchar_pk_table")
     end
 
     it 'skips reset without logging a warning (no sequence attached to varchar PK)' do
       expect(Tapsoob.log).not_to receive(:warn)
       Tapsoob::Schema.reset_db_sequences(@src_url)
     end
+
+    it 'round-trips the table without identity column errors' do
+      dump_dir = Dir.mktmpdir
+      begin
+        pull(src_url, dump_dir)
+        expect { push(dst_url, dump_dir) }.not_to raise_error
+        expect(dst_db[:varchar_pk_table].where(id: 'abc-123').count).to eq(1)
+      ensure
+        FileUtils.rm_rf(dump_dir)
+      end
+    end
   end
 
   context 'when reset_primary_key_sequence raises a DatabaseError' do

data/spec/spec_helper.rb

@@ -45,6 +45,7 @@ RSpec.configure do |config|
   # Integration tests require a real DB — skip unless env vars are set.
   config.filter_run_excluding :integration unless ENV['INTEGRATION_TESTS'] || ENV['SRC_DATABASE_URL']
 
-  config.include DbHelpers,       :integration
-  config.include RoundTripHelper, :integration
+  config.include DbHelpers,        :integration
+  config.include RoundTripHelper,  :integration
+  config.include OperationHelpers
 end

data/spec/support/operation_helpers.rb

@@ -0,0 +1,43 @@
+require 'tapsoob/operation/pull'
+require 'tapsoob/operation/push'
+
+# Shared helpers for unit specs that exercise Pull / Push / Base.
+# Included automatically via spec_helper for all unit specs.
+module OperationHelpers
+  # Default opts used across pull/push/base unit tests.
+  UNIT_OPTS = {
+    data:              true,
+    schema:            true,
+    indexes:           false,
+    progress:          false,
+    default_chunksize: 1000,
+    no_split:          true,
+  }.freeze
+
+  # A pre-seeded in-memory SQLite DB with :users (5 rows) and :widgets (3 rows).
+  # Returns a new connection each call — callers own disconnection.
+  def seeded_sqlite_db
+    d = connect_sqlite
+    d.create_table(:users)   { primary_key :id; String :name }
+    d.create_table(:widgets) { primary_key :id; Integer :qty }
+    5.times { |i| d[:users].insert(name: "user_#{i}") }
+    3.times { |i| d[:widgets].insert(qty: i * 10) }
+    d
+  end
+
+  def build_pull(db, dump_dir, extra_opts = {})
+    op = Tapsoob::Operation::Pull.new(sqlite_memory_url, dump_dir, UNIT_OPTS.merge(extra_opts))
+    op.instance_variable_set(:@db, db)
+    op
+  end
+
+  def build_push(db_url, dump_dir, extra_opts = {})
+    Tapsoob::Operation::Push.new(db_url, dump_dir, UNIT_OPTS.merge(extra_opts))
+  end
+
+  def build_base(db, dump_dir, extra_opts = {})
+    op = Tapsoob::Operation::Pull.new(sqlite_memory_url, dump_dir, UNIT_OPTS.merge(extra_opts))
+    op.instance_variable_set(:@db, db)
+    op
+  end
+end

data/spec/unit/tapsoob/base_spec.rb

@@ -0,0 +1,222 @@
+require 'spec_helper'
+require 'tapsoob/operation/base'
+require 'tapsoob/operation/pull'
+require 'tapsoob/operation/push'
+
+RSpec.describe Tapsoob::Operation::Base do
+  let(:db)       { seeded_sqlite_db }
+  let(:dump_dir) { Dir.mktmpdir("tapsoob_base_") }
+
+  after do
+    db.disconnect
+    FileUtils.rm_rf(dump_dir)
+  end
+
+  # ── format_number ────────────────────────────────────────────────────────────
+
+  describe '#format_number' do
+    it 'formats numbers with commas' do
+      op = build_base(db, dump_dir)
+      expect(op.format_number(1_000_000)).to eq("1,000,000")
+      expect(op.format_number(1234)).to eq("1,234")
+      expect(op.format_number(999)).to eq("999")
+    end
+  end
+
+  # ── resuming? ────────────────────────────────────────────────────────────────
+
+  describe '#resuming?' do
+    it 'returns false by default' do
+      expect(build_base(db, dump_dir).resuming?).to be false
+    end
+
+    it 'returns true when :resume is set' do
+      expect(build_base(db, dump_dir, resume: true).resuming?).to be true
+    end
+  end
+
+  # ── parallel? / parallel_workers ─────────────────────────────────────────────
+
+  describe '#parallel?' do
+    it 'returns false when parallel is 1' do
+      expect(build_base(db, dump_dir, parallel: 1).parallel?).to be false
+    end
+
+    it 'returns true when parallel > 1' do
+      expect(build_base(db, dump_dir, parallel: 2).parallel?).to be true
+    end
+  end
+
+  describe '#parallel_workers' do
+    it 'defaults to 1' do
+      expect(build_base(db, dump_dir).parallel_workers).to eq(1)
+    end
+
+    it 'returns the requested count' do
+      expect(build_base(db, dump_dir, parallel: 4).parallel_workers).to eq(4)
+    end
+  end
+
+  # ── table_parallel_workers ───────────────────────────────────────────────────
+
+  describe '#table_parallel_workers' do
+    it 'returns 1 when no_split is set' do
+      expect(build_base(db, dump_dir, no_split: true).table_parallel_workers(:users, 5_000_000)).to eq(1)
+    end
+
+    it 'returns 1 when dump_path is nil' do
+      op = Tapsoob::Operation::Pull.new(sqlite_memory_url, nil, { default_chunksize: 1000 })
+      expect(op.table_parallel_workers(:users, 5_000_000)).to eq(1)
+    end
+
+    it 'returns 1 when row_count is below threshold' do
+      expect(build_base(db, dump_dir, no_split: false).table_parallel_workers(:users, 50_000)).to eq(1)
+    end
+
+    it 'returns >= 2 for a very large table' do
+      expect(build_base(db, dump_dir, no_split: false).table_parallel_workers(:users, 5_000_000)).to be >= 2
+    end
+
+    it 'returns >= 2 for a 1M+ row table' do
+      expect(build_base(db, dump_dir, no_split: false).table_parallel_workers(:users, 1_000_000)).to be >= 2
+    end
+
+    it 'returns >= 2 for a 500K+ row table' do
+      expect(build_base(db, dump_dir, no_split: false).table_parallel_workers(:users, 500_000)).to be >= 2
+    end
+
+    it 'returns 2 for a table just over the 100K threshold' do
+      expect(build_base(db, dump_dir, no_split: false).table_parallel_workers(:users, 150_000)).to eq(2)
+    end
+  end
+
+  # ── stream_state ─────────────────────────────────────────────────────────────
+
+  describe '#stream_state / #stream_state=' do
+    it 'defaults to empty hash' do
+      expect(build_base(db, dump_dir).stream_state).to eq({})
+    end
+
+    it 'stores and retrieves state' do
+      op = build_base(db, dump_dir)
+      op.stream_state = { table_name: :users }
+      expect(op.stream_state).to eq({ table_name: :users })
+    end
+  end
+
+  # ── add_completed_table ──────────────────────────────────────────────────────
+
+  describe '#add_completed_table' do
+    it 'appends to completed_tables thread-safely' do
+      op = build_base(db, dump_dir)
+      op.add_completed_table(:users)
+      op.add_completed_table(:widgets)
+      expect(op.completed_tables).to include("users", "widgets")
+    end
+  end
+
+  # ── max_intra_table_workers ──────────────────────────────────────────────────
+
+  describe '#max_intra_table_workers' do
+    it 'returns at least 2' do
+      expect(build_base(db, dump_dir).max_intra_table_workers).to be >= 2
+    end
+  end
+
+  # ── catch_errors ─────────────────────────────────────────────────────────────
+
+  describe '#catch_errors' do
+    it 'yields and returns the block result' do
+      expect(build_base(db, dump_dir).send(:catch_errors) { 42 }).to eq(42)
+    end
+
+    it 're-raises exceptions' do
+      op = build_base(db, dump_dir)
+      expect { op.send(:catch_errors) { raise ArgumentError, "boom" } }.to raise_error(ArgumentError, "boom")
+    end
+  end
+
+  # ── apply_table_filter (array form) ──────────────────────────────────────────
+
+  describe '#apply_table_filter' do
+    it 'filters an array by table_filter' do
+      op = build_base(db, dump_dir, tables: ["users"])
+      expect(op.apply_table_filter(["users", "widgets"])).to eq(["users"])
+    end
+
+    it 'excludes tables from an array' do
+      op = build_base(db, dump_dir, exclude_tables: ["widgets"])
+      expect(op.apply_table_filter(["users", "widgets"])).to eq(["users"])
+    end
+  end
+
+  # ── Base.factory ─────────────────────────────────────────────────────────────
+
+  describe '.factory' do
+    it 'returns a Pull instance for :pull type' do
+      expect(described_class.factory(:pull, sqlite_memory_url, dump_dir, { default_chunksize: 1000 })).to be_a(Tapsoob::Operation::Pull)
+    end
+
+    it 'returns a Push instance for :push type' do
+      expect(described_class.factory(:push, sqlite_memory_url, dump_dir, { default_chunksize: 1000 })).to be_a(Tapsoob::Operation::Push)
+    end
+
+    it 'raises for unknown type' do
+      expect { described_class.factory(:unknown, sqlite_memory_url, dump_dir, {}) }
+        .to raise_error(RuntimeError, /Unknown Operation Type/)
+    end
+
+    it 'returns a resume instance when opts[:resume] is true' do
+      op = build_pull(db, dump_dir)
+      op.initialize_dump_directory
+      op.pull_schema
+
+      # Pull#to_hash calls remote_tables_info which requires an active pull run;
+      # use the base to_hash binding to get just the serializable fields.
+      hash = Tapsoob::Operation::Base.instance_method(:to_hash).bind(op).call
+      resumed = described_class.factory(:pull, sqlite_memory_url, dump_dir,
+        hash.merge(resume: true, klass: "Tapsoob::Operation::Pull", default_chunksize: 1000))
+      expect(resumed).to be_a(Tapsoob::Operation::Pull)
+    end
+  end
+
+  # ── exiting? / setup_signal_trap ─────────────────────────────────────────────
+
+  describe '#exiting?' do
+    it 'returns false initially' do
+      expect(build_base(db, dump_dir).exiting?).to be false
+    end
+  end
+
+  describe '#setup_signal_trap' do
+    it 'registers signal handlers without error' do
+      op = build_base(db, dump_dir)
+      expect { op.setup_signal_trap }.not_to raise_error
+    end
+  end
+
+  # ── can_use_pk_partitioning? ─────────────────────────────────────────────────
+
+  describe '#can_use_pk_partitioning?' do
+    it 'returns true for a table with a single integer PK' do
+      op = build_base(db, dump_dir)
+      expect(op.can_use_pk_partitioning?(:users)).to be true
+    end
+  end
+
+  # ── db / default_chunksize ───────────────────────────────────────────────────
+
+  describe '#default_chunksize' do
+    it 'returns the value from opts' do
+      expect(build_base(db, dump_dir, default_chunksize: 500).default_chunksize).to eq(500)
+    end
+  end
+
+  describe '#table_filter / #exclude_tables' do
+    it 'returns empty arrays by default' do
+      op = build_base(db, dump_dir)
+      expect(op.table_filter).to eq([])
+      expect(op.exclude_tables).to eq([])
+    end
+  end
+end