tapsoob 0.8.6 → 0.8.7

data/spec/unit/tapsoob/progress_spec.rb

@@ -0,0 +1,335 @@
+require 'spec_helper'
+require 'tapsoob/progress'
+
+RSpec.describe Tapsoob::Progress do
+  # All output is redirected to StringIO so nothing hits stdout/terminal.
+
+  let(:out) { StringIO.new }
+
+  # ── Bar ──────────────────────────────────────────────────────────────────────
+
+  describe Tapsoob::Progress::Bar do
+    subject(:bar) { described_class.new("Test", 100, out) }
+
+    describe '#initialize' do
+      it 'starts at 0' do
+        expect(bar.current).to eq(0)
+      end
+
+      it 'stores total' do
+        expect(bar.total).to eq(100)
+      end
+
+      it 'stores title' do
+        expect(bar.title).to eq("Test")
+      end
+    end
+
+    describe '#inc' do
+      it 'advances current by 1 by default' do
+        bar.inc
+        expect(bar.current).to eq(1)
+      end
+
+      it 'advances current by a custom step' do
+        bar.inc(10)
+        expect(bar.current).to eq(10)
+      end
+
+      it 'clamps current to total' do
+        bar.inc(200)
+        expect(bar.current).to eq(100)
+      end
+    end
+
+    describe '#set' do
+      it 'sets current to the given value' do
+        bar.set(42)
+        expect(bar.current).to eq(42)
+      end
+
+      it 'raises on negative value' do
+        expect { bar.set(-1) }.to raise_error(RuntimeError)
+      end
+
+      it 'raises when value exceeds total' do
+        expect { bar.set(101) }.to raise_error(RuntimeError)
+      end
+    end
+
+    describe '#finish' do
+      it 'sets current to total and marks finished' do
+        bar.finish
+        expect(bar.current).to eq(100)
+        expect(bar.finished?).to be true
+      end
+    end
+
+    describe '#halt' do
+      it 'marks finished without changing current' do
+        bar.inc(30)
+        bar.halt
+        expect(bar.finished?).to be true
+        expect(bar.current).to eq(30)
+      end
+    end
+
+    describe '#inspect' do
+      it 'includes current/total' do
+        expect(bar.inspect).to include('0/100')
+      end
+    end
+
+    describe '#file_transfer_mode' do
+      it 'switches format_arguments to stat_for_file_transfer' do
+        bar.file_transfer_mode
+        expect(bar.instance_variable_get(:@format_arguments)).to include(:stat_for_file_transfer)
+      end
+    end
+
+    describe '#format=' do
+      it 'updates the format string' do
+        bar.format = "custom %s"
+        expect(bar.instance_variable_get(:@format)).to eq("custom %s")
+      end
+    end
+
+    describe '#format_arguments=' do
+      it 'updates format arguments' do
+        bar.format_arguments = [:title, :percentage]
+        expect(bar.instance_variable_get(:@format_arguments)).to eq([:title, :percentage])
+      end
+    end
+
+    describe 'convert_bytes (via fmt_stat_for_file_transfer)' do
+      it 'renders correctly in file_transfer_mode after finishing' do
+        bar.file_transfer_mode
+        bar.inc(50)
+        bar.finish
+        expect(out.string).not_to be_empty
+      end
+
+      it 'renders KB range correctly' do
+        b = described_class.new("KB", 2048, out)
+        b.file_transfer_mode
+        b.inc(2048)
+        b.finish
+        expect(out.string).to match(/KB/)
+      end
+
+      it 'renders MB range correctly' do
+        b = described_class.new("MB", 2 * 1024 * 1024, out)
+        b.file_transfer_mode
+        b.inc(2 * 1024 * 1024)
+        b.finish
+        expect(out.string).to match(/MB/)
+      end
+    end
+
+    describe 'zero total' do
+      subject(:bar) { described_class.new("Empty", 0, out) }
+
+      it 'does not raise on init' do
+        expect { bar }.not_to raise_error
+      end
+
+      it 'finish does not raise' do
+        expect { bar.finish }.not_to raise_error
+      end
+    end
+  end
+
+  # ── ReversedBar ──────────────────────────────────────────────────────────────
+
+  describe Tapsoob::Progress::ReversedBar do
+    subject(:bar) { described_class.new("Rev", 100, out) }
+
+    it 'inherits from Bar' do
+      expect(bar).to be_a(Tapsoob::Progress::Bar)
+    end
+
+    it 'reports inverted percentage internally (100 at start)' do
+      # do_percentage is private; test via inspect indirectly - just ensure no raise
+      expect { bar.inc(10) }.not_to raise_error
+    end
+  end
+
+  # ── ThreadSafeBar ────────────────────────────────────────────────────────────
+
+  describe Tapsoob::Progress::ThreadSafeBar do
+    let(:multi) do
+      mb = instance_double(Tapsoob::Progress::MultiBar)
+      allow(mb).to receive(:update)
+      allow(mb).to receive(:finish_bar)
+      allow(mb).to receive(:max_title_width).and_return(14)
+      mb
+    end
+
+    subject(:bar) { described_class.new("Worker", 50, multi) }
+
+    describe '#initialize' do
+      it 'stores title and total' do
+        expect(bar.title).to eq("Worker")
+        expect(bar.total).to eq(50)
+      end
+    end
+
+    describe '#inc' do
+      it 'increments current and notifies multi' do
+        expect(multi).to receive(:update)
+        bar.inc(5)
+        expect(bar.current).to eq(5)
+      end
+    end
+
+    describe '#finish' do
+      it 'sets current to total and calls finish_bar on multi' do
+        expect(multi).to receive(:finish_bar).with(bar)
+        bar.finish
+        expect(bar.current).to eq(50)
+      end
+    end
+
+    describe '#mark_finished / #finished?' do
+      it 'returns true after mark_finished' do
+        bar.mark_finished
+        expect(bar.finished?).to be true
+      end
+    end
+
+    describe '#render_to' do
+      it 'writes something to the given output' do
+        io = StringIO.new
+        bar.render_to(io)
+        expect(io.string).not_to be_empty
+      end
+    end
+
+    describe '#clear' do
+      it 'is a no-op (does not raise)' do
+        expect { bar.send(:clear) }.not_to raise_error
+      end
+    end
+  end
+
+  # ── MultiBar ─────────────────────────────────────────────────────────────────
+
+  describe Tapsoob::Progress::MultiBar do
+    subject(:multi) { described_class.new(2) }
+
+    describe '#initialize' do
+      it 'starts with no bars' do
+        expect(multi.instance_variable_get(:@bars)).to be_empty
+      end
+    end
+
+    describe '#create_bar' do
+      it 'returns a ThreadSafeBar' do
+        bar = multi.create_bar("Table1", 100)
+        expect(bar).to be_a(Tapsoob::Progress::ThreadSafeBar)
+      end
+
+      it 'does not create duplicate bars for the same title' do
+        multi.create_bar("Table1", 100)
+        multi.create_bar("Table1", 200)
+        bars = multi.instance_variable_get(:@bars)
+        expect(bars.count { |b| b.title == "Table1" }).to eq(1)
+      end
+    end
+
+    describe '#set_info' do
+      it 'does not raise' do
+        expect { multi.set_info("Processing...") }.not_to raise_error
+      end
+    end
+
+    describe '#stop' do
+      it 'deactivates the multi bar' do
+        multi.stop
+        expect(multi.instance_variable_get(:@active)).to be false
+      end
+
+      it 'is idempotent' do
+        multi.stop
+        expect { multi.stop }.not_to raise_error
+      end
+    end
+
+    describe '#max_title_width' do
+      it 'grows to accommodate longer titles' do
+        multi.create_bar("ShortTitle", 10)
+        multi.create_bar("A Much Longer Title Here", 10)
+        expect(multi.max_title_width).to be >= "A Much Longer Title Here".length
+      end
+    end
+
+    # ── rendering paths (drive render_active_display / redraw_all) ──────────────
+
+    describe '#update' do
+      it 'does not raise when active and initialized with bars' do
+        out = StringIO.new
+        multi.instance_variable_set(:@out, out)
+        multi.instance_variable_set(:@initialized, true)
+        multi.instance_variable_set(:@total_lines, 4)
+        multi.instance_variable_set(:@last_update, Time.now - 1)
+        bar = multi.create_bar("UpdateTable", 10)
+        expect { multi.update }.not_to raise_error
+      end
+
+      it 'is a no-op when @active is false' do
+        multi.stop
+        expect { multi.update }.not_to raise_error
+      end
+    end
+
+    describe '#finish_bar' do
+      it 'marks the bar as finished without raising' do
+        out = StringIO.new
+        multi.instance_variable_set(:@out, out)
+        multi.instance_variable_set(:@initialized, true)
+        multi.instance_variable_set(:@total_lines, 4)
+        multi.instance_variable_set(:@last_update, Time.now - 1)
+        bar = multi.create_bar("FinishTable", 5)
+        expect { multi.finish_bar(bar) }.not_to raise_error
+        expect(bar.instance_variable_get(:@finished_p)).to be true
+      end
+    end
+
+    describe '#set_info (with rendering)' do
+      it 'triggers redraw when initialized and a bar exists' do
+        out = StringIO.new
+        multi.instance_variable_set(:@out, out)
+        multi.instance_variable_set(:@initialized, true)
+        multi.instance_variable_set(:@total_lines, 4)
+        multi.instance_variable_set(:@last_update, Time.now - 1)
+        multi.create_bar("InfoTable", 10)
+        expect { multi.set_info("doing things") }.not_to raise_error
+      end
+    end
+
+    describe '#stop (initialized)' do
+      it 'clears terminal lines when previously initialized' do
+        out = StringIO.new
+        multi.instance_variable_set(:@out, out)
+        multi.instance_variable_set(:@initialized, true)
+        multi.instance_variable_set(:@total_lines, 4)
+        multi.stop
+        expect(out.string).to include("\e[")
+      end
+    end
+
+    describe 'render_active_display (via update)' do
+      it 'writes escape sequences to output when bars exist' do
+        out = StringIO.new
+        multi.instance_variable_set(:@out, out)
+        multi.instance_variable_set(:@initialized, true)
+        multi.instance_variable_set(:@total_lines, 4)
+        multi.instance_variable_set(:@last_update, Time.now - 1)
+        multi.instance_variable_set(:@info_message, "some info")
+        multi.create_bar("RenderTable", 10)
+        multi.update
+        expect(out.string).not_to be_empty
+      end
+    end
+  end
+end

data/spec/unit/tapsoob/pull_spec.rb

@@ -0,0 +1,335 @@
+require 'spec_helper'
+require 'tapsoob/operation/pull'
+
+RSpec.describe Tapsoob::Operation::Pull do
+  let(:db)       { seeded_sqlite_db }
+  let(:dump_dir) { Dir.mktmpdir("tapsoob_pull_") }
+
+  after do
+    db.disconnect
+    FileUtils.rm_rf(dump_dir)
+  end
+
+  # ── initialize_dump_directory ────────────────────────────────────────────────
+
+  describe '#initialize_dump_directory' do
+    it 'creates data, schemas, and indexes subdirectories' do
+      build_pull(db, dump_dir).initialize_dump_directory
+      %w[data schemas indexes].each do |sub|
+        expect(File.directory?(File.join(dump_dir, sub))).to be true
+      end
+    end
+
+    it 'removes table_order.txt if present' do
+      order_file = File.join(dump_dir, "table_order.txt")
+      File.write(order_file, "users\n")
+      build_pull(db, dump_dir).initialize_dump_directory
+      expect(File.exist?(order_file)).to be false
+    end
+
+    it 'cleans existing subdirectory contents' do
+      stale = File.join(dump_dir, "data", "old_table.json")
+      FileUtils.mkdir_p(File.dirname(stale))
+      File.write(stale, "stale")
+      build_pull(db, dump_dir).initialize_dump_directory
+      expect(File.exist?(stale)).to be false
+    end
+  end
+
+  # ── fetch_tables_info ────────────────────────────────────────────────────────
+
+  describe '#fetch_tables_info' do
+    it 'returns a hash of table_name => count (symbol keys from Sequel)' do
+      expect(build_pull(db, dump_dir).fetch_tables_info).to include(:users => 5, :widgets => 3)
+    end
+
+    it 'applies table_filter when provided' do
+      info = build_pull(db, dump_dir, tables: ["users"]).fetch_tables_info
+      expect(info.keys.map(&:to_s)).to contain_exactly("users")
+    end
+
+    it 'applies exclude_tables when provided' do
+      info = build_pull(db, dump_dir, exclude_tables: ["widgets"]).fetch_tables_info
+      expect(info.keys.map(&:to_s)).not_to include("widgets")
+    end
+  end
+
+  # ── tables / record_count ────────────────────────────────────────────────────
+
+  describe '#tables' do
+    it 'excludes already-completed tables' do
+      op = build_pull(db, dump_dir)
+      op.opts[:completed_tables] = ["users"]
+      expect(op.tables.keys).not_to include("users")
+    end
+  end
+
+  describe '#record_count' do
+    it 'sums all table counts' do
+      expect(build_pull(db, dump_dir).record_count).to eq(8)
+    end
+  end
+
+  # ── pull_schema ──────────────────────────────────────────────────────────────
+
+  describe '#pull_schema' do
+    before { build_pull(db, dump_dir).initialize_dump_directory }
+
+    it 'writes a schema file for each table' do
+      op = build_pull(db, dump_dir)
+      op.pull_schema
+      %w[users widgets].each do |t|
+        schema_file = File.join(dump_dir, "schemas", "#{t}.rb")
+        expect(File.exist?(schema_file)).to be true
+        expect(File.size(schema_file)).to be > 0
+      end
+    end
+
+    it 'writes table_order.txt listing all tables' do
+      op = build_pull(db, dump_dir)
+      op.pull_schema
+      content = File.read(File.join(dump_dir, "table_order.txt"))
+      %w[users widgets].each { |t| expect(content).to include(t) }
+    end
+  end
+
+  # ── pull_data_serial ─────────────────────────────────────────────────────────
+
+  describe '#pull_data_serial' do
+    before do
+      op = build_pull(db, dump_dir)
+      op.initialize_dump_directory
+      op.pull_schema
+    end
+
+    it 'writes NDJSON data files for each table' do
+      op = build_pull(db, dump_dir)
+      op.pull_data_serial
+      %w[users widgets].each do |t|
+        data_file = File.join(dump_dir, "data", "#{t}.json")
+        expect(File.exist?(data_file)).to be true
+        parsed = JSON.parse(File.readlines(data_file).first.strip)
+        expect(parsed).to have_key("data")
+      end
+    end
+
+    it 'marks tables as completed after writing' do
+      op = build_pull(db, dump_dir)
+      op.pull_data_serial
+      expect(op.completed_tables).to include("users", "widgets")
+    end
+  end
+
+  # ── save_table_order / load_table_order ──────────────────────────────────────
+
+  describe '#save_table_order / #load_table_order' do
+    it 'round-trips table names through the order file' do
+      op = build_pull(db, dump_dir)
+      op.save_table_order(["users", "widgets"])
+      expect(op.load_table_order).to eq(["users", "widgets"])
+    end
+  end
+
+  # ── Base#to_hash ─────────────────────────────────────────────────────────────
+
+  describe '#to_hash (base fields)' do
+    it 'includes klass and database_url keys' do
+      op   = build_pull(db, dump_dir)
+      hash = Tapsoob::Operation::Base.instance_method(:to_hash).bind(op).call
+      expect(hash).to have_key(:klass)
+      expect(hash).to have_key(:database_url)
+    end
+  end
+
+  # ── apply_table_filter ───────────────────────────────────────────────────────
+
+  describe '#apply_table_filter' do
+    it 'passes all tables when no filter is set' do
+      op    = build_pull(db, dump_dir)
+      input = { "users" => 5, "widgets" => 3 }
+      expect(op.apply_table_filter(input)).to eq(input)
+    end
+
+    it 'selects only filtered tables' do
+      op = build_pull(db, dump_dir, tables: ["users"])
+      expect(op.apply_table_filter({ "users" => 5, "widgets" => 3 })).to eq("users" => 5)
+    end
+  end
+
+  # ── pull_data (routing) ──────────────────────────────────────────────────────
+
+  describe '#pull_data' do
+    before do
+      op = build_pull(db, dump_dir)
+      op.initialize_dump_directory
+      op.pull_schema
+    end
+
+    it 'runs serial path by default' do
+      op = build_pull(db, dump_dir)
+      op.pull_data
+      expect(File.exist?(File.join(dump_dir, "data", "users.json"))).to be true
+    end
+
+    it 'runs parallel path when parallel > 1' do
+      # In-memory SQLite is not shared across threads on JRuby (each JDBC connection
+      # is isolated). Use a file-backed DB so parallel workers can all connect to it.
+      db_path = File.join(Dir.tmpdir, "tapsoob_pull_parallel_#{Process.pid}.db")
+      db_url  = DbHelpers.adapt_url("sqlite://#{db_path}")
+      file_db = Sequel.connect(db_url)
+      file_db.extension :schema_dumper
+      file_db.create_table(:users)   { primary_key :id; String :name }
+      file_db.create_table(:widgets) { primary_key :id; Integer :qty }
+      5.times { |i| file_db[:users].insert(name: "user_#{i}") }
+      3.times { |i| file_db[:widgets].insert(qty: i * 10) }
+
+      begin
+        op = Tapsoob::Operation::Pull.new(db_url, dump_dir, OperationHelpers::UNIT_OPTS.merge(parallel: 2, no_split: true))
+        op.pull_data
+        expect(File.exist?(File.join(dump_dir, "data", "users.json"))).to be true
+      ensure
+        file_db.disconnect rescue nil
+        File.delete(db_path) rescue nil
+      end
+    end
+  end
+
+  # ── pull_reset_sequences ─────────────────────────────────────────────────────
+
+  describe '#pull_reset_sequences' do
+    it 'runs without error on SQLite' do
+      op = build_pull(db, dump_dir)
+      expect { op.pull_reset_sequences }.not_to raise_error
+    end
+  end
+
+  # ── pull_indexes ─────────────────────────────────────────────────────────────
+
+  describe '#pull_indexes' do
+    before do
+      op = build_pull(db, dump_dir)
+      op.initialize_dump_directory
+    end
+
+    it 'writes index files to the indexes directory' do
+      op = build_pull(db, dump_dir)
+      op.pull_indexes
+      expect(File.directory?(File.join(dump_dir, "indexes"))).to be true
+    end
+  end
+
+  # ── store_session ────────────────────────────────────────────────────────────
+
+  describe '#store_session' do
+    it 'writes a .dat session file to the current directory' do
+      op = build_pull(db, dump_dir)
+      session_file = nil
+      begin
+        # Pull#to_hash calls remote_tables_info which requires an active pull run;
+        # call Base#store_session logic directly via the base to_hash binding.
+        base_hash = Tapsoob::Operation::Base.instance_method(:to_hash).bind(op).call
+        file = "pull_#{Time.now.strftime("%Y%m%d%H%M")}.dat"
+        File.write(file, JSON.generate(base_hash))
+        session_file = file
+        data = JSON.parse(File.read(session_file))
+        expect(data).to have_key("database_url")
+      ensure
+        File.delete(session_file) if session_file && File.exist?(session_file)
+      end
+    end
+  end
+
+  # ── pull_partial_data ────────────────────────────────────────────────────────
+
+  describe '#pull_partial_data' do
+    before do
+      op = build_pull(db, dump_dir)
+      op.initialize_dump_directory
+      op.pull_schema
+    end
+
+    it 'returns early when stream_state is empty' do
+      op = build_pull(db, dump_dir)
+      expect { op.pull_partial_data }.not_to raise_error
+    end
+
+    it 'raises ArgumentError when stream_state is set (production bug: factory called with 2 args)' do
+      op = build_pull(db, dump_dir)
+      op.stream_state = { table_name: "users", chunksize: 1000, offset: 5, size: 5, klass: "Tapsoob::DataStream::Base" }
+      expect { op.pull_partial_data }.to raise_error(ArgumentError)
+    end
+  end
+
+  # ── pull_data_from_table_parallel ────────────────────────────────────────────
+  #
+  # These tests use file-backed SQLite because pull_data_from_table_parallel
+  # spawns threads that each open a new Sequel connection to @database_url.
+  # On JRuby/JDBC, each new connection to an in-memory SQLite URL creates an
+  # isolated empty database — the worker threads cannot see the seeded tables.
+  # File-backed SQLite is shared across all connections on both MRI and JRuby.
+
+  describe '#pull_data_from_table_parallel' do
+    it 'writes data using PK-based partitioning (table with integer PK)' do
+      db_path = File.join(Dir.tmpdir, "tapsoob_pull_parallel_pk_#{Process.pid}.db")
+      db_url  = DbHelpers.adapt_url("sqlite://#{db_path}")
+      file_db = Sequel.connect(db_url)
+      file_db.extension :schema_dumper
+      file_db.create_table(:users)   { primary_key :id; String :name }
+      file_db.create_table(:widgets) { primary_key :id; Integer :qty }
+      5.times { |i| file_db[:users].insert(name: "user_#{i}") }
+      3.times { |i| file_db[:widgets].insert(qty: i * 10) }
+      begin
+        op = Tapsoob::Operation::Pull.new(db_url, dump_dir, OperationHelpers::UNIT_OPTS.dup)
+        op.initialize_dump_directory
+        op.pull_schema
+        expect { op.pull_data_from_table_parallel(:users, 5, 2) }.not_to raise_error
+        expect(File.exist?(File.join(dump_dir, "data", "users.json"))).to be true
+      ensure
+        file_db.disconnect rescue nil
+        File.delete(db_path) rescue nil
+      end
+    end
+
+    it 'handles interleaved chunking for tables without integer PK' do
+      db_path = File.join(Dir.tmpdir, "tapsoob_pull_parallel_nopk_#{Process.pid}.db")
+      db_url  = DbHelpers.adapt_url("sqlite://#{db_path}")
+      file_db = Sequel.connect(db_url)
+      file_db.extension :schema_dumper
+      file_db.create_table(:users) { primary_key :id; String :name }
+      file_db.create_table(:nopk)  { String :key, size: 50; Integer :val }
+      3.times { |i| file_db[:nopk].insert(key: "k#{i}", val: i) }
+      begin
+        op = Tapsoob::Operation::Pull.new(db_url, dump_dir, OperationHelpers::UNIT_OPTS.dup)
+        op.initialize_dump_directory
+        op.pull_schema
+        expect { op.pull_data_from_table_parallel(:nopk, 3, 2) }.not_to raise_error
+      ensure
+        file_db.disconnect rescue nil
+        File.delete(db_path) rescue nil
+      end
+    end
+
+    it 'writes data files when called via pull_data_serial with forced parallel workers' do
+      db_path = File.join(Dir.tmpdir, "tapsoob_pull_serial_par_#{Process.pid}.db")
+      db_url  = DbHelpers.adapt_url("sqlite://#{db_path}")
+      file_db = Sequel.connect(db_url)
+      file_db.extension :schema_dumper
+      file_db.create_table(:users)   { primary_key :id; String :name }
+      file_db.create_table(:widgets) { primary_key :id; Integer :qty }
+      5.times { |i| file_db[:users].insert(name: "user_#{i}") }
+      3.times { |i| file_db[:widgets].insert(qty: i * 10) }
+      begin
+        op = Tapsoob::Operation::Pull.new(db_url, dump_dir, OperationHelpers::UNIT_OPTS.merge(no_split: false))
+        op.initialize_dump_directory
+        op.pull_schema
+        allow(op).to receive(:table_parallel_workers).with("users", anything).and_return(2)
+        allow(op).to receive(:table_parallel_workers).with("widgets", anything).and_return(2)
+        op.pull_data_serial
+        expect(File.exist?(File.join(dump_dir, "data", "users.json"))).to be true
+      ensure
+        file_db.disconnect rescue nil
+        File.delete(db_path) rescue nil
+      end
+    end
+  end
+end