tapsoob 0.8.6 → 0.8.7

data/spec/unit/tapsoob/config_spec.rb

@@ -0,0 +1,54 @@
+require 'spec_helper'
+require 'tapsoob/config'
+
+RSpec.describe Tapsoob::Config do
+  describe '.verify_database_url' do
+    it 'connects and lists tables without error for a valid URL' do
+      expect { described_class.verify_database_url(sqlite_memory_url) }.not_to raise_error
+    end
+
+    it 'uses self.database_url when no argument is given' do
+      described_class.database_url = sqlite_memory_url
+      expect { described_class.verify_database_url }.not_to raise_error
+    end
+
+    it 'prints an error and exits on an invalid URL' do
+      expect {
+        expect { described_class.verify_database_url("sqlite:///nonexistent/bad/path.db") }.to raise_error(SystemExit)
+      }.to output(/Failed to connect/).to_stdout
+    end
+  end
+end
+
+RSpec.describe Tapsoob do
+  describe '.exiting= / #exiting?' do
+    it 'sets and reads the exiting flag' do
+      obj = Object.new
+      obj.extend(Tapsoob)
+      Tapsoob.exiting = true
+      expect(obj.exiting?).to be true
+      Tapsoob.exiting = false
+      expect(obj.exiting?).to be false
+    end
+  end
+end
+
+RSpec.describe Tapsoob do
+  describe '.log' do
+    it 'returns a Logger instance' do
+      expect(Tapsoob.log).to be_a(Logger)
+    end
+
+    it 'returns the same instance on repeated calls' do
+      expect(Tapsoob.log).to equal(Tapsoob.log)
+    end
+
+    it 'accepts a custom logger' do
+      custom = Logger.new(StringIO.new)
+      Tapsoob.log = custom
+      expect(Tapsoob.log).to equal(custom)
+      # Reset to default for other tests
+      Tapsoob.log = nil
+    end
+  end
+end

data/spec/unit/tapsoob/data_stream_spec.rb

@@ -83,6 +83,54 @@ RSpec.describe Tapsoob::DataStream do
       end
     end
 
+    describe '#error / #error=' do
+      it 'defaults to false' do
+        expect(stream.error).to be false
+      end
+
+      it 'stores and retrieves error state' do
+        stream.error = true
+        expect(stream.error).to be true
+      end
+    end
+
+    describe '#to_json' do
+      it 'returns a JSON string including the class name' do
+        json = stream.to_json
+        expect(json).to be_a(String)
+        parsed = JSON.parse(json)
+        expect(parsed).to have_key("klass")
+      end
+    end
+
+    describe '#increment' do
+      it 'advances the state offset by the row count' do
+        initial = stream.state[:offset]
+        stream.increment(5)
+        expect(stream.state[:offset]).to eq(initial + 5)
+      end
+    end
+
+    describe '#verify_stream' do
+      it 'sets offset to the current table row count' do
+        stream.verify_stream
+        expect(stream.state[:offset]).to eq(db[:stream_test].count)
+      end
+    end
+
+    describe '#import_rows with extra columns' do
+      it 'drops unknown columns and imports the rest' do
+        rows = {
+          table_name: "stream_test",
+          header: ["id", "label", "value", "ghost_column"],
+          types: ["integer", "string", "integer", "string"],
+          data: [[999, "imported", 42, "extra"]]
+        }
+        expect { stream.import_rows(rows) }.not_to raise_error
+        expect(db[:stream_test].where(id: 999).first).not_to be_nil
+      end
+    end
+
     describe '#parse_encoded_data' do
       it 'raises CorruptedData on checksum mismatch' do
         encoded, _, _ = stream.fetch

data/spec/unit/tapsoob/file_partition_spec.rb

@@ -0,0 +1,117 @@
+require 'spec_helper'
+require 'tapsoob/data_stream/file_partition'
+
+RSpec.describe Tapsoob::DataStream::FilePartition do
+  let(:db) do
+    d = connect_sqlite
+    d.extension :schema_dumper
+    d.create_table(:widgets) { primary_key :id; String :name }
+    d
+  end
+
+  after { db.disconnect }
+
+  def make_ndjson(dir, table, chunks)
+    FileUtils.mkdir_p(File.join(dir, 'data'))
+    path = File.join(dir, 'data', "#{table}.json")
+    File.open(path, 'w') do |f|
+      chunks.each { |chunk| f.puts(JSON.generate(chunk)) }
+    end
+    path
+  end
+
+  def new_stream(state_overrides = {})
+    base_state = { table_name: :widgets, chunksize: 10 }
+    described_class.new(db, base_state.merge(state_overrides))
+  end
+
+  # ── initialize ────────────────────────────────────────────────────────────────
+
+  describe '#initialize' do
+    it 'sets current_line to start_line when line_range is provided' do
+      stream = new_stream(line_range: [3, 7])
+      expect(stream.state[:current_line]).to eq(3)
+    end
+
+    it 'leaves current_line unset when no line_range' do
+      stream = new_stream
+      expect(stream.state[:current_line]).to be_nil
+    end
+  end
+
+  # ── complete? ────────────────────────────────────────────────────────────────
+
+  describe '#complete?' do
+    it 'returns true when line_range is nil' do
+      expect(new_stream.complete?).to be true
+    end
+
+    it 'returns false when current_line is at start' do
+      stream = new_stream(line_range: [0, 4])
+      expect(stream.complete?).to be false
+    end
+
+    it 'returns true when current_line exceeds end_line' do
+      stream = new_stream(line_range: [0, 2])
+      stream.state[:current_line] = 3
+      expect(stream.complete?).to be true
+    end
+  end
+
+  # ── fetch_file ───────────────────────────────────────────────────────────────
+
+  describe '#fetch_file' do
+    let(:dir) { Dir.mktmpdir }
+    after { FileUtils.rm_rf(dir) }
+
+    it 'returns {} when line_range is nil' do
+      stream = new_stream
+      expect(stream.fetch_file(dir)).to eq({})
+    end
+
+    it 'reads rows within the assigned line range' do
+      chunks = [
+        { "table_name" => "widgets", "header" => ["id", "name"], "types" => ["integer", "string"], "data" => [[1, "a"]] },
+        { "table_name" => "widgets", "header" => ["id", "name"], "types" => ["integer", "string"], "data" => [[2, "b"]] },
+        { "table_name" => "widgets", "header" => ["id", "name"], "types" => ["integer", "string"], "data" => [[3, "c"]] },
+      ]
+      make_ndjson(dir, :widgets, chunks)
+
+      stream = new_stream(line_range: [0, 1])
+      result = stream.fetch_file(dir)
+
+      expect(result[:table_name]).to eq("widgets")
+      expect(result[:header]).to eq(["id", "name"])
+      expect(result[:data]).to eq([[1, "a"], [2, "b"]])
+    end
+
+    it 'advances current_line after each fetch' do
+      chunks = 3.times.map { |i| { "table_name" => "widgets", "header" => ["id"], "types" => ["integer"], "data" => [[i]] } }
+      make_ndjson(dir, :widgets, chunks)
+
+      stream = new_stream(line_range: [0, 2], chunksize: 2)
+      stream.fetch_file(dir)
+      expect(stream.state[:current_line]).to eq(2)
+    end
+
+    it 'marks complete after reading all lines in range' do
+      chunks = 2.times.map { |i| { "table_name" => "widgets", "header" => ["id"], "types" => ["integer"], "data" => [[i]] } }
+      make_ndjson(dir, :widgets, chunks)
+
+      stream = new_stream(line_range: [0, 1], chunksize: 10)
+      stream.fetch_file(dir)
+      expect(stream.complete?).to be true
+    end
+
+    it 'respects skip-duplicates option' do
+      chunks = [
+        { "table_name" => "widgets", "header" => ["id"], "types" => ["integer"], "data" => [[1], [1], [2]] },
+      ]
+      make_ndjson(dir, :widgets, chunks)
+
+      stream = described_class.new(db, { table_name: :widgets, chunksize: 10, line_range: [0, 0] }, { "skip-duplicates": true })
+      result = stream.fetch_file(dir)
+      expect(result[:data].uniq).to eq(result[:data])
+    end
+  end
+end

data/spec/unit/tapsoob/keyed_spec.rb

@@ -0,0 +1,121 @@
+require 'spec_helper'
+require 'tapsoob/data_stream/keyed'
+
+Sequel.extension :core_extensions
+
+RSpec.describe Tapsoob::DataStream::Keyed do
+  let(:db) do
+    d = connect_sqlite
+    d.extension :schema_dumper
+    d.create_table(:keyed_items) { primary_key :id; String :label }
+    10.times { |i| d[:keyed_items].insert(label: "item_#{i}") }
+    d
+  end
+
+  after { db.disconnect }
+
+  subject(:stream) { described_class.new(db, { table_name: :keyed_items, chunksize: 4 }) }
+
+  # ── primary_key / buffer_limit ───────────────────────────────────────────────
+
+  describe '#primary_key' do
+    it 'returns :id for this table' do
+      expect(stream.primary_key).to eq(:id)
+    end
+  end
+
+  describe '#buffer_limit' do
+    it 'returns filter when buffer is non-empty' do
+      stream.state[:filter] = 5
+      stream.buffer << { id: 1 }
+      expect(stream.buffer_limit).to eq(5)
+    end
+
+    it 'returns last_fetched when it is less than filter and buffer is empty' do
+      stream.state[:filter] = 10
+      stream.state[:last_fetched] = 3
+      stream.buffer.clear
+      expect(stream.buffer_limit).to eq(3)
+    end
+
+    it 'returns filter when last_fetched >= filter' do
+      stream.state[:filter] = 5
+      stream.state[:last_fetched] = 5
+      stream.buffer.clear
+      expect(stream.buffer_limit).to eq(5)
+    end
+  end
+
+  # ── calc_limit ───────────────────────────────────────────────────────────────
+
+  describe '#calc_limit' do
+    it 'returns chunksize * 3 outside Sinatra' do
+      expect(stream.calc_limit(100)).to eq(300)
+    end
+  end
+
+  # ── load_buffer ──────────────────────────────────────────────────────────────
+
+  describe '#load_buffer' do
+    it 'loads rows into the buffer' do
+      stream.load_buffer(4)
+      expect(stream.buffer.size).to be >= 4
+    end
+
+    it 'updates state[:filter] to the last PK in the buffer' do
+      stream.load_buffer(4)
+      expect(stream.state[:filter]).to eq(stream.buffer.last[:id])
+    end
+
+    it 'stops early when the table is exhausted' do
+      stream.load_buffer(1000)
+      expect(stream.buffer.size).to eq(10)
+    end
+  end
+
+  # ── fetch_buffered ───────────────────────────────────────────────────────────
+
+  describe '#fetch_buffered' do
+    it 'returns up to chunksize rows' do
+      rows = stream.fetch_buffered(4)
+      expect(rows.size).to eq(4)
+    end
+
+    it 'records last_fetched as the PK of the last returned row' do
+      rows = stream.fetch_buffered(4)
+      expect(stream.state[:last_fetched]).to eq(rows.last[:id])
+    end
+
+    it 'returns nil last_fetched when no rows available' do
+      # drain the table
+      stream.load_buffer(100)
+      stream.buffer.clear
+      rows = stream.fetch_buffered(4)
+      expect(rows).to be_empty
+      expect(stream.state[:last_fetched]).to be_nil
+    end
+  end
+
+  # ── increment ────────────────────────────────────────────────────────────────
+
+  describe '#increment' do
+    it 'removes n rows from the front of the buffer' do
+      stream.load_buffer(6)
+      original_fourth = stream.buffer[3]
+      stream.increment(3)
+      expect(stream.buffer.first).to eq(original_fourth)
+    end
+  end
+
+  # ── verify_stream ────────────────────────────────────────────────────────────
+
+  describe '#verify_stream' do
+    it 'sets filter to the max PK and clears last_fetched' do
+      stream.state[:last_fetched] = 3
+      stream.verify_stream
+      max_id = db[:keyed_items].max(:id)
+      expect(stream.state[:filter]).to eq(max_id)
+      expect(stream.state[:last_fetched]).to be_nil
+    end
+  end
+end

data/spec/unit/tapsoob/progress_event_spec.rb

@@ -0,0 +1,136 @@
+require 'spec_helper'
+require 'tapsoob/progress_event'
+
+RSpec.describe Tapsoob::ProgressEvent do
+  before do
+    described_class.enabled = true
+    # Reset throttle state between examples
+    described_class.instance_variable_set(:@last_progress_time, {})
+  end
+
+  after { described_class.enabled = false }
+
+  # ── enabled? ─────────────────────────────────────────────────────────────────
+
+  describe '.enabled?' do
+    it 'reflects the value set by .enabled=' do
+      described_class.enabled = false
+      expect(described_class.enabled?).to be false
+      described_class.enabled = true
+      expect(described_class.enabled?).to be true
+    end
+  end
+
+  # ── emit ─────────────────────────────────────────────────────────────────────
+
+  describe '.emit' do
+    it 'writes a PROGRESS: JSON line to STDERR when enabled' do
+      err = StringIO.new
+      stub_const('STDERR', err)
+      described_class.emit('test_event', foo: 'bar')
+      output = err.string
+      expect(output).to include('PROGRESS:')
+      data = JSON.parse(output.sub('PROGRESS: ', ''))
+      expect(data['event']).to eq('test_event')
+      expect(data['foo']).to eq('bar')
+    end
+
+    it 'does nothing when disabled' do
+      described_class.enabled = false
+      err = StringIO.new
+      stub_const('STDERR', err)
+      described_class.emit('ignored')
+      expect(err.string).to be_empty
+    end
+  end
+
+  # ── throttle helpers ─────────────────────────────────────────────────────────
+
+  describe '.should_emit_progress?' do
+    it 'returns true on the first call for a table' do
+      expect(described_class.should_emit_progress?(:my_table)).to be true
+    end
+
+    it 'returns false when called again immediately' do
+      described_class.should_emit_progress?(:my_table)
+      expect(described_class.should_emit_progress?(:my_table)).to be false
+    end
+  end
+
+  describe '.clear_throttle' do
+    it 'resets state so the next call returns true' do
+      described_class.should_emit_progress?(:my_table)
+      described_class.clear_throttle(:my_table)
+      expect(described_class.should_emit_progress?(:my_table)).to be true
+    end
+  end
+
+  # ── high-level event helpers ──────────────────────────────────────────────────
+
+  {
+    schema_start:     [3],
+    schema_complete:  [3],
+    data_start:       [3, 100],
+    data_complete:    [3, 100],
+    indexes_start:    [3],
+    indexes_complete: [3],
+    sequences_start:  [],
+    sequences_complete: [],
+  }.each do |method, args|
+    describe ".#{method}" do
+      it 'does not raise' do
+        expect { described_class.send(method, *args) }.not_to raise_error
+      end
+    end
+  end
+
+  describe '.table_start' do
+    it 'resets throttle and emits without error' do
+      expect { described_class.table_start(:users, 500, workers: 2) }.not_to raise_error
+    end
+  end
+
+  describe '.table_progress' do
+    it 'emits when throttle allows' do
+      described_class.clear_throttle(:users)
+      err = StringIO.new
+      stub_const('STDERR', err)
+      described_class.table_progress(:users, 50, 200)
+      expect(err.string).to include('table_progress')
+    end
+
+    it 'skips emission when throttled' do
+      described_class.should_emit_progress?(:users)  # consume the token
+      err = StringIO.new
+      stub_const('STDERR', err)
+      described_class.table_progress(:users, 50, 200)
+      expect(err.string).to be_empty
+    end
+
+    it 'emits 0% when total is 0' do
+      described_class.clear_throttle(:empty_table)
+      err = StringIO.new
+      stub_const('STDERR', err)
+      described_class.table_progress(:empty_table, 0, 0)
+      data = JSON.parse(err.string.sub('PROGRESS: ', '').strip)
+      expect(data['percentage']).to eq(0)
+    end
+  end
+
+  describe '.table_complete' do
+    it 'clears throttle and emits without error' do
+      expect { described_class.table_complete(:users, 500) }.not_to raise_error
+    end
+  end
+
+  describe '.error' do
+    it 'emits an error event with the message' do
+      err = StringIO.new
+      stub_const('STDERR', err)
+      described_class.error('something broke', table: 'users')
+      data = JSON.parse(err.string.sub('PROGRESS: ', '').strip)
+      expect(data['event']).to eq('error')
+      expect(data['message']).to eq('something broke')
+    end
+  end
+end