tapsoob 0.7.17 → 0.8.0

data/spec/spec_helper.rb

@@ -1,91 +1,53 @@
 require 'simplecov'
-SimpleCov.start
+SimpleCov.start do
+  add_filter '/spec/'
+  add_group 'Operation',   'lib/tapsoob/operation'
+  add_group 'DataStream',  'lib/tapsoob/data_stream'
+  add_group 'CLI',         'lib/tapsoob/cli'
+end
+
+require 'tapsoob'
+require 'sequel'
+require 'fileutils'
+require 'tmpdir'
+
+Dir[File.join(__dir__, 'support', '**', '*.rb')].sort.each { |f| require f }
+
+# ── JRuby-aware SQLite helpers ───────────────────────────────────────────────
+# Use these everywhere instead of Sequel.sqlite / 'sqlite::memory:' so the
+# unit specs run identically under MRI and JRuby.
+
+def sqlite_memory_url
+  DbHelpers.adapt_url('sqlite::memory:')
+end
+
+def connect_sqlite
+  db = Sequel.connect(sqlite_memory_url)
+  db.extension :schema_dumper
+  db
+end
 
-# This file was generated by the `rspec --init` command. Conventionally, all
-# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
-# The generated `.rspec` file contains
-# `--require spec_helper` which will cause this
-# file to always be loaded, without
-# a need to explicitly require it in any files.
-#
-# Given that it is always loaded, you are encouraged to keep this file as
-# light-weight as possible. Requiring heavyweight dependencies from this file
-# will add to the boot time of your test suite on EVERY test run, even for an
-# individual file that may not need all of that loaded. Instead, consider making
-# a separate helper file that requires the additional dependencies and performs
-# the additional setup,
-# and require it from the spec files that actually need it.
-#
-# The `.rspec` file also contains a few flags that are not defaults but that
-# users commonly want.
-#
-# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
 RSpec.configure do |config|
-  # rspec-expectations config goes here. You can use an alternate
-  # assertion/expectation library such as wrong or the stdlib/minitest
-  # assertions if you prefer.
   config.expect_with :rspec do |expectations|
-    # This option will default to `true` in RSpec 4. It makes the `description`
-    # and `failure_message` of custom matchers include text for helper methods
-    # defined using `chain`, e.g.:
-    # be_bigger_than(2).and_smaller_than(4).description
-    #   # => "be bigger than 2 and smaller than 4"
-    # ...rather than:
-    #   # => "be bigger than 2"
     expectations.include_chain_clauses_in_custom_matcher_descriptions = true
   end
 
-  # rspec-mocks config goes here. You can use an alternate test double
-  # library (such as bogus or mocha) by changing the `mock_with` option here.
   config.mock_with :rspec do |mocks|
-    # Prevents you from mocking or stubbing a method that does not exist on
-    # a real object. This is generally recommended, and will default to
-    # `true` in RSpec 4.
     mocks.verify_partial_doubles = true
   end
 
-  # The settings below are suggested to provide a good initial experience
-  # with RSpec, but feel free to customize to your heart's content.
-  #   # These two settings work together to allow you to limit a spec run
-  #   # to individual examples or groups you care about by tagging them with
-  #   # `:focus` metadata. When nothing is tagged with `:focus`, all examples
-  #   # get run.
-  #   config.filter_run :focus
-  #   config.run_all_when_everything_filtered = true
-  #
+  config.shared_context_metadata_behavior = :apply_to_host_groups
+  config.filter_run_when_matching :focus
+  config.disable_monkey_patching!
+  config.order = :random
+  Kernel.srand config.seed
+
+  # Integration/system tests require a real DB — skip unless env vars are set.
+  config.filter_run_excluding :integration unless ENV['INTEGRATION_TESTS'] || ENV['SRC_DATABASE_URL']
+  config.filter_run_excluding :system      unless ENV['SYSTEM_TESTS']      || ENV['SRC_DATABASE_URL']
 
-  #   config.disable_monkey_patching!
-  #
-  #   # This setting enables warnings. It's recommended, but in some cases may
-  #   # be too noisy due to issues in dependencies.
-  #   config.warnings = true
-  #
-  #   # Many RSpec users commonly either run the entire suite or an individual
-  #   # file, and it's useful to allow more verbose output when running an
-  #   # individual spec file.
-  #   if config.files_to_run.one?
-  #     # Use the documentation formatter for detailed output,
-  #     # unless a formatter has already been configured
-  #     # (e.g. via a command-line flag).
-  #     config.default_formatter = 'doc'
-  #   end
-  #
-  #   # Print the 10 slowest examples and example groups at the
-  #   # end of the spec run, to help surface which specs are running
-  #   # particularly slow.
-  #   config.profile_examples = 10
-  #
-  #   # Run specs in random order to surface order dependencies. If you find an
-  #   # order dependency and want to debug it,
-  #   # you can fix the order by providing
-  #   # the seed, which is printed after each run.
-  #   #     --seed 1234
-  #   config.order = :random
-  #
-  #   # Seed global randomization in this process using the `--seed` CLI option.
-  #   # Setting this allows you to use `--seed` to deterministically reproduce
-  #   # test failures
-  #   # related to randomization by passing the same `--seed` value
-  #   # as the one that triggered the failure.
-  #   Kernel.srand config.seed
+  config.include DbHelpers,      :integration
+  config.include DbHelpers,      :system
+  config.include RoundTripHelper, :integration
+  config.include RoundTripHelper, :system
 end

data/spec/support/db_helpers.rb

@@ -0,0 +1,115 @@
+require 'sequel'
+require 'database_cleaner/sequel'
+
+# Helpers included in all :integration and :system examples.
+# Provides connection management, table creation, and DatabaseCleaner wiring.
+module DbHelpers
+  def self.included(base)
+    base.instance_eval do
+      # dump_dir is per-example (let is fine here — it's never used in before/after(:all))
+      let(:dump_dir) { Dir.mktmpdir('tapsoob_dump_') }
+      after(:each) { FileUtils.rm_rf(dump_dir) }
+    end
+  end
+
+  # Instance-method accessors used by examples and shared examples.
+  # Host specs must assign @src_url / @dst_url / @src_db / @dst_db in before(:all).
+  def src_url; @src_url; end
+  def dst_url; @dst_url; end
+  def src_db;  @src_db;  end
+  def dst_db;  @dst_db;  end
+
+  # ── URL normalisation ────────────────────────────────────────────────────────
+  #
+  # CI and local env vars always carry MRI-style URLs (mysql2://, postgres://, sqlite://).
+  # Under JRuby, Sequel requires JDBC-style URLs (jdbc:mysql://, jdbc:postgresql://, jdbc:sqlite:).
+  # This method rewrites the URL transparently so every caller gets the right scheme.
+  #
+  # Mapping:
+  #   sqlite://path/to/file   → jdbc:sqlite:path/to/file       (JRuby)
+  #   sqlite::memory:         → jdbc:sqlite::memory:            (JRuby)
+  #   mysql2://host/db        → jdbc:mysql://host/db            (JRuby)
+  #   postgres://host/db      → jdbc:postgresql://host/db       (JRuby)
+  #   postgresql://host/db    → jdbc:postgresql://host/db       (JRuby)
+  #   anything jdbc:*         → left unchanged (already JDBC)
+  #
+  JRUBY = (RUBY_PLATFORM =~ /java/)
+
+  def self.adapt_url(url)
+    return url unless JRUBY
+    return url if url.start_with?('jdbc:')
+
+    case url
+    when /\Asqlite:(?:\/\/)?(.*)\z/           then "jdbc:sqlite:#{$1}"
+    when /\Amysql2?:\/\/(.*)\z/               then "jdbc:mysql://#{$1}"
+    when /\Apostgres(?:ql)?:\/\/(.*)\z/       then "jdbc:postgresql://#{$1}"
+    else url
+    end
+  end
+
+  # ── connection pool ──────────────────────────────────────────────────────────
+
+  CONNECTIONS = {}
+  CONNECTIONS_MUTEX = Mutex.new
+
+  def self.connect(url)
+    CONNECTIONS_MUTEX.synchronize do
+      CONNECTIONS[url] ||= begin
+        db = Sequel.connect(url, max_connections: 10)
+        db.extension :schema_dumper
+        db
+      end
+    end
+  end
+
+  def self.disconnect_all
+    CONNECTIONS_MUTEX.synchronize do
+      CONNECTIONS.each_value(&:disconnect)
+      CONNECTIONS.clear
+    end
+  end
+
+  # ── table lifecycle helpers ──────────────────────────────────────────────────
+
+  def drop_and_create(db, &block)
+    db.instance_eval(&block)
+  end
+
+  # Truncate every test table on db after each example (faster than drop/create).
+  def truncate_tables(db, *tables)
+    tables.each do |t|
+      next unless db.table_exists?(t)
+      if [:mysql, :mysql2].include?(db.adapter_scheme)
+        db.run("SET foreign_key_checks = 0")
+        db[t].truncate
+        db.run("SET foreign_key_checks = 1")
+      else
+        db[t].truncate(cascade: true)
+      end
+    end
+  end
+
+  # ── generic row-count assertion helpers ─────────────────────────────────────
+
+  def row_count(db, table)
+    db[table].count
+  end
+
+  def table_exists?(db, table)
+    db.table_exists?(table)
+  end
+
+  # ── schema helpers ───────────────────────────────────────────────────────────
+
+  # Drop a table if it exists, accounting for FK checks on MySQL.
+  def safe_drop(db, table)
+    return unless db.table_exists?(table)
+    if [:mysql, :mysql2].include?(db.adapter_scheme)
+      db.run("SET foreign_key_checks = 0")
+      db.drop_table(table)
+      db.run("SET foreign_key_checks = 1")
+    else
+      db.drop_table(table, cascade: true)
+    end
+  end
+end

data/spec/support/fixtures.rb

@@ -0,0 +1,304 @@
+require 'faker'
+require 'sequel'
+
+# Fixtures.setup_source(db) creates all test tables and seeds them with
+# realistic data. Each table targets a distinct edge-case scenario.
+#
+# Tables created:
+#   users          – common mixed-type table (name, email, timestamps)
+#   orders         – FK relationship to users, decimal amounts
+#   products       – nullable fields, boolean flag, float price
+#   documents      – large TEXT body (up to 64 KB per row)
+#   attachments    – BLOB payloads (up to 256 KB per row)
+#   events         – no primary key (exercises Base stream, not Keyed)
+#   large_table    – 150 000+ integer rows (triggers intra-table parallelization)
+#   null_heavy     – every nullable column is NULL for half the rows
+#
+module Fixtures
+  LARGE_TABLE_ROWS   = 150_000
+  DOCUMENT_ROWS      = 500
+  ATTACHMENT_ROWS    = 200
+  STANDARD_ROWS      = 1_000
+  NULL_HEAVY_ROWS    = 400
+
+  # ── schema ───────────────────────────────────────────────────────────────────
+
+  def self.create_tables(db)
+    db.create_table!(:users) do
+      primary_key :id
+      String  :name,       null: false, size: 100
+      String  :email,      null: false, size: 255
+      String  :locale,     size: 10
+      Integer :age
+      Date    :birthday
+      DateTime :created_at
+      DateTime :updated_at
+    end
+
+    db.create_table!(:orders) do
+      primary_key :id
+      foreign_key :user_id, :users, null: false
+      String  :reference,  null: false, size: 50
+      Float   :amount
+      String  :status,     size: 20,    default: 'pending'
+      DateTime :placed_at
+    end
+
+    db.create_table!(:products) do
+      primary_key :id
+      String  :sku,        null: false, size: 50
+      String  :name,       null: false, size: 200
+      Float   :price
+      Integer :stock,      default: 0
+      TrueClass :available, default: true
+      String  :description, text: true
+    end
+
+    body_type = db.database_type == :mysql ? :mediumtext : :text
+    db.create_table!(:documents) do
+      primary_key :id
+      String   :title,      null: false, size: 255
+      column   :body,       body_type
+      String   :author,     size: 100
+      DateTime :published_at
+    end
+
+    payload_type = case db.database_type
+                   when :mysql  then :mediumblob
+                   when :postgres then :bytea
+                   else :blob
+                   end
+    db.create_table!(:attachments) do
+      primary_key :id
+      String  :filename,   null: false, size: 255
+      String  :mime_type,  size: 100
+      Integer :size_bytes
+      column  :payload,    payload_type
+    end
+
+    db.create_table!(:events) do
+      String  :event_type, null: false, size: 50
+      String  :actor,      size: 100
+      String  :target,     size: 100
+      DateTime :occurred_at
+    end
+
+    db.create_table!(:large_table) do
+      primary_key :id
+      String  :data,       null: false, size: 100
+      Integer :sequence
+      Float   :value
+    end
+
+    db.create_table!(:null_heavy) do
+      primary_key :id
+      String  :maybe_name,   size: 100
+      Integer :maybe_number
+      Float   :maybe_score
+      Date    :maybe_date
+      String  :maybe_text,   text: true
+    end
+  end
+
+  def self.drop_tables(db)
+    # Drop in reverse FK order
+    [:null_heavy, :large_table, :events, :attachments,
+     :documents, :products, :orders, :users].each do |t|
+      db.drop_table(t, if_exists: true)
+    end
+  end
+
+  # ── seeding ──────────────────────────────────────────────────────────────────
+
+  def self.seed(db)
+    seed_users(db)
+    seed_orders(db)
+    seed_products(db)
+    seed_documents(db)
+    seed_attachments(db)
+    seed_events(db)
+    seed_large_table(db)
+    seed_null_heavy(db)
+  end
+
+  # ── users ────────────────────────────────────────────────────────────────────
+  # Realistic person records: names from multiple locales, valid emails,
+  # age range 18–80, random birthdays, ISO timestamps.
+
+  def self.seed_users(db)
+    Faker::Config.locale = :en
+    rows = STANDARD_ROWS.times.map do
+      now = Faker::Time.between(from: DateTime.new(2020, 1, 1), to: DateTime.now)
+      {
+        name:       Faker::Name.name,
+        email:      Faker::Internet.unique.email,
+        locale:     %w[en fr de es ja pt it nl].sample,
+        age:        Faker::Number.between(from: 18, to: 80),
+        birthday:   Faker::Date.birthday(min_age: 18, max_age: 80).to_s,
+        created_at: now.strftime('%Y-%m-%d %H:%M:%S'),
+        updated_at: now.strftime('%Y-%m-%d %H:%M:%S')
+      }
+    end
+    db[:users].multi_insert(rows)
+    Faker::UniqueGenerator.clear
+  end
+
+  # ── orders ───────────────────────────────────────────────────────────────────
+  # Each order references a real user. Amounts use realistic e-commerce prices
+  # (0.99 – 9999.99). Statuses mirror a typical order lifecycle.
+
+  STATUSES = %w[pending processing shipped delivered cancelled refunded].freeze
+
+  def self.seed_orders(db)
+    user_ids = db[:users].select_map(:id)
+    rows = STANDARD_ROWS.times.map do
+      placed = Faker::Time.between(from: DateTime.new(2022, 1, 1), to: DateTime.now)
+      {
+        user_id:    user_ids.sample,
+        reference:  "ORD-#{Faker::Alphanumeric.unique.alphanumeric(number: 10).upcase}",
+        amount:     (Faker::Commerce.price(range: 0.99..9999.99) * 100).round / 100.0,
+        status:     STATUSES.sample,
+        placed_at:  placed.strftime('%Y-%m-%d %H:%M:%S')
+      }
+    end
+    db[:orders].multi_insert(rows)
+    Faker::UniqueGenerator.clear
+  end
+
+  # ── products ─────────────────────────────────────────────────────────────────
+  # SKUs follow a realistic pattern (3-letter category + 6-digit number).
+  # Descriptions are nullable Markdown-ish paragraphs (~200–800 chars).
+
+  def self.seed_products(db)
+    categories = %w[ELC CLT FRN SPT HOM KIT OFC]
+    rows = STANDARD_ROWS.times.map do
+      {
+        sku:         "#{categories.sample}-#{Faker::Number.number(digits: 6)}",
+        name:        Faker::Commerce.product_name,
+        price:       (Faker::Commerce.price * 100).round / 100.0,
+        stock:       Faker::Number.between(from: 0, to: 5_000),
+        available:   [true, true, true, false].sample,
+        description: [nil, Faker::Lorem.paragraphs(number: rand(1..3)).join("\n\n")].sample
+      }
+    end
+    db[:products].multi_insert(rows)
+  end
+
+  # ── documents ────────────────────────────────────────────────────────────────
+  # Bodies range from a few hundred bytes up to ~64 KB to stress TEXT columns
+  # and the adaptive chunksize logic.
+
+  LOREM_BASE = Faker::Lorem.characters(number: 1_000)
+
+  def self.seed_documents(db)
+    rows = DOCUMENT_ROWS.times.map do
+      target_bytes = [256, 1_024, 4_096, 16_384, 65_536].sample
+      body = (LOREM_BASE * ((target_bytes / LOREM_BASE.size) + 2))[0, target_bytes]
+      published = Faker::Time.between(from: DateTime.new(2010, 1, 1), to: DateTime.now)
+      {
+        title:        Faker::Lorem.sentence(word_count: rand(4..10)).chomp('.'),
+        body:         body,
+        author:       Faker::Name.name,
+        published_at: published.strftime('%Y-%m-%d %H:%M:%S')
+      }
+    end
+    db[:documents].multi_insert(rows)
+  end
+
+  # ── attachments ──────────────────────────────────────────────────────────────
+  # Binary payloads: realistic mix of small thumbnails (1–4 KB), medium images
+  # (~50 KB), and occasional large files (~256 KB). Uses Random.bytes so the
+  # data is genuinely binary and exercises base64 encode/decode faithfully.
+
+  MIME_TYPES = {
+    'image/png'       => '.png',
+    'image/jpeg'      => '.jpg',
+    'application/pdf' => '.pdf',
+    'application/zip' => '.zip',
+    'video/mp4'       => '.mp4'
+  }.freeze
+
+  def self.seed_attachments(db)
+    rows = ATTACHMENT_ROWS.times.map do
+      size = [
+        rand(1_024..4_096),       # thumbnail  (1–4 KB)
+        rand(40_000..60_000),     # image      (~50 KB)
+        rand(200_000..262_144)    # large file (~256 KB)
+      ].sample
+      mime, ext = MIME_TYPES.to_a.sample
+      {
+        filename:   "#{Faker::File.file_name(dir: '', ext: ext.delete('.'))}",
+        mime_type:  mime,
+        size_bytes: size,
+        payload:    Sequel::SQL::Blob.new(Random.bytes(size))
+      }
+    end
+    db[:attachments].multi_insert(rows)
+  end
+
+  # ── events ───────────────────────────────────────────────────────────────────
+  # No primary key — exercises DataStream::Base (non-keyed path) for both
+  # pull and push.
+
+  EVENT_TYPES = %w[login logout purchase refund view search click signup].freeze
+
+  def self.seed_events(db)
+    rows = STANDARD_ROWS.times.map do
+      {
+        event_type:  EVENT_TYPES.sample,
+        actor:       Faker::Internet.username,
+        target:      "/#{Faker::Internet.slug}",
+        occurred_at: Faker::Time.between(
+          from: DateTime.new(2023, 1, 1),
+          to:   DateTime.now
+        ).strftime('%Y-%m-%d %H:%M:%S')
+      }
+    end
+    db[:events].multi_insert(rows)
+  end
+
+  # ── large_table ──────────────────────────────────────────────────────────────
+  # 150 000 rows to push past the 100 K intra-table parallelization threshold.
+  # Inserted in batches of 5 000 to keep memory low.
+
+  LARGE_TABLE_BATCH = 5_000
+
+  def self.seed_large_table(db)
+    total = LARGE_TABLE_ROWS
+    batches = (total.to_f / LARGE_TABLE_BATCH).ceil
+    batches.times do |b|
+      count = [LARGE_TABLE_BATCH, total - b * LARGE_TABLE_BATCH].min
+      rows = count.times.map do |i|
+        seq = b * LARGE_TABLE_BATCH + i
+        {
+          data:     Faker::Lorem.characters(number: rand(20..80)),
+          sequence: seq,
+          value:    rand * 10_000.0
+        }
+      end
+      db[:large_table].multi_insert(rows)
+    end
+  end
+
+  # ── null_heavy ───────────────────────────────────────────────────────────────
+  # Half the rows have every nullable column set to NULL.
+  # Ensures NULL survives Marshal encode/decode and DB round-trips.
+
+  def self.seed_null_heavy(db)
+    rows = NULL_HEAVY_ROWS.times.map do |i|
+      if i.even?
+        { maybe_name: nil, maybe_number: nil, maybe_score: nil,
+          maybe_date: nil, maybe_text: nil }
+      else
+        {
+          maybe_name:   Faker::Name.name,
+          maybe_number: rand(-1_000_000..1_000_000),
+          maybe_score:  rand * 100.0,
+          maybe_date:   Faker::Date.between(from: '2000-01-01', to: '2030-12-31').to_s,
+          maybe_text:   Faker::Lorem.paragraph
+        }
+      end
+    end
+    db[:null_heavy].multi_insert(rows)
+  end
+end

data/spec/support/round_trip_helper.rb

@@ -0,0 +1,70 @@
+require 'tapsoob/operation/pull'
+require 'tapsoob/operation/push'
+
+# Helpers for running pull and push operations in specs.
+# Included automatically in :integration and :system examples.
+module RoundTripHelper
+  DEFAULT_OPTS = {
+    data:              true,
+    schema:            true,
+    indexes:           true,
+    indexes_first:     false,
+    progress:          false,
+    default_chunksize: 1000
+  }.freeze
+
+  # Run a full pull from +url+ into +dir+.
+  def pull(url, dir, opts = {})
+    op = Tapsoob::Operation::Pull.new(url, dir, DEFAULT_OPTS.merge(opts))
+    op.run
+    op
+  end
+
+  # Run a full push from +dir+ into +url+.
+  def push(url, dir, opts = {})
+    op = Tapsoob::Operation::Push.new(url, dir, DEFAULT_OPTS.merge(opts))
+    op.run
+    op
+  end
+
+  # Pull from +src_url+ into +dir+, then push from +dir+ into +dst_url+.
+  # Returns [pull_op, push_op].
+  def round_trip(src_url, dst_url, dir, opts = {})
+    p = pull(src_url, dir, opts)
+    q = push(dst_url, dir, opts)
+    [p, q]
+  end
+
+  # Assert that every table present in src_db also exists in dst_db with the
+  # same number of rows.
+  def expect_same_counts(src_db, dst_db)
+    src_db.tables.each do |table|
+      expect(dst_db.table_exists?(table)).to be(true),
+        "expected table #{table} to exist in destination"
+      src_count = src_db[table].count
+      dst_count = dst_db[table].count
+      expect(dst_count).to eq(src_count),
+        "row count mismatch for #{table}: src=#{src_count} dst=#{dst_count}"
+    end
+  end
+
+  # Assert that every row in src_db[table] has an identical row in dst_db[table]
+  # when compared by ordered primary key. Only usable for small tables.
+  def expect_identical_rows(src_db, dst_db, table, order_col: :id)
+    src_rows = src_db[table].order(order_col).all
+    dst_rows = dst_db[table].order(order_col).all
+    expect(dst_rows.size).to eq(src_rows.size)
+    src_rows.zip(dst_rows).each_with_index do |(src, dst), i|
+      src.each do |col, val|
+        if val.is_a?(String) && val.encoding == Encoding::ASCII_8BIT
+          # binary – compare as bytes
+          expect(dst[col].to_s.bytes).to eq(val.bytes),
+            "blob mismatch in #{table}[#{i}].#{col}"
+        else
+          expect(dst[col]).to eq(val),
+            "value mismatch in #{table}[#{i}].#{col}: expected #{val.inspect}, got #{dst[col].inspect}"
+        end
+      end
+    end
+  end
+end