tapsoob 0.5.29 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 748c9cf97135cae0c72f0ba298f7c3f197e65764ce4c44c0670d8ef2fe5181ed
-  data.tar.gz: 7bebc431915dcf8dd5bff0ee341cf22765646d1854485c23f80b5b9c642fc142
+  metadata.gz: 682ebd67a432169d8f72eb47412aaface0df4ba058be6849fba6e91ab96888c1
+  data.tar.gz: 3aba7ecf086fc98b21a9c2c6b2e9ae3413fd71d98106de415a3e3223732b0947
 SHA512:
-  metadata.gz: 404478581e61ebdd2875c21d0af865acd5e9dba010292819046cd5e195c9c78cad675bdd50216ea26c5bba8923c0e532a8bd2593b4fe7eb0e71f85db865ef067
-  data.tar.gz: 3310489ebd6dcfd7efc76c4fd5d9b2ec6ffa9da12c6cb4161ffd02267f376927b957f5142dcf42ba81fd544c470c36e07f7e0263b66bcae6168b6c9dfe309b61
+  metadata.gz: d3b49e8f114d4c2b693a524b60ee3959b36cdddb3dd7e2f6378a9f72a6f86a1ca44ab26b609cf4a4d88747813fffdd233a70efa34465df613ceea0af18af6aa2
+  data.tar.gz: 9793f681463d9f92ef4eae1015ed11b35d351a7f5938b2ad79e5d3cc67793fc421a02e195536957a56acde9b8d8cc81ab429a5a0852ae6b2d2eddcf38445222b

data/lib/tapsoob/cli/root.rb

@@ -22,6 +22,7 @@ module Tapsoob
       option :"exclude-tables", desc: "Shortcut to exclude a list of tables", type: :array, aliases: "-e"
       option :"indexes", type: :boolean, default: false
       option :"same-db", type: :boolean, default: false
+      option :parallel, desc: "Number of parallel workers for table processing (default: 1)", default: 1, type: :numeric, aliases: "-j"
       option :progress, desc: "Show progress", default: true, type: :boolean
       option :debug, desc: "Enable debug messages", default: false, type: :boolean
       def pull(dump_path, database_url)
@@ -47,6 +48,7 @@ module Tapsoob
       option :purge, desc: "Purge data in tables prior to performing the import", default: false, type: :boolean, aliases: "-p"
       option :"skip-duplicates", desc: "Remove duplicates when loading data", default: false, type: :boolean
       option :"discard-identity", desc: "Remove identity when pushing data (may result in creating duplicates)", default: false, type: :boolean
+      option :parallel, desc: "Number of parallel workers for table processing (default: 1)", default: 1, type: :numeric, aliases: "-j"
       option :progress, desc: "Show progress", default: true, type: :boolean
       option :debug, desc: "Enable debug messages", default: false, type: :boolean
       def push(dump_path, database_url)
@@ -82,6 +84,7 @@ module Tapsoob
             indexes_first: options[:"indexes_first"],
             disable_compression: options[:"disable-compression"],
             tables: options[:tables],
+            parallel: options[:parallel],
             progress: options[:progress],
             debug: options[:debug]
           })

data/lib/tapsoob/multi_progress_bar.rb

@@ -0,0 +1,234 @@
+# -*- encoding : utf-8 -*-
+require 'tapsoob/progress_bar'
+
+# MultiProgressBar manages multiple progress bars in parallel
+# Each bar gets its own line in the terminal
+class MultiProgressBar
+  def initialize(max_bars = 4)
+    @max_bars = max_bars
+    @bars = []
+    @mutex = Mutex.new
+    @active = true
+    @out = STDOUT
+    @last_update = Time.now
+    @reserved_lines = 0  # Track how many lines we've actually reserved
+    @max_title_width = 14  # Minimum width, will grow with longer titles
+  end
+
+  # Create a new progress bar and return it
+  def create_bar(title, total)
+    @mutex.synchronize do
+      # Remove any existing bar with the same title to prevent duplicates
+      @bars.reject! { |b| b.title == title }
+
+      # Update max title width to accommodate longer titles
+      @max_title_width = [@max_title_width, title.length].max
+
+      bar = ThreadSafeProgressBar.new(title, total, self)
+
+      # Reserve a line for this new bar during active updates
+      # Cap at 2 * max_bars to show active workers + some recent finished bars
+      if @reserved_lines < @max_bars * 2
+        @out.print "\n"
+        @out.flush
+        @reserved_lines += 1
+      end
+
+      @bars << bar
+      bar
+    end
+  end
+
+  # Get the current maximum title width for alignment
+  # Note: Always called from within synchronized methods, so no mutex needed
+  def max_title_width
+    @max_title_width
+  end
+
+  # Called by individual bars when they update
+  def update
+    @mutex.synchronize do
+      return unless @active
+      return unless should_redraw?
+
+      @last_update = Time.now
+      redraw_all
+    end
+  end
+
+  # Finish a specific bar - mark it as completed
+  def finish_bar(bar)
+    @mutex.synchronize do
+      return unless @active
+
+      bar.mark_finished
+
+      # Respect throttle when finishing to avoid spamming redraws
+      if should_redraw?
+        @last_update = Time.now
+        redraw_all
+      end
+      # If throttled, the next regular update will show the finished state
+    end
+  end
+
+  # Stop all progress bars and keep them visible
+  def stop
+    @mutex.synchronize do
+      @active = false
+
+      # Final cleanup: remove any duplicate titles (keep the last occurrence of each unique title)
+      @bars = @bars.reverse.uniq { |bar| bar.title }.reverse
+
+      # Final redraw to show completed state (skip active check)
+      redraw_all(true)
+      # Move cursor past all bars
+      @out.print "\n"
+      @out.flush
+    end
+  end
+
+  private
+
+  # Check if enough time has passed to redraw (throttle to 10 updates/sec)
+  def should_redraw?
+    Time.now - @last_update >= 0.1
+  end
+
+  def redraw_all(force = false)
+    return unless force || @active
+    return if @bars.empty?
+
+    if force && !@active
+      render_final_display
+    else
+      render_active_display
+    end
+  end
+
+  # Final display: show all completed bars
+  def render_final_display
+    # Clear the reserved lines first
+    if @reserved_lines > 0
+      @out.print "\r\e[#{@reserved_lines}A"
+      @reserved_lines.times { @out.print "\r\e[K\n" }
+    end
+
+    # Print all bars (adds new lines as needed)
+    @bars.each do |bar|
+      @out.print "\r\e[K"
+      bar.render_to(@out)
+      @out.print "\n"
+    end
+
+    @out.flush
+  end
+
+  # Normal operation: show active bars + recent finished in reserved space
+  def render_active_display
+    return if @reserved_lines == 0
+
+    # Partition bars in a single pass for efficiency
+    active_bars, finished_bars = @bars.partition { |b| !b.finished? }
+
+    # Build display: active bars first, then recent finished to fill remaining space
+    # Ensure we don't request negative count from .last()
+    remaining_space = [@reserved_lines - active_bars.length, 0].max
+    bars_to_draw = active_bars + finished_bars.last(remaining_space)
+
+    # If we have more bars than reserved lines, show only the most recent
+    bars_to_draw = bars_to_draw.last(@reserved_lines) if bars_to_draw.length > @reserved_lines
+
+    # Move up and redraw in reserved space
+    @out.print "\r\e[#{@reserved_lines}A"
+    @reserved_lines.times do |i|
+      @out.print "\r\e[K"
+      bars_to_draw[i].render_to(@out) if i < bars_to_draw.length
+      @out.print "\n"
+    end
+
+    @out.flush
+  end
+end
+
+# Thread-safe progress bar that reports to a MultiProgressBar
+class ThreadSafeProgressBar < ProgressBar
+  attr_reader :title
+
+  def initialize(title, total, multi_progress_bar)
+    @multi_progress_bar = multi_progress_bar
+    @out = STDOUT  # Need this for get_width to work
+    # Don't call parent initialize, we'll manage output ourselves
+    @title = title
+    @total = total
+    @terminal_width = 80
+    @bar_mark = "="
+    @current = 0
+    @previous = 0
+    @finished_p = false
+    @start_time = ::Time.now
+    @previous_time = @start_time
+    @format_arguments = [:title, :percentage, :bar, :stat]
+  end
+
+  # Override show to notify multi-progress instead of direct output
+  def show
+    @previous_time = ::Time.now  # Update to prevent time-based refresh spam
+    @multi_progress_bar.update
+  end
+
+  # Render this bar to the given output stream
+  def render_to(out)
+    # Get dynamic title width from MultiProgressBar for consistent alignment
+    # Store as instance variable so parent class fmt_* methods can use it
+    @title_width = @multi_progress_bar.max_title_width
+
+    # Recalculate terminal width to handle resizes and use full width
+    width = get_width
+    # Calculate bar width: total_width - fixed_elements - padding
+    # Fixed: title(variable) + " "(1) + percentage(4) + " "(1) + "|"(1) + "|"(1) + " "(1) + timer(15) = title_width + 25
+    # Padding: +3 for timer fluctuations and safety
+    fixed_chars = @title_width + 28
+    @terminal_width = [width - fixed_chars, 20].max
+
+    # Build format string with dynamic title width
+    format = "%-#{@title_width}s %3d%% %s %s"
+    arguments = @format_arguments.map { |method| send("fmt_#{method}") }
+    line = sprintf(format, *arguments)
+
+    # Ensure line doesn't exceed terminal width to prevent wrapping
+    # Leave 2 chars margin for safety
+    line = line[0, width - 2] if line.length > width - 2
+
+    out.print(line)
+  end
+
+  # Override clear to do nothing (managed by MultiProgressBar)
+  def clear
+    # no-op
+  end
+
+  # Mark this bar as finished (for tracking)
+  def mark_finished
+    @finished_p = true
+  end
+
+  # Override to use the same @finished_p flag
+  def finished?
+    @finished_p
+  end
+
+  # Override finish to notify multi-progress
+  def finish
+    @current = @total
+    @multi_progress_bar.finish_bar(self)
+  end
+
+  # Override inc to check if we need to update
+  def inc(step = 1)
+    @current += step
+    @current = @total if @current > @total
+    show_if_needed
+    @previous = @current
+  end
+end

data/lib/tapsoob/operation.rb

@@ -1,9 +1,11 @@
 # -*- encoding : utf-8 -*-
 require 'sequel'
+require 'thread'
 
 require 'tapsoob/data_stream'
 require 'tapsoob/log'
 require 'tapsoob/progress_bar'
+require 'tapsoob/multi_progress_bar'
 require 'tapsoob/schema'
 
 module Tapsoob
@@ -117,7 +119,7 @@ module Tapsoob
     end
 
     def db
-      @db ||= Sequel.connect(database_url)
+      @db ||= Sequel.connect(database_url, max_connections: parallel_workers * 2)
       @db.extension :schema_dumper
       @db.loggers << Tapsoob.log if opts[:debug]
 
@@ -130,6 +132,24 @@ module Tapsoob
       @db
     end
 
+    def parallel?
+      parallel_workers > 1
+    end
+
+    def parallel_workers
+      @parallel_workers ||= [opts[:parallel].to_i, 1].max
+    end
+
+    def completed_tables_mutex
+      @completed_tables_mutex ||= Mutex.new
+    end
+
+    def add_completed_table(table_name)
+      completed_tables_mutex.synchronize do
+        completed_tables << table_name.to_s
+      end
+    end
+
     def format_number(num)
       num.to_s.gsub(/(\d)(?=(\d\d\d)+(?!\d))/, "\\1,")
     end
@@ -183,7 +203,8 @@ module Tapsoob
 
       progress = ProgressBar.new('Schema', tables.size)
       tables.each do |table_name, count|
-        schema_data = Tapsoob::Schema.dump_table(database_url, table_name, @opts.slice(:indexes, :same_db))
+        # Reuse existing db connection for better performance
+        schema_data = Tapsoob::Schema.dump_table(db, table_name, @opts.slice(:indexes, :same_db))
         log.debug "Table: #{table_name}\n#{schema_data}\n"
         output = Tapsoob::Utils.export_schema(dump_path, table_name, schema_data)
         puts output if dump_path.nil? && output
@@ -197,6 +218,14 @@ module Tapsoob
 
       log.info "#{tables.size} tables, #{format_number(record_count)} records"
 
+      if parallel?
+        pull_data_parallel
+      else
+        pull_data_serial
+      end
+    end
+
+    def pull_data_serial
       tables.each do |table_name, count|
         stream   = Tapsoob::DataStream.factory(db, {
           :chunksize  => default_chunksize,
@@ -208,6 +237,38 @@ module Tapsoob
       end
     end
 
+    def pull_data_parallel
+      log.info "Using #{parallel_workers} parallel workers"
+
+      multi_progress = opts[:progress] ? MultiProgressBar.new(parallel_workers) : nil
+      table_queue = Queue.new
+      tables.each { |table_name, count| table_queue << [table_name, count] }
+
+      workers = (1..parallel_workers).map do
+        Thread.new do
+          loop do
+            break if table_queue.empty?
+
+            table_name, count = table_queue.pop(true) rescue break
+
+            # Each thread gets its own connection from the pool
+            stream = Tapsoob::DataStream.factory(db, {
+              :chunksize  => default_chunksize,
+              :table_name => table_name
+            }, { :debug => opts[:debug] })
+
+            estimated_chunks = [(count.to_f / default_chunksize).ceil, 1].max
+            progress = multi_progress ? multi_progress.create_bar(table_name.to_s, estimated_chunks) : nil
+
+            pull_data_from_table(stream, progress)
+          end
+        end
+      end
+
+      workers.each(&:join)
+      multi_progress.stop if multi_progress
+    end
+
     def pull_partial_data
       return if stream_state == {}
 
@@ -279,7 +340,7 @@ module Tapsoob
       end
 
       progress.finish if progress
-      completed_tables << stream.table_name.to_s
+      add_completed_table(stream.table_name)
       self.stream_state = {}
     end
 
@@ -332,9 +393,12 @@ module Tapsoob
       raw_idxs = Tapsoob::Schema.indexes_individual(database_url)
       idxs     = (raw_idxs && raw_idxs.length >= 2 ? JSON.parse(raw_idxs) : {})
 
-      apply_table_filter(idxs).each do |table, indexes|
-        next unless indexes.size > 0
-        progress = ProgressBar.new(table, indexes.size)
+      # Calculate max title width for consistent alignment
+      filtered_idxs = apply_table_filter(idxs).select { |table, indexes| indexes.size > 0 }
+      max_title_width = filtered_idxs.keys.map { |table| "#{table} indexes".length }.max || 14
+
+      filtered_idxs.each do |table, indexes|
+        progress = ProgressBar.new("#{table} indexes", indexes.size, STDOUT, max_title_width)
         indexes.each do |idx|
           output = Tapsoob::Utils.export_indexes(dump_path, table, idx)
           puts output if dump_path.nil? && output
@@ -388,9 +452,12 @@ module Tapsoob
 
       log.info "Sending indexes"
 
-      apply_table_filter(idxs).each do |table, indexes|
-        next unless indexes.size > 0
-        progress = ProgressBar.new(table, indexes.size)
+      # Calculate max title width for consistent alignment
+      filtered_idxs = apply_table_filter(idxs).select { |table, indexes| indexes.size > 0 }
+      max_title_width = filtered_idxs.keys.map { |table| "#{table} indexes".length }.max || 14
+
+      filtered_idxs.each do |table, indexes|
+        progress = ProgressBar.new("#{table} indexes", indexes.size, STDOUT, max_title_width)
         indexes.each do |idx|
           Tapsoob::Utils.load_indexes(database_url, idx)
           progress.inc(1)
@@ -405,7 +472,8 @@ module Tapsoob
       progress = ProgressBar.new('Schema', tables.size)
       tables.each do |table, count|
         log.debug "Loading '#{table}' schema\n"
-        Tapsoob::Utils.load_schema(dump_path, database_url, table)
+        # Reuse existing db connection for better performance
+        Tapsoob::Utils.load_schema(dump_path, db, table)
         progress.inc(1)
       end
       progress.finish
@@ -435,6 +503,14 @@ module Tapsoob
 
       log.info "#{tables.size} tables, #{format_number(record_count)} records"
 
+      if parallel?
+        push_data_parallel
+      else
+        push_data_serial
+      end
+    end
+
+    def push_data_serial
       tables.each do |table_name, count|
         # Skip if data file doesn't exist or has no data
         data_file = File.join(dump_path, "data", "#{table_name}.json")
@@ -455,6 +531,49 @@ module Tapsoob
       end
     end
 
+    def push_data_parallel
+      log.info "Using #{parallel_workers} parallel workers"
+
+      multi_progress = opts[:progress] ? MultiProgressBar.new(parallel_workers) : nil
+      table_queue = Queue.new
+
+      tables.each do |table_name, count|
+        data_file = File.join(dump_path, "data", "#{table_name}.json")
+        next unless File.exist?(data_file) && count > 0
+        table_queue << [table_name, count]
+      end
+
+      workers = (1..parallel_workers).map do
+        Thread.new do
+          loop do
+            break if table_queue.empty?
+
+            table_name, count = table_queue.pop(true) rescue break
+
+            # Each thread gets its own connection from the pool
+            db[table_name.to_sym].truncate if @opts[:purge]
+            stream = Tapsoob::DataStream.factory(db, {
+              :table_name => table_name,
+              :chunksize => default_chunksize
+            }, {
+              :"skip-duplicates" => opts[:"skip-duplicates"] || false,
+              :"discard-identity" => opts[:"discard-identity"] || false,
+              :purge => opts[:purge] || false,
+              :debug => opts[:debug]
+            })
+
+            estimated_chunks = [(count.to_f / default_chunksize).ceil, 1].max
+            progress = multi_progress ? multi_progress.create_bar(table_name.to_s, estimated_chunks) : nil
+
+            push_data_from_file(stream, progress)
+          end
+        end
+      end
+
+      workers.each(&:join)
+      multi_progress.stop if multi_progress
+    end
+
     def push_data_from_file(stream, progress)
       loop do
         if exiting?
@@ -508,7 +627,7 @@ module Tapsoob
       end
 
       progress.finish if progress
-      completed_tables << stream.table_name.to_s
+      add_completed_table(stream.table_name)
       self.stream_state = {}
     end
 

data/lib/tapsoob/progress_bar.rb

@@ -13,7 +13,7 @@
 class ProgressBar
   VERSION = "0.9"
 
-  def initialize (title, total, out = STDOUT)
+  def initialize (title, total, out = STDOUT, title_width = nil)
     @title = title
     @total = total
     @out = out
@@ -24,7 +24,8 @@ class ProgressBar
     @finished_p = false
     @start_time = ::Time.now
     @previous_time = @start_time
-    @title_width = 14
+    # Set title width: use provided width, or accommodate the title, with a minimum of 14
+    @title_width = title_width || [title.length, 14].max
     @format = "%-#{@title_width}s %3d%% %s %s"
     @format_arguments = [:title, :percentage, :bar, :stat]
     clear

data/lib/tapsoob/schema.rb

@@ -31,9 +31,11 @@ END_MIG
       template.result(binding)
     end
 
-    def dump_table(database_url, table, options)
+    def dump_table(database_url_or_db, table, options)
       table = table.to_sym
-      Sequel.connect(database_url) do |db|
+      # Accept either a database URL or an existing connection object
+      if database_url_or_db.is_a?(Sequel::Database)
+        db = database_url_or_db
         db.extension :schema_dumper
         <<END_MIG
 Class.new(Sequel::Migration) do
@@ -46,6 +48,21 @@ Class.new(Sequel::Migration) do
   end
 end
 END_MIG
+      else
+        Sequel.connect(database_url_or_db) do |db|
+          db.extension :schema_dumper
+          <<END_MIG
+Class.new(Sequel::Migration) do
+  def up
+    #{db.dump_table_schema(table, options)}
+  end
+
+  def down
+    drop_table("#{table}", if_exists: true)
+  end
+end
+END_MIG
+        end
       end
     end
 
@@ -86,8 +103,10 @@ END_MIG
       JSON.generate(idxs)
     end
 
-    def load(database_url, schema, options = { drop: false })
-      Sequel.connect(database_url) do |db|
+    def load(database_url_or_db, schema, options = { drop: false })
+      # Accept either a database URL or an existing connection object
+      if database_url_or_db.is_a?(Sequel::Database)
+        db = database_url_or_db
         db.extension :schema_dumper
         klass = eval(schema)
         if options[:drop]
@@ -101,6 +120,22 @@ END_MIG
           db.run("SET foreign_key_checks = 1") if [:mysql, :mysql2].include?(db.adapter_scheme)
         end
         klass.apply(db, :up)
+      else
+        Sequel.connect(database_url_or_db) do |db|
+          db.extension :schema_dumper
+          klass = eval(schema)
+          if options[:drop]
+            # Start special hack for MySQL
+            db.run("SET foreign_key_checks = 0") if [:mysql, :mysql2].include?(db.adapter_scheme)
+
+            # Run down migration
+            klass.apply(db, :down)
+
+            # End special hack for MySQL
+            db.run("SET foreign_key_checks = 1") if [:mysql, :mysql2].include?(db.adapter_scheme)
+          end
+          klass.apply(db, :up)
+        end
       end
     end
 

data/lib/tapsoob/utils.rb

@@ -161,9 +161,16 @@ Data : #{data}
       end
     end
 
-    def load_schema(dump_path, database_url, table)
-      schema = File.join(dump_path, "schemas", "#{table}.rb")
-      schema_bin(:load, database_url, schema.to_s)
+    def load_schema(dump_path, database_url_or_db, table)
+      schema_file = File.join(dump_path, "schemas", "#{table}.rb")
+      schema_content = File.read(schema_file)
+
+      # If we have a connection object, use it directly for better performance
+      if database_url_or_db.is_a?(Sequel::Database)
+        Tapsoob::Schema.load(database_url_or_db, schema_content)
+      else
+        schema_bin(:load, database_url_or_db, schema_file)
+      end
     end
 
     def load_indexes(database_url, index)

data/lib/tapsoob/version.rb

@@ -1,4 +1,4 @@
 # -*- encoding : utf-8 -*-
 module Tapsoob
-  VERSION = "0.5.29".freeze
+  VERSION = "0.6.0".freeze
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: tapsoob
 version: !ruby/object:Gem::Version
-  version: 0.5.29
+  version: 0.6.0
 platform: ruby
 authors:
 - Félix Bellanger
@@ -104,6 +104,7 @@ files:
 - lib/tapsoob/data_stream.rb
 - lib/tapsoob/errors.rb
 - lib/tapsoob/log.rb
+- lib/tapsoob/multi_progress_bar.rb
 - lib/tapsoob/operation.rb
 - lib/tapsoob/progress_bar.rb
 - lib/tapsoob/railtie.rb