tapsoob 0.5.31 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bff0c57641151d932d224ec7b245a23c4b993eac3c8da25f10724208b46f5953
-  data.tar.gz: 186003a5f0156be1ebaac8f80ef44a3f985b23544a33a1e127067b776cf50d88
+  metadata.gz: 682ebd67a432169d8f72eb47412aaface0df4ba058be6849fba6e91ab96888c1
+  data.tar.gz: 3aba7ecf086fc98b21a9c2c6b2e9ae3413fd71d98106de415a3e3223732b0947
 SHA512:
-  metadata.gz: c78e9d29d6247823deae5f9f1a86c59792b4e52c34f4d1536cdd16bc6168a2d38b3bfba434ec96b2be72d633e927f672578e70904387435a4152bc212ac544d5
-  data.tar.gz: 30a3507296d49349a3049fc2e24b61cbc98f49fdd0c38ce05f1714aefa0a598eed832ac5826c213f982b549708b566cc263053acf085b86ecc9fa7beb4ac6837
+  metadata.gz: d3b49e8f114d4c2b693a524b60ee3959b36cdddb3dd7e2f6378a9f72a6f86a1ca44ab26b609cf4a4d88747813fffdd233a70efa34465df613ceea0af18af6aa2
+  data.tar.gz: 9793f681463d9f92ef4eae1015ed11b35d351a7f5938b2ad79e5d3cc67793fc421a02e195536957a56acde9b8d8cc81ab429a5a0852ae6b2d2eddcf38445222b

data/lib/tapsoob/cli/root.rb

@@ -22,6 +22,7 @@ module Tapsoob
       option :"exclude-tables", desc: "Shortcut to exclude a list of tables", type: :array, aliases: "-e"
       option :"indexes", type: :boolean, default: false
       option :"same-db", type: :boolean, default: false
+      option :parallel, desc: "Number of parallel workers for table processing (default: 1)", default: 1, type: :numeric, aliases: "-j"
       option :progress, desc: "Show progress", default: true, type: :boolean
       option :debug, desc: "Enable debug messages", default: false, type: :boolean
       def pull(dump_path, database_url)
@@ -47,6 +48,7 @@ module Tapsoob
       option :purge, desc: "Purge data in tables prior to performing the import", default: false, type: :boolean, aliases: "-p"
       option :"skip-duplicates", desc: "Remove duplicates when loading data", default: false, type: :boolean
       option :"discard-identity", desc: "Remove identity when pushing data (may result in creating duplicates)", default: false, type: :boolean
+      option :parallel, desc: "Number of parallel workers for table processing (default: 1)", default: 1, type: :numeric, aliases: "-j"
       option :progress, desc: "Show progress", default: true, type: :boolean
       option :debug, desc: "Enable debug messages", default: false, type: :boolean
       def push(dump_path, database_url)
@@ -82,6 +84,7 @@ module Tapsoob
             indexes_first: options[:"indexes_first"],
             disable_compression: options[:"disable-compression"],
             tables: options[:tables],
+            parallel: options[:parallel],
             progress: options[:progress],
             debug: options[:debug]
           })

data/lib/tapsoob/multi_progress_bar.rb

@@ -0,0 +1,234 @@
+# -*- encoding : utf-8 -*-
+require 'tapsoob/progress_bar'
+
+# MultiProgressBar manages multiple progress bars in parallel
+# Each bar gets its own line in the terminal
+class MultiProgressBar
+  def initialize(max_bars = 4)
+    @max_bars = max_bars
+    @bars = []
+    @mutex = Mutex.new
+    @active = true
+    @out = STDOUT
+    @last_update = Time.now
+    @reserved_lines = 0  # Track how many lines we've actually reserved
+    @max_title_width = 14  # Minimum width, will grow with longer titles
+  end
+
+  # Create a new progress bar and return it
+  def create_bar(title, total)
+    @mutex.synchronize do
+      # Remove any existing bar with the same title to prevent duplicates
+      @bars.reject! { |b| b.title == title }
+
+      # Update max title width to accommodate longer titles
+      @max_title_width = [@max_title_width, title.length].max
+
+      bar = ThreadSafeProgressBar.new(title, total, self)
+
+      # Reserve a line for this new bar during active updates
+      # Cap at 2 * max_bars to show active workers + some recent finished bars
+      if @reserved_lines < @max_bars * 2
+        @out.print "\n"
+        @out.flush
+        @reserved_lines += 1
+      end
+
+      @bars << bar
+      bar
+    end
+  end
+
+  # Get the current maximum title width for alignment
+  # Note: Always called from within synchronized methods, so no mutex needed
+  def max_title_width
+    @max_title_width
+  end
+
+  # Called by individual bars when they update
+  def update
+    @mutex.synchronize do
+      return unless @active
+      return unless should_redraw?
+
+      @last_update = Time.now
+      redraw_all
+    end
+  end
+
+  # Finish a specific bar - mark it as completed
+  def finish_bar(bar)
+    @mutex.synchronize do
+      return unless @active
+
+      bar.mark_finished
+
+      # Respect throttle when finishing to avoid spamming redraws
+      if should_redraw?
+        @last_update = Time.now
+        redraw_all
+      end
+      # If throttled, the next regular update will show the finished state
+    end
+  end
+
+  # Stop all progress bars and keep them visible
+  def stop
+    @mutex.synchronize do
+      @active = false
+
+      # Final cleanup: remove any duplicate titles (keep the last occurrence of each unique title)
+      @bars = @bars.reverse.uniq { |bar| bar.title }.reverse
+
+      # Final redraw to show completed state (skip active check)
+      redraw_all(true)
+      # Move cursor past all bars
+      @out.print "\n"
+      @out.flush
+    end
+  end
+
+  private
+
+  # Check if enough time has passed to redraw (throttle to 10 updates/sec)
+  def should_redraw?
+    Time.now - @last_update >= 0.1
+  end
+
+  def redraw_all(force = false)
+    return unless force || @active
+    return if @bars.empty?
+
+    if force && !@active
+      render_final_display
+    else
+      render_active_display
+    end
+  end
+
+  # Final display: show all completed bars
+  def render_final_display
+    # Clear the reserved lines first
+    if @reserved_lines > 0
+      @out.print "\r\e[#{@reserved_lines}A"
+      @reserved_lines.times { @out.print "\r\e[K\n" }
+    end
+
+    # Print all bars (adds new lines as needed)
+    @bars.each do |bar|
+      @out.print "\r\e[K"
+      bar.render_to(@out)
+      @out.print "\n"
+    end
+
+    @out.flush
+  end
+
+  # Normal operation: show active bars + recent finished in reserved space
+  def render_active_display
+    return if @reserved_lines == 0
+
+    # Partition bars in a single pass for efficiency
+    active_bars, finished_bars = @bars.partition { |b| !b.finished? }
+
+    # Build display: active bars first, then recent finished to fill remaining space
+    # Ensure we don't request negative count from .last()
+    remaining_space = [@reserved_lines - active_bars.length, 0].max
+    bars_to_draw = active_bars + finished_bars.last(remaining_space)
+
+    # If we have more bars than reserved lines, show only the most recent
+    bars_to_draw = bars_to_draw.last(@reserved_lines) if bars_to_draw.length > @reserved_lines
+
+    # Move up and redraw in reserved space
+    @out.print "\r\e[#{@reserved_lines}A"
+    @reserved_lines.times do |i|
+      @out.print "\r\e[K"
+      bars_to_draw[i].render_to(@out) if i < bars_to_draw.length
+      @out.print "\n"
+    end
+
+    @out.flush
+  end
+end
+
+# Thread-safe progress bar that reports to a MultiProgressBar
+class ThreadSafeProgressBar < ProgressBar
+  attr_reader :title
+
+  def initialize(title, total, multi_progress_bar)
+    @multi_progress_bar = multi_progress_bar
+    @out = STDOUT  # Need this for get_width to work
+    # Don't call parent initialize, we'll manage output ourselves
+    @title = title
+    @total = total
+    @terminal_width = 80
+    @bar_mark = "="
+    @current = 0
+    @previous = 0
+    @finished_p = false
+    @start_time = ::Time.now
+    @previous_time = @start_time
+    @format_arguments = [:title, :percentage, :bar, :stat]
+  end
+
+  # Override show to notify multi-progress instead of direct output
+  def show
+    @previous_time = ::Time.now  # Update to prevent time-based refresh spam
+    @multi_progress_bar.update
+  end
+
+  # Render this bar to the given output stream
+  def render_to(out)
+    # Get dynamic title width from MultiProgressBar for consistent alignment
+    # Store as instance variable so parent class fmt_* methods can use it
+    @title_width = @multi_progress_bar.max_title_width
+
+    # Recalculate terminal width to handle resizes and use full width
+    width = get_width
+    # Calculate bar width: total_width - fixed_elements - padding
+    # Fixed: title(variable) + " "(1) + percentage(4) + " "(1) + "|"(1) + "|"(1) + " "(1) + timer(15) = title_width + 25
+    # Padding: +3 for timer fluctuations and safety
+    fixed_chars = @title_width + 28
+    @terminal_width = [width - fixed_chars, 20].max
+
+    # Build format string with dynamic title width
+    format = "%-#{@title_width}s %3d%% %s %s"
+    arguments = @format_arguments.map { |method| send("fmt_#{method}") }
+    line = sprintf(format, *arguments)
+
+    # Ensure line doesn't exceed terminal width to prevent wrapping
+    # Leave 2 chars margin for safety
+    line = line[0, width - 2] if line.length > width - 2
+
+    out.print(line)
+  end
+
+  # Override clear to do nothing (managed by MultiProgressBar)
+  def clear
+    # no-op
+  end
+
+  # Mark this bar as finished (for tracking)
+  def mark_finished
+    @finished_p = true
+  end
+
+  # Override to use the same @finished_p flag
+  def finished?
+    @finished_p
+  end
+
+  # Override finish to notify multi-progress
+  def finish
+    @current = @total
+    @multi_progress_bar.finish_bar(self)
+  end
+
+  # Override inc to check if we need to update
+  def inc(step = 1)
+    @current += step
+    @current = @total if @current > @total
+    show_if_needed
+    @previous = @current
+  end
+end

data/lib/tapsoob/operation.rb

@@ -1,9 +1,11 @@
 # -*- encoding : utf-8 -*-
 require 'sequel'
+require 'thread'
 
 require 'tapsoob/data_stream'
 require 'tapsoob/log'
 require 'tapsoob/progress_bar'
+require 'tapsoob/multi_progress_bar'
 require 'tapsoob/schema'
 
 module Tapsoob
@@ -117,7 +119,7 @@ module Tapsoob
     end
 
     def db
-      @db ||= Sequel.connect(database_url)
+      @db ||= Sequel.connect(database_url, max_connections: parallel_workers * 2)
       @db.extension :schema_dumper
       @db.loggers << Tapsoob.log if opts[:debug]
 
@@ -130,6 +132,24 @@ module Tapsoob
       @db
     end
 
+    def parallel?
+      parallel_workers > 1
+    end
+
+    def parallel_workers
+      @parallel_workers ||= [opts[:parallel].to_i, 1].max
+    end
+
+    def completed_tables_mutex
+      @completed_tables_mutex ||= Mutex.new
+    end
+
+    def add_completed_table(table_name)
+      completed_tables_mutex.synchronize do
+        completed_tables << table_name.to_s
+      end
+    end
+
     def format_number(num)
       num.to_s.gsub(/(\d)(?=(\d\d\d)+(?!\d))/, "\\1,")
     end
@@ -198,6 +218,14 @@ module Tapsoob
 
       log.info "#{tables.size} tables, #{format_number(record_count)} records"
 
+      if parallel?
+        pull_data_parallel
+      else
+        pull_data_serial
+      end
+    end
+
+    def pull_data_serial
       tables.each do |table_name, count|
         stream   = Tapsoob::DataStream.factory(db, {
           :chunksize  => default_chunksize,
@@ -209,6 +237,38 @@ module Tapsoob
       end
     end
 
+    def pull_data_parallel
+      log.info "Using #{parallel_workers} parallel workers"
+
+      multi_progress = opts[:progress] ? MultiProgressBar.new(parallel_workers) : nil
+      table_queue = Queue.new
+      tables.each { |table_name, count| table_queue << [table_name, count] }
+
+      workers = (1..parallel_workers).map do
+        Thread.new do
+          loop do
+            break if table_queue.empty?
+
+            table_name, count = table_queue.pop(true) rescue break
+
+            # Each thread gets its own connection from the pool
+            stream = Tapsoob::DataStream.factory(db, {
+              :chunksize  => default_chunksize,
+              :table_name => table_name
+            }, { :debug => opts[:debug] })
+
+            estimated_chunks = [(count.to_f / default_chunksize).ceil, 1].max
+            progress = multi_progress ? multi_progress.create_bar(table_name.to_s, estimated_chunks) : nil
+
+            pull_data_from_table(stream, progress)
+          end
+        end
+      end
+
+      workers.each(&:join)
+      multi_progress.stop if multi_progress
+    end
+
     def pull_partial_data
       return if stream_state == {}
 
@@ -280,7 +340,7 @@ module Tapsoob
       end
 
       progress.finish if progress
-      completed_tables << stream.table_name.to_s
+      add_completed_table(stream.table_name)
       self.stream_state = {}
     end
 
@@ -333,9 +393,12 @@ module Tapsoob
       raw_idxs = Tapsoob::Schema.indexes_individual(database_url)
       idxs     = (raw_idxs && raw_idxs.length >= 2 ? JSON.parse(raw_idxs) : {})
 
-      apply_table_filter(idxs).each do |table, indexes|
-        next unless indexes.size > 0
-        progress = ProgressBar.new(table, indexes.size)
+      # Calculate max title width for consistent alignment
+      filtered_idxs = apply_table_filter(idxs).select { |table, indexes| indexes.size > 0 }
+      max_title_width = filtered_idxs.keys.map { |table| "#{table} indexes".length }.max || 14
+
+      filtered_idxs.each do |table, indexes|
+        progress = ProgressBar.new("#{table} indexes", indexes.size, STDOUT, max_title_width)
         indexes.each do |idx|
           output = Tapsoob::Utils.export_indexes(dump_path, table, idx)
           puts output if dump_path.nil? && output
@@ -389,9 +452,12 @@ module Tapsoob
 
       log.info "Sending indexes"
 
-      apply_table_filter(idxs).each do |table, indexes|
-        next unless indexes.size > 0
-        progress = ProgressBar.new(table, indexes.size)
+      # Calculate max title width for consistent alignment
+      filtered_idxs = apply_table_filter(idxs).select { |table, indexes| indexes.size > 0 }
+      max_title_width = filtered_idxs.keys.map { |table| "#{table} indexes".length }.max || 14
+
+      filtered_idxs.each do |table, indexes|
+        progress = ProgressBar.new("#{table} indexes", indexes.size, STDOUT, max_title_width)
         indexes.each do |idx|
           Tapsoob::Utils.load_indexes(database_url, idx)
           progress.inc(1)
@@ -437,6 +503,14 @@ module Tapsoob
 
       log.info "#{tables.size} tables, #{format_number(record_count)} records"
 
+      if parallel?
+        push_data_parallel
+      else
+        push_data_serial
+      end
+    end
+
+    def push_data_serial
       tables.each do |table_name, count|
         # Skip if data file doesn't exist or has no data
         data_file = File.join(dump_path, "data", "#{table_name}.json")
@@ -457,6 +531,49 @@ module Tapsoob
       end
     end
 
+    def push_data_parallel
+      log.info "Using #{parallel_workers} parallel workers"
+
+      multi_progress = opts[:progress] ? MultiProgressBar.new(parallel_workers) : nil
+      table_queue = Queue.new
+
+      tables.each do |table_name, count|
+        data_file = File.join(dump_path, "data", "#{table_name}.json")
+        next unless File.exist?(data_file) && count > 0
+        table_queue << [table_name, count]
+      end
+
+      workers = (1..parallel_workers).map do
+        Thread.new do
+          loop do
+            break if table_queue.empty?
+
+            table_name, count = table_queue.pop(true) rescue break
+
+            # Each thread gets its own connection from the pool
+            db[table_name.to_sym].truncate if @opts[:purge]
+            stream = Tapsoob::DataStream.factory(db, {
+              :table_name => table_name,
+              :chunksize => default_chunksize
+            }, {
+              :"skip-duplicates" => opts[:"skip-duplicates"] || false,
+              :"discard-identity" => opts[:"discard-identity"] || false,
+              :purge => opts[:purge] || false,
+              :debug => opts[:debug]
+            })
+
+            estimated_chunks = [(count.to_f / default_chunksize).ceil, 1].max
+            progress = multi_progress ? multi_progress.create_bar(table_name.to_s, estimated_chunks) : nil
+
+            push_data_from_file(stream, progress)
+          end
+        end
+      end
+
+      workers.each(&:join)
+      multi_progress.stop if multi_progress
+    end
+
     def push_data_from_file(stream, progress)
       loop do
         if exiting?
@@ -510,7 +627,7 @@ module Tapsoob
       end
 
       progress.finish if progress
-      completed_tables << stream.table_name.to_s
+      add_completed_table(stream.table_name)
       self.stream_state = {}
     end
 

data/lib/tapsoob/progress_bar.rb

@@ -13,7 +13,7 @@
 class ProgressBar
   VERSION = "0.9"
 
-  def initialize (title, total, out = STDOUT)
+  def initialize (title, total, out = STDOUT, title_width = nil)
     @title = title
     @total = total
     @out = out
@@ -24,7 +24,8 @@ class ProgressBar
     @finished_p = false
     @start_time = ::Time.now
     @previous_time = @start_time
-    @title_width = 14
+    # Set title width: use provided width, or accommodate the title, with a minimum of 14
+    @title_width = title_width || [title.length, 14].max
     @format = "%-#{@title_width}s %3d%% %s %s"
     @format_arguments = [:title, :percentage, :bar, :stat]
     clear

data/lib/tapsoob/version.rb

@@ -1,4 +1,4 @@
 # -*- encoding : utf-8 -*-
 module Tapsoob
-  VERSION = "0.5.31".freeze
+  VERSION = "0.6.0".freeze
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: tapsoob
 version: !ruby/object:Gem::Version
-  version: 0.5.31
+  version: 0.6.0
 platform: ruby
 authors:
 - Félix Bellanger
@@ -104,6 +104,7 @@ files:
 - lib/tapsoob/data_stream.rb
 - lib/tapsoob/errors.rb
 - lib/tapsoob/log.rb
+- lib/tapsoob/multi_progress_bar.rb
 - lib/tapsoob/operation.rb
 - lib/tapsoob/progress_bar.rb
 - lib/tapsoob/railtie.rb