tapsoob 0.6.0-java → 0.6.2-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b2d30be0a3150a1592ece7ced339c7bf56c1de3f3b4c0fe9468b2370e8e6b855
-  data.tar.gz: e7ceee5ebe56f0fa375bb76e5ab360604b9231ce8a5c1263fabe24c84abc15e5
+  metadata.gz: 4a60a9f569ac31e7a9390328af8974ee95a056a3d35b510eedafb13fd9ab3331
+  data.tar.gz: 5f6a3ca22a9b95bff31adf8a9921bfc79e3768047fbe769a0c9c3adbfe72c7f7
 SHA512:
-  metadata.gz: d19b9ad09c465356dcfa49af92ded16dcf496c9743f1a345a5ca3b6afb46570ae8c1d646400a98b504e3bc86ff8e59e083fb673b960da6b6b616ff04ca6d218f
-  data.tar.gz: 15ce04aad9422147b81b2b51fee2e9d270ab824680bb777e8ea5014c9f11da1b83cfc426352a9db5cfb9b47fc0fb9ce355e2eee43ad68d3103c4d6e220d042dc
+  metadata.gz: c7f62ebad5ce2449ae283a22f5ae0e63faaa84cdce808537eda8e35f60346ff782be1826c3614fd4636787b67df8e1cc78f1ce1d74103d33c78b2ff41146dbec
+  data.tar.gz: 91911b166a9bae36b504dfa2112088fc11033f36f8b100ded1e9e9d94ad409c9bfb23a7dffe5afb5ec0d6e1adcc619a7923cf774bc9856c0013e6a78b2ae16fd

data/README.md

@@ -36,7 +36,7 @@ You can list all available options using the command:
     tapsoob push -h
 
 
-## NEW : Piping your schema/indexes/data
+## Piping your schema/indexes/data
 
 Due to some needs we added ways to pipe your schema/indexes/data directly from one database to another, here's an equivalent of the export/import process described above using this technique :
 
@@ -60,6 +60,16 @@ If you're using Rails, there's also two Rake tasks provided:
 * `tapsoob:pull` which dumps the database into a new folder under the `db` folder
 * `tapsoob:push` which reads the last dump you made from `tapsoob:pull` from the `db` folder
 
+## NEW : Full parallelization support from 0.6.1 onwards
+
+You can now dump/load a full database or data using parallelization to speed up the process at memory cost and database load like so :
+
+```
+tapsoob pull [OPTIONS] <dump_path> <database_url> -j <number_of_threads>
+tapsoob push [OPTIONS] <dump_path> <database_url> -j <number_of_threads>
+```
+
+It defaults to a single thread as per pre 0.6.1, it is also appliable to `tapsoob data pull/push` but only when dumping to files, you can't parallelize and pipe for obvious reasons, it'll fall back to a single thread with a warning if you try to do this for safety.
 
 ## Notes
 

data/lib/tapsoob/cli/data_stream.rb

@@ -14,10 +14,20 @@ module Tapsoob
       option :chunksize, desc: "Initial chunksize", default: 1000, type: :numeric, aliases: "-c"
       option :tables, desc: "Shortcut to filter on a list of tables", type: :array, aliases: "-t"
       option :"exclude-tables", desc: "Shortcut to exclude a list of tables", type: :array, aliases: "-e"
+      option :parallel, desc: "Number of parallel workers for table processing (default: 1)", default: 1, type: :numeric, aliases: "-j"
       option :progress, desc: "Show progress", default: true, type: :boolean, aliases: "-p"
       option :debug, desc: "Enable debug messages", default: false, type: :boolean, aliases: "-d"
       def pull(database_url, dump_path = nil)
-        op = Tapsoob::Operation.factory(:pull, database_url, dump_path, parse_opts(options))
+        opts = parse_opts(options)
+
+        # Force serial mode when outputting to STDOUT (for piping)
+        # Parallel mode would interleave output and corrupt the JSON stream
+        if dump_path.nil? && opts[:parallel] && opts[:parallel] > 1
+          STDERR.puts "Warning: Parallel mode disabled when outputting to STDOUT (for piping)"
+          opts[:parallel] = 1
+        end
+
+        op = Tapsoob::Operation.factory(:pull, database_url, dump_path, opts)
         op.pull_data
       end
 
@@ -25,36 +35,48 @@ module Tapsoob
       option :chunksize, desc: "Initial chunksize", default: 1000, type: :numeric, aliases: "-c"
       option :tables, desc: "Shortcut to filter on a list of tables", type: :array, aliases: "-t"
       option :"exclude-tables", desc: "Shortcut to exclude a list of tables", type: :array, aliases: "-e"
+      option :parallel, desc: "Number of parallel workers for table processing (default: 1)", default: 1, type: :numeric, aliases: "-j"
       option :progress, desc: "Show progress", default: true, type: :boolean, aliases: "-p"
       option :purge, desc: "Purge data in tables prior to performing the import", default: false, type: :boolean
       option :"skip-duplicates", desc: "Remove duplicates when loading data", default: false, type: :boolean
       option :"discard-identity", desc: "Remove identity when pushing data (may result in creating duplicates)", default: false, type: :boolean
       option :debug, desc: "Enable debug messages", default: false, type: :boolean, aliases: "-d"
       def push(database_url, dump_path = nil)
-        # instantiate stuff
-        data = []
         opts = parse_opts(options)
 
-        # read data from dump_path or from STDIN
+        # If dump_path is provided, use the Operation class for proper parallel support
         if dump_path && Dir.exist?(dump_path)
-          files = Dir[Pathname.new(dump_path).join("*.json")]
-          files.each { |file| data << JSON.parse(File.read(file), symbolize_names: true) }
+          op = Tapsoob::Operation.factory(:push, database_url, dump_path, opts)
+          op.push_data
         else
+          # STDIN mode: read and import data directly (no parallel support for STDIN)
+          if opts[:parallel] && opts[:parallel] > 1
+            STDERR.puts "Warning: Parallel mode not supported when reading from STDIN"
+          end
+
+          data = []
           STDIN.each_line { |line| data << JSON.parse(line, symbolize_names: true) }
-        end
 
-        # import data
-        data.each do |table|
-          stream = Tapsoob::DataStream.factory(db(database_url, opts), {
-            table_name: table[:table_name],
-            chunksize: opts[:default_chunksize]
-          }, { :"discard-identity" => opts[:"discard-identity"] || false, :purge => opts[:purge] || false, :debug => opts[:debug] })
-
-          begin
-            stream.import_rows(table)
-          rescue Exception => e
-            stream.log.debug e.message
-            STDERR.puts "Error loading data in #{table[:table_name]} : #{e.message}"
+          # import data
+          data.each do |table|
+            table_name = table[:table_name]
+
+            # Truncate table if purge option is enabled
+            if opts[:purge]
+              db(database_url, opts)[table_name.to_sym].truncate
+            end
+
+            stream = Tapsoob::DataStream.factory(db(database_url, opts), {
+              table_name: table_name,
+              chunksize: opts[:default_chunksize]
+            }, { :"discard-identity" => opts[:"discard-identity"] || false, :purge => opts[:purge] || false, :debug => opts[:debug] })
+
+            begin
+              stream.import_rows(table)
+            rescue Exception => e
+              stream.log.debug e.message
+              STDERR.puts "Error loading data in #{table_name} : #{e.message}"
+            end
           end
         end
       end
@@ -65,6 +87,7 @@ module Tapsoob
           opts = {
             progress: options[:progress],
             tables: options[:tables],
+            parallel: options[:parallel],
             debug: options[:debug]
           }
 
@@ -85,7 +108,9 @@ module Tapsoob
         end
 
         def db(database_url, opts = {})
-          @db ||= Sequel.connect(database_url)
+          # Support connection pooling for parallel operations
+          parallel_workers = opts[:parallel] || 1
+          @db ||= Sequel.connect(database_url, max_connections: parallel_workers * 2)
           @db.loggers << Tapsoob.log if opts[:debug]
 
           # Set parameters

data/lib/tapsoob/operation.rb

@@ -4,8 +4,7 @@ require 'thread'
 
 require 'tapsoob/data_stream'
 require 'tapsoob/log'
-require 'tapsoob/progress_bar'
-require 'tapsoob/multi_progress_bar'
+require 'tapsoob/progress'
 require 'tapsoob/schema'
 
 module Tapsoob

data/lib/tapsoob/progress/bar.rb

@@ -0,0 +1,245 @@
+# -*- encoding : utf-8 -*-
+#
+# Ruby/ProgressBar - a text progress bar library
+#
+# Copyright (C) 2001-2005 Satoru Takabayashi <satoru@namazu.org>
+#     All rights reserved.
+#     This is free software with ABSOLUTELY NO WARRANTY.
+#
+# You can redistribute it and/or modify it under the terms
+# of Ruby's license.
+#
+
+module Tapsoob
+  module Progress
+    class Bar
+      VERSION = "0.9"
+
+      def initialize (title, total, out = STDOUT, title_width = nil)
+        @title = title
+        @total = total
+        @out = out
+        @terminal_width = 80
+        @bar_mark = "="
+        @current = 0
+        @previous = 0
+        @finished_p = false
+        @start_time = ::Time.now
+        @previous_time = @start_time
+        # Set title width: use provided width, or accommodate the title, with a minimum of 14
+        @title_width = title_width || [title.length, 14].max
+        @format = "%-#{@title_width}s %3d%% %s %s"
+        @format_arguments = [:title, :percentage, :bar, :stat]
+        clear
+        show
+      end
+      attr_reader   :title
+      attr_reader   :current
+      attr_reader   :total
+      attr_accessor :start_time
+
+      private
+      def fmt_bar
+        bar_width = do_percentage * @terminal_width / 100
+        sprintf("|%s%s|",
+                @bar_mark * bar_width,
+                " " *  (@terminal_width - bar_width))
+      end
+
+      def fmt_percentage
+        do_percentage
+      end
+
+      def fmt_stat
+        if @finished_p then elapsed else eta end
+      end
+
+      def fmt_stat_for_file_transfer
+        if @finished_p then
+          sprintf("%s %s %s", bytes, transfer_rate, elapsed)
+        else
+          sprintf("%s %s %s", bytes, transfer_rate, eta)
+        end
+      end
+
+      def fmt_title
+        @title[0,(@title_width - 1)] + ":"
+      end
+
+      def convert_bytes (bytes)
+        if bytes < 1024
+          sprintf("%6dB", bytes)
+        elsif bytes < 1024 * 1000 # 1000kb
+          sprintf("%5.1fKB", bytes.to_f / 1024)
+        elsif bytes < 1024 * 1024 * 1000  # 1000mb
+          sprintf("%5.1fMB", bytes.to_f / 1024 / 1024)
+        else
+          sprintf("%5.1fGB", bytes.to_f / 1024 / 1024 / 1024)
+        end
+      end
+
+      def transfer_rate
+        bytes_per_second = @current.to_f / (::Time.now - @start_time)
+        sprintf("%s/s", convert_bytes(bytes_per_second))
+      end
+
+      def bytes
+        convert_bytes(@current)
+      end
+
+      def format_time (t)
+        t = t.to_i
+        sec = t % 60
+        min  = (t / 60) % 60
+        hour = t / 3600
+        sprintf("%02d:%02d:%02d", hour, min, sec);
+      end
+
+      # ETA stands for Estimated Time of Arrival.
+      def eta
+        if @current == 0
+          "ETA:  --:--:--"
+        else
+          elapsed = ::Time.now - @start_time
+          eta = elapsed * @total / @current - elapsed;
+          sprintf("ETA:  %s", format_time(eta))
+        end
+      end
+
+      def elapsed
+        elapsed = ::Time.now - @start_time
+        sprintf("Time: %s", format_time(elapsed))
+      end
+
+      def eol
+        if @finished_p then "\n" else "\r" end
+      end
+
+      def do_percentage
+        if @total.zero?
+          100
+        else
+          @current  * 100 / @total
+        end
+      end
+
+      def get_width
+        # FIXME: I don't know how portable it is.
+        default_width = 80
+        begin
+          tiocgwinsz = 0x5413
+          data = [0, 0, 0, 0].pack("SSSS")
+          if @out.ioctl(tiocgwinsz, data) >= 0 then
+            rows, cols, xpixels, ypixels = data.unpack("SSSS")
+            if cols > 0 then cols else default_width end
+          else
+            default_width
+          end
+        rescue Exception
+          default_width
+        end
+      end
+
+      def show
+        arguments = @format_arguments.map {|method|
+          method = sprintf("fmt_%s", method)
+          send(method)
+        }
+        line = sprintf(@format, *arguments)
+
+        width = get_width
+        if line.length == width - 1
+          @out.print(line + eol)
+          @out.flush
+        elsif line.length >= width
+          @terminal_width = [@terminal_width - (line.length - width + 1), 0].max
+          if @terminal_width == 0 then @out.print(line + eol) else show end
+        else # line.length < width - 1
+          @terminal_width += width - line.length + 1
+          show
+        end
+        @previous_time = ::Time.now
+      end
+
+      def show_if_needed
+        if @total.zero?
+          cur_percentage = 100
+          prev_percentage = 0
+        else
+          cur_percentage  = (@current  * 100 / @total).to_i
+          prev_percentage = (@previous * 100 / @total).to_i
+        end
+
+        # Use "!=" instead of ">" to support negative changes
+        if cur_percentage != prev_percentage ||
+            ::Time.now - @previous_time >= 1 || @finished_p
+          show
+        end
+      end
+
+      public
+      def clear
+        @out.print "\r"
+        @out.print(" " * (get_width - 1))
+        @out.print "\r"
+      end
+
+      def finish
+        @current = @total
+        @finished_p = true
+        show
+      end
+
+      def finished?
+        @finished_p
+      end
+
+      def file_transfer_mode
+        @format_arguments = [:title, :percentage, :bar, :stat_for_file_transfer]
+      end
+
+      def format= (format)
+        @format = format
+      end
+
+      def format_arguments= (arguments)
+        @format_arguments = arguments
+      end
+
+      def halt
+        @finished_p = true
+        show
+      end
+
+      def inc (step = 1)
+        @current += step
+        @current = @total if @current > @total
+        show_if_needed
+        @previous = @current
+      end
+
+      def set (count)
+        if count < 0 || count > @total
+          raise "invalid count: #{count} (total: #{@total})"
+        end
+        @current = count
+        show_if_needed
+        @previous = @current
+      end
+
+      def inspect
+        "#<Tapsoob::Progress::Bar:#{@current}/#{@total}>"
+      end
+    end
+
+    class ReversedBar < Bar
+      def do_percentage
+        100 - super
+      end
+    end
+  end
+end
+
+# Backward compatibility aliases
+ProgressBar = Tapsoob::Progress::Bar
+ReversedProgressBar = Tapsoob::Progress::ReversedBar

data/lib/tapsoob/progress/multi_bar.rb

@@ -0,0 +1,158 @@
+# -*- encoding : utf-8 -*-
+
+module Tapsoob
+  module Progress
+    # MultiBar manages multiple progress bars in parallel
+    # Each bar gets its own line in the terminal
+    class MultiBar
+      def initialize(max_bars = 4)
+        @max_bars = max_bars
+        @bars = []
+        @mutex = Mutex.new
+        @active = true
+        @out = STDOUT
+        @last_update = Time.now
+        @reserved_lines = 0  # Track how many lines we've actually reserved
+        @max_title_width = 14  # Minimum width, will grow with longer titles
+      end
+
+      # Create a new progress bar and return it
+      def create_bar(title, total)
+        @mutex.synchronize do
+          # Remove any existing bar with the same title to prevent duplicates
+          @bars.reject! { |b| b.title == title }
+
+          # Update max title width to accommodate longer titles
+          @max_title_width = [@max_title_width, title.length].max
+
+          bar = ThreadSafeBar.new(title, total, self)
+
+          # Reserve a line for this new bar during active updates
+          # Cap at 2 * max_bars to show active workers + some recent finished bars
+          if @reserved_lines < @max_bars * 2
+            @out.print "\n"
+            @out.flush
+            @reserved_lines += 1
+          end
+
+          @bars << bar
+          bar
+        end
+      end
+
+      # Get the current maximum title width for alignment
+      # Note: Always called from within synchronized methods, so no mutex needed
+      def max_title_width
+        @max_title_width
+      end
+
+      # Called by individual bars when they update
+      def update
+        @mutex.synchronize do
+          return unless @active
+          return unless should_redraw?
+
+          @last_update = Time.now
+          redraw_all
+        end
+      end
+
+      # Finish a specific bar - mark it as completed
+      def finish_bar(bar)
+        @mutex.synchronize do
+          return unless @active
+
+          bar.mark_finished
+
+          # Respect throttle when finishing to avoid spamming redraws
+          if should_redraw?
+            @last_update = Time.now
+            redraw_all
+          end
+          # If throttled, the next regular update will show the finished state
+        end
+      end
+
+      # Stop all progress bars and keep them visible
+      def stop
+        @mutex.synchronize do
+          @active = false
+
+          # Final cleanup: remove any duplicate titles (keep the last occurrence of each unique title)
+          @bars = @bars.reverse.uniq { |bar| bar.title }.reverse
+
+          # Final redraw to show completed state (skip active check)
+          redraw_all(true)
+          # Move cursor past all bars
+          @out.print "\n"
+          @out.flush
+        end
+      end
+
+      private
+
+      # Check if enough time has passed to redraw (throttle to 10 updates/sec)
+      def should_redraw?
+        Time.now - @last_update >= 0.1
+      end
+
+      def redraw_all(force = false)
+        return unless force || @active
+        return if @bars.empty?
+
+        if force && !@active
+          render_final_display
+        else
+          render_active_display
+        end
+      end
+
+      # Final display: show all completed bars
+      def render_final_display
+        # Clear the reserved lines first
+        if @reserved_lines > 0
+          @out.print "\r\e[#{@reserved_lines}A"
+          @reserved_lines.times { @out.print "\r\e[K\n" }
+        end
+
+        # Print all bars (adds new lines as needed)
+        @bars.each do |bar|
+          @out.print "\r\e[K"
+          bar.render_to(@out)
+          @out.print "\n"
+        end
+
+        @out.flush
+      end
+
+      # Normal operation: show active bars + recent finished in reserved space
+      def render_active_display
+        return if @reserved_lines == 0
+
+        # Partition bars in a single pass for efficiency
+        active_bars, finished_bars = @bars.partition { |b| !b.finished? }
+
+        # Build display: active bars first, then recent finished to fill remaining space
+        # Ensure we don't request negative count from .last()
+        remaining_space = [@reserved_lines - active_bars.length, 0].max
+        bars_to_draw = active_bars + finished_bars.last(remaining_space)
+
+        # If we have more bars than reserved lines, show only the most recent
+        bars_to_draw = bars_to_draw.last(@reserved_lines) if bars_to_draw.length > @reserved_lines
+
+        # Move up and redraw in reserved space
+        @out.print "\r\e[#{@reserved_lines}A"
+        @reserved_lines.times do |i|
+          @out.print "\r\e[K"
+          bars_to_draw[i].render_to(@out) if i < bars_to_draw.length
+          @out.print "\n"
+        end
+
+        @out.flush
+      end
+    end
+  end
+end
+
+# Backward compatibility alias
+MultiProgressBar = Tapsoob::Progress::MultiBar

data/lib/tapsoob/progress/thread_safe_bar.rb

@@ -0,0 +1,90 @@
+# -*- encoding : utf-8 -*-
+
+module Tapsoob
+  module Progress
+    # Thread-safe progress bar that reports to a MultiBar
+    class ThreadSafeBar < Bar
+      attr_reader :title
+
+      def initialize(title, total, multi_progress_bar)
+        @multi_progress_bar = multi_progress_bar
+        @out = STDOUT  # Need this for get_width to work
+        # Don't call parent initialize, we'll manage output ourselves
+        @title = title
+        @total = total
+        @terminal_width = 80
+        @bar_mark = "="
+        @current = 0
+        @previous = 0
+        @finished_p = false
+        @start_time = ::Time.now
+        @previous_time = @start_time
+        @format_arguments = [:title, :percentage, :bar, :stat]
+      end
+
+      # Override show to notify multi-progress instead of direct output
+      def show
+        @previous_time = ::Time.now  # Update to prevent time-based refresh spam
+        @multi_progress_bar.update
+      end
+
+      # Render this bar to the given output stream
+      def render_to(out)
+        # Get dynamic title width from MultiBar for consistent alignment
+        # Store as instance variable so parent class fmt_* methods can use it
+        @title_width = @multi_progress_bar.max_title_width
+
+        # Recalculate terminal width to handle resizes and use full width
+        width = get_width
+        # Calculate bar width: total_width - fixed_elements - padding
+        # Fixed: title(variable) + " "(1) + percentage(4) + " "(1) + "|"(1) + "|"(1) + " "(1) + timer(15) = title_width + 25
+        # Padding: +3 for timer fluctuations and safety
+        fixed_chars = @title_width + 28
+        @terminal_width = [width - fixed_chars, 20].max
+
+        # Build format string with dynamic title width
+        format = "%-#{@title_width}s %3d%% %s %s"
+        arguments = @format_arguments.map { |method| send("fmt_#{method}") }
+        line = sprintf(format, *arguments)
+
+        # Ensure line doesn't exceed terminal width to prevent wrapping
+        # Leave 2 chars margin for safety
+        line = line[0, width - 2] if line.length > width - 2
+
+        out.print(line)
+      end
+
+      # Override clear to do nothing (managed by MultiBar)
+      def clear
+        # no-op
+      end
+
+      # Mark this bar as finished (for tracking)
+      def mark_finished
+        @finished_p = true
+      end
+
+      # Override to use the same @finished_p flag
+      def finished?
+        @finished_p
+      end
+
+      # Override finish to notify multi-progress
+      def finish
+        @current = @total
+        @multi_progress_bar.finish_bar(self)
+      end
+
+      # Override inc to check if we need to update
+      def inc(step = 1)
+        @current += step
+        @current = @total if @current > @total
+        show_if_needed
+        @previous = @current
+      end
+    end
+  end
+end
+
+# Backward compatibility alias
+ThreadSafeProgressBar = Tapsoob::Progress::ThreadSafeBar

data/lib/tapsoob/progress.rb

@@ -0,0 +1,11 @@
+# -*- encoding : utf-8 -*-
+
+# Progress bar module for Tapsoob
+# Provides progress tracking for database operations with support for:
+# - Single progress bars (Bar)
+# - Multiple parallel progress bars (MultiBar)
+# - Thread-safe progress bars (ThreadSafeBar)
+
+require 'tapsoob/progress/bar'
+require 'tapsoob/progress/multi_bar'
+require 'tapsoob/progress/thread_safe_bar'

data/lib/tapsoob/version.rb

@@ -1,4 +1,4 @@
 # -*- encoding : utf-8 -*-
 module Tapsoob
-  VERSION = "0.6.0".freeze
+  VERSION = "0.6.2".freeze
 end

data/lib/tasks/tapsoob.rake

@@ -1,8 +1,14 @@
 namespace :tapsoob do
-  desc "Pulls a database to your filesystem"
+  desc "Pulls a database to your filesystem (PARALLEL=4 for 4 workers)"
   task :pull => :environment do
     # Default options
-    opts={:default_chunksize => 1000, :debug => false, :resume_filename => nil, :disable_compression => false, :schema => true, :data => true, :indexes_first => false, :progress => true}
+    opts={:default_chunksize => 1000, :debug => false, :resume_filename => nil, :disable_compression => false, :schema => true, :data => true, :indexes_first => false, :progress => true, :parallel => 1}
+
+    # Allow overriding parallel workers via PARALLEL environment variable
+    if ENV['PARALLEL'] && ENV['PARALLEL'].to_i > 0
+      opts[:parallel] = ENV['PARALLEL'].to_i
+      puts "Using #{opts[:parallel]} parallel workers"
+    end
 
     # Get the dump_path
     dump_path = File.expand_path(Rails.root.join("db", Time.now.strftime("%Y%m%d%I%M%S%p"))).to_s
@@ -20,10 +26,16 @@ namespace :tapsoob do
     Rake::Task["tapsoob:clean"].invoke
   end
 
-  desc "Push a compatible dump on your filesystem to a database"
+  desc "Push a compatible dump on your filesystem to a database (PARALLEL=4 for 4 workers)"
   task :push, [:timestamp] => :environment do |t, args|
     # Default options
-    opts={:default_chunksize => 1000, :debug => false, :resume_filename => nil, :disable_compression => false, :schema => true, :data => true, :indexes_first => false, :progress => true}
+    opts={:default_chunksize => 1000, :debug => false, :resume_filename => nil, :disable_compression => false, :schema => true, :data => true, :indexes_first => false, :progress => true, :parallel => 1}
+
+    # Allow overriding parallel workers via PARALLEL environment variable
+    if ENV['PARALLEL'] && ENV['PARALLEL'].to_i > 0
+      opts[:parallel] = ENV['PARALLEL'].to_i
+      puts "Using #{opts[:parallel]} parallel workers"
+    end
 
     # Get the dumps
     dumps = Dir[Rails.root.join("db", "*/")].select { |e| e =~ /([0-9]{14})([A-Z]{2})/ }.sort

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: tapsoob
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.6.2
 platform: java
 authors:
 - Félix Bellanger
@@ -104,9 +104,11 @@ files:
 - lib/tapsoob/data_stream.rb
 - lib/tapsoob/errors.rb
 - lib/tapsoob/log.rb
-- lib/tapsoob/multi_progress_bar.rb
 - lib/tapsoob/operation.rb
-- lib/tapsoob/progress_bar.rb
+- lib/tapsoob/progress.rb
+- lib/tapsoob/progress/bar.rb
+- lib/tapsoob/progress/multi_bar.rb
+- lib/tapsoob/progress/thread_safe_bar.rb
 - lib/tapsoob/railtie.rb
 - lib/tapsoob/schema.rb
 - lib/tapsoob/utils.rb

data/lib/tapsoob/multi_progress_bar.rb

@@ -1,234 +0,0 @@
-# -*- encoding : utf-8 -*-
-require 'tapsoob/progress_bar'
-
-# MultiProgressBar manages multiple progress bars in parallel
-# Each bar gets its own line in the terminal
-class MultiProgressBar
-  def initialize(max_bars = 4)
-    @max_bars = max_bars
-    @bars = []
-    @mutex = Mutex.new
-    @active = true
-    @out = STDOUT
-    @last_update = Time.now
-    @reserved_lines = 0  # Track how many lines we've actually reserved
-    @max_title_width = 14  # Minimum width, will grow with longer titles
-  end
-
-  # Create a new progress bar and return it
-  def create_bar(title, total)
-    @mutex.synchronize do
-      # Remove any existing bar with the same title to prevent duplicates
-      @bars.reject! { |b| b.title == title }
-
-      # Update max title width to accommodate longer titles
-      @max_title_width = [@max_title_width, title.length].max
-
-      bar = ThreadSafeProgressBar.new(title, total, self)
-
-      # Reserve a line for this new bar during active updates
-      # Cap at 2 * max_bars to show active workers + some recent finished bars
-      if @reserved_lines < @max_bars * 2
-        @out.print "\n"
-        @out.flush
-        @reserved_lines += 1
-      end
-
-      @bars << bar
-      bar
-    end
-  end
-
-  # Get the current maximum title width for alignment
-  # Note: Always called from within synchronized methods, so no mutex needed
-  def max_title_width
-    @max_title_width
-  end
-
-  # Called by individual bars when they update
-  def update
-    @mutex.synchronize do
-      return unless @active
-      return unless should_redraw?
-
-      @last_update = Time.now
-      redraw_all
-    end
-  end
-
-  # Finish a specific bar - mark it as completed
-  def finish_bar(bar)
-    @mutex.synchronize do
-      return unless @active
-
-      bar.mark_finished
-
-      # Respect throttle when finishing to avoid spamming redraws
-      if should_redraw?
-        @last_update = Time.now
-        redraw_all
-      end
-      # If throttled, the next regular update will show the finished state
-    end
-  end
-
-  # Stop all progress bars and keep them visible
-  def stop
-    @mutex.synchronize do
-      @active = false
-
-      # Final cleanup: remove any duplicate titles (keep the last occurrence of each unique title)
-      @bars = @bars.reverse.uniq { |bar| bar.title }.reverse
-
-      # Final redraw to show completed state (skip active check)
-      redraw_all(true)
-      # Move cursor past all bars
-      @out.print "\n"
-      @out.flush
-    end
-  end
-
-  private
-
-  # Check if enough time has passed to redraw (throttle to 10 updates/sec)
-  def should_redraw?
-    Time.now - @last_update >= 0.1
-  end
-
-  def redraw_all(force = false)
-    return unless force || @active
-    return if @bars.empty?
-
-    if force && !@active
-      render_final_display
-    else
-      render_active_display
-    end
-  end
-
-  # Final display: show all completed bars
-  def render_final_display
-    # Clear the reserved lines first
-    if @reserved_lines > 0
-      @out.print "\r\e[#{@reserved_lines}A"
-      @reserved_lines.times { @out.print "\r\e[K\n" }
-    end
-
-    # Print all bars (adds new lines as needed)
-    @bars.each do |bar|
-      @out.print "\r\e[K"
-      bar.render_to(@out)
-      @out.print "\n"
-    end
-
-    @out.flush
-  end
-
-  # Normal operation: show active bars + recent finished in reserved space
-  def render_active_display
-    return if @reserved_lines == 0
-
-    # Partition bars in a single pass for efficiency
-    active_bars, finished_bars = @bars.partition { |b| !b.finished? }
-
-    # Build display: active bars first, then recent finished to fill remaining space
-    # Ensure we don't request negative count from .last()
-    remaining_space = [@reserved_lines - active_bars.length, 0].max
-    bars_to_draw = active_bars + finished_bars.last(remaining_space)
-
-    # If we have more bars than reserved lines, show only the most recent
-    bars_to_draw = bars_to_draw.last(@reserved_lines) if bars_to_draw.length > @reserved_lines
-
-    # Move up and redraw in reserved space
-    @out.print "\r\e[#{@reserved_lines}A"
-    @reserved_lines.times do |i|
-      @out.print "\r\e[K"
-      bars_to_draw[i].render_to(@out) if i < bars_to_draw.length
-      @out.print "\n"
-    end
-
-    @out.flush
-  end
-end
-
-# Thread-safe progress bar that reports to a MultiProgressBar
-class ThreadSafeProgressBar < ProgressBar
-  attr_reader :title
-
-  def initialize(title, total, multi_progress_bar)
-    @multi_progress_bar = multi_progress_bar
-    @out = STDOUT  # Need this for get_width to work
-    # Don't call parent initialize, we'll manage output ourselves
-    @title = title
-    @total = total
-    @terminal_width = 80
-    @bar_mark = "="
-    @current = 0
-    @previous = 0
-    @finished_p = false
-    @start_time = ::Time.now
-    @previous_time = @start_time
-    @format_arguments = [:title, :percentage, :bar, :stat]
-  end
-
-  # Override show to notify multi-progress instead of direct output
-  def show
-    @previous_time = ::Time.now  # Update to prevent time-based refresh spam
-    @multi_progress_bar.update
-  end
-
-  # Render this bar to the given output stream
-  def render_to(out)
-    # Get dynamic title width from MultiProgressBar for consistent alignment
-    # Store as instance variable so parent class fmt_* methods can use it
-    @title_width = @multi_progress_bar.max_title_width
-
-    # Recalculate terminal width to handle resizes and use full width
-    width = get_width
-    # Calculate bar width: total_width - fixed_elements - padding
-    # Fixed: title(variable) + " "(1) + percentage(4) + " "(1) + "|"(1) + "|"(1) + " "(1) + timer(15) = title_width + 25
-    # Padding: +3 for timer fluctuations and safety
-    fixed_chars = @title_width + 28
-    @terminal_width = [width - fixed_chars, 20].max
-
-    # Build format string with dynamic title width
-    format = "%-#{@title_width}s %3d%% %s %s"
-    arguments = @format_arguments.map { |method| send("fmt_#{method}") }
-    line = sprintf(format, *arguments)
-
-    # Ensure line doesn't exceed terminal width to prevent wrapping
-    # Leave 2 chars margin for safety
-    line = line[0, width - 2] if line.length > width - 2
-
-    out.print(line)
-  end
-
-  # Override clear to do nothing (managed by MultiProgressBar)
-  def clear
-    # no-op
-  end
-
-  # Mark this bar as finished (for tracking)
-  def mark_finished
-    @finished_p = true
-  end
-
-  # Override to use the same @finished_p flag
-  def finished?
-    @finished_p
-  end
-
-  # Override finish to notify multi-progress
-  def finish
-    @current = @total
-    @multi_progress_bar.finish_bar(self)
-  end
-
-  # Override inc to check if we need to update
-  def inc(step = 1)
-    @current += step
-    @current = @total if @current > @total
-    show_if_needed
-    @previous = @current
-  end
-end

data/lib/tapsoob/progress_bar.rb

@@ -1,237 +0,0 @@
-# -*- encoding : utf-8 -*-
-#
-# Ruby/ProgressBar - a text progress bar library
-#
-# Copyright (C) 2001-2005 Satoru Takabayashi <satoru@namazu.org>
-#     All rights reserved.
-#     This is free software with ABSOLUTELY NO WARRANTY.
-#
-# You can redistribute it and/or modify it under the terms
-# of Ruby's license.
-#
-
-class ProgressBar
-  VERSION = "0.9"
-
-  def initialize (title, total, out = STDOUT, title_width = nil)
-    @title = title
-    @total = total
-    @out = out
-    @terminal_width = 80
-    @bar_mark = "="
-    @current = 0
-    @previous = 0
-    @finished_p = false
-    @start_time = ::Time.now
-    @previous_time = @start_time
-    # Set title width: use provided width, or accommodate the title, with a minimum of 14
-    @title_width = title_width || [title.length, 14].max
-    @format = "%-#{@title_width}s %3d%% %s %s"
-    @format_arguments = [:title, :percentage, :bar, :stat]
-    clear
-    show
-  end
-  attr_reader   :title
-  attr_reader   :current
-  attr_reader   :total
-  attr_accessor :start_time
-
-  private
-  def fmt_bar
-    bar_width = do_percentage * @terminal_width / 100
-    sprintf("|%s%s|",
-            @bar_mark * bar_width,
-            " " *  (@terminal_width - bar_width))
-  end
-
-  def fmt_percentage
-    do_percentage
-  end
-
-  def fmt_stat
-    if @finished_p then elapsed else eta end
-  end
-
-  def fmt_stat_for_file_transfer
-    if @finished_p then
-      sprintf("%s %s %s", bytes, transfer_rate, elapsed)
-    else
-      sprintf("%s %s %s", bytes, transfer_rate, eta)
-    end
-  end
-
-  def fmt_title
-    @title[0,(@title_width - 1)] + ":"
-  end
-
-  def convert_bytes (bytes)
-    if bytes < 1024
-      sprintf("%6dB", bytes)
-    elsif bytes < 1024 * 1000 # 1000kb
-      sprintf("%5.1fKB", bytes.to_f / 1024)
-    elsif bytes < 1024 * 1024 * 1000  # 1000mb
-      sprintf("%5.1fMB", bytes.to_f / 1024 / 1024)
-    else
-      sprintf("%5.1fGB", bytes.to_f / 1024 / 1024 / 1024)
-    end
-  end
-
-  def transfer_rate
-    bytes_per_second = @current.to_f / (::Time.now - @start_time)
-    sprintf("%s/s", convert_bytes(bytes_per_second))
-  end
-
-  def bytes
-    convert_bytes(@current)
-  end
-
-  def format_time (t)
-    t = t.to_i
-    sec = t % 60
-    min  = (t / 60) % 60
-    hour = t / 3600
-    sprintf("%02d:%02d:%02d", hour, min, sec);
-  end
-
-  # ETA stands for Estimated Time of Arrival.
-  def eta
-    if @current == 0
-      "ETA:  --:--:--"
-    else
-      elapsed = ::Time.now - @start_time
-      eta = elapsed * @total / @current - elapsed;
-      sprintf("ETA:  %s", format_time(eta))
-    end
-  end
-
-  def elapsed
-    elapsed = ::Time.now - @start_time
-    sprintf("Time: %s", format_time(elapsed))
-  end
-
-  def eol
-    if @finished_p then "\n" else "\r" end
-  end
-
-  def do_percentage
-    if @total.zero?
-      100
-    else
-      @current  * 100 / @total
-    end
-  end
-
-  def get_width
-    # FIXME: I don't know how portable it is.
-    default_width = 80
-    begin
-      tiocgwinsz = 0x5413
-      data = [0, 0, 0, 0].pack("SSSS")
-      if @out.ioctl(tiocgwinsz, data) >= 0 then
-        rows, cols, xpixels, ypixels = data.unpack("SSSS")
-        if cols > 0 then cols else default_width end
-      else
-        default_width
-      end
-    rescue Exception
-      default_width
-    end
-  end
-
-  def show
-    arguments = @format_arguments.map {|method|
-      method = sprintf("fmt_%s", method)
-      send(method)
-    }
-    line = sprintf(@format, *arguments)
-
-    width = get_width
-    if line.length == width - 1
-      @out.print(line + eol)
-      @out.flush
-    elsif line.length >= width
-      @terminal_width = [@terminal_width - (line.length - width + 1), 0].max
-      if @terminal_width == 0 then @out.print(line + eol) else show end
-    else # line.length < width - 1
-      @terminal_width += width - line.length + 1
-      show
-    end
-    @previous_time = ::Time.now
-  end
-
-  def show_if_needed
-    if @total.zero?
-      cur_percentage = 100
-      prev_percentage = 0
-    else
-      cur_percentage  = (@current  * 100 / @total).to_i
-      prev_percentage = (@previous * 100 / @total).to_i
-    end
-
-    # Use "!=" instead of ">" to support negative changes
-    if cur_percentage != prev_percentage ||
-        ::Time.now - @previous_time >= 1 || @finished_p
-      show
-    end
-  end
-
-  public
-  def clear
-    @out.print "\r"
-    @out.print(" " * (get_width - 1))
-    @out.print "\r"
-  end
-
-  def finish
-    @current = @total
-    @finished_p = true
-    show
-  end
-
-  def finished?
-    @finished_p
-  end
-
-  def file_transfer_mode
-    @format_arguments = [:title, :percentage, :bar, :stat_for_file_transfer]
-  end
-
-  def format= (format)
-    @format = format
-  end
-
-  def format_arguments= (arguments)
-    @format_arguments = arguments
-  end
-
-  def halt
-    @finished_p = true
-    show
-  end
-
-  def inc (step = 1)
-    @current += step
-    @current = @total if @current > @total
-    show_if_needed
-    @previous = @current
-  end
-
-  def set (count)
-    if count < 0 || count > @total
-      raise "invalid count: #{count} (total: #{@total})"
-    end
-    @current = count
-    show_if_needed
-    @previous = @current
-  end
-
-  def inspect
-    "#<ProgressBar:#{@current}/#{@total}>"
-  end
-end
-
-class ReversedProgressBar < ProgressBar
-  def do_percentage
-    100 - super
-  end
-end