tapsoob 0.5.31 → 0.6.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bff0c57641151d932d224ec7b245a23c4b993eac3c8da25f10724208b46f5953
-  data.tar.gz: 186003a5f0156be1ebaac8f80ef44a3f985b23544a33a1e127067b776cf50d88
+  metadata.gz: d509bbdc385a715e9784dc63e823d50d11cb179f3cf4c78fa229643550b6bd84
+  data.tar.gz: c6d37e2166cbf5be914945cec18e166f5a2ba8d51f7ea71ef85088def0499256
 SHA512:
-  metadata.gz: c78e9d29d6247823deae5f9f1a86c59792b4e52c34f4d1536cdd16bc6168a2d38b3bfba434ec96b2be72d633e927f672578e70904387435a4152bc212ac544d5
-  data.tar.gz: 30a3507296d49349a3049fc2e24b61cbc98f49fdd0c38ce05f1714aefa0a598eed832ac5826c213f982b549708b566cc263053acf085b86ecc9fa7beb4ac6837
+  metadata.gz: ed60501ee636716fd3f34e3d6291725ab418c0a18bc5aba52cfd736ab937bb83b61e3070a140de49650966c2244aeb0d195c13f4b6f4e1ea48f60db8e8572f69
+  data.tar.gz: e117b807f9a425034928c758ea8bd1937702cacaf9d26a88124f4da02fd1587f5970be9e748f4ddff509ad030304ad15722d904d7cc38925661f9526a6c3227a

data/README.md

@@ -36,7 +36,7 @@ You can list all available options using the command:
     tapsoob push -h
 
 
-## NEW : Piping your schema/indexes/data
+## Piping your schema/indexes/data
 
 Due to some needs we added ways to pipe your schema/indexes/data directly from one database to another, here's an equivalent of the export/import process described above using this technique :
 
@@ -60,6 +60,16 @@ If you're using Rails, there's also two Rake tasks provided:
 * `tapsoob:pull` which dumps the database into a new folder under the `db` folder
 * `tapsoob:push` which reads the last dump you made from `tapsoob:pull` from the `db` folder
 
+## NEW : Full parallelization support from 0.6.1 onwards
+
+You can now dump/load a full database or data using parallelization to speed up the process at memory cost and database load like so :
+
+```
+tapsoob pull [OPTIONS] <dump_path> <database_url> -j <number_of_threads>
+tapsoob push [OPTIONS] <dump_path> <database_url> -j <number_of_threads>
+```
+
+It defaults to a single thread as per pre 0.6.1, it is also appliable to `tapsoob data pull/push` but only when dumping to files, you can't parallelize and pipe for obvious reasons, it'll fall back to a single thread with a warning if you try to do this for safety.
 
 ## Notes
 

data/lib/tapsoob/cli/data_stream.rb

@@ -14,10 +14,20 @@ module Tapsoob
       option :chunksize, desc: "Initial chunksize", default: 1000, type: :numeric, aliases: "-c"
       option :tables, desc: "Shortcut to filter on a list of tables", type: :array, aliases: "-t"
       option :"exclude-tables", desc: "Shortcut to exclude a list of tables", type: :array, aliases: "-e"
+      option :parallel, desc: "Number of parallel workers for table processing (default: 1)", default: 1, type: :numeric, aliases: "-j"
       option :progress, desc: "Show progress", default: true, type: :boolean, aliases: "-p"
       option :debug, desc: "Enable debug messages", default: false, type: :boolean, aliases: "-d"
       def pull(database_url, dump_path = nil)
-        op = Tapsoob::Operation.factory(:pull, database_url, dump_path, parse_opts(options))
+        opts = parse_opts(options)
+
+        # Force serial mode when outputting to STDOUT (for piping)
+        # Parallel mode would interleave output and corrupt the JSON stream
+        if dump_path.nil? && opts[:parallel] && opts[:parallel] > 1
+          STDERR.puts "Warning: Parallel mode disabled when outputting to STDOUT (for piping)"
+          opts[:parallel] = 1
+        end
+
+        op = Tapsoob::Operation.factory(:pull, database_url, dump_path, opts)
         op.pull_data
       end
 
@@ -25,36 +35,48 @@ module Tapsoob
       option :chunksize, desc: "Initial chunksize", default: 1000, type: :numeric, aliases: "-c"
       option :tables, desc: "Shortcut to filter on a list of tables", type: :array, aliases: "-t"
       option :"exclude-tables", desc: "Shortcut to exclude a list of tables", type: :array, aliases: "-e"
+      option :parallel, desc: "Number of parallel workers for table processing (default: 1)", default: 1, type: :numeric, aliases: "-j"
       option :progress, desc: "Show progress", default: true, type: :boolean, aliases: "-p"
       option :purge, desc: "Purge data in tables prior to performing the import", default: false, type: :boolean
       option :"skip-duplicates", desc: "Remove duplicates when loading data", default: false, type: :boolean
       option :"discard-identity", desc: "Remove identity when pushing data (may result in creating duplicates)", default: false, type: :boolean
       option :debug, desc: "Enable debug messages", default: false, type: :boolean, aliases: "-d"
       def push(database_url, dump_path = nil)
-        # instantiate stuff
-        data = []
         opts = parse_opts(options)
 
-        # read data from dump_path or from STDIN
+        # If dump_path is provided, use the Operation class for proper parallel support
         if dump_path && Dir.exist?(dump_path)
-          files = Dir[Pathname.new(dump_path).join("*.json")]
-          files.each { |file| data << JSON.parse(File.read(file), symbolize_names: true) }
+          op = Tapsoob::Operation.factory(:push, database_url, dump_path, opts)
+          op.push_data
         else
+          # STDIN mode: read and import data directly (no parallel support for STDIN)
+          if opts[:parallel] && opts[:parallel] > 1
+            STDERR.puts "Warning: Parallel mode not supported when reading from STDIN"
+          end
+
+          data = []
           STDIN.each_line { |line| data << JSON.parse(line, symbolize_names: true) }
-        end
 
-        # import data
-        data.each do |table|
-          stream = Tapsoob::DataStream.factory(db(database_url, opts), {
-            table_name: table[:table_name],
-            chunksize: opts[:default_chunksize]
-          }, { :"discard-identity" => opts[:"discard-identity"] || false, :purge => opts[:purge] || false, :debug => opts[:debug] })
-
-          begin
-            stream.import_rows(table)
-          rescue Exception => e
-            stream.log.debug e.message
-            STDERR.puts "Error loading data in #{table[:table_name]} : #{e.message}"
+          # import data
+          data.each do |table|
+            table_name = table[:table_name]
+
+            # Truncate table if purge option is enabled
+            if opts[:purge]
+              db(database_url, opts)[table_name.to_sym].truncate
+            end
+
+            stream = Tapsoob::DataStream.factory(db(database_url, opts), {
+              table_name: table_name,
+              chunksize: opts[:default_chunksize]
+            }, { :"discard-identity" => opts[:"discard-identity"] || false, :purge => opts[:purge] || false, :debug => opts[:debug] })
+
+            begin
+              stream.import_rows(table)
+            rescue Exception => e
+              stream.log.debug e.message
+              STDERR.puts "Error loading data in #{table_name} : #{e.message}"
+            end
           end
         end
       end
@@ -65,6 +87,7 @@ module Tapsoob
           opts = {
             progress: options[:progress],
             tables: options[:tables],
+            parallel: options[:parallel],
             debug: options[:debug]
           }
 
@@ -85,7 +108,9 @@ module Tapsoob
         end
 
         def db(database_url, opts = {})
-          @db ||= Sequel.connect(database_url)
+          # Support connection pooling for parallel operations
+          parallel_workers = opts[:parallel] || 1
+          @db ||= Sequel.connect(database_url, max_connections: parallel_workers * 2)
           @db.loggers << Tapsoob.log if opts[:debug]
 
           # Set parameters

data/lib/tapsoob/cli/root.rb

@@ -22,6 +22,7 @@ module Tapsoob
       option :"exclude-tables", desc: "Shortcut to exclude a list of tables", type: :array, aliases: "-e"
       option :"indexes", type: :boolean, default: false
       option :"same-db", type: :boolean, default: false
+      option :parallel, desc: "Number of parallel workers for table processing (default: 1)", default: 1, type: :numeric, aliases: "-j"
       option :progress, desc: "Show progress", default: true, type: :boolean
       option :debug, desc: "Enable debug messages", default: false, type: :boolean
       def pull(dump_path, database_url)
@@ -47,6 +48,7 @@ module Tapsoob
       option :purge, desc: "Purge data in tables prior to performing the import", default: false, type: :boolean, aliases: "-p"
       option :"skip-duplicates", desc: "Remove duplicates when loading data", default: false, type: :boolean
       option :"discard-identity", desc: "Remove identity when pushing data (may result in creating duplicates)", default: false, type: :boolean
+      option :parallel, desc: "Number of parallel workers for table processing (default: 1)", default: 1, type: :numeric, aliases: "-j"
       option :progress, desc: "Show progress", default: true, type: :boolean
       option :debug, desc: "Enable debug messages", default: false, type: :boolean
       def push(dump_path, database_url)
@@ -82,6 +84,7 @@ module Tapsoob
             indexes_first: options[:"indexes_first"],
             disable_compression: options[:"disable-compression"],
             tables: options[:tables],
+            parallel: options[:parallel],
             progress: options[:progress],
             debug: options[:debug]
           })

data/lib/tapsoob/multi_progress_bar.rb

@@ -0,0 +1,234 @@
+# -*- encoding : utf-8 -*-
+require 'tapsoob/progress_bar'
+
+# MultiProgressBar manages multiple progress bars in parallel
+# Each bar gets its own line in the terminal
+class MultiProgressBar
+  def initialize(max_bars = 4)
+    @max_bars = max_bars
+    @bars = []
+    @mutex = Mutex.new
+    @active = true
+    @out = STDOUT
+    @last_update = Time.now
+    @reserved_lines = 0  # Track how many lines we've actually reserved
+    @max_title_width = 14  # Minimum width, will grow with longer titles
+  end
+
+  # Create a new progress bar and return it
+  def create_bar(title, total)
+    @mutex.synchronize do
+      # Remove any existing bar with the same title to prevent duplicates
+      @bars.reject! { |b| b.title == title }
+
+      # Update max title width to accommodate longer titles
+      @max_title_width = [@max_title_width, title.length].max
+
+      bar = ThreadSafeProgressBar.new(title, total, self)
+
+      # Reserve a line for this new bar during active updates
+      # Cap at 2 * max_bars to show active workers + some recent finished bars
+      if @reserved_lines < @max_bars * 2
+        @out.print "\n"
+        @out.flush
+        @reserved_lines += 1
+      end
+
+      @bars << bar
+      bar
+    end
+  end
+
+  # Get the current maximum title width for alignment
+  # Note: Always called from within synchronized methods, so no mutex needed
+  def max_title_width
+    @max_title_width
+  end
+
+  # Called by individual bars when they update
+  def update
+    @mutex.synchronize do
+      return unless @active
+      return unless should_redraw?
+
+      @last_update = Time.now
+      redraw_all
+    end
+  end
+
+  # Finish a specific bar - mark it as completed
+  def finish_bar(bar)
+    @mutex.synchronize do
+      return unless @active
+
+      bar.mark_finished
+
+      # Respect throttle when finishing to avoid spamming redraws
+      if should_redraw?
+        @last_update = Time.now
+        redraw_all
+      end
+      # If throttled, the next regular update will show the finished state
+    end
+  end
+
+  # Stop all progress bars and keep them visible
+  def stop
+    @mutex.synchronize do
+      @active = false
+
+      # Final cleanup: remove any duplicate titles (keep the last occurrence of each unique title)
+      @bars = @bars.reverse.uniq { |bar| bar.title }.reverse
+
+      # Final redraw to show completed state (skip active check)
+      redraw_all(true)
+      # Move cursor past all bars
+      @out.print "\n"
+      @out.flush
+    end
+  end
+
+  private
+
+  # Check if enough time has passed to redraw (throttle to 10 updates/sec)
+  def should_redraw?
+    Time.now - @last_update >= 0.1
+  end
+
+  def redraw_all(force = false)
+    return unless force || @active
+    return if @bars.empty?
+
+    if force && !@active
+      render_final_display
+    else
+      render_active_display
+    end
+  end
+
+  # Final display: show all completed bars
+  def render_final_display
+    # Clear the reserved lines first
+    if @reserved_lines > 0
+      @out.print "\r\e[#{@reserved_lines}A"
+      @reserved_lines.times { @out.print "\r\e[K\n" }
+    end
+
+    # Print all bars (adds new lines as needed)
+    @bars.each do |bar|
+      @out.print "\r\e[K"
+      bar.render_to(@out)
+      @out.print "\n"
+    end
+
+    @out.flush
+  end
+
+  # Normal operation: show active bars + recent finished in reserved space
+  def render_active_display
+    return if @reserved_lines == 0
+
+    # Partition bars in a single pass for efficiency
+    active_bars, finished_bars = @bars.partition { |b| !b.finished? }
+
+    # Build display: active bars first, then recent finished to fill remaining space
+    # Ensure we don't request negative count from .last()
+    remaining_space = [@reserved_lines - active_bars.length, 0].max
+    bars_to_draw = active_bars + finished_bars.last(remaining_space)
+
+    # If we have more bars than reserved lines, show only the most recent
+    bars_to_draw = bars_to_draw.last(@reserved_lines) if bars_to_draw.length > @reserved_lines
+
+    # Move up and redraw in reserved space
+    @out.print "\r\e[#{@reserved_lines}A"
+    @reserved_lines.times do |i|
+      @out.print "\r\e[K"
+      bars_to_draw[i].render_to(@out) if i < bars_to_draw.length
+      @out.print "\n"
+    end
+
+    @out.flush
+  end
+end
+
+# Thread-safe progress bar that reports to a MultiProgressBar
+class ThreadSafeProgressBar < ProgressBar
+  attr_reader :title
+
+  def initialize(title, total, multi_progress_bar)
+    @multi_progress_bar = multi_progress_bar
+    @out = STDOUT  # Need this for get_width to work
+    # Don't call parent initialize, we'll manage output ourselves
+    @title = title
+    @total = total
+    @terminal_width = 80
+    @bar_mark = "="
+    @current = 0
+    @previous = 0
+    @finished_p = false
+    @start_time = ::Time.now
+    @previous_time = @start_time
+    @format_arguments = [:title, :percentage, :bar, :stat]
+  end
+
+  # Override show to notify multi-progress instead of direct output
+  def show
+    @previous_time = ::Time.now  # Update to prevent time-based refresh spam
+    @multi_progress_bar.update
+  end
+
+  # Render this bar to the given output stream
+  def render_to(out)
+    # Get dynamic title width from MultiProgressBar for consistent alignment
+    # Store as instance variable so parent class fmt_* methods can use it
+    @title_width = @multi_progress_bar.max_title_width
+
+    # Recalculate terminal width to handle resizes and use full width
+    width = get_width
+    # Calculate bar width: total_width - fixed_elements - padding
+    # Fixed: title(variable) + " "(1) + percentage(4) + " "(1) + "|"(1) + "|"(1) + " "(1) + timer(15) = title_width + 25
+    # Padding: +3 for timer fluctuations and safety
+    fixed_chars = @title_width + 28
+    @terminal_width = [width - fixed_chars, 20].max
+
+    # Build format string with dynamic title width
+    format = "%-#{@title_width}s %3d%% %s %s"
+    arguments = @format_arguments.map { |method| send("fmt_#{method}") }
+    line = sprintf(format, *arguments)
+
+    # Ensure line doesn't exceed terminal width to prevent wrapping
+    # Leave 2 chars margin for safety
+    line = line[0, width - 2] if line.length > width - 2
+
+    out.print(line)
+  end
+
+  # Override clear to do nothing (managed by MultiProgressBar)
+  def clear
+    # no-op
+  end
+
+  # Mark this bar as finished (for tracking)
+  def mark_finished
+    @finished_p = true
+  end
+
+  # Override to use the same @finished_p flag
+  def finished?
+    @finished_p
+  end
+
+  # Override finish to notify multi-progress
+  def finish
+    @current = @total
+    @multi_progress_bar.finish_bar(self)
+  end
+
+  # Override inc to check if we need to update
+  def inc(step = 1)
+    @current += step
+    @current = @total if @current > @total
+    show_if_needed
+    @previous = @current
+  end
+end

data/lib/tapsoob/operation.rb

@@ -1,9 +1,11 @@
 # -*- encoding : utf-8 -*-
 require 'sequel'
+require 'thread'
 
 require 'tapsoob/data_stream'
 require 'tapsoob/log'
 require 'tapsoob/progress_bar'
+require 'tapsoob/multi_progress_bar'
 require 'tapsoob/schema'
 
 module Tapsoob
@@ -117,7 +119,7 @@ module Tapsoob
     end
 
     def db
-      @db ||= Sequel.connect(database_url)
+      @db ||= Sequel.connect(database_url, max_connections: parallel_workers * 2)
       @db.extension :schema_dumper
       @db.loggers << Tapsoob.log if opts[:debug]
 
@@ -130,6 +132,24 @@ module Tapsoob
       @db
     end
 
+    def parallel?
+      parallel_workers > 1
+    end
+
+    def parallel_workers
+      @parallel_workers ||= [opts[:parallel].to_i, 1].max
+    end
+
+    def completed_tables_mutex
+      @completed_tables_mutex ||= Mutex.new
+    end
+
+    def add_completed_table(table_name)
+      completed_tables_mutex.synchronize do
+        completed_tables << table_name.to_s
+      end
+    end
+
     def format_number(num)
       num.to_s.gsub(/(\d)(?=(\d\d\d)+(?!\d))/, "\\1,")
     end
@@ -198,6 +218,14 @@ module Tapsoob
 
       log.info "#{tables.size} tables, #{format_number(record_count)} records"
 
+      if parallel?
+        pull_data_parallel
+      else
+        pull_data_serial
+      end
+    end
+
+    def pull_data_serial
       tables.each do |table_name, count|
         stream   = Tapsoob::DataStream.factory(db, {
           :chunksize  => default_chunksize,
@@ -209,6 +237,38 @@ module Tapsoob
       end
     end
 
+    def pull_data_parallel
+      log.info "Using #{parallel_workers} parallel workers"
+
+      multi_progress = opts[:progress] ? MultiProgressBar.new(parallel_workers) : nil
+      table_queue = Queue.new
+      tables.each { |table_name, count| table_queue << [table_name, count] }
+
+      workers = (1..parallel_workers).map do
+        Thread.new do
+          loop do
+            break if table_queue.empty?
+
+            table_name, count = table_queue.pop(true) rescue break
+
+            # Each thread gets its own connection from the pool
+            stream = Tapsoob::DataStream.factory(db, {
+              :chunksize  => default_chunksize,
+              :table_name => table_name
+            }, { :debug => opts[:debug] })
+
+            estimated_chunks = [(count.to_f / default_chunksize).ceil, 1].max
+            progress = multi_progress ? multi_progress.create_bar(table_name.to_s, estimated_chunks) : nil
+
+            pull_data_from_table(stream, progress)
+          end
+        end
+      end
+
+      workers.each(&:join)
+      multi_progress.stop if multi_progress
+    end
+
     def pull_partial_data
       return if stream_state == {}
 
@@ -280,7 +340,7 @@ module Tapsoob
       end
 
       progress.finish if progress
-      completed_tables << stream.table_name.to_s
+      add_completed_table(stream.table_name)
       self.stream_state = {}
     end
 
@@ -333,9 +393,12 @@ module Tapsoob
       raw_idxs = Tapsoob::Schema.indexes_individual(database_url)
       idxs     = (raw_idxs && raw_idxs.length >= 2 ? JSON.parse(raw_idxs) : {})
 
-      apply_table_filter(idxs).each do |table, indexes|
-        next unless indexes.size > 0
-        progress = ProgressBar.new(table, indexes.size)
+      # Calculate max title width for consistent alignment
+      filtered_idxs = apply_table_filter(idxs).select { |table, indexes| indexes.size > 0 }
+      max_title_width = filtered_idxs.keys.map { |table| "#{table} indexes".length }.max || 14
+
+      filtered_idxs.each do |table, indexes|
+        progress = ProgressBar.new("#{table} indexes", indexes.size, STDOUT, max_title_width)
         indexes.each do |idx|
           output = Tapsoob::Utils.export_indexes(dump_path, table, idx)
           puts output if dump_path.nil? && output
@@ -389,9 +452,12 @@ module Tapsoob
 
       log.info "Sending indexes"
 
-      apply_table_filter(idxs).each do |table, indexes|
-        next unless indexes.size > 0
-        progress = ProgressBar.new(table, indexes.size)
+      # Calculate max title width for consistent alignment
+      filtered_idxs = apply_table_filter(idxs).select { |table, indexes| indexes.size > 0 }
+      max_title_width = filtered_idxs.keys.map { |table| "#{table} indexes".length }.max || 14
+
+      filtered_idxs.each do |table, indexes|
+        progress = ProgressBar.new("#{table} indexes", indexes.size, STDOUT, max_title_width)
         indexes.each do |idx|
           Tapsoob::Utils.load_indexes(database_url, idx)
           progress.inc(1)
@@ -437,6 +503,14 @@ module Tapsoob
 
       log.info "#{tables.size} tables, #{format_number(record_count)} records"
 
+      if parallel?
+        push_data_parallel
+      else
+        push_data_serial
+      end
+    end
+
+    def push_data_serial
       tables.each do |table_name, count|
         # Skip if data file doesn't exist or has no data
         data_file = File.join(dump_path, "data", "#{table_name}.json")
@@ -457,6 +531,49 @@ module Tapsoob
       end
     end
 
+    def push_data_parallel
+      log.info "Using #{parallel_workers} parallel workers"
+
+      multi_progress = opts[:progress] ? MultiProgressBar.new(parallel_workers) : nil
+      table_queue = Queue.new
+
+      tables.each do |table_name, count|
+        data_file = File.join(dump_path, "data", "#{table_name}.json")
+        next unless File.exist?(data_file) && count > 0
+        table_queue << [table_name, count]
+      end
+
+      workers = (1..parallel_workers).map do
+        Thread.new do
+          loop do
+            break if table_queue.empty?
+
+            table_name, count = table_queue.pop(true) rescue break
+
+            # Each thread gets its own connection from the pool
+            db[table_name.to_sym].truncate if @opts[:purge]
+            stream = Tapsoob::DataStream.factory(db, {
+              :table_name => table_name,
+              :chunksize => default_chunksize
+            }, {
+              :"skip-duplicates" => opts[:"skip-duplicates"] || false,
+              :"discard-identity" => opts[:"discard-identity"] || false,
+              :purge => opts[:purge] || false,
+              :debug => opts[:debug]
+            })
+
+            estimated_chunks = [(count.to_f / default_chunksize).ceil, 1].max
+            progress = multi_progress ? multi_progress.create_bar(table_name.to_s, estimated_chunks) : nil
+
+            push_data_from_file(stream, progress)
+          end
+        end
+      end
+
+      workers.each(&:join)
+      multi_progress.stop if multi_progress
+    end
+
     def push_data_from_file(stream, progress)
       loop do
         if exiting?
@@ -510,7 +627,7 @@ module Tapsoob
       end
 
       progress.finish if progress
-      completed_tables << stream.table_name.to_s
+      add_completed_table(stream.table_name)
       self.stream_state = {}
     end
 

data/lib/tapsoob/progress_bar.rb

@@ -13,7 +13,7 @@
 class ProgressBar
   VERSION = "0.9"
 
-  def initialize (title, total, out = STDOUT)
+  def initialize (title, total, out = STDOUT, title_width = nil)
     @title = title
     @total = total
     @out = out
@@ -24,7 +24,8 @@ class ProgressBar
     @finished_p = false
     @start_time = ::Time.now
     @previous_time = @start_time
-    @title_width = 14
+    # Set title width: use provided width, or accommodate the title, with a minimum of 14
+    @title_width = title_width || [title.length, 14].max
     @format = "%-#{@title_width}s %3d%% %s %s"
     @format_arguments = [:title, :percentage, :bar, :stat]
     clear

data/lib/tapsoob/version.rb

@@ -1,4 +1,4 @@
 # -*- encoding : utf-8 -*-
 module Tapsoob
-  VERSION = "0.5.31".freeze
+  VERSION = "0.6.1".freeze
 end

data/lib/tasks/tapsoob.rake

@@ -1,8 +1,14 @@
 namespace :tapsoob do
-  desc "Pulls a database to your filesystem"
+  desc "Pulls a database to your filesystem (PARALLEL=4 for 4 workers)"
   task :pull => :environment do
     # Default options
-    opts={:default_chunksize => 1000, :debug => false, :resume_filename => nil, :disable_compression => false, :schema => true, :data => true, :indexes_first => false, :progress => true}
+    opts={:default_chunksize => 1000, :debug => false, :resume_filename => nil, :disable_compression => false, :schema => true, :data => true, :indexes_first => false, :progress => true, :parallel => 1}
+
+    # Allow overriding parallel workers via PARALLEL environment variable
+    if ENV['PARALLEL'] && ENV['PARALLEL'].to_i > 0
+      opts[:parallel] = ENV['PARALLEL'].to_i
+      puts "Using #{opts[:parallel]} parallel workers"
+    end
 
     # Get the dump_path
     dump_path = File.expand_path(Rails.root.join("db", Time.now.strftime("%Y%m%d%I%M%S%p"))).to_s
@@ -20,10 +26,16 @@ namespace :tapsoob do
     Rake::Task["tapsoob:clean"].invoke
   end
 
-  desc "Push a compatible dump on your filesystem to a database"
+  desc "Push a compatible dump on your filesystem to a database (PARALLEL=4 for 4 workers)"
   task :push, [:timestamp] => :environment do |t, args|
     # Default options
-    opts={:default_chunksize => 1000, :debug => false, :resume_filename => nil, :disable_compression => false, :schema => true, :data => true, :indexes_first => false, :progress => true}
+    opts={:default_chunksize => 1000, :debug => false, :resume_filename => nil, :disable_compression => false, :schema => true, :data => true, :indexes_first => false, :progress => true, :parallel => 1}
+
+    # Allow overriding parallel workers via PARALLEL environment variable
+    if ENV['PARALLEL'] && ENV['PARALLEL'].to_i > 0
+      opts[:parallel] = ENV['PARALLEL'].to_i
+      puts "Using #{opts[:parallel]} parallel workers"
+    end
 
     # Get the dumps
     dumps = Dir[Rails.root.join("db", "*/")].select { |e| e =~ /([0-9]{14})([A-Z]{2})/ }.sort

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: tapsoob
 version: !ruby/object:Gem::Version
-  version: 0.5.31
+  version: 0.6.1
 platform: ruby
 authors:
 - Félix Bellanger
@@ -104,6 +104,7 @@ files:
 - lib/tapsoob/data_stream.rb
 - lib/tapsoob/errors.rb
 - lib/tapsoob/log.rb
+- lib/tapsoob/multi_progress_bar.rb
 - lib/tapsoob/operation.rb
 - lib/tapsoob/progress_bar.rb
 - lib/tapsoob/railtie.rb