tapsoob 0.6.1 → 0.7.0

data/lib/tapsoob/progress/bar.rb

@@ -0,0 +1,241 @@
+# -*- encoding : utf-8 -*-
+#
+# Ruby/ProgressBar - a text progress bar library
+#
+# Copyright (C) 2001-2005 Satoru Takabayashi <satoru@namazu.org>
+#     All rights reserved.
+#     This is free software with ABSOLUTELY NO WARRANTY.
+#
+# You can redistribute it and/or modify it under the terms
+# of Ruby's license.
+#
+
+module Tapsoob
+  module Progress
+    class Bar
+      VERSION = "0.9"
+
+      def initialize (title, total, out = STDOUT, title_width = nil)
+        @title = title
+        @total = total
+        @out = out
+        @terminal_width = 80
+        @bar_mark = "="
+        @current = 0
+        @previous = 0
+        @finished_p = false
+        @start_time = ::Time.now
+        @previous_time = @start_time
+        # Set title width: use provided width, or accommodate the title, with a minimum of 14
+        @title_width = title_width || [title.length, 14].max
+        @format = "%-#{@title_width}s %3d%% %s %s"
+        @format_arguments = [:title, :percentage, :bar, :stat]
+        clear
+        show
+      end
+      attr_reader   :title
+      attr_reader   :current
+      attr_reader   :total
+      attr_accessor :start_time
+
+      private
+      def fmt_bar
+        bar_width = do_percentage * @terminal_width / 100
+        sprintf("|%s%s|",
+                @bar_mark * bar_width,
+                " " *  (@terminal_width - bar_width))
+      end
+
+      def fmt_percentage
+        do_percentage
+      end
+
+      def fmt_stat
+        if @finished_p then elapsed else eta end
+      end
+
+      def fmt_stat_for_file_transfer
+        if @finished_p then
+          sprintf("%s %s %s", bytes, transfer_rate, elapsed)
+        else
+          sprintf("%s %s %s", bytes, transfer_rate, eta)
+        end
+      end
+
+      def fmt_title
+        @title[0,(@title_width - 1)] + ":"
+      end
+
+      def convert_bytes (bytes)
+        if bytes < 1024
+          sprintf("%6dB", bytes)
+        elsif bytes < 1024 * 1000 # 1000kb
+          sprintf("%5.1fKB", bytes.to_f / 1024)
+        elsif bytes < 1024 * 1024 * 1000  # 1000mb
+          sprintf("%5.1fMB", bytes.to_f / 1024 / 1024)
+        else
+          sprintf("%5.1fGB", bytes.to_f / 1024 / 1024 / 1024)
+        end
+      end
+
+      def transfer_rate
+        bytes_per_second = @current.to_f / (::Time.now - @start_time)
+        sprintf("%s/s", convert_bytes(bytes_per_second))
+      end
+
+      def bytes
+        convert_bytes(@current)
+      end
+
+      def format_time (t)
+        t = t.to_i
+        sec = t % 60
+        min  = (t / 60) % 60
+        hour = t / 3600
+        sprintf("%02d:%02d:%02d", hour, min, sec);
+      end
+
+      # ETA stands for Estimated Time of Arrival.
+      def eta
+        if @current == 0
+          "ETA:  --:--:--"
+        else
+          elapsed = ::Time.now - @start_time
+          eta = elapsed * @total / @current - elapsed;
+          sprintf("ETA:  %s", format_time(eta))
+        end
+      end
+
+      def elapsed
+        elapsed = ::Time.now - @start_time
+        sprintf("Time: %s", format_time(elapsed))
+      end
+
+      def eol
+        if @finished_p then "\n" else "\r" end
+      end
+
+      def do_percentage
+        if @total.zero?
+          100
+        else
+          @current  * 100 / @total
+        end
+      end
+
+      def get_width
+        # FIXME: I don't know how portable it is.
+        default_width = 80
+        begin
+          tiocgwinsz = 0x5413
+          data = [0, 0, 0, 0].pack("SSSS")
+          if @out.ioctl(tiocgwinsz, data) >= 0 then
+            rows, cols, xpixels, ypixels = data.unpack("SSSS")
+            if cols > 0 then cols else default_width end
+          else
+            default_width
+          end
+        rescue Exception
+          default_width
+        end
+      end
+
+      def show
+        arguments = @format_arguments.map {|method|
+          method = sprintf("fmt_%s", method)
+          send(method)
+        }
+        line = sprintf(@format, *arguments)
+
+        width = get_width
+        if line.length == width - 1
+          @out.print(line + eol)
+          @out.flush
+        elsif line.length >= width
+          @terminal_width = [@terminal_width - (line.length - width + 1), 0].max
+          if @terminal_width == 0 then @out.print(line + eol) else show end
+        else # line.length < width - 1
+          @terminal_width += width - line.length + 1
+          show
+        end
+        @previous_time = ::Time.now
+      end
+
+      def show_if_needed
+        if @total.zero?
+          cur_percentage = 100
+          prev_percentage = 0
+        else
+          cur_percentage  = (@current  * 100 / @total).to_i
+          prev_percentage = (@previous * 100 / @total).to_i
+        end
+
+        # Use "!=" instead of ">" to support negative changes
+        if cur_percentage != prev_percentage ||
+            ::Time.now - @previous_time >= 1 || @finished_p
+          show
+        end
+      end
+
+      public
+      def clear
+        @out.print "\r"
+        @out.print(" " * (get_width - 1))
+        @out.print "\r"
+      end
+
+      def finish
+        @current = @total
+        @finished_p = true
+        show
+      end
+
+      def finished?
+        @finished_p
+      end
+
+      def file_transfer_mode
+        @format_arguments = [:title, :percentage, :bar, :stat_for_file_transfer]
+      end
+
+      def format= (format)
+        @format = format
+      end
+
+      def format_arguments= (arguments)
+        @format_arguments = arguments
+      end
+
+      def halt
+        @finished_p = true
+        show
+      end
+
+      def inc (step = 1)
+        @current += step
+        @current = @total if @current > @total
+        show_if_needed
+        @previous = @current
+      end
+
+      def set (count)
+        if count < 0 || count > @total
+          raise "invalid count: #{count} (total: #{@total})"
+        end
+        @current = count
+        show_if_needed
+        @previous = @current
+      end
+
+      def inspect
+        "#<Tapsoob::Progress::Bar:#{@current}/#{@total}>"
+      end
+    end
+
+    class ReversedBar < Bar
+      def do_percentage
+        100 - super
+      end
+    end
+  end
+end

data/lib/tapsoob/progress/multi_bar.rb

@@ -0,0 +1,190 @@
+# -*- encoding : utf-8 -*-
+
+module Tapsoob
+  module Progress
+    # MultiBar manages multiple progress bars in parallel with a clean interface:
+    # - N progress bar lines (constantly updating)
+    # - 1 separator line
+    # - 1 info message line (shows latest INFO, gets replaced)
+    class MultiBar
+      def initialize(max_bars = 4)
+        @max_bars = max_bars
+        @bars = []
+        @mutex = Mutex.new
+        @active = true
+        @out = STDOUT
+        @last_update = Time.now
+        @max_title_width = 14  # Minimum width, will grow with longer titles
+        @initialized = false
+        @total_lines = 0  # Total lines: max_bars + separator + info line
+        @info_message = ""  # Current info message to display
+        @start_time = Time.now  # Track total elapsed time
+        @terminal_width = get_terminal_width
+      end
+
+      # Get terminal width, default to 160 if can't detect
+      def get_terminal_width
+        require 'io/console'
+        IO.console&.winsize&.[](1) || 160
+      rescue
+        160
+      end
+
+      # Create a new progress bar and return it
+      def create_bar(title, total)
+        @mutex.synchronize do
+          # Initialize display area on first bar creation
+          unless @initialized
+            @total_lines = @max_bars + 2  # bars + separator + info line
+            @total_lines.times { @out.print "\n" }
+            @out.flush
+            @initialized = true
+          end
+
+          # Remove any existing bar with the same title to prevent duplicates
+          @bars.reject! { |b| b.title == title }
+
+          # Update max title width to accommodate longer titles
+          @max_title_width = [@max_title_width, title.length].max
+
+          bar = ThreadSafeBar.new(title, total, self)
+          @bars << bar
+          bar
+        end
+      end
+
+      # Update the info message line (called from outside for INFO logs)
+      def set_info(message)
+        @mutex.synchronize do
+          return unless @active
+          @info_message = message
+          redraw_all if @initialized
+        end
+      end
+
+      # Get the current maximum title width for alignment
+      # Note: Always called from within synchronized methods, so no mutex needed
+      def max_title_width
+        @max_title_width
+      end
+
+      # Called by individual bars when they update
+      def update
+        @mutex.synchronize do
+          return unless @active
+          return unless should_redraw?
+
+          @last_update = Time.now
+          redraw_all
+        end
+      end
+
+      # Finish a specific bar - mark it as completed
+      def finish_bar(bar)
+        @mutex.synchronize do
+          return unless @active
+
+          bar.mark_finished
+
+          # Respect throttle when finishing to avoid spamming redraws
+          if should_redraw?
+            @last_update = Time.now
+            redraw_all
+          end
+          # If throttled, the next regular update will show the finished state
+        end
+      end
+
+      # Stop all progress bars and clear them from display
+      def stop
+        @mutex.synchronize do
+          return unless @active  # Already stopped
+          @active = false
+
+          # Clear all lines (progress bars + separator + info line)
+          if @total_lines > 0 && @initialized
+            # Move cursor up to first line
+            @out.print "\e[#{@total_lines}A"
+
+            # Clear each line
+            @total_lines.times do
+              @out.print "\r\e[2K\n"
+            end
+
+            # Move cursor back to start
+            @out.print "\e[#{@total_lines}A\r"
+          end
+
+          @out.flush
+        end
+      end
+
+      private
+
+      # Check if enough time has passed to redraw (throttle to 10 updates/sec)
+      def should_redraw?
+        Time.now - @last_update >= 0.1
+      end
+
+      def redraw_all(force = false)
+        return unless @active
+        return if @bars.empty?
+
+        render_active_display
+      end
+
+      # Format elapsed time as HH:MM:SS
+      def format_elapsed_time
+        elapsed = Time.now - @start_time
+        hours = (elapsed / 3600).to_i
+        minutes = ((elapsed % 3600) / 60).to_i
+        seconds = (elapsed % 60).to_i
+        sprintf("%02d:%02d:%02d", hours, minutes, seconds)
+      end
+
+      # Render the complete display: progress bars + separator + info line
+      def render_active_display
+        return if @total_lines == 0
+
+        # Show the last N bars (finished or not) - creates a rolling window effect
+        # As new tables start, old completed ones scroll off the top
+        bars_to_draw = @bars.last(@max_bars)
+
+        # Move cursor up to first line
+        @out.print "\e[#{@total_lines}A"
+
+        # Draw progress bars (they handle their own width)
+        @max_bars.times do |i|
+          @out.print "\r\e[K"
+          bars_to_draw[i].render_to(@out) if i < bars_to_draw.length
+          @out.print "\n"
+        end
+
+        # Draw separator line using box drawing character
+        @out.print "\r\e[K"
+        @out.print "─" * @terminal_width
+        @out.print "\n"
+
+        # Draw info message line with elapsed time on the right
+        @out.print "\r\e[K"
+        unless @info_message.empty?
+          elapsed_str = "Elapsed: #{format_elapsed_time}"
+          # Calculate space to right-align elapsed time
+          available_width = @terminal_width - @info_message.length - elapsed_str.length - 2
+          if available_width > 0
+            @out.print @info_message
+            @out.print " " * available_width
+            @out.print elapsed_str
+          else
+            # If too long, just show message
+            @out.print @info_message[0...(@terminal_width - elapsed_str.length - 2)]
+            @out.print "  " + elapsed_str
+          end
+        end
+        @out.print "\n"
+
+        @out.flush
+      end
+    end
+  end
+end

data/lib/tapsoob/progress/thread_safe_bar.rb

@@ -0,0 +1,87 @@
+# -*- encoding : utf-8 -*-
+
+module Tapsoob
+  module Progress
+    # Thread-safe progress bar that reports to a MultiBar
+    class ThreadSafeBar < Bar
+      attr_reader :title
+
+      def initialize(title, total, multi_progress_bar)
+        @multi_progress_bar = multi_progress_bar
+        @out = STDOUT  # Need this for get_width to work
+        # Don't call parent initialize, we'll manage output ourselves
+        @title = title
+        @total = total
+        @terminal_width = 80
+        @bar_mark = "="
+        @current = 0
+        @previous = 0
+        @finished_p = false
+        @start_time = ::Time.now
+        @previous_time = @start_time
+        @format_arguments = [:title, :percentage, :bar, :stat]
+      end
+
+      # Override show to notify multi-progress instead of direct output
+      def show
+        @previous_time = ::Time.now  # Update to prevent time-based refresh spam
+        @multi_progress_bar.update
+      end
+
+      # Render this bar to the given output stream
+      def render_to(out)
+        # Get dynamic title width from MultiBar for consistent alignment
+        # Store as instance variable so parent class fmt_* methods can use it
+        @title_width = @multi_progress_bar.max_title_width
+
+        # Recalculate terminal width to handle resizes and use full width
+        width = get_width
+        # Calculate bar width: total_width - fixed_elements - padding
+        # Fixed: title(variable) + " "(1) + percentage(4) + " "(1) + "|"(1) + "|"(1) + " "(1) + timer(15) = title_width + 25
+        # Padding: +3 for timer fluctuations and safety
+        fixed_chars = @title_width + 28
+        @terminal_width = [width - fixed_chars, 20].max
+
+        # Build format string with dynamic title width
+        format = "%-#{@title_width}s %3d%% %s %s"
+        arguments = @format_arguments.map { |method| send("fmt_#{method}") }
+        line = sprintf(format, *arguments)
+
+        # Ensure line doesn't exceed terminal width to prevent wrapping
+        # Leave 2 chars margin for safety
+        line = line[0, width - 2] if line.length > width - 2
+
+        out.print(line)
+      end
+
+      # Override clear to do nothing (managed by MultiBar)
+      def clear
+        # no-op
+      end
+
+      # Mark this bar as finished (for tracking)
+      def mark_finished
+        @finished_p = true
+      end
+
+      # Override to use the same @finished_p flag
+      def finished?
+        @finished_p
+      end
+
+      # Override finish to notify multi-progress
+      def finish
+        @current = @total
+        @multi_progress_bar.finish_bar(self)
+      end
+
+      # Override inc to check if we need to update
+      def inc(step = 1)
+        @current += step
+        @current = @total if @current > @total
+        show_if_needed
+        @previous = @current
+      end
+    end
+  end
+end

data/lib/tapsoob/progress.rb

@@ -0,0 +1,11 @@
+# -*- encoding : utf-8 -*-
+
+# Progress bar module for Tapsoob
+# Provides progress tracking for database operations with support for:
+# - Single progress bars (Bar)
+# - Multiple parallel progress bars (MultiBar)
+# - Thread-safe progress bars (ThreadSafeBar)
+
+require 'tapsoob/progress/bar'
+require 'tapsoob/progress/multi_bar'
+require 'tapsoob/progress/thread_safe_bar'

data/lib/tapsoob/progress_event.rb

@@ -0,0 +1,109 @@
+# -*- encoding : utf-8 -*-
+require 'json'
+
+module Tapsoob
+  module ProgressEvent
+    @last_progress_time = {}
+    @progress_throttle = 0.5  # Emit progress at most every 0.5 seconds per table
+    @enabled = false  # Only emit when CLI progress bars are disabled
+
+    def self.enabled=(value)
+      @enabled = value
+    end
+
+    def self.enabled?
+      @enabled
+    end
+
+    # Emit structured JSON progress events to STDERR for machine parsing
+    # Only emits when enabled (typically when CLI progress bars are disabled)
+    def self.emit(event_type, data = {})
+      return unless @enabled
+      event = {
+        event: event_type,
+        timestamp: Time.now.utc.iso8601
+      }.merge(data)
+
+      STDERR.puts "PROGRESS: #{JSON.generate(event)}"
+      STDERR.flush
+    end
+
+    # Check if enough time has passed to emit a progress event for this table
+    def self.should_emit_progress?(table_name)
+      now = Time.now
+      last_time = @last_progress_time[table_name]
+
+      if last_time.nil? || (now - last_time) >= @progress_throttle
+        @last_progress_time[table_name] = now
+        true
+      else
+        false
+      end
+    end
+
+    # Clear throttle state for a table (call when table completes)
+    def self.clear_throttle(table_name)
+      @last_progress_time.delete(table_name)
+    end
+
+    # Schema events
+    def self.schema_start(table_count)
+      emit('schema_start', tables: table_count)
+    end
+
+    def self.schema_complete(table_count)
+      emit('schema_complete', tables: table_count)
+    end
+
+    # Data events
+    def self.data_start(table_count, record_count)
+      emit('data_start', tables: table_count, records: record_count)
+    end
+
+    def self.data_complete(table_count, record_count)
+      emit('data_complete', tables: table_count, records: record_count)
+    end
+
+    # Table-level events
+    def self.table_start(table_name, record_count, workers: 1)
+      clear_throttle(table_name)  # Reset throttle for new table
+      emit('table_start', table: table_name, records: record_count, workers: workers)
+    end
+
+    def self.table_progress(table_name, current, total)
+      # Throttle progress events to avoid spam
+      return unless should_emit_progress?(table_name)
+
+      percentage = total > 0 ? ((current.to_f / total) * 100).round(1) : 0
+      emit('table_progress', table: table_name, current: current, total: total, percentage: percentage)
+    end
+
+    def self.table_complete(table_name, record_count)
+      clear_throttle(table_name)  # Clean up throttle state
+      emit('table_complete', table: table_name, records: record_count)
+    end
+
+    # Index events
+    def self.indexes_start(table_count)
+      emit('indexes_start', tables: table_count)
+    end
+
+    def self.indexes_complete(table_count)
+      emit('indexes_complete', tables: table_count)
+    end
+
+    # Sequence events
+    def self.sequences_start
+      emit('sequences_start')
+    end
+
+    def self.sequences_complete
+      emit('sequences_complete')
+    end
+
+    # Error events
+    def self.error(message, context = {})
+      emit('error', { message: message }.merge(context))
+    end
+  end
+end

data/lib/tapsoob/version.rb

@@ -1,4 +1,4 @@
 # -*- encoding : utf-8 -*-
 module Tapsoob
-  VERSION = "0.6.1".freeze
+  VERSION = "0.7.0".freeze
 end

data/lib/tasks/tapsoob.rake

@@ -19,7 +19,7 @@ namespace :tapsoob do
     FileUtils.mkpath "#{dump_path}/indexes"
 
     # Run operation
-    Tapsoob::Operation.factory(:pull, database_uri, dump_path, opts).run
+    Tapsoob::Operation::Base.factory(:pull, database_uri, dump_path, opts).run
 
     # Invoke cleanup task
     Rake::Task["tapsoob:clean"].reenable
@@ -53,7 +53,7 @@ namespace :tapsoob do
     end
 
     # Run operation
-    Tapsoob::Operation.factory(:push, database_uri, dump_path, opts).run
+    Tapsoob::Operation::Base.factory(:push, database_uri, dump_path, opts).run
   end
 
   desc "Cleanup old dumps"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: tapsoob
 version: !ruby/object:Gem::Version
-  version: 0.6.1
+  version: 0.7.0
 platform: ruby
 authors:
 - Félix Bellanger
@@ -102,11 +102,22 @@ files:
 - lib/tapsoob/cli/schema.rb
 - lib/tapsoob/config.rb
 - lib/tapsoob/data_stream.rb
+- lib/tapsoob/data_stream/base.rb
+- lib/tapsoob/data_stream/file_partition.rb
+- lib/tapsoob/data_stream/interleaved.rb
+- lib/tapsoob/data_stream/keyed.rb
+- lib/tapsoob/data_stream/keyed_partition.rb
 - lib/tapsoob/errors.rb
 - lib/tapsoob/log.rb
-- lib/tapsoob/multi_progress_bar.rb
 - lib/tapsoob/operation.rb
-- lib/tapsoob/progress_bar.rb
+- lib/tapsoob/operation/base.rb
+- lib/tapsoob/operation/pull.rb
+- lib/tapsoob/operation/push.rb
+- lib/tapsoob/progress.rb
+- lib/tapsoob/progress/bar.rb
+- lib/tapsoob/progress/multi_bar.rb
+- lib/tapsoob/progress/thread_safe_bar.rb
+- lib/tapsoob/progress_event.rb
 - lib/tapsoob/railtie.rb
 - lib/tapsoob/schema.rb
 - lib/tapsoob/utils.rb