parallel_sftp 0.3.0

data/lib/parallel_sftp/download.rb

@@ -0,0 +1,164 @@
+# frozen_string_literal: true
+
+require "open3"
+require "fileutils"
+
+module ParallelSftp
+  # Executes lftp downloads with progress tracking
+  class Download
+    attr_reader :lftp_command, :on_progress, :on_segment_progress, :output_buffer
+
+    # Default interval for polling the status file (in seconds)
+    DEFAULT_POLL_INTERVAL = 1
+
+    def initialize(lftp_command, on_progress: nil, on_segment_progress: nil)
+      @lftp_command = lftp_command
+      @on_progress = on_progress
+      @on_segment_progress = on_segment_progress
+      @output_buffer = []
+      @progress_parser = ProgressParser.new
+      @segment_parser = SegmentProgressParser.new
+      @time_estimator = TimeEstimator.new
+      @polling_thread = nil
+      @stop_polling = false
+    end
+
+    # Execute the download
+    # Returns the local file path on success
+    # Raises DownloadError on failure
+    def execute
+      ParallelSftp.ensure_lftp_available!
+
+      # Ensure local directory exists
+      FileUtils.mkdir_p(File.dirname(lftp_command.local_path))
+
+      run_lftp
+    end
+
+    private
+
+    def run_lftp
+      exit_status = nil
+      status_file = status_file_path
+
+      Open3.popen2e(*lftp_command.to_command) do |stdin, stdout_stderr, wait_thr|
+        stdin.close
+
+        # Start background polling for segment progress
+        start_segment_polling(status_file) if on_segment_progress
+
+        stdout_stderr.each_line do |line|
+          @output_buffer << line
+          process_output_line(line)
+        end
+
+        exit_status = wait_thr.value
+      end
+
+      stop_segment_polling
+      handle_result(exit_status)
+    end
+
+    def status_file_path
+      "#{lftp_command.local_path}.lftp-pget-status"
+    end
+
+    def start_segment_polling(status_file)
+      @stop_polling = false
+      @polling_thread = Thread.new do
+        poll_segment_progress(status_file)
+      end
+    end
+
+    def stop_segment_polling
+      @stop_polling = true
+      if @polling_thread&.alive?
+        @polling_thread.join(2) # Wait up to 2 seconds for clean shutdown
+        @polling_thread.kill if @polling_thread.alive?
+      end
+    end
+
+    def poll_segment_progress(status_file)
+      until @stop_polling
+        sleep DEFAULT_POLL_INTERVAL
+
+        begin
+          next unless File.exist?(status_file)
+
+          if @segment_parser.parse(status_file)
+            progress = build_segment_progress
+            on_segment_progress&.call(progress)
+          end
+        rescue StandardError
+          # Silently continue on parse errors - the file may be mid-write
+        end
+      end
+    end
+
+    def build_segment_progress
+      progress = @segment_parser.to_h
+      total_downloaded = progress[:total_downloaded]
+      total_size = progress[:total_size]
+
+      # Record sample for time estimation
+      @time_estimator.record(total_downloaded)
+
+      # Add calculated time estimates
+      progress[:speed] = @time_estimator.speed_bytes_per_second
+      progress[:eta] = @time_estimator.eta_formatted(total_size, total_downloaded) if total_size && total_size > 0
+      progress[:elapsed] = @time_estimator.elapsed_seconds
+      progress[:average_speed] = @time_estimator.average_speed
+
+      progress
+    end
+
+    def process_output_line(line)
+      return unless on_progress
+
+      if @progress_parser.parse(line)
+        on_progress.call(@progress_parser.to_h)
+      end
+    end
+
+    def handle_result(exit_status)
+      if exit_status.success?
+        verify_download
+        lftp_command.local_path
+      else
+        raise DownloadError.new(
+          "lftp exited with status #{exit_status.exitstatus}",
+          remote_path: lftp_command.remote_path,
+          exit_status: exit_status.exitstatus,
+          output: @output_buffer.join
+        )
+      end
+    end
+
+    def verify_download
+      unless File.exist?(lftp_command.local_path)
+        raise DownloadError.new(
+          "Downloaded file not found at expected location",
+          remote_path: lftp_command.remote_path,
+          output: @output_buffer.join
+        )
+      end
+
+      # Verify zip integrity if applicable
+      verify_zip_integrity if lftp_command.local_path.end_with?(".zip")
+    end
+
+    def verify_zip_integrity
+      path = lftp_command.local_path
+      output, status = Open3.capture2e("unzip", "-t", path)
+
+      unless status.success?
+        error_lines = output.lines.grep(/error:|bad zipfile|invalid compressed/).first(5).join
+        raise ZipIntegrityError.new(
+          "Zip file corrupted (possible segment boundary issue)",
+          path: path,
+          output: error_lines
+        )
+      end
+    end
+  end
+end

data/lib/parallel_sftp/errors.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module ParallelSftp
+  # Base error class for all ParallelSftp errors
+  class Error < StandardError; end
+
+  # Raised when lftp is not installed or not found in PATH
+  class LftpNotFoundError < Error
+    def initialize(msg = "lftp is not installed or not found in PATH. Install with: brew install lftp (macOS) or apt install lftp (Linux)")
+      super
+    end
+  end
+
+  # Raised when SFTP connection fails
+  class ConnectionError < Error
+    attr_reader :host, :exit_status
+
+    def initialize(msg = nil, host: nil, exit_status: nil)
+      @host = host
+      @exit_status = exit_status
+      super(msg || "Failed to connect to SFTP server#{host ? ": #{host}" : ""}")
+    end
+  end
+
+  # Raised when file download fails
+  class DownloadError < Error
+    attr_reader :remote_path, :exit_status, :output
+
+    def initialize(msg = nil, remote_path: nil, exit_status: nil, output: nil)
+      @remote_path = remote_path
+      @exit_status = exit_status
+      @output = output
+      super(msg || "Failed to download file#{remote_path ? ": #{remote_path}" : ""}")
+    end
+  end
+
+  # Raised when downloaded file integrity check fails
+  class IntegrityError < Error
+    attr_reader :expected_size, :actual_size
+
+    def initialize(msg = nil, expected_size: nil, actual_size: nil)
+      @expected_size = expected_size
+      @actual_size = actual_size
+      super(msg || "File integrity check failed. Expected: #{expected_size}, Got: #{actual_size}")
+    end
+  end
+
+  # Raised when zip file integrity check fails (segment boundary corruption)
+  class ZipIntegrityError < IntegrityError
+    attr_reader :path, :output
+
+    def initialize(msg = nil, path: nil, output: nil)
+      @path = path
+      @output = output
+      super(msg || "Zip integrity check failed#{path ? ": #{path}" : ""}")
+    end
+  end
+end

data/lib/parallel_sftp/lftp_command.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module ParallelSftp
+  # Builds lftp command scripts for SFTP downloads
+  class LftpCommand
+    attr_reader :host, :user, :password, :port, :remote_path, :local_path,
+                :segments, :timeout, :max_retries, :reconnect_interval, :resume,
+                :sftp_connect_program
+
+    def initialize(options = {})
+      @host = options.fetch(:host)
+      @user = options.fetch(:user)
+      @password = options.fetch(:password)
+      @port = options.fetch(:port, ParallelSftp.configuration.default_port)
+      @remote_path = options.fetch(:remote_path)
+      @local_path = options.fetch(:local_path)
+      @segments = options.fetch(:segments, ParallelSftp.configuration.default_segments)
+      @timeout = options.fetch(:timeout, ParallelSftp.configuration.timeout)
+      @max_retries = options.fetch(:max_retries, ParallelSftp.configuration.max_retries)
+      @reconnect_interval = options.fetch(:reconnect_interval, ParallelSftp.configuration.reconnect_interval)
+      @resume = options.fetch(:resume, true)
+      @sftp_connect_program = options.fetch(:sftp_connect_program,
+        ParallelSftp.configuration.sftp_connect_program)
+    end
+
+    # Generate the lftp script for download
+    def to_script
+      lines = [
+        "set net:timeout #{timeout}",
+        "set net:max-retries #{max_retries}",
+        "set net:reconnect-interval-base #{reconnect_interval}",
+        "set sftp:auto-confirm yes",
+        "set ssl:verify-certificate no"
+      ]
+
+      # Add custom SSH connect program if configured (for legacy host key algorithms)
+      if sftp_connect_program
+        lines << "set sftp:connect-program \"#{sftp_connect_program}\""
+      end
+
+      lines << "open -p #{port} sftp://#{user}:#{escaped_password}@#{host}"
+      lines << "pget -n #{segments}#{resume_flag} \"#{remote_path}\" -o \"#{local_path}\""
+      lines << "quit"
+
+      lines.join("\n") + "\n"
+    end
+
+    # Generate the full lftp command with script
+    def to_command
+      ["lftp", "-c", to_script]
+    end
+
+    private
+
+    def escaped_password
+      # Escape special characters in password for URL
+      password.gsub(/[^a-zA-Z0-9_.-]/) { |c| format("%%%02X", c.ord) }
+    end
+
+    def resume_flag
+      resume ? " -c" : ""
+    end
+  end
+end

data/lib/parallel_sftp/progress_parser.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+module ParallelSftp
+  # Parses lftp output to extract progress information
+  class ProgressParser
+    # Regex patterns for parsing lftp pget output
+    PROGRESS_PATTERN = /(\d+(?:\.\d+)?)\s*([KMGT]?B?)\/s/.freeze
+    BYTES_PATTERN = /(\d+(?:\.\d+)?)\s*([KMGT]?)B?\s+(?:of\s+)?(\d+(?:\.\d+)?)\s*([KMGT]?)B?/.freeze
+    PERCENT_PATTERN = /(\d+)%/.freeze
+    ETA_PATTERN = /eta:?\s*(\d+[hms](?:\d+[ms])?|\d+:\d+(?::\d+)?)/.freeze
+
+    attr_reader :bytes_transferred, :total_bytes, :speed, :percent, :eta
+
+    def initialize
+      @bytes_transferred = 0
+      @total_bytes = 0
+      @speed = 0
+      @percent = 0
+      @eta = nil
+    end
+
+    # Parse a line of lftp output and update progress info
+    # Returns true if progress was updated, false otherwise
+    def parse(line)
+      return false if line.nil? || line.strip.empty?
+
+      updated = false
+
+      # Extract percentage
+      if (match = line.match(PERCENT_PATTERN))
+        @percent = match[1].to_i
+        updated = true
+      end
+
+      # Extract speed
+      if (match = line.match(PROGRESS_PATTERN))
+        @speed = parse_size(match[1], match[2])
+        updated = true
+      end
+
+      # Extract bytes transferred and total
+      if (match = line.match(BYTES_PATTERN))
+        @bytes_transferred = parse_size(match[1], match[2])
+        @total_bytes = parse_size(match[3], match[4])
+        updated = true
+      end
+
+      # Extract ETA
+      if (match = line.match(ETA_PATTERN))
+        @eta = match[1]
+        updated = true
+      end
+
+      updated
+    end
+
+    # Return progress info as a hash
+    def to_h
+      {
+        bytes_transferred: bytes_transferred,
+        total_bytes: total_bytes,
+        speed: speed,
+        percent: percent,
+        eta: eta
+      }
+    end
+
+    private
+
+    def parse_size(value, unit)
+      base = value.to_f
+      multiplier = case unit.upcase.gsub("B", "")
+                   when "K" then 1024
+                   when "M" then 1024**2
+                   when "G" then 1024**3
+                   when "T" then 1024**4
+                   else 1
+                   end
+      (base * multiplier).to_i
+    end
+  end
+end

data/lib/parallel_sftp/segment_progress_parser.rb

@@ -0,0 +1,153 @@
+# frozen_string_literal: true
+
+module ParallelSftp
+  # Parses lftp's .lftp-pget-status file to extract per-segment progress
+  #
+  # The status file format is:
+  #   size=20955686931
+  #   0.pos=57442304
+  #   0.limit=2619460869
+  #   1.pos=2670611717
+  #   1.limit=5238921735
+  #   ...
+  #
+  # Where:
+  #   - size: Total file size in bytes
+  #   - N.pos: Current position (bytes downloaded) for segment N
+  #   - N.limit: End position (byte limit) for segment N
+  class SegmentProgressParser
+    # Represents a single download segment
+    Segment = Struct.new(:index, :pos, :limit, :start, keyword_init: true) do
+      def downloaded
+        pos - start
+      end
+
+      def segment_size
+        limit - start
+      end
+
+      def percent
+        return 0.0 if segment_size.zero?
+        ((downloaded.to_f / segment_size) * 100).round(1)
+      end
+    end
+
+    attr_reader :total_size, :segments
+
+    def initialize
+      @total_size = nil
+      @segments = []
+    end
+
+    # Parse a status file from disk
+    #
+    # @param status_file_path [String] Path to the .lftp-pget-status file
+    # @return [Boolean] true if file was parsed successfully, false otherwise
+    def parse(status_file_path)
+      return false unless File.exist?(status_file_path)
+
+      content = File.read(status_file_path)
+      parse_content(content)
+    end
+
+    # Parse status file content directly
+    #
+    # @param content [String] Content of the status file
+    # @return [Boolean] true if content was parsed successfully
+    def parse_content(content)
+      @segments = []
+      @total_size = nil
+      segment_data = {}
+
+      lines = content.strip.split("\n")
+
+      lines.each do |line|
+        case line
+        when /^size=(-?\d+)/
+          @total_size = ::Regexp.last_match(1).to_i
+        when /^(\d+)\.pos=(\d+)/
+          idx = ::Regexp.last_match(1).to_i
+          pos = ::Regexp.last_match(2).to_i
+          segment_data[idx] ||= {}
+          segment_data[idx][:pos] = pos
+        when /^(\d+)\.limit=(\d+)/
+          idx = ::Regexp.last_match(1).to_i
+          limit = ::Regexp.last_match(2).to_i
+          segment_data[idx] ||= {}
+          segment_data[idx][:limit] = limit
+        end
+      end
+
+      build_segments(segment_data)
+      true
+    end
+
+    # Convert to hash representation
+    #
+    # @return [Hash] Hash with total_size, segments array, total_downloaded, and overall_percent
+    def to_h
+      {
+        total_size: total_size,
+        segments: segments.map { |s| segment_to_h(s) },
+        total_downloaded: total_downloaded,
+        overall_percent: overall_percent
+      }
+    end
+
+    # Total bytes downloaded across all segments
+    #
+    # @return [Integer] Total bytes downloaded
+    def total_downloaded
+      segments.sum(&:downloaded)
+    end
+
+    # Overall download percentage
+    #
+    # @return [Float] Percentage complete (0.0 to 100.0)
+    def overall_percent
+      return 0.0 if total_size.nil? || total_size <= 0
+      ((total_downloaded.to_f / total_size) * 100).round(1)
+    end
+
+    private
+
+    def build_segments(segment_data)
+      return if segment_data.empty?
+
+      # Sort by index to ensure correct order
+      sorted_indices = segment_data.keys.sort
+
+      sorted_indices.each_with_index do |idx, position|
+        data = segment_data[idx]
+        next unless data[:pos] && data[:limit]
+
+        # Calculate start position: segment 0 starts at 0, others start at previous segment's limit
+        start = if position.zero?
+                  0
+                else
+                  prev_idx = sorted_indices[position - 1]
+                  segment_data[prev_idx][:limit]
+                end
+
+        @segments << Segment.new(
+          index: idx,
+          pos: data[:pos],
+          limit: data[:limit],
+          start: start
+        )
+      end
+    end
+
+    def segment_to_h(segment)
+      {
+        index: segment.index,
+        pos: segment.pos,
+        limit: segment.limit,
+        start: segment.start,
+        downloaded: segment.downloaded,
+        segment_size: segment.segment_size,
+        percent: segment.percent
+      }
+    end
+  end
+end

data/lib/parallel_sftp/time_estimator.rb

@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+
+module ParallelSftp
+  # Calculates download speed and time estimates using a moving window of samples
+  class TimeEstimator
+    # Represents a single progress sample
+    Sample = Struct.new(:bytes, :time, keyword_init: true)
+
+    attr_reader :window_size
+
+    # Initialize a new TimeEstimator
+    #
+    # @param window_size [Integer] Number of samples to keep for speed calculation (default: 10)
+    def initialize(window_size: 10)
+      @samples = []
+      @window_size = window_size
+      @start_time = nil
+      @start_bytes = nil
+    end
+
+    # Record a progress sample
+    #
+    # @param bytes_downloaded [Integer] Total bytes downloaded so far
+    # @param timestamp [Time] Time of the sample (default: Time.now)
+    def record(bytes_downloaded, timestamp = Time.now)
+      @start_time ||= timestamp
+      @start_bytes ||= bytes_downloaded
+
+      @samples << Sample.new(bytes: bytes_downloaded, time: timestamp)
+      @samples.shift if @samples.size > @window_size
+    end
+
+    # Calculate current download speed based on recent samples
+    #
+    # @return [Integer, nil] Speed in bytes per second, or nil if insufficient data
+    def speed_bytes_per_second
+      return nil if @samples.size < 2
+
+      first = @samples.first
+      last = @samples.last
+
+      bytes_delta = last.bytes - first.bytes
+      time_delta = last.time - first.time
+
+      return nil if time_delta <= 0
+      (bytes_delta / time_delta).round
+    end
+
+    # Calculate estimated time remaining
+    #
+    # @param total_bytes [Integer] Total file size in bytes
+    # @param current_bytes [Integer] Current bytes downloaded
+    # @return [Integer, nil] Estimated seconds remaining, or nil if cannot calculate
+    def eta_seconds(total_bytes, current_bytes)
+      speed = speed_bytes_per_second
+      return nil if speed.nil? || speed <= 0
+
+      remaining = total_bytes - current_bytes
+      return 0 if remaining <= 0
+
+      (remaining.to_f / speed).round
+    end
+
+    # Calculate estimated time remaining as a formatted string
+    #
+    # @param total_bytes [Integer] Total file size in bytes
+    # @param current_bytes [Integer] Current bytes downloaded
+    # @return [String, nil] Formatted duration (e.g., "1h25m", "5m30s"), or nil
+    def eta_formatted(total_bytes, current_bytes)
+      seconds = eta_seconds(total_bytes, current_bytes)
+      return nil if seconds.nil?
+
+      format_duration(seconds)
+    end
+
+    # Elapsed time since first sample
+    #
+    # @return [Integer] Elapsed seconds
+    def elapsed_seconds
+      return 0 if @start_time.nil?
+      (Time.now - @start_time).round
+    end
+
+    # Average speed since start of download
+    #
+    # @return [Integer, nil] Average speed in bytes per second, or nil
+    def average_speed
+      return nil if @start_time.nil? || @samples.empty?
+
+      elapsed = Time.now - @start_time
+      return nil if elapsed <= 0
+
+      bytes = @samples.last.bytes - @start_bytes
+      (bytes / elapsed).round
+    end
+
+    # Clear all recorded samples and reset state
+    def reset!
+      @samples = []
+      @start_time = nil
+      @start_bytes = nil
+    end
+
+    private
+
+    def format_duration(seconds)
+      return "0s" if seconds <= 0
+
+      hours = seconds / 3600
+      minutes = (seconds % 3600) / 60
+      secs = seconds % 60
+
+      if hours > 0
+        "#{hours}h#{minutes}m"
+      elsif minutes > 0
+        "#{minutes}m#{secs}s"
+      else
+        "#{secs}s"
+      end
+    end
+  end
+end

data/lib/parallel_sftp/version.rb

@@ -0,0 +1,3 @@
+module ParallelSftp
+  VERSION = "0.3.0"
+end