http_mimic 0.3.0

data/lib/http_mimic/downloader.rb

@@ -0,0 +1,272 @@
+# frozen_string_literal: true
+
+require 'rbconfig'
+require 'net/http'
+require 'uri'
+require 'fileutils'
+require 'zlib'
+require 'rubygems/package'
+require 'open3'
+
+module HttpMimic
+  class Downloader
+    DEFAULT_GITHUB_REPO = 'lexiforest/curl-impersonate'
+    DEFAULT_VERSION     = 'v2.1.1'
+
+    class DownloadError < HttpMimic::Error; end
+    class UnsupportedPlatformError < HttpMimic::Error; end
+
+    class << self
+      def download!(version: nil, install_dir: nil, repo: nil, force: false)
+        target_version = normalize_version(version || HttpMimic.configuration.driver_version || DEFAULT_VERSION)
+        target_dir     = File.expand_path(install_dir || HttpMimic.configuration.install_dir)
+        target_repo    = repo || HttpMimic.configuration.github_repo || DEFAULT_GITHUB_REPO
+
+        FileUtils.mkdir_p(target_dir)
+
+        # Check if the target version is already installed and intact
+        if !force && installed?(version: target_version, install_dir: target_dir)
+          log_info("curl-impersonate #{target_version} is already installed in #{target_dir}")
+          return target_dir
+        end
+
+        slug = platform_slug
+        asset_name = "curl-impersonate-#{target_version}.#{slug}.tar.gz"
+        download_url = "https://github.com/#{target_repo}/releases/download/#{target_version}/#{asset_name}"
+
+        log_info("Downloading curl-impersonate (#{target_version}) [#{asset_name}]...")
+
+        archive_data = fetch_binary(download_url)
+
+        log_info("Extracting archive to #{target_dir}...")
+        extract_tar_gz(archive_data, target_dir)
+
+        # Write version marker file
+        File.write(version_file_path(target_dir), target_version)
+
+        log_info("curl-impersonate #{target_version} installation complete!")
+        target_dir
+      end
+
+      def installed?(version: nil, install_dir: nil)
+        target_dir = File.expand_path(install_dir || HttpMimic.configuration.install_dir)
+        core_binary = File.join(target_dir, binary_name_for_platform('curl-impersonate'))
+        return false unless File.file?(core_binary) && File.executable?(core_binary)
+
+        if version
+          target_version = normalize_version(version)
+          v_file = version_file_path(target_dir)
+          return false unless File.file?(v_file)
+          return File.read(v_file).strip == target_version
+        end
+
+        true
+      end
+
+      def binary_path(name, install_dir: nil)
+        target_dir = File.expand_path(install_dir || HttpMimic.configuration.install_dir)
+        candidate = File.join(target_dir, binary_name_for_platform(name))
+        return candidate if File.file?(candidate) && File.executable?(candidate)
+
+        nil
+      end
+
+      def available_binaries(install_dir: nil)
+        target_dir = File.expand_path(install_dir || HttpMimic.configuration.install_dir)
+        return [] unless Dir.exist?(target_dir)
+
+        Dir.glob(File.join(target_dir, '*')).select do |file|
+          File.file?(file) && File.executable?(file)
+        end.map { |f| File.basename(f) }
+      end
+
+      def platform_slug
+        os  = host_os
+        cpu = host_cpu
+
+        case os
+        when :macos
+          case cpu
+          when :arm64 then 'arm64-macos'
+          when :x86_64 then 'x86_64-macos'
+          else
+            raise UnsupportedPlatformError, "Unsupported macOS CPU architecture: #{cpu}"
+          end
+        when :linux
+          libc = linux_libc
+          case cpu
+          when :x86_64
+            libc == :musl ? 'x86_64-linux-musl' : 'x86_64-linux-gnu'
+          when :aarch64, :arm64
+            libc == :musl ? 'aarch64-linux-musl' : 'aarch64-linux-gnu'
+          when :i386, :i686
+            'i386-linux-gnu'
+          when :arm
+            'arm-linux-gnueabihf'
+          when :riscv64
+            'riscv64-linux-gnu'
+          when :loongarch64
+            'loongarch64-linux-gnu'
+          else
+            raise UnsupportedPlatformError, "Unsupported Linux CPU architecture: #{cpu}"
+          end
+        when :windows
+          case cpu
+          when :x86_64 then 'x86_64-win32'
+          when :arm64 then 'arm64-win32'
+          when :i386, :i686 then 'i686-win32'
+          else
+            raise UnsupportedPlatformError, "Unsupported Windows CPU architecture: #{cpu}"
+          end
+        when :freebsd
+          case cpu
+          when :x86_64 then 'x86_64-freebsd'
+          when :arm64, :aarch64 then 'aarch64-freebsd'
+          else
+            raise UnsupportedPlatformError, "Unsupported FreeBSD CPU architecture: #{cpu}"
+          end
+        else
+          raise UnsupportedPlatformError, "Unsupported operating system: #{RbConfig::CONFIG['host_os']}"
+        end
+      end
+
+      private
+
+      def host_os
+        os_str = RbConfig::CONFIG['host_os'].downcase
+        case os_str
+        when /darwin|mac os/
+          :macos
+        when /linux/
+          :linux
+        when /mswin|msys|mingw|cygwin|bccwin|wince|emc/
+          :windows
+        when /freebsd/
+          :freebsd
+        else
+          :unknown
+        end
+      end
+
+      def host_cpu
+        cpu_str = RbConfig::CONFIG['host_cpu'].downcase
+        case cpu_str
+        when /arm64|aarch64/
+          host_os == :macos || host_os == :windows ? :arm64 : :aarch64
+        when /x86_64|amd64|x64/
+          :x86_64
+        when /i[3-6]86|x86/
+          :i386
+        when /arm/
+          :arm
+        when /riscv64/
+          :riscv64
+        when /loongarch64/
+          :loongarch64
+        else
+          cpu_str.to_sym
+        end
+      end
+
+      def linux_libc
+        if File.exist?('/etc/alpine-release') || `ldd --version 2>&1` =~ /musl/i
+          :musl
+        else
+          :gnu
+        end
+      rescue StandardError
+        :gnu
+      end
+
+      def binary_name_for_platform(name)
+        host_os == :windows ? "#{name}.exe" : name
+      end
+
+      def normalize_version(ver)
+        v = ver.to_s.strip
+        v.start_with?('v') ? v : "v#{v}"
+      end
+
+      def version_file_path(dir)
+        File.join(dir, '.version')
+      end
+
+      def fetch_binary(url, limit = 5)
+        raise DownloadError, "Download failed: too many redirects (#{url})" if limit <= 0
+
+        uri = URI.parse(url)
+        http = Net::HTTP.new(uri.host, uri.port)
+        http.use_ssl = (uri.scheme == 'https')
+        http.read_timeout = 60
+        http.open_timeout = 30
+
+        req = Net::HTTP::Get.new(uri.request_uri)
+        req['User-Agent'] = "Ruby-HttpMimic/#{HttpMimic::VERSION} (#{RUBY_PLATFORM})"
+
+        res = http.request(req)
+
+        case res
+        when Net::HTTPSuccess
+          res.body
+        when Net::HTTPRedirection
+          location = res['location']
+          fetch_binary(location, limit - 1)
+        else
+          raise DownloadError, "Download failed with HTTP status #{res.code}: #{url}"
+        end
+      rescue StandardError => e
+        raise DownloadError, "Failed to download curl-impersonate release: #{e.message}"
+      end
+
+      def extract_tar_gz(data, dest_dir)
+        # 1. Try system tar command first
+        begin
+          tmp_tar = File.join(dest_dir, ".temp_download_#{Process.pid}.tar.gz")
+          File.binwrite(tmp_tar, data)
+          
+          _stdout, _stderr, status = Open3.capture3('tar', '-xzf', tmp_tar, '-C', dest_dir)
+          FileUtils.rm_f(tmp_tar)
+
+          if status && status.success?
+            chmod_all_binaries(dest_dir)
+            return
+          end
+        rescue StandardError => e
+          log_info("System tar command failed (#{e.message}), falling back to Ruby extraction...")
+        end
+
+        # 2. Pure Ruby tar.gz extraction fallback
+        gz = Zlib::GzipReader.new(StringIO.new(data))
+        tar = Gem::Package::TarReader.new(gz)
+
+        tar.each do |entry|
+          target_path = File.join(dest_dir, entry.full_name)
+          if entry.directory?
+            FileUtils.mkdir_p(target_path)
+          elsif entry.file?
+            FileUtils.mkdir_p(File.dirname(target_path))
+            File.binwrite(target_path, entry.read)
+            File.chmod(0755, target_path)
+          end
+        end
+        tar.close
+        gz.close
+        chmod_all_binaries(dest_dir)
+      end
+
+      def chmod_all_binaries(dir)
+        Dir.glob(File.join(dir, '*')).each do |f|
+          File.chmod(0755, f) if File.file?(f)
+        end
+      end
+
+      def log_info(msg)
+        if HttpMimic.configuration.logger
+          HttpMimic.configuration.logger.info("[HttpMimic::Downloader] #{msg}")
+        elsif HttpMimic.configuration.debug
+          puts "[HttpMimic::Downloader] #{msg}"
+        end
+      end
+    end
+  end
+end

data/lib/http_mimic/exceptions.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module HttpMimic
+  # Base exception class
+  class Error < StandardError; end
+
+  # Raised when the curl-impersonate binary cannot be found
+  class BinaryNotFoundError < Error; end
+
+  # Raised when the curl command fails or returns a non-zero exit status
+  class CommandError < Error
+    attr_reader :exit_code, :stderr, :command, :response
+
+    def initialize(message, exit_code: nil, stderr: nil, command: nil, response: nil)
+      super(message)
+      @exit_code = exit_code
+      @stderr = stderr
+      @command = command
+      @response = response
+    end
+  end
+
+  # Connection timeout (Curl exit code 28)
+  class TimeoutError < CommandError; end
+
+  # Connection failure or DNS resolution error (Curl exit code 6, 7, 52, etc.)
+  class ConnectionError < CommandError; end
+
+  # SSL/TLS handshake or certificate error (Curl exit code 35, 51, 60, etc.)
+  class SSLError < CommandError; end
+
+  # Raised when response parsing fails
+  class ResponseParseError < Error; end
+end

data/lib/http_mimic/headers.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+module HttpMimic
+  # Case-insensitive HTTP Headers wrapper that preserves original casing
+  class Headers
+    include Enumerable
+
+    def initialize(headers_hash = {})
+      @headers = {}
+      @names = {}
+      @multi_values = {}
+
+      headers_hash.each do |key, val|
+        self[key] = val
+      end
+    end
+
+    def [](key)
+      return nil if key.nil?
+      canonical = @names[normalize_key(key)]
+      return nil unless canonical
+
+      norm_key = normalize_key(key)
+      if norm_key == 'set-cookie' && @multi_values.key?(norm_key)
+        # For Set-Cookie, return an Array if multiple values exist
+        @multi_values[norm_key].size > 1 ? @multi_values[norm_key] : @multi_values[norm_key].first
+      else
+        @headers[canonical]
+      end
+    end
+
+    def []=(key, value)
+      return if key.nil?
+      norm = normalize_key(key)
+      if existing_key = @names[norm]
+        @headers.delete(existing_key)
+      end
+      orig_key = key.to_s
+      @names[norm] = orig_key
+      @headers[orig_key] = value
+
+      @multi_values[norm] = value.is_a?(Array) ? value.dup : [value]
+    end
+
+    # Add header (supports duplicate headers with the same name, e.g., multiple Set-Cookie entries)
+    def add(key, value)
+      return if key.nil?
+      norm = normalize_key(key)
+      orig_key = @names[norm] || key.to_s
+      @names[norm] = orig_key
+
+      @multi_values[norm] ||= []
+      @multi_values[norm] << value
+
+      @headers[orig_key] = value
+    end
+
+    def get_all(key)
+      return [] if key.nil?
+      @multi_values[normalize_key(key)] || []
+    end
+
+    def key?(key)
+      return false if key.nil?
+      @names.key?(normalize_key(key))
+    end
+    alias has_key? key?
+    alias include? key?
+
+    def each(&block)
+      @headers.each(&block)
+    end
+
+    def keys
+      @headers.keys
+    end
+
+    def values
+      @headers.values
+    end
+
+    def to_h
+      @headers.dup
+    end
+    alias to_hash to_h
+
+    def inspect
+      "#<#{self.class.name} #{@headers.inspect}>"
+    end
+
+    def to_s
+      @headers.to_s
+    end
+
+    private
+
+    def normalize_key(key)
+      key.to_s.downcase.tr('_', '-')
+    end
+  end
+end

data/lib/http_mimic/module_methods.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+module HttpMimic
+  module ModuleMethods
+    def base_uri(uri = nil)
+      return default_options[:base_uri] if uri.nil?
+      default_options[:base_uri] = uri
+    end
+
+    def headers(h = nil)
+      return default_options[:headers] if h.nil?
+      default_options[:headers] ||= {}
+      default_options[:headers].merge!(h)
+    end
+
+    def default_params(params = nil)
+      return default_options[:query] if params.nil?
+      default_options[:query] ||= {}
+      default_options[:query].merge!(params)
+    end
+    alias default_query default_params
+
+    def default_timeout(seconds = nil)
+      return default_options[:timeout] if seconds.nil?
+      default_options[:timeout] = seconds
+    end
+
+    def impersonate(target = nil)
+      return default_options[:impersonate] if target.nil?
+      default_options[:impersonate] = target
+    end
+
+    def proxy(proxy_str = nil)
+      return default_options[:proxy] if proxy_str.nil?
+      default_options[:proxy] = proxy_str
+    end
+
+    def cookies(c = nil)
+      return default_options[:cookies] if c.nil?
+      default_options[:cookies] ||= {}
+      default_options[:cookies].merge!(c)
+    end
+
+    def default_options
+      @default_options ||= {}
+    end
+
+    def get(url, options = {})
+      request(:get, url, options)
+    end
+
+    def post(url, options = {})
+      request(:post, url, options)
+    end
+
+    def put(url, options = {})
+      request(:put, url, options)
+    end
+
+    def patch(url, options = {})
+      request(:patch, url, options)
+    end
+
+    def delete(url, options = {})
+      request(:delete, url, options)
+    end
+
+    def head(url, options = {})
+      request(:head, url, options)
+    end
+
+    def options(url, options = {})
+      request(:options, url, options)
+    end
+
+    def request(method, url, options = {})
+      merged = default_options.merge(options)
+
+      # Deep merge headers
+      if default_options[:headers] || options[:headers]
+        base_h = default_options[:headers] || {}
+        opt_h = options[:headers] || {}
+        merged[:headers] = base_h.merge(opt_h)
+      end
+
+      # Deep merge query
+      base_q = default_options[:query] || {}
+      opt_q = options[:query] || options[:params] || {}
+      if !base_q.empty? || !opt_q.empty?
+        merged[:query] = base_q.merge(opt_q)
+      end
+
+      # Deep merge cookies
+      if default_options[:cookies].is_a?(Hash) && options[:cookies].is_a?(Hash)
+        merged[:cookies] = default_options[:cookies].merge(options[:cookies])
+      end
+
+      Request.new(method, url, merged).perform
+    end
+  end
+end

data/lib/http_mimic/request.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+require 'open3'
+
+module HttpMimic
+  class Request
+    attr_reader :method, :url, :options, :config
+
+    def initialize(method, url, options = {}, config = nil)
+      @method  = method
+      @url     = url
+      @options = options.dup
+      @config  = config || HttpMimic.configuration
+    end
+
+    def perform
+      builder = CommandBuilder.new(method, url, options, config)
+      binary, args, stdin_data, final_url = builder.build
+
+      full_command = [binary] + args
+
+      log_debug("Executing HttpMimic command: #{full_command.join(' ')}")
+      log_debug("Stdin data: #{stdin_data}") if stdin_data
+
+      stdout, stderr, status = execute_open3(full_command, stdin_data)
+
+      log_debug("Curl exit status: #{status.exitstatus}")
+      log_debug("Curl stderr: #{stderr}") unless stderr.empty?
+
+      response = ResponseParser.new(
+        stdout,
+        exit_status: status,
+        stderr: stderr,
+        command: full_command,
+        request_url: final_url
+      ).parse
+
+      handle_errors(status, stderr, full_command, response)
+
+      response
+    end
+
+    private
+
+    def execute_open3(command_array, stdin_data)
+      if stdin_data
+        Open3.capture3(*command_array, stdin_data: stdin_data)
+      else
+        Open3.capture3(*command_array)
+      end
+    rescue Errno::ENOENT => e
+      raise BinaryNotFoundError, "Executable not found #{command_array.first}: #{e.message}"
+    end
+
+    def handle_errors(status, stderr, command, response)
+      raise_error = options.fetch(:raise_on_error, config.raise_on_error)
+      exit_code = status.exitstatus
+
+      if exit_code != 0
+        message = "curl execution error (exit code #{exit_code}): #{stderr.strip}"
+        error_class = case exit_code
+                      when 28
+                        TimeoutError
+                      when 6, 7, 52
+                        ConnectionError
+                      when 35, 51, 60
+                        SSLError
+                      else
+                        CommandError
+                      end
+
+        if raise_error
+          raise error_class.new(message, exit_code: exit_code, stderr: stderr, command: command, response: response)
+        end
+      elsif raise_error && response.error?
+        raise CommandError.new("HTTP request failed (status #{response.code})", exit_code: exit_code, stderr: stderr, command: command, response: response)
+      end
+    end
+
+    def log_debug(message)
+      if config.logger
+        config.logger.debug(message)
+      elsif config.debug || options[:debug]
+        puts "[HttpMimic DEBUG] #{message}"
+      end
+    end
+  end
+end

data/lib/http_mimic/response.rb

@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+module HttpMimic
+  class Response
+    attr_reader :code,
+                :http_version,
+                :status_message,
+                :headers,
+                :cookies,
+                :body,
+                :parsed_response,
+                :raw_headers,
+                :history,
+                :request_url,
+                :stderr,
+                :exit_code,
+                :command
+
+    alias status code
+
+    def initialize(attributes = {})
+      @code            = attributes[:code] || 0
+      @http_version    = attributes[:http_version]
+      @status_message  = attributes[:status_message]
+      @headers         = attributes[:headers] || Headers.new
+      @cookies         = attributes[:cookies] || Cookies.new
+      @body            = attributes[:body] || ''
+      @parsed_response = attributes[:parsed_response]
+      @raw_headers     = attributes[:raw_headers] || ''
+      @history         = attributes[:history] || []
+      @request_url     = attributes[:request_url]
+      @stderr          = attributes[:stderr] || ''
+      @exit_code       = attributes[:exit_code] || 0
+      @command         = attributes[:command]
+    end
+
+    def success?
+      code >= 200 && code < 300
+    end
+    alias ok? success?
+
+    def redirect?
+      code >= 300 && code < 400
+    end
+
+    def client_error?
+      code >= 400 && code < 500
+    end
+
+    def server_error?
+      code >= 500 && code < 600
+    end
+
+    def error?
+      client_error? || server_error?
+    end
+
+    def [](key)
+      if parsed_response.is_a?(Hash) || parsed_response.is_a?(Array)
+        parsed_response[key]
+      else
+        nil
+      end
+    end
+
+    def to_s
+      body.to_s
+    end
+    alias to_str to_s
+
+    def blank?
+      body.nil? || body.strip.empty?
+    end
+
+    def present?
+      !blank?
+    end
+
+    def inspect
+      "#<#{self.class.name}:0x#{object_id.to_s(16)} @code=#{code} @status_message=#{status_message.inspect} @headers=#{headers.to_h.inspect} @parsed_response=#{parsed_response.inspect}>"
+    end
+
+    def method_missing(method_name, *args, &block)
+      if parsed_response.respond_to?(method_name)
+        parsed_response.public_send(method_name, *args, &block)
+      elsif body.respond_to?(method_name)
+        body.public_send(method_name, *args, &block)
+      else
+        super
+      end
+    end
+
+    def respond_to_missing?(method_name, include_private = false)
+      parsed_response.respond_to?(method_name, include_private) ||
+        body.respond_to?(method_name, include_private) ||
+        super
+    end
+  end
+end