http_mimic 0.3.1 → 0.4.0

data/lib/http_mimic/cookies.rb

@@ -47,6 +47,44 @@ module HttpMimic
       @store.values
     end
 
+    def empty?
+      @store.empty?
+    end
+
+    def size
+      @store.size
+    end
+    alias length size
+
+    def delete(name)
+      @cookie_items.delete(name.to_s)
+      @store.delete(name.to_s)
+    end
+
+    def clear
+      @cookie_items.clear
+      @store.clear
+    end
+
+    def merge(other)
+      dup_cookies = self.class.new(@store)
+      if other.is_a?(Cookies)
+        other.each { |k, v| dup_cookies[k] = v }
+      elsif other.is_a?(Hash)
+        other.each { |k, v| dup_cookies[k] = v }
+      end
+      dup_cookies
+    end
+
+    def merge!(other)
+      if other.is_a?(Cookies)
+        other.each { |k, v| self[k] = v }
+      elsif other.is_a?(Hash)
+        other.each { |k, v| self[k] = v }
+      end
+      self
+    end
+
     def to_h
       @store.dup
     end

data/lib/http_mimic/downloader.rb

@@ -13,6 +13,9 @@ module HttpMimic
     DEFAULT_GITHUB_REPO = 'lexiforest/curl-impersonate'
     DEFAULT_VERSION     = 'v2.1.1'
 
+    DEFAULT_QJS_REPO    = 'quickjs-ng/quickjs'
+    DEFAULT_QJS_VERSION = 'v0.16.2'
+
     class DownloadError < HttpMimic::Error; end
     class UnsupportedPlatformError < HttpMimic::Error; end
 
@@ -63,6 +66,105 @@ module HttpMimic
         true
       end
 
+      def download_qjs!(version: nil, install_dir: nil, repo: nil, force: false)
+        target_version = normalize_version(version || HttpMimic.configuration.qjs_version || DEFAULT_QJS_VERSION)
+        target_dir     = File.expand_path(install_dir || HttpMimic.configuration.install_dir)
+        target_repo    = repo || HttpMimic.configuration.qjs_github_repo || DEFAULT_QJS_REPO
+
+        FileUtils.mkdir_p(target_dir)
+
+        dest_binary = File.join(target_dir, binary_name_for_platform('qjs'))
+
+        if !force && qjs_installed?(version: target_version, install_dir: target_dir)
+          log_info("qjs #{target_version} is already installed in #{target_dir}")
+          return dest_binary
+        end
+
+        asset_name = qjs_platform_asset
+        download_url = "https://github.com/#{target_repo}/releases/download/#{target_version}/#{asset_name}"
+
+        log_info("Downloading QuickJS (#{target_version}) [#{asset_name}]...")
+        binary_data = fetch_binary(download_url)
+
+        File.binwrite(dest_binary, binary_data)
+        File.chmod(0755, dest_binary)
+
+        File.write(qjs_version_file_path(target_dir), target_version)
+
+        log_info("QuickJS #{target_version} installation complete! (#{dest_binary})")
+        dest_binary
+      end
+
+      def qjs_installed?(version: nil, install_dir: nil)
+        target_dir = File.expand_path(install_dir || HttpMimic.configuration.install_dir)
+        qjs_bin = File.join(target_dir, binary_name_for_platform('qjs'))
+        return false unless File.file?(qjs_bin) && File.executable?(qjs_bin)
+
+        if version
+          target_version = normalize_version(version)
+          v_file = qjs_version_file_path(target_dir)
+          return false unless File.file?(v_file)
+          return File.read(v_file).strip == target_version
+        end
+
+        true
+      end
+
+      def qjs_path(install_dir: nil)
+        target_dir = File.expand_path(install_dir || HttpMimic.configuration.install_dir)
+        candidate = File.join(target_dir, binary_name_for_platform('qjs'))
+        return candidate if File.file?(candidate) && File.executable?(candidate)
+
+        # Fallback to system PATH
+        sys_qjs = `which qjs 2>/dev/null`.strip
+        return sys_qjs if !sys_qjs.empty? && File.executable?(sys_qjs)
+
+        nil
+      end
+
+      def qjs_platform_asset
+        os  = host_os
+        cpu = host_cpu
+
+        case os
+        when :macos
+          case cpu
+          when :arm64 then 'qjs-darwin-arm64'
+          when :x86_64 then 'qjs-darwin-x86_64'
+          else
+            raise UnsupportedPlatformError, "Unsupported macOS CPU architecture for QuickJS: #{cpu}"
+          end
+        when :linux
+          case cpu
+          when :x86_64
+            'qjs-linux-x86_64'
+          when :aarch64, :arm64
+            'qjs-linux-aarch64'
+          when :arm
+            'qjs-linux-armv7'
+          when :i386, :i686
+            'qjs-linux-x86'
+          when :riscv64
+            'qjs-linux-riscv64'
+          else
+            raise UnsupportedPlatformError, "Unsupported Linux CPU architecture for QuickJS: #{cpu}"
+          end
+        when :windows
+          case cpu
+          when :x86_64 then 'qjs-windows-x86_64.exe'
+          when :i386, :i686 then 'qjs-windows-x86.exe'
+          else
+            raise UnsupportedPlatformError, "Unsupported Windows CPU architecture for QuickJS: #{cpu}"
+          end
+        else
+          raise UnsupportedPlatformError, "Unsupported operating system for QuickJS: #{RbConfig::CONFIG['host_os']}"
+        end
+      end
+
+      def qjs_version_file_path(dir)
+        File.join(dir, '.qjs_version')
+      end
+
       def binary_path(name, install_dir: nil)
         target_dir = File.expand_path(install_dir || HttpMimic.configuration.install_dir)
         candidate = File.join(target_dir, binary_name_for_platform(name))

data/lib/http_mimic/exceptions.rb

@@ -31,4 +31,19 @@ module HttpMimic
 
   # Raised when response parsing fails
   class ResponseParseError < Error; end
+
+  # Raised when JavaScript execution fails in JSRuntime
+  class JSError < Error
+    attr_reader :stderr, :exit_code
+
+    def initialize(message, stderr: nil, exit_code: nil)
+      super(message)
+      @stderr = stderr
+      @exit_code = exit_code
+    end
+  end
+
+  # Raised when JavaScript execution times out
+  class JSTimeoutError < JSError; end
 end
+

data/lib/http_mimic/js_runtime.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+require 'open3'
+require 'json'
+require 'timeout'
+
+module HttpMimic
+  class JSRuntime
+    STDIN_RUNNER = <<~JS
+      const input = std.in.readAsString();
+      const result = eval(input);
+      if (result !== undefined) {
+        if (typeof result === "object" && result !== null) {
+          console.log(JSON.stringify(result));
+        } else {
+          console.log(result);
+        }
+      }
+    JS
+
+    class << self
+      def eval(js_code, timeout: nil, install_dir: nil, auto_download: nil)
+        ensure_qjs_installed!(install_dir: install_dir, auto_download: auto_download)
+
+        qjs_bin = Downloader.qjs_path(install_dir: install_dir)
+        raise BinaryNotFoundError, "QuickJS binary 'qjs' not found. Run HttpMimic.download_qjs! to install." unless qjs_bin
+
+        cmd = [qjs_bin, '--std', '-e', STDIN_RUNNER]
+        effective_timeout = timeout || HttpMimic.configuration.default_timeout
+
+        stdout = nil
+        stderr = nil
+        status = nil
+
+        begin
+          if effective_timeout && effective_timeout > 0
+            Timeout.timeout(effective_timeout) do
+              stdout, stderr, status = Open3.capture3(*cmd, stdin_data: js_code.to_s)
+            end
+          else
+            stdout, stderr, status = Open3.capture3(*cmd, stdin_data: js_code.to_s)
+          end
+        rescue Timeout::Error
+          raise JSTimeoutError.new("JavaScript execution timed out after #{effective_timeout}s")
+        end
+
+        unless status && status.success?
+          raise JSError.new(
+            "JavaScript execution failed: #{stderr.to_s.strip}",
+            stderr: stderr,
+            exit_code: status ? status.exitstatus : nil
+          )
+        end
+
+        stdout.to_s.strip
+      end
+
+      def eval_json(js_code, timeout: nil, install_dir: nil, auto_download: nil)
+        output = eval(js_code, timeout: timeout, install_dir: install_dir, auto_download: auto_download)
+        return nil if output.nil? || output.empty?
+
+        JSON.parse(output)
+      rescue JSON::ParserError => e
+        raise ResponseParseError, "Failed to parse JavaScript JSON output: #{e.message} (Output: #{output.inspect})"
+      end
+
+      private
+
+      def ensure_qjs_installed!(install_dir: nil, auto_download: nil)
+        should_download = auto_download.nil? ? HttpMimic.configuration.auto_download : auto_download
+        return unless should_download
+        return if Downloader.qjs_installed?(install_dir: install_dir)
+
+        Downloader.download_qjs!(install_dir: install_dir)
+      end
+    end
+  end
+end

data/lib/http_mimic/module_methods.rb

@@ -41,6 +41,11 @@ module HttpMimic
       default_options[:cookies].merge!(c)
     end
 
+    def persist_cookies(enabled = nil)
+      return default_options[:persist_cookies] if enabled.nil?
+      default_options[:persist_cookies] = enabled
+    end
+
     def mode(m = nil)
       return default_options[:mode] if m.nil?
       default_options[:mode] = m

data/lib/http_mimic/request.rb

@@ -18,8 +18,22 @@ module HttpMimic
       auto_fallback = options.fetch(:auto_fallback, config.auto_fallback)
       retry_statuses = options[:retry_statuses] || config.retry_statuses || [403, 429, 503]
 
+      # Persistent CookieStore integration
+      should_persist_cookie = options.fetch(:persist_cookies, config.persist_cookies)
+      host = CookieStore.extract_host(url)
+
+      if should_persist_cookie && host
+        stored_cookies = CookieStore.load(host)
+        if stored_cookies && !stored_cookies.empty?
+          log_debug("[HttpMimic::CookieStore] Loaded #{stored_cookies.size} persistent cookies for #{host}")
+          user_cookies = options[:cookies] ? (options[:cookies].is_a?(Hash) ? options[:cookies] : options[:cookies].to_h) : {}
+          options[:cookies] = stored_cookies.merge(user_cookies)
+        end
+      end
+
       attempts = []
       profiles_to_try = determine_profiles(mode, auto_fallback)
+      waf_solve_attempted = false
 
       response = nil
       final_status = nil
@@ -66,7 +80,32 @@ module HttpMimic
         final_stderr = stderr
         final_command = full_command
 
-        is_blocked = (status.exitstatus != 0) || retry_statuses.include?(response.code)
+        # Forward any received cookies to subsequent attempts
+        if response.cookies && !response.cookies.empty?
+          existing_cookies = options[:cookies] ? (options[:cookies].is_a?(Hash) ? options[:cookies] : options[:cookies].to_h) : {}
+          options[:cookies] = response.cookies.to_h.merge(existing_cookies)
+        end
+
+        auto_solve_waf = options.fetch(:solve_waf, config.auto_solve_waf)
+        is_blocked = (status.exitstatus != 0) || retry_statuses.include?(response.code) || Waf::Detector.challenge_page?(response)
+
+        # Only attempt WAF resolution once per request to avoid unnecessary latency on subsequent fallbacks
+        if is_blocked && auto_solve_waf && !waf_solve_attempted && method.to_s.upcase == 'GET'
+          waf_type = Waf::Detector.detect(response)
+          if waf_type
+            waf_solve_attempted = true
+            log_debug("[HttpMimic] Detected #{waf_type.to_s.capitalize} WAF challenge. Attempting to solve with QuickJS...")
+            solved_resp = Waf.solve(url, response, current_opts)
+            if solved_resp
+              response = solved_resp
+              is_blocked = (response.code != 0 && retry_statuses.include?(response.code))
+              if response.cookies && !response.cookies.empty?
+                options[:cookies] = (options[:cookies] || {}).merge(response.cookies.to_h)
+              end
+            end
+          end
+        end
+
         if !is_blocked || (index == profiles_to_try.size - 1)
           break
         end
@@ -74,6 +113,12 @@ module HttpMimic
         log_debug("[HttpMimic] Attempt #{index + 1} with #{profile} resulted in status #{response.code}. Triggering smart fallback to next profile...")
       end
 
+      # Persist cookies back to store if enabled
+      if should_persist_cookie && host && response && response.cookies && !response.cookies.empty?
+        CookieStore.save(host, response.cookies)
+        log_debug("[HttpMimic::CookieStore] Saved #{response.cookies.size} cookies for #{host}")
+      end
+
       handle_errors(final_status, final_stderr, final_command, response)
       response
     end
@@ -83,17 +128,29 @@ module HttpMimic
     def determine_profiles(mode, auto_fallback)
       case mode
       when :curl, :curl_first
-        auto_fallback ? [:curl, :impersonate] : [:curl]
+        auto_fallback ? [:curl, :impersonate, :android, :ios] : [:curl]
       when :curl_only
         [:curl]
       when :impersonate_only
         [:impersonate]
+      when :mobile_only
+        [:mobile]
+      when :android_only
+        [:android]
+      when :ios_only
+        [:ios]
+      when :mobile_first
+        auto_fallback ? [:android, :ios, :impersonate, :curl] : [:mobile]
+      when :android_first
+        auto_fallback ? [:android, :ios, :impersonate, :curl] : [:android]
+      when :ios_first
+        auto_fallback ? [:ios, :android, :impersonate, :curl] : [:ios]
       when :impersonate_first
-        auto_fallback ? [:impersonate, :curl] : [:impersonate]
+        auto_fallback ? [:impersonate, :android, :ios, :curl] : [:impersonate]
       when :auto, :smart
-        auto_fallback ? [:impersonate, :curl] : [:impersonate]
+        auto_fallback ? [:impersonate, :android, :ios, :curl] : [:impersonate]
       else
-        auto_fallback ? [:impersonate, :curl] : [:impersonate]
+        auto_fallback ? [:impersonate, :android, :ios, :curl] : [:impersonate]
       end
     end
 

data/lib/http_mimic/response.rb

@@ -67,6 +67,76 @@ module HttpMimic
       client_error? || server_error?
     end
 
+    def binary?
+      content_type = headers['content-type'].to_s.downcase
+      ResponseParser::BINARY_MIME_KEYWORDS.any? { |kw| content_type.include?(kw) } ||
+        body.to_s.b[0, 1024]&.include?("\x00".b)
+    end
+
+    def save_to_file(filepath)
+      require 'fileutils'
+      FileUtils.mkdir_p(File.dirname(filepath))
+      File.binwrite(filepath, body)
+      filepath
+    end
+    alias save save_to_file
+
+    def title
+      return nil if binary?
+      body[/<title[^>]*>(.*?)<\/title>/im, 1]&.strip
+    end
+
+    def og_image
+      return nil if binary?
+      body[/<meta\s+[^>]*property=["']og:image["'][^>]*content=["']([^"']+)["']/im, 1] ||
+        body[/<meta\s+[^>]*content=["']([^"']+)["'][^>]*property=["']og:image["']/im, 1]
+    end
+
+    def meta_description
+      return nil if binary?
+      body[/<meta\s+[^>]*name=["']description["'][^>]*content=["']([^"']+)["']/im, 1] ||
+        body[/<meta\s+[^>]*content=["']([^"']+)["'][^>]*name=["']description["']/im, 1]
+    end
+
+    def extract_images(base_url: nil)
+      return [] if binary?
+
+      base = base_url || request_url || ''
+      images = []
+
+      # 1. Match img tags (src, data-src, data-zoom-image, data-original, srcset)
+      body.scan(/<img\s+[^>]*>/i).each do |img_tag|
+        %w[src data-src data-zoom-image data-original data-high-res-src data-full-size-image-url].each do |attr|
+          if match = img_tag.match(/#{attr}=["']([^"']+)["']/i)
+            images << match[1].strip
+          end
+        end
+
+        if match = img_tag.match(/srcset=["']([^"']+)["']/i)
+          match[1].split(',').each do |item|
+            url = item.strip.split(/\s+/).first
+            images << url if url && !url.empty?
+          end
+        end
+      end
+
+      # 2. Match og:image meta tag
+      if og = og_image
+        images << og
+      end
+
+      # 3. Match raw image URLs found in document / script payloads
+      body.scan(/https?:[^\s"'<>]+\.(?:jpg|jpeg|png|webp|avif|gif)/i).each do |raw_url|
+        images << raw_url
+      end
+
+      # Clean, resolve relative URLs to absolute, and deduplicate
+      images.map do |img|
+        clean_url = img.gsub(/&amp;/, '&').strip
+        resolve_url(clean_url, base)
+      end.compact.reject(&:empty?).uniq
+    end
+
     def [](key)
       if parsed_response.is_a?(Hash) || parsed_response.is_a?(Array)
         parsed_response[key]
@@ -92,6 +162,20 @@ module HttpMimic
       "#<#{self.class.name}:0x#{object_id.to_s(16)} @code=#{code} @status_message=#{status_message.inspect} @headers=#{headers.to_h.inspect} @parsed_response=#{parsed_response.inspect}>"
     end
 
+    private
+
+    def resolve_url(url, base)
+      return nil if url.nil? || url.empty? || url.start_with?('data:', 'javascript:', 'blob:', '#')
+      return url if url =~ /\Ahttps?:\/\//i
+      return "https:#{url}" if url.start_with?('//')
+
+      return url if base.nil? || base.empty?
+      require 'uri'
+      URI.join(base, url).to_s
+    rescue URI::InvalidURIError
+      url
+    end
+
     def method_missing(method_name, *args, &block)
       if parsed_response.respond_to?(method_name)
         parsed_response.public_send(method_name, *args, &block)

data/lib/http_mimic/response_parser.rb

@@ -5,6 +5,8 @@ require 'json'
 module HttpMimic
   class ResponseParser
     HTTP_STATUS_LINE_REGEX = /\AHTTP\/(?<version>[\d\.]+)\s+(?<code>\d{3})(?:\s+(?<message>.*))?/i
+    HTTP_STATUS_LINE_B     = /\AHTTP\/(?:[\d\.]+)\s+\d{3}/i
+    BINARY_MIME_KEYWORDS   = %w[image/ audio/ video/ pdf octet-stream zip gzip tar compressed stream font wasm].freeze
 
     attr_reader :raw_output, :exit_status, :stderr, :command, :request_url
 
@@ -17,8 +19,8 @@ module HttpMimic
     end
 
     def parse
-      # Split header blocks and body
-      header_blocks, body = split_headers_and_body(@raw_output)
+      # Split header blocks and body in binary-safe manner
+      header_blocks, raw_body = split_headers_and_body(@raw_output)
 
       final_header_block = header_blocks.last || ''
       history_header_blocks = header_blocks.size > 1 ? header_blocks[0...-1] : []
@@ -39,6 +41,14 @@ module HttpMimic
         }
       end
 
+      # Process body: retain raw bytes for binary, or UTF-8 encode for text
+      content_type = headers['content-type'].to_s.downcase
+      body = if binary_content?(content_type, raw_body)
+               raw_body
+             else
+               raw_body.dup.force_encoding('UTF-8').scrub
+             end
+
       # Parse body (auto-detect JSON)
       parsed_body = parse_body(body, headers)
 
@@ -61,32 +71,36 @@ module HttpMimic
 
     private
 
-    # Split raw_output into header blocks and body by \r?\n\r?\n
+    # Split raw_output into header blocks and body in a binary-safe manner
     def split_headers_and_body(text)
-      return [[], ''] if text.nil? || text.empty?
-
-      # Normalize line endings
-      normalized = text.gsub(/\r\n/, "\n")
-      parts = normalized.split("\n\n")
+      return [[], ''.b] if text.nil? || text.empty?
 
+      remaining = text.to_s.b
       header_blocks = []
-      body_index = 0
 
-      parts.each_with_index do |part, idx|
-        trimmed = part.strip
-        if trimmed =~ HTTP_STATUS_LINE_REGEX
-          header_blocks << part
-          body_index = idx + 1
+      while remaining =~ HTTP_STATUS_LINE_B
+        crlf_idx = remaining.index("\r\n\r\n".b)
+        lf_idx   = remaining.index("\n\n".b)
+
+        break unless crlf_idx || lf_idx
+
+        if crlf_idx && lf_idx
+          delim_pos = [crlf_idx, lf_idx].min
+          delim_len = (delim_pos == crlf_idx) ? 4 : 2
+        elsif crlf_idx
+          delim_pos = crlf_idx
+          delim_len = 4
         else
-          # Once a non-HTTP status block is encountered, the rest is body
-          break
+          delim_pos = lf_idx
+          delim_len = 2
         end
-      end
 
-      # Combine remaining parts as body
-      body = parts[body_index..-1] ? parts[body_index..-1].join("\n\n") : ''
+        header_block = remaining[0...delim_pos].force_encoding('UTF-8').scrub
+        header_blocks << header_block
+        remaining = remaining[(delim_pos + delim_len)..-1] || ''.b
+      end
 
-      [header_blocks, body]
+      [header_blocks, remaining]
     end
 
     def parse_header_block(block)
@@ -129,12 +143,22 @@ module HttpMimic
       [code, http_version, status_message, headers, cookies]
     end
 
+    def binary_content?(content_type, body)
+      return true if BINARY_MIME_KEYWORDS.any? { |kw| content_type.include?(kw) }
+
+      # Check for null bytes in the initial portion of body
+      sample = body[0, 1024]
+      sample&.include?("\x00".b)
+    end
+
     def parse_body(body, headers)
       return nil if body.nil? || body.empty?
 
       content_type = headers['content-type'].to_s.downcase
+      return body if binary_content?(content_type, body)
 
-      if content_type.include?('json') || body.strip.start_with?('{', '[')
+      trimmed = body.strip
+      if content_type.include?('json') || trimmed.start_with?('{', '[')
         begin
           JSON.parse(body)
         rescue JSON::ParserError

data/lib/http_mimic/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module HttpMimic
-  VERSION = "0.3.1"
+  VERSION = "0.4.0"
 end