http_mimic 0.4.0 → 0.5.3

data/lib/http_mimic/obscura.rb

@@ -0,0 +1,158 @@
+# frozen_string_literal: true
+
+require 'open3'
+require 'uri'
+require 'json'
+
+module HttpMimic
+  module Obscura
+    class ExecutionError < HttpMimic::Error; end
+    class BinaryNotFoundError < HttpMimic::Error; end
+
+    class << self
+      # Renders a web page using the Obscura headless SPA engine and returns an HttpMimic::Response
+      #
+      # @param url [String, URI] The target URL to render
+      # @param options [Hash] Rendering options
+      # @option options [String] :wait_until ('networkidle0') Wait condition (e.g. 'load', 'domcontentloaded', 'networkidle0')
+      # @option options [Integer] :timeout Per-command timeout in seconds
+      # @option options [String] :proxy Proxy URL (e.g. 'socks5://127.0.0.1:1080' or 'http://...')
+      # @option options [String] :eval JavaScript expression to evaluate on the page
+      # @option options [String] :dump Output format ('html', 'text', 'links')
+      # @return [HttpMimic::Response]
+      def render(url, options = {})
+        bin = resolve_binary
+        raise BinaryNotFoundError, "Obscura binary not found. Run HttpMimic.download_obscura! or configure obscura_path." unless bin
+
+        args = [bin, 'fetch', url.to_s]
+
+        # Dump format: default to html (can be 'html', 'text', 'links', 'markdown', 'original', 'cookies')
+        dump_format = options[:dump] || 'html'
+        args << '--dump' << dump_format.to_s
+
+        # Stealth mode (enabled by default)
+        args << '--stealth' if options.fetch(:stealth, true)
+
+        # Wait condition
+        if options[:wait_until]
+          args << '--wait-until' << options[:wait_until].to_s
+        end
+
+        # Wait delay
+        if options[:wait]
+          args << '--wait' << options[:wait].to_i.to_s
+        end
+
+        # Timeout in seconds (integer)
+        timeout = options[:timeout] || HttpMimic.configuration.default_timeout
+        args << '--timeout' << timeout.to_i.to_s if timeout
+
+        # User Agent
+        if options[:user_agent]
+          args << '--user-agent' << options[:user_agent].to_s
+        end
+
+        # Proxy
+        if options[:proxy]
+          args << '--proxy' << options[:proxy].to_s
+        end
+
+        # Cookies: pass cookies string or hash (Two-Phase Pipeline support)
+        if options[:cookies] || options[:cookie]
+          cookie_val = options[:cookies] || options[:cookie]
+          cookie_str = if cookie_val.is_a?(Hash)
+            cookie_val.map { |k, v| "#{k}=#{v}" }.join('; ')
+          elsif cookie_val.is_a?(Array)
+            cookie_val.join('; ')
+          else
+            cookie_val.to_s
+          end
+          args << '--cookie' << cookie_str unless cookie_str.empty?
+        end
+
+        # Optional JS evaluation
+        args << '--eval' << options[:eval].to_s if options[:eval]
+
+        # Selector
+        args << '--selector' << options[:selector].to_s if options[:selector]
+
+        log_debug("Executing Obscura SPA render: #{args.join(' ')}")
+
+        stdout, stderr, status = Open3.capture3(*args)
+
+        code = (status && status.success?) ? 200 : (status ? status.exitstatus : 500)
+
+        # Detect if the rendered output contains an explicit HTTP error block page
+        if code == 200 && stdout
+          if stdout.include?('HTTP 403 - Forbidden') ||
+             (stdout.include?('Reference Error:') && stdout.include?('Akamai')) ||
+             stdout.include?('"page_name": "403 ERROR"')
+            code = 403
+          elsif stdout.include?('404 Not Found') && stdout.include?('<title>404')
+            code = 404
+          end
+        end
+
+        status_msg = if code == 200
+          'OK (Obscura SPA Rendered)'
+        elsif code == 403
+          'Forbidden (WAF Blocked)'
+        else
+          'Error'
+        end
+
+        raw_headers = "HTTP/2 #{code} #{status_msg}\r\ncontent-type: text/html; charset=utf-8\r\nx-rendered-by: obscura\r\n\r\n"
+        headers = Headers.new({ 'content-type' => 'text/html; charset=utf-8', 'x-rendered-by' => 'obscura' })
+
+        Response.new(
+          code: code,
+          http_version: 'HTTP/2',
+          status_message: status_msg,
+          headers: headers,
+          cookies: Cookies.new,
+          body: stdout,
+          parsed_response: nil,
+          raw_headers: raw_headers,
+          history: [],
+          request_url: url.to_s,
+          stderr: stderr,
+          exit_code: status ? status.exitstatus : 0,
+          command: args.join(' ')
+        )
+      end
+
+      def resolve_binary
+        # 1. Configured custom path
+        custom_path = HttpMimic.configuration.obscura_path
+        if custom_path && (File.file?(custom_path) || File.executable?(custom_path))
+          return custom_path
+        end
+
+        # 2. Check install directory or system PATH
+        installed_path = Downloader.obscura_path
+        return installed_path if installed_path
+
+        # 3. Auto-download if enabled
+        if HttpMimic.configuration.auto_download
+          begin
+            return Downloader.download_obscura!
+          rescue StandardError => e
+            HttpMimic.configuration.logger&.warn("[HttpMimic::Obscura] Auto-download failed: #{e.message}")
+          end
+        end
+
+        nil
+      end
+
+      private
+
+      def log_debug(msg)
+        if HttpMimic.configuration.logger
+          HttpMimic.configuration.logger.debug("[HttpMimic::Obscura] #{msg}")
+        elsif HttpMimic.configuration.debug
+          puts "[HttpMimic::Obscura] #{msg}"
+        end
+      end
+    end
+  end
+end

data/lib/http_mimic/proxy_pool.rb

@@ -0,0 +1,224 @@
+# frozen_string_literal: true
+
+require 'net/http'
+require 'uri'
+require 'set'
+
+module HttpMimic
+  class ProxyPool
+    DEFAULT_SOURCES = [
+      'https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/http.txt',
+      'https://api.proxyscrape.com/v2/?request=displayproxies&protocol=http&timeout=5000&country=all&ssl=all&anonymity=all',
+      'https://raw.githubusercontent.com/TheSpeedX/SOCKS-List/master/http.txt'
+    ].freeze
+
+    class << self
+      def instance
+        @instance ||= new
+      end
+
+      def reset_instance!
+        @instance = nil
+      end
+
+      def get
+        instance.get
+      end
+      alias sample get
+      alias next_proxy get
+
+      def mark_dead(proxy)
+        instance.mark_dead(proxy)
+      end
+
+      def mark_alive(proxy)
+        instance.mark_alive(proxy)
+      end
+
+      def refresh!(force: true)
+        instance.refresh!(force: force)
+      end
+
+      def load(proxies)
+        instance.load(proxies)
+      end
+
+      def all
+        instance.all
+      end
+
+      def available
+        instance.available
+      end
+
+      def dead_proxies
+        instance.dead_proxies
+      end
+
+      def size
+        instance.size
+      end
+
+      def clear!
+        instance.clear!
+      end
+    end
+
+    attr_reader :sources, :ttl, :timeout
+    attr_accessor :proxies
+
+    def initialize(options = {})
+      config = HttpMimic.configuration rescue nil
+      @sources = options[:sources] || config&.proxy_sources || DEFAULT_SOURCES.dup
+      @ttl = options[:ttl] || config&.proxy_pool_ttl || 1800
+      @timeout = options[:timeout] || config&.proxy_timeout || 5
+      @proxies = []
+      @dead_proxies = Set.new
+      @last_fetched_at = nil
+      @mutex = Mutex.new
+    end
+
+    # Retrieve an available proxy from the pool
+    #
+    # @return [String, nil] Proxy URL (e.g. 'http://1.2.3.4:8080') or nil if none available
+    def get
+      @mutex.synchronize do
+        refresh_unlocked(force: false) if should_refresh_unlocked?
+        avail = @proxies - @dead_proxies.to_a
+        if avail.empty? && !@proxies.empty?
+          # If all current proxies are exhausted/dead, clear dead set and try one more refresh
+          @dead_proxies.clear
+          refresh_unlocked(force: true)
+          avail = @proxies - @dead_proxies.to_a
+        end
+        avail.sample
+      end
+    end
+    alias sample get
+    alias next_proxy get
+
+    # Mark a proxy as dead/unusable
+    #
+    # @param proxy [String]
+    def mark_dead(proxy)
+      return unless proxy
+      @mutex.synchronize do
+        @dead_proxies.add(normalize_proxy(proxy))
+      end
+    end
+
+    # Mark a proxy as active/usable
+    #
+    # @param proxy [String]
+    def mark_alive(proxy)
+      return unless proxy
+      @mutex.synchronize do
+        @dead_proxies.delete(normalize_proxy(proxy))
+      end
+    end
+
+    # Explicitly refresh proxy pool from configured sources
+    def refresh!(force: true)
+      @mutex.synchronize do
+        refresh_unlocked(force: force)
+      end
+    end
+
+    # Manually load a custom array of proxies
+    #
+    # @param proxy_list [Array<String>]
+    def load(proxy_list)
+      @mutex.synchronize do
+        @proxies = Array(proxy_list).map { |p| normalize_proxy(p) }.compact.uniq
+        @dead_proxies.clear
+        @last_fetched_at = Time.now
+      end
+    end
+
+    # Returns all loaded proxies
+    def all
+      @mutex.synchronize { @proxies.dup }
+    end
+
+    # Returns all non-dead available proxies
+    def available
+      @mutex.synchronize { (@proxies - @dead_proxies.to_a).dup }
+    end
+
+    # Returns dead proxies
+    def dead_proxies
+      @mutex.synchronize { @dead_proxies.to_a }
+    end
+
+    # Number of available proxies
+    def size
+      @mutex.synchronize { (@proxies - @dead_proxies.to_a).size }
+    end
+
+    # Reset proxy pool
+    def clear!
+      @mutex.synchronize do
+        @proxies.clear
+        @dead_proxies.clear
+        @last_fetched_at = nil
+      end
+    end
+
+    private
+
+    def should_refresh_unlocked?
+      @proxies.empty? || @last_fetched_at.nil? || (Time.now - @last_fetched_at > @ttl)
+    end
+
+    def refresh_unlocked(force: false)
+      return if !force && !should_refresh_unlocked?
+
+      new_proxies = fetch_from_sources
+      if !new_proxies.empty?
+        @proxies = new_proxies
+        @last_fetched_at = Time.now
+      end
+    end
+
+    def fetch_from_sources
+      collected = []
+      sources_to_try = Array(@sources).empty? ? DEFAULT_SOURCES : @sources
+
+      sources_to_try.each do |source_url|
+        begin
+          uri = URI.parse(source_url)
+          http = Net::HTTP.new(uri.host, uri.port)
+          http.use_ssl = (uri.scheme == 'https')
+          http.open_timeout = @timeout
+          http.read_timeout = @timeout
+
+          req = Net::HTTP::Get.new(uri.request_uri)
+          req['User-Agent'] = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36'
+
+          res = http.request(req)
+          if res.is_a?(Net::HTTPSuccess) && res.body
+            # Extract IP:PORT matches
+            ips = res.body.scan(/\b(?:\d{1,3}\.){3}\d{1,3}:\d{2,5}\b/)
+            if !ips.empty?
+              collected.concat(ips.take(150))
+              # Stop early if we have collected enough proxies
+              break if collected.size >= 50
+            end
+          end
+        rescue StandardError => e
+          if HttpMimic.configuration.debug
+            puts "[HttpMimic::ProxyPool] Failed to fetch proxy list from #{source_url}: #{e.message}"
+          end
+        end
+      end
+
+      collected.uniq.map { |p| normalize_proxy(p) }
+    end
+
+    def normalize_proxy(proxy)
+      p = proxy.to_s.strip
+      return nil if p.empty?
+      p.start_with?('http://', 'https://', 'socks5://', 'socks4://') ? p : "http://#{p}"
+    end
+  end
+end

data/lib/http_mimic/request.rb

@@ -15,6 +15,22 @@ module HttpMimic
 
     def perform
       mode = (options[:mode] || config.mode || :auto).to_sym
+
+      # Automatic Free Proxy Pool integration
+      should_auto_proxy = options.fetch(:auto_proxy, config.auto_proxy)
+      if should_auto_proxy && !options[:proxy]
+        selected_proxy = ProxyPool.get
+        if selected_proxy
+          options[:proxy] = selected_proxy
+          log_debug("[HttpMimic::ProxyPool] Assigned proxy from pool: #{selected_proxy}")
+        end
+      end
+
+      # Delegate directly to Obscura headless SPA renderer if requested
+      if options[:render] == :spa || options[:render] == :obscura || mode == :spa || mode == :obscura
+        return Obscura.render(url, options)
+      end
+
       auto_fallback = options.fetch(:auto_fallback, config.auto_fallback)
       retry_statuses = options[:retry_statuses] || config.retry_statuses || [403, 429, 503]
 
@@ -34,6 +50,8 @@ module HttpMimic
       attempts = []
       profiles_to_try = determine_profiles(mode, auto_fallback)
       waf_solve_attempted = false
+      best_response = nil
+      proxy_retries_left = should_auto_proxy ? (options[:proxy_retries] || config.proxy_retries || 3) : 0
 
       response = nil
       final_status = nil
@@ -41,6 +59,12 @@ module HttpMimic
       final_command = nil
 
       profiles_to_try.each_with_index do |profile, index|
+        # Never fall back to plain curl if a WAF challenge has already been detected/attempted
+        if profile == :curl && waf_solve_attempted
+          log_debug("[HttpMimic] Skipping plain :curl fallback for protected WAF target.")
+          next
+        end
+
         current_opts = options.merge(profile: profile)
 
         builder = CommandBuilder.new(method, url, current_opts, config)
@@ -55,6 +79,25 @@ module HttpMimic
         log_debug("Curl exit status: #{status.exitstatus}")
         log_debug("Curl stderr: #{stderr}") unless stderr.empty?
 
+        # Handle proxy failure retry when auto_proxy is enabled
+        if should_auto_proxy && current_opts[:proxy] && (status.exitstatus != 0)
+          is_proxy_err = [5, 7, 28, 35, 56].include?(status.exitstatus) ||
+                         stderr.include?('Failed to connect to') ||
+                         stderr.include?('tunneling failed') ||
+                         stderr.downcase.include?('proxy')
+
+          if is_proxy_err
+            ProxyPool.mark_dead(current_opts[:proxy])
+            if proxy_retries_left > 0
+              proxy_retries_left -= 1
+              new_proxy = ProxyPool.get
+              log_debug("[HttpMimic::ProxyPool] Proxy #{current_opts[:proxy]} failed (exit #{status.exitstatus}). Retrying with new proxy #{new_proxy} (#{proxy_retries_left} retries left)...")
+              options[:proxy] = new_proxy
+              redo
+            end
+          end
+        end
+
         response = ResponseParser.new(
           stdout,
           exit_status: status,
@@ -63,6 +106,10 @@ module HttpMimic
           request_url: final_url
         ).parse
 
+        if should_auto_proxy && current_opts[:proxy] && response.success?
+          ProxyPool.mark_alive(current_opts[:proxy])
+        end
+
         response.mode_used = profile
         response.fallback_triggered = (index > 0)
 
@@ -80,6 +127,17 @@ module HttpMimic
         final_stderr = stderr
         final_command = full_command
 
+        # Track best response so a 200 (or challenge page) isn't wiped out by a failed fallback
+        if response
+          if best_response.nil?
+            best_response = response
+          elsif response.success? && !best_response.success?
+            best_response = response
+          elsif response.code == 200 && best_response.code != 200
+            best_response = response
+          end
+        end
+
         # Forward any received cookies to subsequent attempts
         if response.cookies && !response.cookies.empty?
           existing_cookies = options[:cookies] ? (options[:cookies].is_a?(Hash) ? options[:cookies] : options[:cookies].to_h) : {}
@@ -98,10 +156,13 @@ module HttpMimic
             solved_resp = Waf.solve(url, response, current_opts)
             if solved_resp
               response = solved_resp
-              is_blocked = (response.code != 0 && retry_statuses.include?(response.code))
+              is_blocked = (response.code != 0 && retry_statuses.include?(response.code)) || Waf::Detector.challenge_page?(response)
               if response.cookies && !response.cookies.empty?
                 options[:cookies] = (options[:cookies] || {}).merge(response.cookies.to_h)
               end
+              if response.success? || (response.code == 200 && best_response&.code != 200)
+                best_response = response
+              end
             end
           end
         end
@@ -113,10 +174,61 @@ module HttpMimic
         log_debug("[HttpMimic] Attempt #{index + 1} with #{profile} resulted in status #{response.code}. Triggering smart fallback to next profile...")
       end
 
+      # Preserve best response if the last attempt resulted in a regression (e.g. 403 / error after getting 200)
+      if best_response && (response.nil? || response.error? || (final_status && final_status.exitstatus != 0))
+        if best_response.success? || (best_response.code == 200 && response&.code != 200)
+          response = best_response
+        end
+      end
+
+      # Automatic SPA Detection & Obscura rendering fallback for SPA shells & Behavioral Challenges
+      auto_render_spa = options.fetch(:auto_render_spa, config.auto_render_spa)
+      auto_solve_waf = options.fetch(:solve_waf, config.auto_solve_waf)
+      should_render_spa = auto_render_spa && response && response.success? && SpaDetector.spa?(response)
+      should_render_cpt = auto_solve_waf && response && Waf::Detector.challenge_page?(response)
+
+      if (should_render_spa || should_render_cpt) && method.to_s.upcase == 'GET'
+        target_reason = should_render_cpt ? 'WAF challenge page' : 'unhydrated SPA shell'
+        log_debug("[HttpMimic] Detected #{target_reason} on #{url}. Automatically rendering with Obscura...")
+        begin
+          spa_opts = options.dup
+          # Forward all validated cookies from Tier 1 (Mode 2: Two-Phase Pipeline)
+          if response.cookies && !response.cookies.empty?
+            tier1_cookies = response.cookies.to_h
+            existing_cookies = spa_opts[:cookies].is_a?(Hash) ? spa_opts[:cookies] : {}
+            spa_opts[:cookies] = existing_cookies.merge(tier1_cookies)
+          end
+          spa_opts[:wait_until] ||= 'load' if should_render_cpt
+          rendered_resp = Obscura.render(url, spa_opts)
+          if rendered_resp && rendered_resp.success?
+            response = rendered_resp
+          end
+        rescue StandardError => e
+          log_debug("[HttpMimic] Automatic Obscura render failed (#{e.message}), keeping Tier 1 response.")
+        end
+      end
+
       # Persist cookies back to store if enabled
-      if should_persist_cookie && host && response && response.cookies && !response.cookies.empty?
-        CookieStore.save(host, response.cookies)
-        log_debug("[HttpMimic::CookieStore] Saved #{response.cookies.size} cookies for #{host}")
+      if should_persist_cookie && host
+        is_success = response && (response.success? || response.redirect?) && !Waf::Detector.challenge_page?(response) && (final_status.nil? || final_status.exitstatus == 0)
+        verification_failed = !is_success && response && (
+          [401, 403].include?(response.code) ||
+          retry_statuses.include?(response.code) ||
+          Waf::Detector.challenge_page?(response)
+        )
+
+        persist_on_failure = options.fetch(:persist_on_failure, config.persist_on_failure)
+        clear_on_failure = options.fetch(:clear_on_failure, config.clear_on_failure)
+
+        if is_success || persist_on_failure
+          if response && response.cookies && !response.cookies.empty?
+            CookieStore.save(host, response.cookies)
+            log_debug("[HttpMimic::CookieStore] Saved #{response.cookies.size} cookies for #{host}")
+          end
+        elsif clear_on_failure && verification_failed
+          CookieStore.clear(host)
+          log_debug("[HttpMimic::CookieStore] Verification failed for #{host} (status: #{response&.code}). Cleared stored cookies.")
+        end
       end
 
       handle_errors(final_status, final_stderr, final_command, response)
@@ -128,7 +240,7 @@ module HttpMimic
     def determine_profiles(mode, auto_fallback)
       case mode
       when :curl, :curl_first
-        auto_fallback ? [:curl, :impersonate, :android, :ios] : [:curl]
+        auto_fallback ? [:curl, :impersonate, :android, :ios, :firefox] : [:curl]
       when :curl_only
         [:curl]
       when :impersonate_only
@@ -139,18 +251,22 @@ module HttpMimic
         [:android]
       when :ios_only
         [:ios]
+      when :firefox_only
+        [:firefox]
+      when :safari_only
+        [:safari]
       when :mobile_first
-        auto_fallback ? [:android, :ios, :impersonate, :curl] : [:mobile]
+        auto_fallback ? [:android, :ios, :impersonate, :firefox] : [:mobile]
       when :android_first
-        auto_fallback ? [:android, :ios, :impersonate, :curl] : [:android]
+        auto_fallback ? [:android, :ios, :impersonate, :firefox] : [:android]
       when :ios_first
-        auto_fallback ? [:ios, :android, :impersonate, :curl] : [:ios]
+        auto_fallback ? [:ios, :android, :impersonate, :firefox] : [:ios]
       when :impersonate_first
-        auto_fallback ? [:impersonate, :android, :ios, :curl] : [:impersonate]
+        auto_fallback ? [:impersonate, :android, :ios, :firefox] : [:impersonate]
       when :auto, :smart
-        auto_fallback ? [:impersonate, :android, :ios, :curl] : [:impersonate]
+        auto_fallback ? [:impersonate, :android, :ios, :firefox] : [:impersonate]
       else
-        auto_fallback ? [:impersonate, :android, :ios, :curl] : [:impersonate]
+        auto_fallback ? [:impersonate, :android, :ios, :firefox] : [:impersonate]
       end
     end
 

data/lib/http_mimic/spa_detector.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+module HttpMimic
+  module SpaDetector
+    # Framework mount points that indicate an unhydrated SPA client shell
+    EMPTY_MOUNT_REGEX = %r{
+      <div[^>]+id=["'](?:root|app|__next|__nuxt|svelte)["'][^>]*>\s*</div>|
+      <app-root[^>]*>\s*</app-root>|
+      <div[^>]+id=["']app["'][^>]*>\s*<!--\s*(?:app-content)?\s*-->\s*</div>
+    }ix
+
+    # Noscript patterns requiring JavaScript execution
+    NOSCRIPT_JS_REQUIRED_REGEX = %r{
+      <noscript[^>]*>[\s\S]*?(?:enable\s+javascript|javascript\s+is\s+required|need\s+to\s+enable\s+javascript|requires\s+javascript)[\s\S]*?</noscript>
+    }ix
+
+    # Google Search Guard dynamic SPA shell indicators
+    GOOGLE_SHELL_REGEX = %r{
+      (?:/httpservice/retry/enablejs|SG_REL|emsg=SG_REL|knitsail)
+    }ix
+
+    class << self
+      # Returns true if the HTTP response represents an unhydrated Single Page Application (SPA) shell
+      #
+      # @param response [HttpMimic::Response]
+      # @return [Boolean]
+      def spa?(response)
+        return false unless response && response.success?
+        return false if response.respond_to?(:binary?) && response.binary?
+
+        content_type = response.headers['content-type'].to_s.downcase
+        return false unless content_type.empty? || content_type.include?('text/html') || content_type.include?('application/xhtml')
+
+        body = response.body.to_s
+        return false if body.strip.empty?
+
+        # 1. Google Dynamic SERP / Search Guard JS Shell
+        if google_serp_shell?(body)
+          return true
+        end
+
+        # 2. Modern SPA framework empty mount points (React, Vue, Angular, Next, Nuxt, Svelte)
+        if empty_mount_point?(body)
+          return true
+        end
+
+        # 3. Noscript requirement message
+        if noscript_js_required?(body)
+          return true
+        end
+
+        # 4. Small body with multiple scripts and virtually no visible text
+        if low_text_js_heavy_shell?(body)
+          return true
+        end
+
+        false
+      end
+
+      def empty_mount_point?(body)
+        body.match?(EMPTY_MOUNT_REGEX)
+      end
+
+      def noscript_js_required?(body)
+        body.match?(NOSCRIPT_JS_REQUIRED_REGEX)
+      end
+
+      def google_serp_shell?(body)
+        return false unless body.match?(GOOGLE_SHELL_REGEX)
+
+        # If it has the Google shell indicators AND lacks rendered organic result elements
+        !body.include?('<h3 class=') && !body.include?('id="search"') && !body.include?('id="rso"')
+      end
+
+      def low_text_js_heavy_shell?(body)
+        return false if body.length > 25_000 # Larger HTML files typically have SSR content
+
+        # Check if there are script tags
+        script_count = body.scan(/<script/i).size
+        return false if script_count.zero?
+
+        # Strip all HTML tags, scripts, and styles to get raw visible text length
+        visible_text = body
+                       .gsub(/<script[\s\S]*?<\/script>/i, '')
+                       .gsub(/<style[\s\S]*?<\/style>/i, '')
+                       .gsub(/<[^>]+>/, ' ')
+                       .gsub(/\s+/, ' ')
+                       .strip
+
+        # If visible text is virtually nonexistent but page has multiple scripts
+        visible_text.length < 80 && script_count >= 2
+      end
+    end
+  end
+end

data/lib/http_mimic/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module HttpMimic
-  VERSION = "0.4.0"
+  VERSION = "0.5.3"
 end