http_mimic 0.3.2 → 0.5.1

data/lib/http_mimic/downloader.rb

@@ -13,6 +13,12 @@ module HttpMimic
     DEFAULT_GITHUB_REPO = 'lexiforest/curl-impersonate'
     DEFAULT_VERSION     = 'v2.1.1'
 
+    DEFAULT_QJS_REPO    = 'quickjs-ng/quickjs'
+    DEFAULT_QJS_VERSION = 'v0.16.2'
+
+    DEFAULT_OBSCURA_REPO    = 'h4ckf0r0day/obscura'
+    DEFAULT_OBSCURA_VERSION = 'v0.2.1'
+
     class DownloadError < HttpMimic::Error; end
     class UnsupportedPlatformError < HttpMimic::Error; end
 
@@ -63,6 +69,199 @@ module HttpMimic
         true
       end
 
+      def download_qjs!(version: nil, install_dir: nil, repo: nil, force: false)
+        target_version = normalize_version(version || HttpMimic.configuration.qjs_version || DEFAULT_QJS_VERSION)
+        target_dir     = File.expand_path(install_dir || HttpMimic.configuration.install_dir)
+        target_repo    = repo || HttpMimic.configuration.qjs_github_repo || DEFAULT_QJS_REPO
+
+        FileUtils.mkdir_p(target_dir)
+
+        dest_binary = File.join(target_dir, binary_name_for_platform('qjs'))
+
+        if !force && qjs_installed?(version: target_version, install_dir: target_dir)
+          log_info("qjs #{target_version} is already installed in #{target_dir}")
+          return dest_binary
+        end
+
+        asset_name = qjs_platform_asset
+        download_url = "https://github.com/#{target_repo}/releases/download/#{target_version}/#{asset_name}"
+
+        log_info("Downloading QuickJS (#{target_version}) [#{asset_name}]...")
+        binary_data = fetch_binary(download_url)
+
+        File.binwrite(dest_binary, binary_data)
+        File.chmod(0755, dest_binary)
+
+        File.write(qjs_version_file_path(target_dir), target_version)
+
+        log_info("QuickJS #{target_version} installation complete! (#{dest_binary})")
+        dest_binary
+      end
+
+      def qjs_installed?(version: nil, install_dir: nil)
+        target_dir = File.expand_path(install_dir || HttpMimic.configuration.install_dir)
+        qjs_bin = File.join(target_dir, binary_name_for_platform('qjs'))
+        return false unless File.file?(qjs_bin) && File.executable?(qjs_bin)
+
+        if version
+          target_version = normalize_version(version)
+          v_file = qjs_version_file_path(target_dir)
+          return false unless File.file?(v_file)
+          return File.read(v_file).strip == target_version
+        end
+
+        true
+      end
+
+      def qjs_path(install_dir: nil)
+        target_dir = File.expand_path(install_dir || HttpMimic.configuration.install_dir)
+        candidate = File.join(target_dir, binary_name_for_platform('qjs'))
+        return candidate if File.file?(candidate) && File.executable?(candidate)
+
+        # Fallback to system PATH
+        sys_qjs = `which qjs 2>/dev/null`.strip
+        return sys_qjs if !sys_qjs.empty? && File.executable?(sys_qjs)
+
+        nil
+      end
+
+      def qjs_platform_asset
+        os  = host_os
+        cpu = host_cpu
+
+        case os
+        when :macos
+          case cpu
+          when :arm64 then 'qjs-darwin-arm64'
+          when :x86_64 then 'qjs-darwin-x86_64'
+          else
+            raise UnsupportedPlatformError, "Unsupported macOS CPU architecture for QuickJS: #{cpu}"
+          end
+        when :linux
+          case cpu
+          when :x86_64
+            'qjs-linux-x86_64'
+          when :aarch64, :arm64
+            'qjs-linux-aarch64'
+          when :arm
+            'qjs-linux-armv7'
+          when :i386, :i686
+            'qjs-linux-x86'
+          when :riscv64
+            'qjs-linux-riscv64'
+          else
+            raise UnsupportedPlatformError, "Unsupported Linux CPU architecture for QuickJS: #{cpu}"
+          end
+        when :windows
+          case cpu
+          when :x86_64 then 'qjs-windows-x86_64.exe'
+          when :i386, :i686 then 'qjs-windows-x86.exe'
+          else
+            raise UnsupportedPlatformError, "Unsupported Windows CPU architecture for QuickJS: #{cpu}"
+          end
+        else
+          raise UnsupportedPlatformError, "Unsupported operating system for QuickJS: #{RbConfig::CONFIG['host_os']}"
+        end
+      end
+
+      def qjs_version_file_path(dir)
+        File.join(dir, '.qjs_version')
+      end
+
+      def download_obscura!(version: nil, install_dir: nil, repo: nil, force: false)
+        target_version = normalize_version(version || HttpMimic.configuration.obscura_version || DEFAULT_OBSCURA_VERSION)
+        target_dir     = File.expand_path(install_dir || HttpMimic.configuration.install_dir)
+        target_repo    = repo || HttpMimic.configuration.obscura_github_repo || DEFAULT_OBSCURA_REPO
+
+        FileUtils.mkdir_p(target_dir)
+
+        dest_binary = File.join(target_dir, binary_name_for_platform('obscura'))
+
+        if !force && obscura_installed?(version: target_version, install_dir: target_dir)
+          log_info("obscura #{target_version} is already installed in #{target_dir}")
+          return dest_binary
+        end
+
+        asset_name = obscura_platform_asset
+        download_url = "https://github.com/#{target_repo}/releases/download/#{target_version}/#{asset_name}"
+
+        log_info("Downloading Obscura (#{target_version}) [#{asset_name}]...")
+        archive_data = fetch_binary(download_url)
+
+        log_info("Extracting Obscura archive to #{target_dir}...")
+        extract_tar_gz(archive_data, target_dir)
+
+        # Ensure executable permissions on extracted binaries
+        %w[obscura obscura-worker].each do |name|
+          p = File.join(target_dir, binary_name_for_platform(name))
+          File.chmod(0755, p) if File.exist?(p)
+        end
+
+        File.write(obscura_version_file_path(target_dir), target_version)
+
+        log_info("Obscura #{target_version} installation complete! (#{dest_binary})")
+        dest_binary
+      end
+
+      def obscura_installed?(version: nil, install_dir: nil)
+        target_dir = File.expand_path(install_dir || HttpMimic.configuration.install_dir)
+        bin = File.join(target_dir, binary_name_for_platform('obscura'))
+        return false unless File.file?(bin) && File.executable?(bin)
+
+        if version
+          target_version = normalize_version(version)
+          v_file = obscura_version_file_path(target_dir)
+          return false unless File.file?(v_file)
+          return File.read(v_file).strip == target_version
+        end
+
+        true
+      end
+
+      def obscura_path(install_dir: nil)
+        return HttpMimic.configuration.obscura_path if HttpMimic.configuration.obscura_path && File.executable?(HttpMimic.configuration.obscura_path)
+
+        target_dir = File.expand_path(install_dir || HttpMimic.configuration.install_dir)
+        candidate = File.join(target_dir, binary_name_for_platform('obscura'))
+        return candidate if File.file?(candidate) && File.executable?(candidate)
+
+        # Fallback to system PATH
+        sys_bin = `which obscura 2>/dev/null`.strip
+        return sys_bin if !sys_bin.empty? && File.executable?(sys_bin)
+
+        nil
+      end
+
+      def obscura_platform_asset
+        os  = host_os
+        cpu = host_cpu
+
+        case os
+        when :macos
+          case cpu
+          when :arm64 then 'obscura-aarch64-macos-stealth.tar.gz'
+          when :x86_64 then 'obscura-x86_64-macos-stealth.tar.gz'
+          else
+            raise UnsupportedPlatformError, "Unsupported macOS CPU architecture for Obscura: #{cpu}"
+          end
+        when :linux
+          case cpu
+          when :x86_64 then 'obscura-x86_64-linux-stealth.tar.gz'
+          when :aarch64, :arm64 then 'obscura-aarch64-linux-stealth.tar.gz'
+          else
+            raise UnsupportedPlatformError, "Unsupported Linux CPU architecture for Obscura: #{cpu}"
+          end
+        when :windows
+          'obscura-x86_64-windows-stealth.zip'
+        else
+          raise UnsupportedPlatformError, "Unsupported operating system for Obscura: #{RbConfig::CONFIG['host_os']}"
+        end
+      end
+
+      def obscura_version_file_path(dir)
+        File.join(dir, '.obscura_version')
+      end
+
       def binary_path(name, install_dir: nil)
         target_dir = File.expand_path(install_dir || HttpMimic.configuration.install_dir)
         candidate = File.join(target_dir, binary_name_for_platform(name))

data/lib/http_mimic/exceptions.rb

@@ -31,4 +31,19 @@ module HttpMimic
 
   # Raised when response parsing fails
   class ResponseParseError < Error; end
+
+  # Raised when JavaScript execution fails in JSRuntime
+  class JSError < Error
+    attr_reader :stderr, :exit_code
+
+    def initialize(message, stderr: nil, exit_code: nil)
+      super(message)
+      @stderr = stderr
+      @exit_code = exit_code
+    end
+  end
+
+  # Raised when JavaScript execution times out
+  class JSTimeoutError < JSError; end
 end
+

data/lib/http_mimic/js_runtime.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+require 'open3'
+require 'json'
+require 'timeout'
+
+module HttpMimic
+  class JSRuntime
+    STDIN_RUNNER = <<~JS
+      const input = std.in.readAsString();
+      const result = eval(input);
+      if (result !== undefined) {
+        if (typeof result === "object" && result !== null) {
+          console.log(JSON.stringify(result));
+        } else {
+          console.log(result);
+        }
+      }
+    JS
+
+    class << self
+      def eval(js_code, timeout: nil, install_dir: nil, auto_download: nil)
+        ensure_qjs_installed!(install_dir: install_dir, auto_download: auto_download)
+
+        qjs_bin = Downloader.qjs_path(install_dir: install_dir)
+        raise BinaryNotFoundError, "QuickJS binary 'qjs' not found. Run HttpMimic.download_qjs! to install." unless qjs_bin
+
+        cmd = [qjs_bin, '--std', '-e', STDIN_RUNNER]
+        effective_timeout = timeout || HttpMimic.configuration.default_timeout
+
+        stdout = nil
+        stderr = nil
+        status = nil
+
+        begin
+          if effective_timeout && effective_timeout > 0
+            Timeout.timeout(effective_timeout) do
+              stdout, stderr, status = Open3.capture3(*cmd, stdin_data: js_code.to_s)
+            end
+          else
+            stdout, stderr, status = Open3.capture3(*cmd, stdin_data: js_code.to_s)
+          end
+        rescue Timeout::Error
+          raise JSTimeoutError.new("JavaScript execution timed out after #{effective_timeout}s")
+        end
+
+        unless status && status.success?
+          raise JSError.new(
+            "JavaScript execution failed: #{stderr.to_s.strip}",
+            stderr: stderr,
+            exit_code: status ? status.exitstatus : nil
+          )
+        end
+
+        stdout.to_s.strip
+      end
+
+      def eval_json(js_code, timeout: nil, install_dir: nil, auto_download: nil)
+        output = eval(js_code, timeout: timeout, install_dir: install_dir, auto_download: auto_download)
+        return nil if output.nil? || output.empty?
+
+        JSON.parse(output)
+      rescue JSON::ParserError => e
+        raise ResponseParseError, "Failed to parse JavaScript JSON output: #{e.message} (Output: #{output.inspect})"
+      end
+
+      private
+
+      def ensure_qjs_installed!(install_dir: nil, auto_download: nil)
+        should_download = auto_download.nil? ? HttpMimic.configuration.auto_download : auto_download
+        return unless should_download
+        return if Downloader.qjs_installed?(install_dir: install_dir)
+
+        Downloader.download_qjs!(install_dir: install_dir)
+      end
+    end
+  end
+end

data/lib/http_mimic/module_methods.rb

@@ -41,6 +41,16 @@ module HttpMimic
       default_options[:cookies].merge!(c)
     end
 
+    def persist_cookies(enabled = nil)
+      return default_options[:persist_cookies] if enabled.nil?
+      default_options[:persist_cookies] = enabled
+    end
+
+    def auto_render_spa(enabled = nil)
+      return default_options[:auto_render_spa] if enabled.nil?
+      default_options[:auto_render_spa] = enabled
+    end
+
     def mode(m = nil)
       return default_options[:mode] if m.nil?
       default_options[:mode] = m
@@ -88,6 +98,12 @@ module HttpMimic
       request(:options, url, options)
     end
 
+    def render(url, options = {})
+      merged = default_options.merge(options)
+      Obscura.render(url, merged)
+    end
+    alias spa render
+
     def request(method, url, options = {})
       merged = default_options.merge(options)
 

data/lib/http_mimic/obscura.rb

@@ -0,0 +1,139 @@
+# frozen_string_literal: true
+
+require 'open3'
+require 'uri'
+require 'json'
+
+module HttpMimic
+  module Obscura
+    class ExecutionError < HttpMimic::Error; end
+    class BinaryNotFoundError < HttpMimic::Error; end
+
+    class << self
+      # Renders a web page using the Obscura headless SPA engine and returns an HttpMimic::Response
+      #
+      # @param url [String, URI] The target URL to render
+      # @param options [Hash] Rendering options
+      # @option options [String] :wait_until ('networkidle0') Wait condition (e.g. 'load', 'domcontentloaded', 'networkidle0')
+      # @option options [Integer] :timeout Per-command timeout in seconds
+      # @option options [String] :proxy Proxy URL (e.g. 'socks5://127.0.0.1:1080' or 'http://...')
+      # @option options [String] :eval JavaScript expression to evaluate on the page
+      # @option options [String] :dump Output format ('html', 'text', 'links')
+      # @return [HttpMimic::Response]
+      def render(url, options = {})
+        bin = resolve_binary
+        raise BinaryNotFoundError, "Obscura binary not found. Run HttpMimic.download_obscura! or configure obscura_path." unless bin
+
+        args = [bin, 'fetch', url.to_s]
+
+        # Dump format: default to html (can be 'html', 'text', 'links', 'markdown', 'original', 'cookies')
+        dump_format = options[:dump] || 'html'
+        args << '--dump' << dump_format.to_s
+
+        # Stealth mode (enabled by default)
+        args << '--stealth' if options.fetch(:stealth, true)
+
+        # Wait condition
+        if options[:wait_until]
+          args << '--wait-until' << options[:wait_until].to_s
+        end
+
+        # Wait delay
+        if options[:wait]
+          args << '--wait' << options[:wait].to_i.to_s
+        end
+
+        # Timeout in seconds (integer)
+        timeout = options[:timeout] || HttpMimic.configuration.default_timeout
+        args << '--timeout' << timeout.to_i.to_s if timeout
+
+        # User Agent
+        if options[:user_agent]
+          args << '--user-agent' << options[:user_agent].to_s
+        end
+
+        # Proxy
+        if options[:proxy]
+          args << '--proxy' << options[:proxy].to_s
+        end
+
+        # Cookies: pass cookies string or hash (Two-Phase Pipeline support)
+        if options[:cookies] || options[:cookie]
+          cookie_val = options[:cookies] || options[:cookie]
+          cookie_str = if cookie_val.is_a?(Hash)
+            cookie_val.map { |k, v| "#{k}=#{v}" }.join('; ')
+          elsif cookie_val.is_a?(Array)
+            cookie_val.join('; ')
+          else
+            cookie_val.to_s
+          end
+          args << '--cookie' << cookie_str unless cookie_str.empty?
+        end
+
+        # Optional JS evaluation
+        args << '--eval' << options[:eval].to_s if options[:eval]
+
+        # Selector
+        args << '--selector' << options[:selector].to_s if options[:selector]
+
+        log_debug("Executing Obscura SPA render: #{args.join(' ')}")
+
+        stdout, stderr, status = Open3.capture3(*args)
+
+        code = (status && status.success?) ? 200 : (status ? status.exitstatus : 500)
+
+        raw_headers = "HTTP/2 200 OK\r\ncontent-type: text/html; charset=utf-8\r\nx-rendered-by: obscura\r\n\r\n"
+        headers = Headers.new({ 'content-type' => 'text/html; charset=utf-8', 'x-rendered-by' => 'obscura' })
+
+        Response.new(
+          code: code,
+          http_version: 'HTTP/2',
+          status_message: status&.success? ? 'OK (Obscura SPA Rendered)' : 'Error',
+          headers: headers,
+          cookies: Cookies.new,
+          body: stdout,
+          parsed_response: nil,
+          raw_headers: raw_headers,
+          history: [],
+          request_url: url.to_s,
+          stderr: stderr,
+          exit_code: status ? status.exitstatus : 0,
+          command: args.join(' ')
+        )
+      end
+
+      def resolve_binary
+        # 1. Configured custom path
+        custom_path = HttpMimic.configuration.obscura_path
+        if custom_path && (File.file?(custom_path) || File.executable?(custom_path))
+          return custom_path
+        end
+
+        # 2. Check install directory or system PATH
+        installed_path = Downloader.obscura_path
+        return installed_path if installed_path
+
+        # 3. Auto-download if enabled
+        if HttpMimic.configuration.auto_download
+          begin
+            return Downloader.download_obscura!
+          rescue StandardError => e
+            HttpMimic.configuration.logger&.warn("[HttpMimic::Obscura] Auto-download failed: #{e.message}")
+          end
+        end
+
+        nil
+      end
+
+      private
+
+      def log_debug(msg)
+        if HttpMimic.configuration.logger
+          HttpMimic.configuration.logger.debug("[HttpMimic::Obscura] #{msg}")
+        elsif HttpMimic.configuration.debug
+          puts "[HttpMimic::Obscura] #{msg}"
+        end
+      end
+    end
+  end
+end

data/lib/http_mimic/request.rb

@@ -15,11 +15,31 @@ module HttpMimic
 
     def perform
       mode = (options[:mode] || config.mode || :auto).to_sym
+
+      # Delegate directly to Obscura headless SPA renderer if requested
+      if options[:render] == :spa || options[:render] == :obscura || mode == :spa || mode == :obscura
+        return Obscura.render(url, options)
+      end
+
       auto_fallback = options.fetch(:auto_fallback, config.auto_fallback)
       retry_statuses = options[:retry_statuses] || config.retry_statuses || [403, 429, 503]
 
+      # Persistent CookieStore integration
+      should_persist_cookie = options.fetch(:persist_cookies, config.persist_cookies)
+      host = CookieStore.extract_host(url)
+
+      if should_persist_cookie && host
+        stored_cookies = CookieStore.load(host)
+        if stored_cookies && !stored_cookies.empty?
+          log_debug("[HttpMimic::CookieStore] Loaded #{stored_cookies.size} persistent cookies for #{host}")
+          user_cookies = options[:cookies] ? (options[:cookies].is_a?(Hash) ? options[:cookies] : options[:cookies].to_h) : {}
+          options[:cookies] = stored_cookies.merge(user_cookies)
+        end
+      end
+
       attempts = []
       profiles_to_try = determine_profiles(mode, auto_fallback)
+      waf_solve_attempted = false
 
       response = nil
       final_status = nil
@@ -66,7 +86,32 @@ module HttpMimic
         final_stderr = stderr
         final_command = full_command
 
-        is_blocked = (status.exitstatus != 0) || retry_statuses.include?(response.code)
+        # Forward any received cookies to subsequent attempts
+        if response.cookies && !response.cookies.empty?
+          existing_cookies = options[:cookies] ? (options[:cookies].is_a?(Hash) ? options[:cookies] : options[:cookies].to_h) : {}
+          options[:cookies] = response.cookies.to_h.merge(existing_cookies)
+        end
+
+        auto_solve_waf = options.fetch(:solve_waf, config.auto_solve_waf)
+        is_blocked = (status.exitstatus != 0) || retry_statuses.include?(response.code) || Waf::Detector.challenge_page?(response)
+
+        # Only attempt WAF resolution once per request to avoid unnecessary latency on subsequent fallbacks
+        if is_blocked && auto_solve_waf && !waf_solve_attempted && method.to_s.upcase == 'GET'
+          waf_type = Waf::Detector.detect(response)
+          if waf_type
+            waf_solve_attempted = true
+            log_debug("[HttpMimic] Detected #{waf_type.to_s.capitalize} WAF challenge. Attempting to solve with QuickJS...")
+            solved_resp = Waf.solve(url, response, current_opts)
+            if solved_resp
+              response = solved_resp
+              is_blocked = (response.code != 0 && retry_statuses.include?(response.code))
+              if response.cookies && !response.cookies.empty?
+                options[:cookies] = (options[:cookies] || {}).merge(response.cookies.to_h)
+              end
+            end
+          end
+        end
+
         if !is_blocked || (index == profiles_to_try.size - 1)
           break
         end
@@ -74,6 +119,35 @@ module HttpMimic
         log_debug("[HttpMimic] Attempt #{index + 1} with #{profile} resulted in status #{response.code}. Triggering smart fallback to next profile...")
       end
 
+      # Automatic SPA Detection & Obscura rendering fallback
+      auto_render_spa = options.fetch(:auto_render_spa, config.auto_render_spa)
+      if auto_render_spa && response && response.success? && method.to_s.upcase == 'GET'
+        if SpaDetector.spa?(response)
+          log_debug("[HttpMimic] Detected unhydrated SPA shell on #{url}. Automatically rendering with Obscura...")
+          begin
+            spa_opts = options.dup
+            # Forward all validated cookies from Tier 1 (Mode 2: Two-Phase Pipeline)
+            if response.cookies && !response.cookies.empty?
+              tier1_cookies = response.cookies.to_h
+              existing_cookies = spa_opts[:cookies].is_a?(Hash) ? spa_opts[:cookies] : {}
+              spa_opts[:cookies] = existing_cookies.merge(tier1_cookies)
+            end
+            rendered_resp = Obscura.render(url, spa_opts)
+            if rendered_resp && rendered_resp.success?
+              response = rendered_resp
+            end
+          rescue StandardError => e
+            log_debug("[HttpMimic] Automatic Obscura SPA render failed (#{e.message}), keeping Tier 1 response.")
+          end
+        end
+      end
+
+      # Persist cookies back to store if enabled
+      if should_persist_cookie && host && response && response.cookies && !response.cookies.empty?
+        CookieStore.save(host, response.cookies)
+        log_debug("[HttpMimic::CookieStore] Saved #{response.cookies.size} cookies for #{host}")
+      end
+
       handle_errors(final_status, final_stderr, final_command, response)
       response
     end

data/lib/http_mimic/spa_detector.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+module HttpMimic
+  module SpaDetector
+    # Framework mount points that indicate an unhydrated SPA client shell
+    EMPTY_MOUNT_REGEX = %r{
+      <div[^>]+id=["'](?:root|app|__next|__nuxt|svelte)["'][^>]*>\s*</div>|
+      <app-root[^>]*>\s*</app-root>|
+      <div[^>]+id=["']app["'][^>]*>\s*<!--\s*(?:app-content)?\s*-->\s*</div>
+    }ix
+
+    # Noscript patterns requiring JavaScript execution
+    NOSCRIPT_JS_REQUIRED_REGEX = %r{
+      <noscript[^>]*>[\s\S]*?(?:enable\s+javascript|javascript\s+is\s+required|need\s+to\s+enable\s+javascript|requires\s+javascript)[\s\S]*?</noscript>
+    }ix
+
+    # Google Search Guard dynamic SPA shell indicators
+    GOOGLE_SHELL_REGEX = %r{
+      (?:/httpservice/retry/enablejs|SG_REL|emsg=SG_REL|knitsail)
+    }ix
+
+    class << self
+      # Returns true if the HTTP response represents an unhydrated Single Page Application (SPA) shell
+      #
+      # @param response [HttpMimic::Response]
+      # @return [Boolean]
+      def spa?(response)
+        return false unless response && response.success?
+        return false if response.respond_to?(:binary?) && response.binary?
+
+        content_type = response.headers['content-type'].to_s.downcase
+        return false unless content_type.empty? || content_type.include?('text/html') || content_type.include?('application/xhtml')
+
+        body = response.body.to_s
+        return false if body.strip.empty?
+
+        # 1. Google Dynamic SERP / Search Guard JS Shell
+        if google_serp_shell?(body)
+          return true
+        end
+
+        # 2. Modern SPA framework empty mount points (React, Vue, Angular, Next, Nuxt, Svelte)
+        if empty_mount_point?(body)
+          return true
+        end
+
+        # 3. Noscript requirement message
+        if noscript_js_required?(body)
+          return true
+        end
+
+        # 4. Small body with multiple scripts and virtually no visible text
+        if low_text_js_heavy_shell?(body)
+          return true
+        end
+
+        false
+      end
+
+      def empty_mount_point?(body)
+        body.match?(EMPTY_MOUNT_REGEX)
+      end
+
+      def noscript_js_required?(body)
+        body.match?(NOSCRIPT_JS_REQUIRED_REGEX)
+      end
+
+      def google_serp_shell?(body)
+        return false unless body.match?(GOOGLE_SHELL_REGEX)
+
+        # If it has the Google shell indicators AND lacks rendered organic result elements
+        !body.include?('<h3 class=') && !body.include?('id="search"') && !body.include?('id="rso"')
+      end
+
+      def low_text_js_heavy_shell?(body)
+        return false if body.length > 25_000 # Larger HTML files typically have SSR content
+
+        # Check if there are script tags
+        script_count = body.scan(/<script/i).size
+        return false if script_count.zero?
+
+        # Strip all HTML tags, scripts, and styles to get raw visible text length
+        visible_text = body
+                       .gsub(/<script[\s\S]*?<\/script>/i, '')
+                       .gsub(/<style[\s\S]*?<\/style>/i, '')
+                       .gsub(/<[^>]+>/, ' ')
+                       .gsub(/\s+/, ' ')
+                       .strip
+
+        # If visible text is virtually nonexistent but page has multiple scripts
+        visible_text.length < 80 && script_count >= 2
+      end
+    end
+  end
+end

data/lib/http_mimic/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module HttpMimic
-  VERSION = "0.3.2"
+  VERSION = "0.5.1"
 end