http_mimic 0.3.2 → 0.5.1

data/lib/http_mimic/waf/akamai_solver.rb

@@ -0,0 +1,123 @@
+# frozen_string_literal: true
+
+require 'uri'
+require 'json'
+
+module HttpMimic
+  module Waf
+    class AkamaiSolver
+      CONTEXT_JS_PATH = File.expand_path('browser_context.js', __dir__)
+
+      class << self
+        def solve(target_url, initial_response, options = {})
+          cookies = initial_response.cookies.dup
+          impersonate = options[:impersonate] || HttpMimic.configuration.default_impersonate || 'chrome131'
+
+          sensor_script_url = extract_sensor_script_url(target_url, initial_response.body)
+          return nil unless sensor_script_url
+
+          # 1. Fetch the dynamic sensor script
+          script_resp = HttpMimic.get(
+            sensor_script_url,
+            impersonate: impersonate,
+            cookies: cookies,
+            headers: { 'Referer' => target_url },
+            auto_fallback: false,
+            solve_waf: false
+          )
+          return nil unless script_resp.success?
+
+          sensor_js = script_resp.body
+          context_js = File.read(CONTEXT_JS_PATH)
+          cookie_str = cookies.to_cookie_string
+
+          doc_title = initial_response.title.to_s
+          user_agent = options[:user_agent]
+          referer = (options[:headers] || {})['Referer'] || (options[:headers] || {})['referer'] || target_url
+
+          # 2. Run virtual browser simulation inside QuickJS
+          driver_script = <<~JS
+            globalThis.__TARGET_URL__ = #{target_url.to_json};
+            globalThis.__DOCUMENT_TITLE__ = #{doc_title.to_json};
+            globalThis.__INITIAL_COOKIES__ = #{cookie_str.to_json};
+            globalThis.__REFERRER__ = #{referer.to_json};
+            #{user_agent ? "globalThis.__USER_AGENT__ = #{user_agent.to_json};" : ""}
+
+            #{context_js}
+            #{sensor_js}
+
+            if (typeof globalThis.__simulateHumanInteractions === "function") {
+              globalThis.__simulateHumanInteractions();
+            }
+            if (typeof globalThis.__drainEventLoop === "function") {
+              globalThis.__drainEventLoop(100);
+            }
+
+            JSON.stringify({
+              sensor_posts: globalThis.__sensor_posts
+            });
+          JS
+
+          result = JSRuntime.eval_json(driver_script)
+          sensor_posts = result && result['sensor_posts']
+          return nil if sensor_posts.nil? || sensor_posts.empty?
+
+          # 3. Post sensor data to Akamai endpoint
+          updated_cookies = cookies.dup
+          sensor_posts.each do |post|
+            post_body = post['body']
+            next if post_body.nil? || post_body.empty?
+
+            post_resp = HttpMimic.post(
+              sensor_script_url,
+              impersonate: impersonate,
+              cookies: updated_cookies,
+              headers: {
+                'Content-Type' => 'text/plain;charset=UTF-8',
+                'Referer' => target_url,
+                'Origin' => URI.parse(target_url).tap { |u| u.path = ''; u.query = nil }.to_s,
+                'Accept' => '*/*'
+              },
+              body: post_body,
+              auto_fallback: false,
+              solve_waf: false
+            )
+
+            post_resp.cookies.each { |k, v| updated_cookies[k] = v }
+          end
+
+          # 4. Retry original target URL with the updated cookies
+          retry_headers = (options[:headers] || {}).merge('Referer' => target_url)
+          HttpMimic.get(
+            target_url,
+            options.merge(
+              cookies: updated_cookies,
+              headers: retry_headers,
+              auto_fallback: false,
+              solve_waf: false
+            )
+          )
+        rescue StandardError => e
+          if HttpMimic.configuration.debug
+            puts "[HttpMimic::Waf::AkamaiSolver] Error solving Akamai challenge: #{e.message}"
+          end
+          nil
+        end
+
+        def extract_sensor_script_url(target_url, html)
+          return nil if html.nil? || html.empty?
+
+          match = html.match(/<script[^>]*src=["\x27]([^"\x27]*\/[a-zA-Z0-9_-]{10,}\?[a-zA-Z0-9_=-]+)["\x27]/i)
+          match ||= html.match(/<script[^>]*src=["\x27]([^"\x27]*akam[^"\x27]*)["\x27]/i)
+          match ||= html.match(/<script[^>]*src=["\x27]([^"\x27]*\/[a-zA-Z0-9_\-\/]+\?v=[a-zA-Z0-9_-]+)["\x27]/i)
+
+          return nil unless match
+
+          URI.join(target_url, match[1]).to_s
+        rescue StandardError
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/http_mimic/waf/detector.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module HttpMimic
+  module Waf
+    class Detector
+      class << self
+        def detect(response)
+          return nil unless response
+
+          if akamai?(response)
+            :akamai
+          elsif google?(response)
+            :google
+          elsif cloudflare?(response)
+            :cloudflare
+          elsif datadome?(response)
+            :datadome
+          else
+            nil
+          end
+        end
+
+        def challenge_page?(response)
+          return false unless response
+          body = response.body.to_s
+          return true if body.include?('sec-if-cpt-container') || body.include?('sec-bc-button-parent')
+          return true if body.include?('challenges.cloudflare.com') || body.include?('cf-turnstile')
+          return true if body.include?('datadome.captcha') || body.include?('geo.captcha-delivery.com')
+          return true if body.include?('/httpservice/retry/enablejs') || (body.include?('knitsail') && body.include?('SG_SS'))
+          false
+        end
+
+        def google?(response)
+          return false unless response
+          body = response.body.to_s
+          return true if body.include?('/httpservice/retry/enablejs')
+          return true if body.include?('knitsail') && body.include?('SG_SS')
+          return true if response.headers['server']&.downcase&.include?('gws') && body.include?('enablejs')
+          false
+        end
+
+        def akamai?(response)
+          return true if response.headers['set-cookie']&.include?('_abck=')
+          return true if response.cookies.key?('_abck') || response.cookies.key?('bm_sz') || response.cookies.key?('ak_bmsc')
+          return true if response.headers['x-akamai-transformed'] || response.headers['x-reference-error']
+
+          body = response.body.to_s
+          return true if body.include?('Reference Error:') && body.include?('Akamai')
+          return true if body.include?('bmak') || body.include?('sensor_data')
+          return true if body.include?('sec-if-cpt-container') || body.include?('sec-bc-button-parent')
+          return true if body =~ /<script[^>]*src=["\x27]([^"\x27]*\/[a-zA-Z0-9_-]{10,}\?[a-zA-Z0-9_=-]+)["\x27]/
+
+          false
+        end
+
+        def cloudflare?(response)
+          return true if response.headers['server']&.downcase&.include?('cloudflare')
+          return true if response.headers['cf-ray'] || response.cookies.key?('cf_clearance')
+          return true if response.body.to_s.include?('challenges.cloudflare.com')
+
+          false
+        end
+
+        def datadome?(response)
+          return true if response.headers['x-datadome'] || response.cookies.key?('datadome')
+          return true if response.headers['server']&.downcase&.include?('datadome')
+
+          false
+        end
+      end
+    end
+  end
+end

data/lib/http_mimic/waf/google_solver.rb

@@ -0,0 +1,124 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'uri'
+
+module HttpMimic
+  module Waf
+    class GoogleSolver
+      class << self
+        def solve(target_url, response, options = {})
+          return nil unless response
+
+          html = response.body.to_s
+          return nil if html.empty?
+
+          uri = URI.parse(target_url) rescue nil
+          base_url = uri ? "#{uri.scheme}://#{uri.host}" : "https://www.google.com"
+
+          impersonate = options[:impersonate] || HttpMimic.configuration.default_impersonate || 'chrome131'
+          current_cookies = (response.cookies || Cookies.new).dup
+
+          # Strategy 1: Session warmup from Google root if cookies are sparse
+          if current_cookies.empty? || !current_cookies.key?('NID')
+            warmup_resp = HttpMimic.get(
+              "#{base_url}/",
+              impersonate: impersonate,
+              cookies: current_cookies,
+              auto_fallback: false,
+              solve_waf: false,
+              persist_cookies: false
+            )
+            warmup_resp.cookies.each { |k, v| current_cookies[k] = v } if warmup_resp&.cookies
+          end
+
+          # Strategy 2: Attempt QuickJS solve if script payload exists
+          scripts = html.scan(/<script[^>]*>([\s\S]*?)<\/script>/i).map(&:first)
+          if scripts.any? { |s| s.include?('knitsail') || s.include?('SG_SS') || s.include?('closureDynamicButton') }
+            sg_ss_cookie = solve_with_quickjs(target_url, scripts, current_cookies)
+            current_cookies['SG_SS'] = sg_ss_cookie if sg_ss_cookie && !sg_ss_cookie.start_with?('E:')
+          end
+
+          # Strategy 3: Retry search query with same-origin referral headers and updated cookies
+          retry_headers = (options[:headers] || {}).dup
+          retry_headers['Referer'] ||= "#{base_url}/"
+          retry_headers['sec-fetch-site'] ||= 'same-origin'
+
+          retry_opts = options.merge(
+            cookies: current_cookies,
+            headers: retry_headers,
+            auto_fallback: false,
+            solve_waf: false
+          )
+
+          HttpMimic.get(target_url, **retry_opts)
+        rescue StandardError => e
+          HttpMimic.logger.debug("[HttpMimic::Waf::GoogleSolver] Failed to solve Google challenge: #{e.message}") if HttpMimic.logger
+          nil
+        end
+
+        private
+
+        def solve_with_quickjs(target_url, scripts, cookies)
+          context_js_path = File.expand_path('browser_context.js', __dir__)
+          context_js = File.read(context_js_path)
+          cookie_str = cookies.respond_to?(:to_cookie_string) ? cookies.to_cookie_string : cookies.to_s
+
+          driver_script = <<~JS
+            globalThis.__TARGET_URL__ = #{target_url.to_json};
+            globalThis.__DOCUMENT_TITLE__ = "Google Search";
+            globalThis.__INITIAL_COOKIES__ = #{cookie_str.to_json};
+            globalThis.__REFERRER__ = "https://www.google.com/";
+
+            #{context_js}
+
+            // Google specific environment enhancements
+            globalThis.sessionStorage = {
+              _data: {},
+              getItem(k) { return this._data[k] || null; },
+              setItem(k, v) { this._data[k] = String(v); },
+              removeItem(k) { delete this._data[k]; },
+              clear() { this._data = {}; }
+            };
+            globalThis.localStorage = globalThis.sessionStorage;
+            globalThis._F_css = function() {};
+            globalThis.google = { c: { c: { a: true } } };
+
+            #{scripts.join("\n;\n")}
+
+            // Extract p token and invoke knitsail if present
+            var allScripts = #{scripts.join("\n").to_json};
+            var pMatch = allScripts.match(/var p=\\x27([^\\x27]+)\\x27/);
+            var pVal = pMatch ? pMatch[1] : null;
+
+            var solved_token = null;
+            if (globalThis.knitsail && typeof globalThis.knitsail.a === "function" && pVal) {
+              try {
+                globalThis.knitsail.a(pVal, function(resultCallback) {
+                  if (typeof resultCallback === "function") {
+                    resultCallback(function(token) {
+                      solved_token = token;
+                    }, [{}]);
+                  }
+                }, false, undefined, undefined, undefined, undefined, true);
+              } catch(e) {}
+            }
+
+            if (typeof globalThis.__drainEventLoop === "function") {
+              globalThis.__drainEventLoop(50);
+            }
+
+            JSON.stringify({
+              token: solved_token
+            });
+          JS
+
+          result = JSRuntime.eval_json(driver_script)
+          result ? result['token'] : nil
+        rescue StandardError
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/http_mimic/waf.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require 'http_mimic/waf/detector'
+require 'http_mimic/waf/akamai_solver'
+require 'http_mimic/waf/google_solver'
+
+module HttpMimic
+  module Waf
+    class << self
+      def solve(url, response, options = {})
+        waf_type = Detector.detect(response)
+        return nil unless waf_type
+
+        case waf_type
+        when :akamai
+          AkamaiSolver.solve(url, response, options)
+        when :google
+          GoogleSolver.solve(url, response, options)
+        else
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/http_mimic.rb

@@ -16,6 +16,11 @@ require 'http_mimic/command_builder'
 require 'http_mimic/request'
 require 'http_mimic/client'
 require 'http_mimic/module_methods'
+require 'http_mimic/js_runtime'
+require 'http_mimic/waf'
+require 'http_mimic/cookie_store'
+require 'http_mimic/obscura'
+require 'http_mimic/spa_detector'
 
 module HttpMimic
   extend ModuleMethods
@@ -43,6 +48,78 @@ module HttpMimic
       Downloader.installed?(version: version)
     end
 
+    # Manually download and install QuickJS binary driver
+    def download_qjs!(version: nil, force: false)
+      Downloader.download_qjs!(version: version, force: force)
+    end
+
+    # Check if QuickJS driver is installed locally
+    def qjs_installed?(version: nil)
+      Downloader.qjs_installed?(version: version)
+    end
+
+    # Return local path to QuickJS binary
+    def qjs_path
+      Downloader.qjs_path
+    end
+
+    # Evaluate JavaScript code using QuickJS
+    def eval_js(code, options = {})
+      JSRuntime.eval(code, timeout: options[:timeout], install_dir: options[:install_dir], auto_download: options[:auto_download])
+    end
+
+    # Evaluate JavaScript and auto-parse JSON result
+    def eval_js_json(code, options = {})
+      JSRuntime.eval_json(code, timeout: options[:timeout], install_dir: options[:install_dir], auto_download: options[:auto_download])
+    end
+
+    # Automatically detect WAF type from response
+    def detect_waf(response)
+      Waf::Detector.detect(response)
+    end
+
+    # Attempt to solve WAF challenge for a blocked response
+    def solve_waf(url, response, options = {})
+      Waf.solve(url, response, options)
+    end
+
+    # Persistent Host CookieStore helpers
+    def load_cookies(host, max_age: nil)
+      CookieStore.load(host, max_age: max_age)
+    end
+
+    def save_cookies(host, cookies, ttl: nil)
+      CookieStore.save(host, cookies, ttl: ttl || CookieStore::DEFAULT_TTL)
+    end
+
+    def clear_cookies!(host = nil)
+      CookieStore.clear(host)
+    end
+
+    # Obscura headless SPA engine helpers
+    def download_obscura!(version: nil, force: false)
+      Downloader.download_obscura!(version: version, force: force)
+    end
+
+    def obscura_installed?(version: nil)
+      Downloader.obscura_installed?(version: version)
+    end
+
+    def obscura_path
+      Downloader.obscura_path
+    end
+
+    # Render SPA page using Obscura headless engine
+    def render(url, options = {})
+      Obscura.render(url, options)
+    end
+    alias spa render
+
+    # Check if a response represents an unhydrated SPA shell
+    def spa?(response)
+      SpaDetector.spa?(response)
+    end
+
     def included(base)
       base.extend(ModuleMethods)
       base.send(:include, InstanceMethods)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: http_mimic
 version: !ruby/object:Gem::Version
-  version: 0.3.2
+  version: 0.5.1
 platform: ruby
 authors:
 - anxgang
@@ -65,15 +65,23 @@ files:
 - lib/http_mimic/client.rb
 - lib/http_mimic/command_builder.rb
 - lib/http_mimic/configuration.rb
+- lib/http_mimic/cookie_store.rb
 - lib/http_mimic/cookies.rb
 - lib/http_mimic/downloader.rb
 - lib/http_mimic/exceptions.rb
 - lib/http_mimic/headers.rb
+- lib/http_mimic/js_runtime.rb
 - lib/http_mimic/module_methods.rb
+- lib/http_mimic/obscura.rb
 - lib/http_mimic/request.rb
 - lib/http_mimic/response.rb
 - lib/http_mimic/response_parser.rb
+- lib/http_mimic/spa_detector.rb
 - lib/http_mimic/version.rb
+- lib/http_mimic/waf.rb
+- lib/http_mimic/waf/akamai_solver.rb
+- lib/http_mimic/waf/detector.rb
+- lib/http_mimic/waf/google_solver.rb
 homepage: https://github.com/anxgang/http_mimic
 licenses:
 - MIT