http_mimic 0.4.0 → 0.5.3

data/lib/http_mimic/waf/akamai_solver.rb

@@ -22,6 +22,7 @@ module HttpMimic
             impersonate: impersonate,
             cookies: cookies,
             headers: { 'Referer' => target_url },
+            proxy: options[:proxy],
             auto_fallback: false,
             solve_waf: false
           )
@@ -29,22 +30,81 @@ module HttpMimic
 
           sensor_js = script_resp.body
           context_js = File.read(CONTEXT_JS_PATH)
-          cookie_str = cookies.to_cookie_string
 
           doc_title = initial_response.title.to_s
           user_agent = options[:user_agent]
           referer = (options[:headers] || {})['Referer'] || (options[:headers] || {})['referer'] || target_url
 
-          # 2. Run virtual browser simulation inside QuickJS
-          driver_script = <<~JS
-            globalThis.__TARGET_URL__ = #{target_url.to_json};
-            globalThis.__DOCUMENT_TITLE__ = #{doc_title.to_json};
-            globalThis.__INITIAL_COOKIES__ = #{cookie_str.to_json};
-            globalThis.__REFERRER__ = #{referer.to_json};
-            #{user_agent ? "globalThis.__USER_AGENT__ = #{user_agent.to_json};" : ""}
+          # 2. Phase 1: Virtual browser simulation inside QuickJS (initial telemetry)
+          driver_script_1 = build_driver_script(
+            target_url: target_url,
+            doc_title: doc_title,
+            cookie_str: cookies.to_cookie_string,
+            referer: referer,
+            user_agent: user_agent,
+            context_js: context_js,
+            sensor_js: sensor_js,
+            phase: 1
+          )
+
+          result1 = JSRuntime.eval_json(driver_script_1)
+          sensor_posts_1 = result1 && result1['sensor_posts']
+          return nil if sensor_posts_1.nil? || sensor_posts_1.empty?
+
+          # 3. Post Phase 1 sensor data to Akamai endpoint
+          updated_cookies = cookies.dup
+          post_sensor_data(sensor_script_url, target_url, impersonate, updated_cookies, sensor_posts_1, proxy: options[:proxy])
+
+          # 4. Phase 2: If _abck is not verified yet (~0~), perform second round after short pause
+          if !verified_abck?(updated_cookies)
+            sleep 0.8
+            driver_script_2 = build_driver_script(
+              target_url: target_url,
+              doc_title: doc_title,
+              cookie_str: updated_cookies.to_cookie_string,
+              referer: referer,
+              user_agent: user_agent,
+              context_js: context_js,
+              sensor_js: sensor_js,
+              phase: 2
+            )
+
+            result2 = JSRuntime.eval_json(driver_script_2)
+            sensor_posts_2 = result2 && result2['sensor_posts']
+            if sensor_posts_2 && !sensor_posts_2.empty?
+              post_sensor_data(sensor_script_url, target_url, impersonate, updated_cookies, sensor_posts_2, proxy: options[:proxy])
+            end
+          end
+
+          # 5. Retry original target URL with the updated cookies
+          retry_headers = (options[:headers] || {}).merge('Referer' => target_url)
+          HttpMimic.get(
+            target_url,
+            options.merge(
+              cookies: updated_cookies,
+              headers: retry_headers,
+              auto_fallback: false,
+              solve_waf: false
+            )
+          )
+        rescue StandardError => e
+          if HttpMimic.configuration.debug
+            puts "[HttpMimic::Waf::AkamaiSolver] Error solving Akamai challenge: #{e.message}"
+          end
+          nil
+        end
+
+        def build_driver_script(params)
+          <<~JS
+            globalThis.__TARGET_URL__ = #{params[:target_url].to_json};
+            globalThis.__DOCUMENT_TITLE__ = #{params[:doc_title].to_json};
+            globalThis.__INITIAL_COOKIES__ = #{params[:cookie_str].to_json};
+            globalThis.__REFERRER__ = #{params[:referer].to_json};
+            #{params[:user_agent] ? "globalThis.__USER_AGENT__ = #{params[:user_agent].to_json};" : ""}
+            globalThis.__TELEMETRY_PHASE__ = #{params[:phase] || 1};
 
-            #{context_js}
-            #{sensor_js}
+            #{params[:context_js]}
+            #{params[:sensor_js]}
 
             if (typeof globalThis.__simulateHumanInteractions === "function") {
               globalThis.__simulateHumanInteractions();
@@ -57,63 +117,61 @@ module HttpMimic
               sensor_posts: globalThis.__sensor_posts
             });
           JS
+        end
 
-          result = JSRuntime.eval_json(driver_script)
-          sensor_posts = result && result['sensor_posts']
-          return nil if sensor_posts.nil? || sensor_posts.empty?
-
-          # 3. Post sensor data to Akamai endpoint
-          updated_cookies = cookies.dup
+        def post_sensor_data(sensor_script_url, target_url, impersonate, updated_cookies, sensor_posts, proxy: nil)
+          origin_url = URI.parse(target_url).tap { |u| u.path = ''; u.query = nil }.to_s rescue target_url
           sensor_posts.each do |post|
             post_body = post['body']
             next if post_body.nil? || post_body.empty?
 
+            post_url = if post['url'].to_s.empty?
+              target_url
+            elsif post['url'].start_with?('http')
+              post['url']
+            else
+              URI.join(target_url, post['url']).to_s
+            end
+
+            content_type = (post['headers'] && (post['headers']['Content-Type'] || post['headers']['content-type'])) || 'application/json'
+
             post_resp = HttpMimic.post(
-              sensor_script_url,
+              post_url,
               impersonate: impersonate,
               cookies: updated_cookies,
               headers: {
-                'Content-Type' => 'text/plain;charset=UTF-8',
+                'Content-Type' => content_type,
                 'Referer' => target_url,
-                'Origin' => URI.parse(target_url).tap { |u| u.path = ''; u.query = nil }.to_s,
+                'Origin' => origin_url,
                 'Accept' => '*/*'
               },
+              proxy: proxy,
               body: post_body,
               auto_fallback: false,
               solve_waf: false
             )
 
-            post_resp.cookies.each { |k, v| updated_cookies[k] = v }
+            post_resp.cookies.each { |k, v| updated_cookies[k] = v } if post_resp&.cookies
           end
+        end
 
-          # 4. Retry original target URL with the updated cookies
-          retry_headers = (options[:headers] || {}).merge('Referer' => target_url)
-          HttpMimic.get(
-            target_url,
-            options.merge(
-              cookies: updated_cookies,
-              headers: retry_headers,
-              auto_fallback: false,
-              solve_waf: false
-            )
-          )
-        rescue StandardError => e
-          if HttpMimic.configuration.debug
-            puts "[HttpMimic::Waf::AkamaiSolver] Error solving Akamai challenge: #{e.message}"
-          end
-          nil
+        def verified_abck?(cookies)
+          return false unless cookies
+          abck = cookies['_abck'] || (cookies.respond_to?(:key?) && cookies['_abck'])
+          abck.to_s.include?('~0~')
         end
 
         def extract_sensor_script_url(target_url, html)
           return nil if html.nil? || html.empty?
 
-          match = html.match(/<script[^>]*src=["\x27]([^"\x27]*\/[a-zA-Z0-9_-]{10,}\?[a-zA-Z0-9_=-]+)["\x27]/i)
+          match = html.match(/<script[^>]*src=["\x27]([^"\x27]*\/[a-zA-Z0-9_\-\/]+\?[^"\x27\s>]+)["\x27]/i)
           match ||= html.match(/<script[^>]*src=["\x27]([^"\x27]*akam[^"\x27]*)["\x27]/i)
-          match ||= html.match(/<script[^>]*src=["\x27]([^"\x27]*\/[a-zA-Z0-9_\-\/]+\?v=[a-zA-Z0-9_-]+)["\x27]/i)
+          match ||= html.match(/<script[^>]*src=["\x27]([^"\x27]*\/[a-zA-Z0-9_-]{10,}\?[a-zA-Z0-9_=-]+)["\x27]/i)
 
           return nil unless match
 
-          URI.join(target_url, match[1]).to_s
+          raw_url = match[1].gsub('&amp;', '&')
+          URI.join(target_url, raw_url).to_s
         rescue StandardError
           nil
         end

data/lib/http_mimic/waf/detector.rb

@@ -9,6 +9,8 @@ module HttpMimic
 
           if akamai?(response)
             :akamai
+          elsif google?(response)
+            :google
           elsif cloudflare?(response)
             :cloudflare
           elsif datadome?(response)
@@ -24,6 +26,16 @@ module HttpMimic
           return true if body.include?('sec-if-cpt-container') || body.include?('sec-bc-button-parent')
           return true if body.include?('challenges.cloudflare.com') || body.include?('cf-turnstile')
           return true if body.include?('datadome.captcha') || body.include?('geo.captcha-delivery.com')
+          return true if body.include?('/httpservice/retry/enablejs') || (body.include?('knitsail') && body.include?('SG_SS'))
+          false
+        end
+
+        def google?(response)
+          return false unless response
+          body = response.body.to_s
+          return true if body.include?('/httpservice/retry/enablejs')
+          return true if body.include?('knitsail') && body.include?('SG_SS')
+          return true if response.headers['server']&.downcase&.include?('gws') && body.include?('enablejs')
           false
         end
 

data/lib/http_mimic/waf/google_solver.rb

@@ -0,0 +1,124 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'uri'
+
+module HttpMimic
+  module Waf
+    class GoogleSolver
+      class << self
+        def solve(target_url, response, options = {})
+          return nil unless response
+
+          html = response.body.to_s
+          return nil if html.empty?
+
+          uri = URI.parse(target_url) rescue nil
+          base_url = uri ? "#{uri.scheme}://#{uri.host}" : "https://www.google.com"
+
+          impersonate = options[:impersonate] || HttpMimic.configuration.default_impersonate || 'chrome131'
+          current_cookies = (response.cookies || Cookies.new).dup
+
+          # Strategy 1: Session warmup from Google root if cookies are sparse
+          if current_cookies.empty? || !current_cookies.key?('NID')
+            warmup_resp = HttpMimic.get(
+              "#{base_url}/",
+              impersonate: impersonate,
+              cookies: current_cookies,
+              auto_fallback: false,
+              solve_waf: false,
+              persist_cookies: false
+            )
+            warmup_resp.cookies.each { |k, v| current_cookies[k] = v } if warmup_resp&.cookies
+          end
+
+          # Strategy 2: Attempt QuickJS solve if script payload exists
+          scripts = html.scan(/<script[^>]*>([\s\S]*?)<\/script>/i).map(&:first)
+          if scripts.any? { |s| s.include?('knitsail') || s.include?('SG_SS') || s.include?('closureDynamicButton') }
+            sg_ss_cookie = solve_with_quickjs(target_url, scripts, current_cookies)
+            current_cookies['SG_SS'] = sg_ss_cookie if sg_ss_cookie && !sg_ss_cookie.start_with?('E:')
+          end
+
+          # Strategy 3: Retry search query with same-origin referral headers and updated cookies
+          retry_headers = (options[:headers] || {}).dup
+          retry_headers['Referer'] ||= "#{base_url}/"
+          retry_headers['sec-fetch-site'] ||= 'same-origin'
+
+          retry_opts = options.merge(
+            cookies: current_cookies,
+            headers: retry_headers,
+            auto_fallback: false,
+            solve_waf: false
+          )
+
+          HttpMimic.get(target_url, **retry_opts)
+        rescue StandardError => e
+          HttpMimic.logger.debug("[HttpMimic::Waf::GoogleSolver] Failed to solve Google challenge: #{e.message}") if HttpMimic.logger
+          nil
+        end
+
+        private
+
+        def solve_with_quickjs(target_url, scripts, cookies)
+          context_js_path = File.expand_path('browser_context.js', __dir__)
+          context_js = File.read(context_js_path)
+          cookie_str = cookies.respond_to?(:to_cookie_string) ? cookies.to_cookie_string : cookies.to_s
+
+          driver_script = <<~JS
+            globalThis.__TARGET_URL__ = #{target_url.to_json};
+            globalThis.__DOCUMENT_TITLE__ = "Google Search";
+            globalThis.__INITIAL_COOKIES__ = #{cookie_str.to_json};
+            globalThis.__REFERRER__ = "https://www.google.com/";
+
+            #{context_js}
+
+            // Google specific environment enhancements
+            globalThis.sessionStorage = {
+              _data: {},
+              getItem(k) { return this._data[k] || null; },
+              setItem(k, v) { this._data[k] = String(v); },
+              removeItem(k) { delete this._data[k]; },
+              clear() { this._data = {}; }
+            };
+            globalThis.localStorage = globalThis.sessionStorage;
+            globalThis._F_css = function() {};
+            globalThis.google = { c: { c: { a: true } } };
+
+            #{scripts.join("\n;\n")}
+
+            // Extract p token and invoke knitsail if present
+            var allScripts = #{scripts.join("\n").to_json};
+            var pMatch = allScripts.match(/var p=\\x27([^\\x27]+)\\x27/);
+            var pVal = pMatch ? pMatch[1] : null;
+
+            var solved_token = null;
+            if (globalThis.knitsail && typeof globalThis.knitsail.a === "function" && pVal) {
+              try {
+                globalThis.knitsail.a(pVal, function(resultCallback) {
+                  if (typeof resultCallback === "function") {
+                    resultCallback(function(token) {
+                      solved_token = token;
+                    }, [{}]);
+                  }
+                }, false, undefined, undefined, undefined, undefined, true);
+              } catch(e) {}
+            }
+
+            if (typeof globalThis.__drainEventLoop === "function") {
+              globalThis.__drainEventLoop(50);
+            }
+
+            JSON.stringify({
+              token: solved_token
+            });
+          JS
+
+          result = JSRuntime.eval_json(driver_script)
+          result ? result['token'] : nil
+        rescue StandardError
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/http_mimic/waf.rb

@@ -2,6 +2,7 @@
 
 require 'http_mimic/waf/detector'
 require 'http_mimic/waf/akamai_solver'
+require 'http_mimic/waf/google_solver'
 
 module HttpMimic
   module Waf
@@ -13,6 +14,8 @@ module HttpMimic
         case waf_type
         when :akamai
           AkamaiSolver.solve(url, response, options)
+        when :google
+          GoogleSolver.solve(url, response, options)
         else
           nil
         end

data/lib/http_mimic.rb

@@ -19,6 +19,9 @@ require 'http_mimic/module_methods'
 require 'http_mimic/js_runtime'
 require 'http_mimic/waf'
 require 'http_mimic/cookie_store'
+require 'http_mimic/obscura'
+require 'http_mimic/spa_detector'
+require 'http_mimic/proxy_pool'
 
 module HttpMimic
   extend ModuleMethods
@@ -94,6 +97,30 @@ module HttpMimic
       CookieStore.clear(host)
     end
 
+    # Obscura headless SPA engine helpers
+    def download_obscura!(version: nil, force: false)
+      Downloader.download_obscura!(version: version, force: force)
+    end
+
+    def obscura_installed?(version: nil)
+      Downloader.obscura_installed?(version: version)
+    end
+
+    def obscura_path
+      Downloader.obscura_path
+    end
+
+    # Render SPA page using Obscura headless engine
+    def render(url, options = {})
+      Obscura.render(url, options)
+    end
+    alias spa render
+
+    # Check if a response represents an unhydrated SPA shell
+    def spa?(response)
+      SpaDetector.spa?(response)
+    end
+
     def included(base)
       base.extend(ModuleMethods)
       base.send(:include, InstanceMethods)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: http_mimic
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.3
 platform: ruby
 authors:
 - anxgang
@@ -72,13 +72,17 @@ files:
 - lib/http_mimic/headers.rb
 - lib/http_mimic/js_runtime.rb
 - lib/http_mimic/module_methods.rb
+- lib/http_mimic/obscura.rb
+- lib/http_mimic/proxy_pool.rb
 - lib/http_mimic/request.rb
 - lib/http_mimic/response.rb
 - lib/http_mimic/response_parser.rb
+- lib/http_mimic/spa_detector.rb
 - lib/http_mimic/version.rb
 - lib/http_mimic/waf.rb
 - lib/http_mimic/waf/akamai_solver.rb
 - lib/http_mimic/waf/detector.rb
+- lib/http_mimic/waf/google_solver.rb
 homepage: https://github.com/anxgang/http_mimic
 licenses:
 - MIT