http_mimic 0.5.3 → 0.5.5

data/lib/http_mimic/waf/akamai_solver.rb

@@ -11,29 +11,34 @@ module HttpMimic
       class << self
         def solve(target_url, initial_response, options = {})
           cookies = initial_response.cookies.dup
-          impersonate = options[:impersonate] || HttpMimic.configuration.default_impersonate || 'chrome131'
-
-          sensor_script_url = extract_sensor_script_url(target_url, initial_response.body)
-          return nil unless sensor_script_url
-
-          # 1. Fetch the dynamic sensor script
-          script_resp = HttpMimic.get(
-            sensor_script_url,
-            impersonate: impersonate,
-            cookies: cookies,
-            headers: { 'Referer' => target_url },
-            proxy: options[:proxy],
-            auto_fallback: false,
-            solve_waf: false
-          )
-          return nil unless script_resp.success?
+          profile = options[:profile]
+          impersonate = resolve_impersonate_target(profile, options[:impersonate])
+          user_agent = resolve_user_agent(impersonate, options[:user_agent])
 
-          sensor_js = script_resp.body
-          context_js = File.read(CONTEXT_JS_PATH)
+          sensor_script_urls = extract_sensor_script_urls(target_url, initial_response.body)
+          return nil if sensor_script_urls.empty?
 
+          # 1. Fetch all dynamic sensor scripts in order
+          scripts_data = []
+          sensor_script_urls.each do |s_url|
+            s_resp = HttpMimic.get(
+              s_url,
+              impersonate: impersonate,
+              cookies: cookies,
+              headers: { 'Referer' => target_url, 'User-Agent' => user_agent },
+              proxy: options[:proxy],
+              auto_fallback: false,
+              solve_waf: false,
+              debug: options[:debug]
+            )
+            scripts_data << { url: s_url, body: s_resp.body } if s_resp.success?
+          end
+          return nil if scripts_data.empty?
+
+          context_js = File.read(CONTEXT_JS_PATH)
           doc_title = initial_response.title.to_s
-          user_agent = options[:user_agent]
           referer = (options[:headers] || {})['Referer'] || (options[:headers] || {})['referer'] || target_url
+          cpt_script_url = sensor_script_urls.find { |u| u.include?('t=') } || sensor_script_urls.first
 
           # 2. Phase 1: Virtual browser simulation inside QuickJS (initial telemetry)
           driver_script_1 = build_driver_script(
@@ -43,7 +48,7 @@ module HttpMimic
             referer: referer,
             user_agent: user_agent,
             context_js: context_js,
-            sensor_js: sensor_js,
+            scripts_data: scripts_data,
             phase: 1
           )
 
@@ -53,7 +58,7 @@ module HttpMimic
 
           # 3. Post Phase 1 sensor data to Akamai endpoint
           updated_cookies = cookies.dup
-          post_sensor_data(sensor_script_url, target_url, impersonate, updated_cookies, sensor_posts_1, proxy: options[:proxy])
+          post_sensor_data(cpt_script_url, target_url, impersonate, updated_cookies, sensor_posts_1, user_agent: user_agent, proxy: options[:proxy], debug: options[:debug], cpt_url: cpt_script_url)
 
           # 4. Phase 2: If _abck is not verified yet (~0~), perform second round after short pause
           if !verified_abck?(updated_cookies)
@@ -65,24 +70,27 @@ module HttpMimic
               referer: referer,
               user_agent: user_agent,
               context_js: context_js,
-              sensor_js: sensor_js,
+              scripts_data: scripts_data,
               phase: 2
             )
 
             result2 = JSRuntime.eval_json(driver_script_2)
             sensor_posts_2 = result2 && result2['sensor_posts']
             if sensor_posts_2 && !sensor_posts_2.empty?
-              post_sensor_data(sensor_script_url, target_url, impersonate, updated_cookies, sensor_posts_2, proxy: options[:proxy])
+              post_sensor_data(cpt_script_url, target_url, impersonate, updated_cookies, sensor_posts_2, user_agent: user_agent, proxy: options[:proxy], debug: options[:debug], cpt_url: cpt_script_url)
             end
           end
 
           # 5. Retry original target URL with the updated cookies
-          retry_headers = (options[:headers] || {}).merge('Referer' => target_url)
+          retry_headers = (options[:headers] || {}).merge('Referer' => target_url, 'User-Agent' => user_agent)
           HttpMimic.get(
             target_url,
             options.merge(
+              profile: profile,
+              impersonate: impersonate,
               cookies: updated_cookies,
               headers: retry_headers,
+              user_agent: user_agent,
               auto_fallback: false,
               solve_waf: false
             )
@@ -94,7 +102,65 @@ module HttpMimic
           nil
         end
 
+        def resolve_impersonate_target(profile, explicit_imp = nil)
+          return explicit_imp.to_s if explicit_imp && !explicit_imp.empty?
+
+          case profile&.to_sym
+          when :android, :mobile
+            'chrome131_android'
+          when :ios
+            'safari260_ios'
+          when :firefox
+            'firefox135'
+          when :safari
+            'safari180'
+          else
+            HttpMimic.configuration.default_impersonate || 'chrome131'
+          end
+        end
+
+        def resolve_user_agent(impersonate, explicit_ua = nil)
+          return explicit_ua if explicit_ua && !explicit_ua.empty?
+
+          imp = impersonate.to_s.downcase
+          if imp.include?('android')
+            ver = imp[/\d+/] || '131'
+            "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/#{ver}.0.0.0 Mobile Safari/537.36"
+          elsif imp.include?('ios') || imp.include?('iphone') || imp.include?('ipad')
+            "Mozilla/5.0 (iPhone; CPU iPhone OS 18_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.0 Mobile/15E148 Safari/604.1"
+          elsif imp.include?('chrome')
+            ver = imp[/\d+/] || '150'
+            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/#{ver}.0.0.0 Safari/537.36"
+          elsif imp.include?('firefox')
+            ver = imp[/\d+/] || '135'
+            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:#{ver}.0) Gecko/20100101 Firefox/#{ver}.0"
+          elsif imp.include?('safari')
+            ver = imp[/\d+/] || '18_0'
+            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/#{ver.tr('_', '.')} Safari/605.1.15"
+          else
+            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/150.0.0.0 Safari/537.36"
+          end
+        end
+
         def build_driver_script(params)
+          scripts_code = if params[:scripts_data] && !params[:scripts_data].empty?
+            params[:scripts_data].map.with_index do |s, idx|
+              tag_var = "sTag_#{idx}"
+              <<~SCRIPT_EVAL
+                const #{tag_var} = document.createElement('script');
+                #{tag_var}.src = #{s[:url].to_json};
+                document.body.appendChild(#{tag_var});
+                document.currentScript = #{tag_var};
+                try {
+                  ;\n#{s[:body]}\n;
+                } catch(e) {}
+              SCRIPT_EVAL
+            end.join("\n")
+          else
+            ";\n#{params[:sensor_js]}\n;"
+          end
+
+
           <<~JS
             globalThis.__TARGET_URL__ = #{params[:target_url].to_json};
             globalThis.__DOCUMENT_TITLE__ = #{params[:doc_title].to_json};
@@ -104,7 +170,9 @@ module HttpMimic
             globalThis.__TELEMETRY_PHASE__ = #{params[:phase] || 1};
 
             #{params[:context_js]}
-            #{params[:sensor_js]}
+            ;
+
+            #{scripts_code}
 
             if (typeof globalThis.__simulateHumanInteractions === "function") {
               globalThis.__simulateHumanInteractions();
@@ -119,37 +187,71 @@ module HttpMimic
           JS
         end
 
-        def post_sensor_data(sensor_script_url, target_url, impersonate, updated_cookies, sensor_posts, proxy: nil)
+        def post_sensor_data(sensor_script_url, target_url, impersonate, updated_cookies, sensor_posts, user_agent: nil, proxy: nil, debug: nil, cpt_url: nil)
           origin_url = URI.parse(target_url).tap { |u| u.path = ''; u.query = nil }.to_s rescue target_url
           sensor_posts.each do |post|
             post_body = post['body']
-            next if post_body.nil? || post_body.empty?
+            method = (post['method'] || 'POST').to_s.upcase
+            next if method == 'POST' && (post_body.nil? || post_body.empty?)
 
             post_url = if post['url'].to_s.empty?
-              target_url
+              cpt_url || target_url
             elsif post['url'].start_with?('http')
-              post['url']
+              u = post['url']
+              if cpt_url && !u.include?('t=') && cpt_url.include?('t=')
+                q = URI.parse(cpt_url).query rescue nil
+                u = "#{u}?#{q}" if q && !q.empty?
+              end
+              u
             else
               URI.join(target_url, post['url']).to_s
             end
 
+
             content_type = (post['headers'] && (post['headers']['Content-Type'] || post['headers']['content-type'])) || 'application/json'
 
-            post_resp = HttpMimic.post(
-              post_url,
-              impersonate: impersonate,
-              cookies: updated_cookies,
-              headers: {
-                'Content-Type' => content_type,
-                'Referer' => target_url,
-                'Origin' => origin_url,
-                'Accept' => '*/*'
-              },
-              proxy: proxy,
-              body: post_body,
-              auto_fallback: false,
-              solve_waf: false
-            )
+            post_headers = {
+              'Referer' => target_url,
+              'Origin' => origin_url,
+              'Accept' => '*/*',
+              'Sec-Fetch-Dest' => 'empty',
+              'Sec-Fetch-Mode' => 'cors',
+              'Sec-Fetch-Site' => 'same-origin'
+            }
+            if user_agent.to_s.include?('Chrome')
+              is_mobile = user_agent.to_s.include?('Mobile') || user_agent.to_s.include?('Android')
+              ver = user_agent[/Chrome\/(\d+)/, 1] || '131'
+              post_headers['sec-ch-ua'] = %("Google Chrome";v="#{ver}", "Chromium";v="#{ver}", "Not_A Brand";v="24")
+              post_headers['sec-ch-ua-mobile'] = is_mobile ? '?1' : '?0'
+              post_headers['sec-ch-ua-platform'] = is_mobile ? '"Android"' : '"macOS"'
+            end
+            post_headers['Content-Type'] = content_type if method == 'POST'
+            post_headers['User-Agent'] = user_agent if user_agent
+
+            post_resp = if method == 'GET'
+              HttpMimic.get(
+                post_url,
+                impersonate: impersonate,
+                cookies: updated_cookies,
+                headers: post_headers,
+                proxy: proxy,
+                auto_fallback: false,
+                solve_waf: false,
+                debug: debug
+              )
+            else
+              HttpMimic.post(
+                post_url,
+                impersonate: impersonate,
+                cookies: updated_cookies,
+                headers: post_headers,
+                proxy: proxy,
+                body: post_body,
+                auto_fallback: false,
+                solve_waf: false,
+                debug: debug
+              )
+            end
 
             post_resp.cookies.each { |k, v| updated_cookies[k] = v } if post_resp&.cookies
           end
@@ -158,7 +260,12 @@ module HttpMimic
         def verified_abck?(cookies)
           return false unless cookies
           abck = cookies['_abck'] || (cookies.respond_to?(:key?) && cookies['_abck'])
-          abck.to_s.include?('~0~')
+          return true if abck.to_s.include?('~0~')
+
+          bm_sc = cookies['bm_sc'] || (cookies.respond_to?(:key?) && cookies['bm_sc'])
+          return true if bm_sc.to_s.include?('~0~0~0') && !bm_sc.to_s.start_with?('3~')
+
+          false
         end
 
         def extract_sensor_script_url(target_url, html)
@@ -175,6 +282,26 @@ module HttpMimic
         rescue StandardError
           nil
         end
+
+        def extract_sensor_script_urls(target_url, html)
+          return [] if html.nil? || html.empty?
+
+          matches = html.scan(/<script[^>]*src=["\x27]([^"\x27]+)["\x27]/i).flatten
+          akamai_scripts = matches.select do |src|
+            src =~ /\/GPv76/i || src =~ /akam/i || src =~ /\/[a-zA-Z0-9_-]{10,}\?[a-zA-Z0-9_=-]+/i || src =~ /_bm\//i
+          end
+
+          if akamai_scripts.empty?
+            single = extract_sensor_script_url(target_url, html)
+            return single ? [single] : []
+          end
+
+          akamai_scripts.map do |raw_src|
+            URI.join(target_url, raw_src.gsub('&amp;', '&')).to_s
+          rescue StandardError
+            nil
+          end.compact
+        end
       end
     end
   end