http_mimic 0.3.1 → 0.4.0

data/lib/http_mimic/waf/akamai_solver.rb

@@ -0,0 +1,123 @@
+# frozen_string_literal: true
+
+require 'uri'
+require 'json'
+
+module HttpMimic
+  module Waf
+    class AkamaiSolver
+      CONTEXT_JS_PATH = File.expand_path('browser_context.js', __dir__)
+
+      class << self
+        def solve(target_url, initial_response, options = {})
+          cookies = initial_response.cookies.dup
+          impersonate = options[:impersonate] || HttpMimic.configuration.default_impersonate || 'chrome131'
+
+          sensor_script_url = extract_sensor_script_url(target_url, initial_response.body)
+          return nil unless sensor_script_url
+
+          # 1. Fetch the dynamic sensor script
+          script_resp = HttpMimic.get(
+            sensor_script_url,
+            impersonate: impersonate,
+            cookies: cookies,
+            headers: { 'Referer' => target_url },
+            auto_fallback: false,
+            solve_waf: false
+          )
+          return nil unless script_resp.success?
+
+          sensor_js = script_resp.body
+          context_js = File.read(CONTEXT_JS_PATH)
+          cookie_str = cookies.to_cookie_string
+
+          doc_title = initial_response.title.to_s
+          user_agent = options[:user_agent]
+          referer = (options[:headers] || {})['Referer'] || (options[:headers] || {})['referer'] || target_url
+
+          # 2. Run virtual browser simulation inside QuickJS
+          driver_script = <<~JS
+            globalThis.__TARGET_URL__ = #{target_url.to_json};
+            globalThis.__DOCUMENT_TITLE__ = #{doc_title.to_json};
+            globalThis.__INITIAL_COOKIES__ = #{cookie_str.to_json};
+            globalThis.__REFERRER__ = #{referer.to_json};
+            #{user_agent ? "globalThis.__USER_AGENT__ = #{user_agent.to_json};" : ""}
+
+            #{context_js}
+            #{sensor_js}
+
+            if (typeof globalThis.__simulateHumanInteractions === "function") {
+              globalThis.__simulateHumanInteractions();
+            }
+            if (typeof globalThis.__drainEventLoop === "function") {
+              globalThis.__drainEventLoop(100);
+            }
+
+            JSON.stringify({
+              sensor_posts: globalThis.__sensor_posts
+            });
+          JS
+
+          result = JSRuntime.eval_json(driver_script)
+          sensor_posts = result && result['sensor_posts']
+          return nil if sensor_posts.nil? || sensor_posts.empty?
+
+          # 3. Post sensor data to Akamai endpoint
+          updated_cookies = cookies.dup
+          sensor_posts.each do |post|
+            post_body = post['body']
+            next if post_body.nil? || post_body.empty?
+
+            post_resp = HttpMimic.post(
+              sensor_script_url,
+              impersonate: impersonate,
+              cookies: updated_cookies,
+              headers: {
+                'Content-Type' => 'text/plain;charset=UTF-8',
+                'Referer' => target_url,
+                'Origin' => URI.parse(target_url).tap { |u| u.path = ''; u.query = nil }.to_s,
+                'Accept' => '*/*'
+              },
+              body: post_body,
+              auto_fallback: false,
+              solve_waf: false
+            )
+
+            post_resp.cookies.each { |k, v| updated_cookies[k] = v }
+          end
+
+          # 4. Retry original target URL with the updated cookies
+          retry_headers = (options[:headers] || {}).merge('Referer' => target_url)
+          HttpMimic.get(
+            target_url,
+            options.merge(
+              cookies: updated_cookies,
+              headers: retry_headers,
+              auto_fallback: false,
+              solve_waf: false
+            )
+          )
+        rescue StandardError => e
+          if HttpMimic.configuration.debug
+            puts "[HttpMimic::Waf::AkamaiSolver] Error solving Akamai challenge: #{e.message}"
+          end
+          nil
+        end
+
+        def extract_sensor_script_url(target_url, html)
+          return nil if html.nil? || html.empty?
+
+          match = html.match(/<script[^>]*src=["\x27]([^"\x27]*\/[a-zA-Z0-9_-]{10,}\?[a-zA-Z0-9_=-]+)["\x27]/i)
+          match ||= html.match(/<script[^>]*src=["\x27]([^"\x27]*akam[^"\x27]*)["\x27]/i)
+          match ||= html.match(/<script[^>]*src=["\x27]([^"\x27]*\/[a-zA-Z0-9_\-\/]+\?v=[a-zA-Z0-9_-]+)["\x27]/i)
+
+          return nil unless match
+
+          URI.join(target_url, match[1]).to_s
+        rescue StandardError
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/http_mimic/waf/detector.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module HttpMimic
+  module Waf
+    class Detector
+      class << self
+        def detect(response)
+          return nil unless response
+
+          if akamai?(response)
+            :akamai
+          elsif cloudflare?(response)
+            :cloudflare
+          elsif datadome?(response)
+            :datadome
+          else
+            nil
+          end
+        end
+
+        def challenge_page?(response)
+          return false unless response
+          body = response.body.to_s
+          return true if body.include?('sec-if-cpt-container') || body.include?('sec-bc-button-parent')
+          return true if body.include?('challenges.cloudflare.com') || body.include?('cf-turnstile')
+          return true if body.include?('datadome.captcha') || body.include?('geo.captcha-delivery.com')
+          false
+        end
+
+        def akamai?(response)
+          return true if response.headers['set-cookie']&.include?('_abck=')
+          return true if response.cookies.key?('_abck') || response.cookies.key?('bm_sz') || response.cookies.key?('ak_bmsc')
+          return true if response.headers['x-akamai-transformed'] || response.headers['x-reference-error']
+
+          body = response.body.to_s
+          return true if body.include?('Reference Error:') && body.include?('Akamai')
+          return true if body.include?('bmak') || body.include?('sensor_data')
+          return true if body.include?('sec-if-cpt-container') || body.include?('sec-bc-button-parent')
+          return true if body =~ /<script[^>]*src=["\x27]([^"\x27]*\/[a-zA-Z0-9_-]{10,}\?[a-zA-Z0-9_=-]+)["\x27]/
+
+          false
+        end
+
+        def cloudflare?(response)
+          return true if response.headers['server']&.downcase&.include?('cloudflare')
+          return true if response.headers['cf-ray'] || response.cookies.key?('cf_clearance')
+          return true if response.body.to_s.include?('challenges.cloudflare.com')
+
+          false
+        end
+
+        def datadome?(response)
+          return true if response.headers['x-datadome'] || response.cookies.key?('datadome')
+          return true if response.headers['server']&.downcase&.include?('datadome')
+
+          false
+        end
+      end
+    end
+  end
+end

data/lib/http_mimic/waf.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require 'http_mimic/waf/detector'
+require 'http_mimic/waf/akamai_solver'
+
+module HttpMimic
+  module Waf
+    class << self
+      def solve(url, response, options = {})
+        waf_type = Detector.detect(response)
+        return nil unless waf_type
+
+        case waf_type
+        when :akamai
+          AkamaiSolver.solve(url, response, options)
+        else
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/http_mimic.rb

@@ -16,6 +16,9 @@ require 'http_mimic/command_builder'
 require 'http_mimic/request'
 require 'http_mimic/client'
 require 'http_mimic/module_methods'
+require 'http_mimic/js_runtime'
+require 'http_mimic/waf'
+require 'http_mimic/cookie_store'
 
 module HttpMimic
   extend ModuleMethods
@@ -43,6 +46,54 @@ module HttpMimic
       Downloader.installed?(version: version)
     end
 
+    # Manually download and install QuickJS binary driver
+    def download_qjs!(version: nil, force: false)
+      Downloader.download_qjs!(version: version, force: force)
+    end
+
+    # Check if QuickJS driver is installed locally
+    def qjs_installed?(version: nil)
+      Downloader.qjs_installed?(version: version)
+    end
+
+    # Return local path to QuickJS binary
+    def qjs_path
+      Downloader.qjs_path
+    end
+
+    # Evaluate JavaScript code using QuickJS
+    def eval_js(code, options = {})
+      JSRuntime.eval(code, timeout: options[:timeout], install_dir: options[:install_dir], auto_download: options[:auto_download])
+    end
+
+    # Evaluate JavaScript and auto-parse JSON result
+    def eval_js_json(code, options = {})
+      JSRuntime.eval_json(code, timeout: options[:timeout], install_dir: options[:install_dir], auto_download: options[:auto_download])
+    end
+
+    # Automatically detect WAF type from response
+    def detect_waf(response)
+      Waf::Detector.detect(response)
+    end
+
+    # Attempt to solve WAF challenge for a blocked response
+    def solve_waf(url, response, options = {})
+      Waf.solve(url, response, options)
+    end
+
+    # Persistent Host CookieStore helpers
+    def load_cookies(host, max_age: nil)
+      CookieStore.load(host, max_age: max_age)
+    end
+
+    def save_cookies(host, cookies, ttl: nil)
+      CookieStore.save(host, cookies, ttl: ttl || CookieStore::DEFAULT_TTL)
+    end
+
+    def clear_cookies!(host = nil)
+      CookieStore.clear(host)
+    end
+
     def included(base)
       base.extend(ModuleMethods)
       base.send(:include, InstanceMethods)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: http_mimic
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.4.0
 platform: ruby
 authors:
 - anxgang
@@ -65,15 +65,20 @@ files:
 - lib/http_mimic/client.rb
 - lib/http_mimic/command_builder.rb
 - lib/http_mimic/configuration.rb
+- lib/http_mimic/cookie_store.rb
 - lib/http_mimic/cookies.rb
 - lib/http_mimic/downloader.rb
 - lib/http_mimic/exceptions.rb
 - lib/http_mimic/headers.rb
+- lib/http_mimic/js_runtime.rb
 - lib/http_mimic/module_methods.rb
 - lib/http_mimic/request.rb
 - lib/http_mimic/response.rb
 - lib/http_mimic/response_parser.rb
 - lib/http_mimic/version.rb
+- lib/http_mimic/waf.rb
+- lib/http_mimic/waf/akamai_solver.rb
+- lib/http_mimic/waf/detector.rb
 homepage: https://github.com/anxgang/http_mimic
 licenses:
 - MIT