http_mimic 0.3.0 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 32ae2902090ddf216e6f33843f1fb53d82d13cda864a3a71fd7318f5f316cdf4
-  data.tar.gz: 907aec51da124505902e104ebf3bce70eb2883490ec8b360f3f0571e6efd98c2
+  metadata.gz: 0b09482a3e618dd6390934e85d83de762cffaa062d5610d606d77bdabfb0ff11
+  data.tar.gz: 55c4dfeee202c24898a1ad5a12bc5a58df9c5a1e0481b7d08be0fc0c4a909e32
 SHA512:
-  metadata.gz: e47bf2078c03dc6a37ecb3478ee5f7a42724375161c63175067fd6df5c8c76fae90b04867eb6f4b8fe03ab2ad2d919ef64dcfd1389ab20befdcb9e2ff245d492
-  data.tar.gz: 31c4ebd49e1180914bd949dc23bbc4bd7580dea84f898c7a97bc4977a8501120b1d3d5e6301b65ef2bf7cfa9e5c9bbecd8a7f47806f0a8cf42e1e1b4e7b2e45c
+  metadata.gz: 1b4a75f41696e4b6a89c73096640ce5f9b3963dff9262aea51ead8271022868397e7c413edfc724cd7c0cbb439a2663bf55f5ea28cd060e491cff0445e88fea7
+  data.tar.gz: cf2f82e990474a02e00aebe5d5b0b0ca019a994266279cd84f7fd4dd966e9260b488e1915f2b7eccdf7452e50a838489d4709181ec195bebbc773e6aff62d6d8

data/CHANGELOG.md

@@ -5,6 +5,15 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.3.1] - 2026-08-25
+
+### Added
+- **Smart Adaptive Multi-Strategy Modes (`:auto`, `:impersonate_first`, `:curl_first`, `:impersonate_only`, `:curl_only`)**:
+  - **Zero-Configuration Protection Bypass (`:auto` mode, enabled by default)**: Seamlessly tries `curl-impersonate` first, and if blocked by WAFs with `403`/`429`/`503` (e.g. Akamai bot challenges requiring JS), automatically falls back to standard `curl` with server client headers to reliably retrieve 200 OK.
+  - Per-request and class-level configurable execution mode (`mode:`, `auto_fallback:`, `retry_statuses:`).
+  - Enhanced `Response` object with `response.mode_used`, `response.fallback_triggered?`, and `response.attempts` metadata.
+  - Added live integration test for Akamai-protected site bypass (`test_smart_auto_fallback_on_akamai_protected_site`).
+
 ## [0.3.0] - 2026-08-24
 
 ### Added

data/README.md

@@ -142,6 +142,7 @@ puts "User-Agent: #{res['user_agent']}"
 ```ruby
 client = HttpMimic::Client.new(
   base_uri: 'https://api.example.com',
+  mode: :auto,
   impersonate: 'firefox135',
   timeout: 15,
   headers: {
@@ -158,12 +159,41 @@ response = client.post('/v1/users', json: { username: 'bob' })
 
 ---
 
+## 🧠 Smart Adaptive Modes (Zero-Configuration Scraping)
+
+`HttpMimic` provides built-in multi-strategy orchestration so you can fetch protected websites without worrying about which specific WAF (Cloudflare, Akamai, DataDome) protects them:
+
+- **`:auto` (Default & Recommended)**: Tries `curl-impersonate` (Chrome 131) first. If blocked by WAFs like Akamai with `403`/`429`/`503` (which require JS telemetry for browsers but allow standard server clients), it **automatically and seamlessly retries with standard curl + server headers**, directly returning `200 OK`.
+- **`:impersonate_first`**: Prefers `curl-impersonate` and automatically falls back to standard `curl` if blocked.
+- **`:curl_first`**: Prefers standard `curl` and automatically upgrades to `curl-impersonate` if blocked.
+- **`:impersonate_only`**: Strictly uses `curl-impersonate` (no retry/fallback).
+- **`:curl_only`**: Strictly uses standard `curl` (no retry/fallback).
+
+```ruby
+# 1. No-Brain Auto Mode (Works automatically for both Cloudflare & Akamai):
+response = HttpMimic.get('https://www.asics.com/us/en-us/gt-2000-15/p/ANA_1011C235-750.html')
+puts response.code                 # => 200
+puts response.mode_used            # => :curl
+puts response.fallback_triggered?  # => true
+
+# 2. Per-request mode override:
+response = HttpMimic.get(url, mode: :curl_first)
+response = HttpMimic.get(url, mode: :impersonate_only)
+```
+
+---
+
 ## ⚙️ Global Configuration
 
 Configure global defaults in an initializer (e.g., `config/initializers/http_mimic.rb`):
 
 ```ruby
 HttpMimic.configure do |config|
+  # Multi-strategy & Smart Fallback
+  config.mode                    = :auto       # :auto (default), :impersonate_first, :curl_first, :impersonate_only, :curl_only
+  config.auto_fallback           = true        # Automatically retry with alternative profile if blocked
+  config.retry_statuses          = [403, 429, 503] # Status codes that trigger auto-fallback
+
   # Browser simulation & request defaults
   config.default_impersonate     = 'chrome131' # Default browser target
   config.default_timeout         = 30          # Request timeout (seconds)
@@ -188,6 +218,9 @@ end
 
 | Option | Type | Description |
 | :--- | :--- | :--- |
+| `:mode` | Symbol | Execution strategy: `:auto` (default), `:impersonate_first`, `:curl_first`, `:impersonate_only`, `:curl_only` |
+| `:auto_fallback` | Boolean | Whether to automatically retry with alternative profile on blocked status (default `true`) |
+| `:retry_statuses`| Array | Status codes that trigger auto-fallback (default `[403, 429, 503]`) |
 | `:impersonate` | String | Target browser to mimic (e.g., `'chrome131'`, `'chrome120'`, `'firefox135'`, `'safari180'`, `'tor145'`) |
 | `:binary` | String | Path to a custom `curl-impersonate` executable |
 | `:query` / `:params` | Hash | URL query parameters (supports nested parameters and encoding) |
@@ -234,10 +267,13 @@ response.headers['content-type'] # => Case-insensitive header access
 response.cookies['session_id']   # => Parsed Set-Cookie store
 response.history                 # => Array of redirect history metadata
 
-# Underlying execution details
-response.exit_code      # => Process exit status (0 for success)
-response.stderr         # => Stderr output from curl
-response.command        # => Array of the exact CLI arguments executed
+# Underlying execution & multi-strategy details
+response.mode_used            # => :impersonate or :curl
+response.fallback_triggered?  # => true if smart fallback was executed
+response.attempts             # => Array of execution metadata for each attempt
+response.exit_code            # => Process exit status (0 for success)
+response.stderr               # => Stderr output from curl
+response.command              # => Array of the exact CLI arguments executed
 ```
 
 ---

data/http_mimic.gemspec

@@ -14,7 +14,6 @@ Gem::Specification.new do |spec|
   spec.license       = "MIT"
 
   spec.metadata = {
-    "homepage_uri"    => "https://github.com/anxgang/http_mimic",
     "source_code_uri" => "https://github.com/anxgang/http_mimic",
     "bug_tracker_uri" => "https://github.com/anxgang/http_mimic/issues",
     "changelog_uri"   => "https://github.com/anxgang/http_mimic/blob/main/CHANGELOG.md"

data/lib/http_mimic/command_builder.rb

@@ -171,6 +171,15 @@ module HttpMimic
         args << '-A' << options[:user_agent].to_s
       end
 
+      # Profile-specific headers & HTTP version defaults (for server/curl mode)
+      if options[:profile] == :curl
+        args << '--http1.1' unless (options[:curl_options] || []).to_s.include?('--http')
+        headers_to_send['User-Agent'] ||= (options[:user_agent] || 'Ruby')
+        headers_to_send['Accept'] ||= '*/*'
+        headers_to_send['Accept-Encoding'] ||= 'gzip;q=1.0,deflate;q=0.6,identity;q=0.3'
+        headers_to_send['Connection'] ||= 'close'
+      end
+
       # Append headers
       headers_to_send.each do |k, v|
         if v.is_a?(Array)
@@ -194,6 +203,13 @@ module HttpMimic
     end
 
     def resolve_binary
+      # 0. If profile is explicitly set to :curl, use system curl
+      if options[:profile] == :curl
+        curl_path = find_executable('curl') || 'curl'
+        return curl_path if executable?(curl_path)
+        raise BinaryNotFoundError, "System curl executable was not found."
+      end
+
       # 1. Use explicit binary_path from options or config if provided
       explicit = options[:binary] || config.binary_path
       if explicit

data/lib/http_mimic/configuration.rb

@@ -13,6 +13,11 @@ module HttpMimic
     attr_accessor :logger
     attr_accessor :debug
 
+    # Multi-strategy and smart auto-fallback settings
+    attr_accessor :mode
+    attr_accessor :auto_fallback
+    attr_accessor :retry_statuses
+
     # Webdrivers-like auto download and driver management settings
     attr_accessor :auto_download
     attr_accessor :driver_version
@@ -31,6 +36,11 @@ module HttpMimic
       @logger                   = nil
       @debug                    = false
 
+      # Multi-strategy defaults: :auto seamlessly falls back between impersonate and curl
+      @mode                     = :auto
+      @auto_fallback            = true
+      @retry_statuses           = [403, 429, 503]
+
       # Enable automatic downloading of curl-impersonate driver (lexiforest) by default
       @auto_download            = true
       @driver_version           = 'v2.1.1'

data/lib/http_mimic/module_methods.rb

@@ -41,6 +41,21 @@ module HttpMimic
       default_options[:cookies].merge!(c)
     end
 
+    def mode(m = nil)
+      return default_options[:mode] if m.nil?
+      default_options[:mode] = m
+    end
+
+    def auto_fallback(enabled = nil)
+      return default_options[:auto_fallback] if enabled.nil?
+      default_options[:auto_fallback] = enabled
+    end
+
+    def retry_statuses(statuses = nil)
+      return default_options[:retry_statuses] if statuses.nil?
+      default_options[:retry_statuses] = statuses
+    end
+
     def default_options
       @default_options ||= {}
     end

data/lib/http_mimic/request.rb

@@ -14,34 +14,89 @@ module HttpMimic
     end
 
     def perform
-      builder = CommandBuilder.new(method, url, options, config)
-      binary, args, stdin_data, final_url = builder.build
-
-      full_command = [binary] + args
-
-      log_debug("Executing HttpMimic command: #{full_command.join(' ')}")
-      log_debug("Stdin data: #{stdin_data}") if stdin_data
-
-      stdout, stderr, status = execute_open3(full_command, stdin_data)
-
-      log_debug("Curl exit status: #{status.exitstatus}")
-      log_debug("Curl stderr: #{stderr}") unless stderr.empty?
-
-      response = ResponseParser.new(
-        stdout,
-        exit_status: status,
-        stderr: stderr,
-        command: full_command,
-        request_url: final_url
-      ).parse
+      mode = (options[:mode] || config.mode || :auto).to_sym
+      auto_fallback = options.fetch(:auto_fallback, config.auto_fallback)
+      retry_statuses = options[:retry_statuses] || config.retry_statuses || [403, 429, 503]
+
+      attempts = []
+      profiles_to_try = determine_profiles(mode, auto_fallback)
+
+      response = nil
+      final_status = nil
+      final_stderr = nil
+      final_command = nil
+
+      profiles_to_try.each_with_index do |profile, index|
+        current_opts = options.merge(profile: profile)
+
+        builder = CommandBuilder.new(method, url, current_opts, config)
+        binary, args, stdin_data, final_url = builder.build
+        full_command = [binary] + args
+
+        log_debug("Executing HttpMimic command (attempt #{index + 1}, profile: #{profile}): #{full_command.join(' ')}")
+        log_debug("Stdin data: #{stdin_data}") if stdin_data
+
+        stdout, stderr, status = execute_open3(full_command, stdin_data)
+
+        log_debug("Curl exit status: #{status.exitstatus}")
+        log_debug("Curl stderr: #{stderr}") unless stderr.empty?
+
+        response = ResponseParser.new(
+          stdout,
+          exit_status: status,
+          stderr: stderr,
+          command: full_command,
+          request_url: final_url
+        ).parse
+
+        response.mode_used = profile
+        response.fallback_triggered = (index > 0)
+
+        attempts << {
+          attempt: index + 1,
+          profile: profile,
+          command: full_command,
+          code: response.code,
+          exit_code: status.exitstatus,
+          success: response.success?
+        }
+        response.attempts = attempts
+
+        final_status = status
+        final_stderr = stderr
+        final_command = full_command
+
+        is_blocked = (status.exitstatus != 0) || retry_statuses.include?(response.code)
+        if !is_blocked || (index == profiles_to_try.size - 1)
+          break
+        end
 
-      handle_errors(status, stderr, full_command, response)
+        log_debug("[HttpMimic] Attempt #{index + 1} with #{profile} resulted in status #{response.code}. Triggering smart fallback to next profile...")
+      end
 
+      handle_errors(final_status, final_stderr, final_command, response)
       response
     end
 
     private
 
+    def determine_profiles(mode, auto_fallback)
+      case mode
+      when :curl, :curl_first
+        auto_fallback ? [:curl, :impersonate] : [:curl]
+      when :curl_only
+        [:curl]
+      when :impersonate_only
+        [:impersonate]
+      when :impersonate_first
+        auto_fallback ? [:impersonate, :curl] : [:impersonate]
+      when :auto, :smart
+        auto_fallback ? [:impersonate, :curl] : [:impersonate]
+      else
+        auto_fallback ? [:impersonate, :curl] : [:impersonate]
+      end
+    end
+
     def execute_open3(command_array, stdin_data)
       if stdin_data
         Open3.capture3(*command_array, stdin_data: stdin_data)
@@ -54,6 +109,7 @@ module HttpMimic
 
     def handle_errors(status, stderr, command, response)
       raise_error = options.fetch(:raise_on_error, config.raise_on_error)
+      return unless status
       exit_code = status.exitstatus
 
       if exit_code != 0
@@ -72,7 +128,7 @@ module HttpMimic
         if raise_error
           raise error_class.new(message, exit_code: exit_code, stderr: stderr, command: command, response: response)
         end
-      elsif raise_error && response.error?
+      elsif raise_error && response&.error?
         raise CommandError.new("HTTP request failed (status #{response.code})", exit_code: exit_code, stderr: stderr, command: command, response: response)
       end
     end

data/lib/http_mimic/response.rb

@@ -16,22 +16,34 @@ module HttpMimic
                 :exit_code,
                 :command
 
+    attr_accessor :mode_used,
+                  :fallback_triggered,
+                  :attempts
+
     alias status code
+    alias strategy_used mode_used
 
     def initialize(attributes = {})
-      @code            = attributes[:code] || 0
-      @http_version    = attributes[:http_version]
-      @status_message  = attributes[:status_message]
-      @headers         = attributes[:headers] || Headers.new
-      @cookies         = attributes[:cookies] || Cookies.new
-      @body            = attributes[:body] || ''
-      @parsed_response = attributes[:parsed_response]
-      @raw_headers     = attributes[:raw_headers] || ''
-      @history         = attributes[:history] || []
-      @request_url     = attributes[:request_url]
-      @stderr          = attributes[:stderr] || ''
-      @exit_code       = attributes[:exit_code] || 0
-      @command         = attributes[:command]
+      @code               = attributes[:code] || 0
+      @http_version       = attributes[:http_version]
+      @status_message     = attributes[:status_message]
+      @headers            = attributes[:headers] || Headers.new
+      @cookies            = attributes[:cookies] || Cookies.new
+      @body               = attributes[:body] || ''
+      @parsed_response    = attributes[:parsed_response]
+      @raw_headers        = attributes[:raw_headers] || ''
+      @history            = attributes[:history] || []
+      @request_url        = attributes[:request_url]
+      @stderr             = attributes[:stderr] || ''
+      @exit_code          = attributes[:exit_code] || 0
+      @command            = attributes[:command]
+      @mode_used          = attributes[:mode_used] || :impersonate
+      @fallback_triggered = attributes[:fallback_triggered] || false
+      @attempts           = attributes[:attempts] || []
+    end
+
+    def fallback_triggered?
+      !!@fallback_triggered
     end
 
     def success?

data/lib/http_mimic/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module HttpMimic
-  VERSION = "0.3.0"
+  VERSION = "0.3.1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: http_mimic
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.1
 platform: ruby
 authors:
 - anxgang
@@ -78,7 +78,6 @@ homepage: https://github.com/anxgang/http_mimic
 licenses:
 - MIT
 metadata:
-  homepage_uri: https://github.com/anxgang/http_mimic
   source_code_uri: https://github.com/anxgang/http_mimic
   bug_tracker_uri: https://github.com/anxgang/http_mimic/issues
   changelog_uri: https://github.com/anxgang/http_mimic/blob/main/CHANGELOG.md