shugoi 0.4.5 → 0.4.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3f5c8d8f2dbb96ef585e0da0079fffc4ccb21cb8d3c9c488ba2e3ba2f6dcccef
-  data.tar.gz: 8dee0a01a95b9a27e34d0d131cfc24e268834f4992993d843302666c4b531b53
+  metadata.gz: 89733ca0ebe0286fdb6818ea06a1127dfd6731cabf8a107163fa7df853a4280f
+  data.tar.gz: 7b4cb8e61a095cf0ddf6cadd8fc9fd194a81405d697387c657320a6ebbd92df7
 SHA512:
-  metadata.gz: db5513f04ab846d0a248b9dd2cb6b497c74f1f72e30cf9f78c7daaaf2194f4a7b23484f03155f4144e74a2ec51df8e1bd8a3b32e415b196cff3e755d12438de5
-  data.tar.gz: b7a01c4200dda93eabe8b43adf26f6b493ff68f2fece74bb88353f78b8fc039e4c7dac3a6cf6bc40590d07c2eea0bc65d9f3ef891905ef7a45f2cde702d4c7d1
+  metadata.gz: 26fdbcfc3d51751c68e31a437539d024e036f5afc8569afd7e4174dec5f76b7939edc3236a83152d7f865e15d025e34efc03e9c664f261767499bf667fcb9db1
+  data.tar.gz: 930b00eb96a31c6400792d092c5af08c2e2a7b57f578ae0714e153241c73c07d316f30bc3183a504763597891def44da03cfc025f74077bfc00faa9644b186a9

data/CHANGELOG.md

@@ -0,0 +1,15 @@
+# Changelog
+
+## Unreleased
+
+### Maintenance
+- Extracted challenge, guard, rate-limit and response responsibilities into focused modules.
+- Added dedicated regression specs and finalized RuboCop validation.
+- Documented the gem architecture and production metadata.
+
+### Security
+- Removed dynamic JavaScript evaluation from the split-render bootstrap.
+- Removed `unsafe-eval` from the default Content Security Policy.
+
+### Performance
+- Removed Unicode-tag expansion from skeleton responses, reducing transfer size and parse work.

data/README.md

@@ -6,7 +6,7 @@ Bloque les bots, scrapers et clients headless avec :
 - **Pre-flight PoW challenge** (307 + tableau ASCII pour curl/view-source, JS de résolution inline pour les navigateurs)
 - **Machine fingerprinting** (guard-detect) + whitelist
 - **Render-grant HMAC** (lié au siteKey + token + IP + TTL) — anti-bypass "token-only"
-- **Split-render** : skeleton unicode injecté dans la page, remplacé par le contenu réel après vérification
+- **Split-render** : bootstrap JavaScript injecté dans la page, remplacé par le contenu réel après vérification
 
 ## Installation
 
@@ -59,12 +59,13 @@ run MyApp
 ```bash
 bundle install
 bundle exec rspec
+bundle exec rake lint
 ```
 
 ## Parité avec le module Node
 
 Le flux réplique exactement `shugoi` 0.3.6 (npm) :
 - `core.evaluate` → challenge 307 / `/__sg_challenge` / blocage headless
-- `injectGuardScripts` → token signé + skeleton unicode
+- `injectGuardScripts` → token signé + bootstrap sans évaluation dynamique
 - `renderResponseData` → vérification grant + token, service du HTML stocké
 - `verifyRenderGrant` → format `base36(ts):HMAC(secret, "render-grant:siteKey:mid:token:ip:ts")`

data/lib/shugoi/api_client.rb

@@ -1,11 +1,8 @@
-# frozen_string_literal: true
-
 require "net/http"
 require "uri"
 require "json"
 
 module Shugoi
-  # Client HTTP vers l'API Shugoi (parité avec fetch dans le module Node).
   class ApiClient
     TIMEOUT = 5.0
 
@@ -14,9 +11,6 @@ module Shugoi
       @debug = debug
     end
 
-    # GET /whitelist?key=...&cb=...&sig=... → { detectionFlags, skipPaths }
-    # La siteKey est publique ; la configuration serveur doit prouver la possession
-    # du secret de signature (parité Node/PHP, F2/F3).
     def fetch_whitelist(site_key, secret = nil)
       cb = SecureRandom.hex(4)
       params = { key: site_key, cb: cb.to_s }
@@ -31,18 +25,15 @@ module Shugoi
       { whitelist: [], flags: {}, skip_paths: [] }
     end
 
-    # GET /guard-detect?key=...&raw=1&cb=...&sig=... → texte du guard
     def fetch_guard_detect(site_key, secret = nil)
       cb = Utils.now_ms
       query = { key: site_key, raw: 1, cb: cb.to_s }
       query[:sig] = Utils.hmac_hex(secret, cb.to_s) if secret && !secret.empty?
-      body = get_raw("/guard-detect", query)
-      body
+      get_raw("/guard-detect", query)
     rescue StandardError
       nil
     end
 
-    # POST /event — enregistrement d'un événement (block, headless…)
     def post_event(site_key, reason, machine_id = "")
       payload = { siteKey: site_key, reason: reason, machineId: machine_id }
       post_json("/event", payload)
@@ -50,16 +41,13 @@ module Shugoi
       nil
     end
 
-    # POST /validate-key — valide la clé
     def validate_key(site_key, secret)
       payload = { siteKey: site_key, secret: secret }
-      json = post_json("/validate-key", payload)
-      json
+      post_json("/validate-key", payload)
     rescue StandardError
       { "valid" => false, "reason" => "network" }
     end
 
-    # POST /rate-limit-check — vérifie le quota edge par IP (parité core.ts).
     def check_rate_limit(site_key, ip, user_agent = "")
       payload = {
         siteKey: site_key,
@@ -77,7 +65,8 @@ module Shugoi
     def get(path, params)
       uri = build_uri(path, params)
       log("GET #{uri}")
-      res = Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https", open_timeout: TIMEOUT, read_timeout: TIMEOUT) do |http|
+      res = Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https", open_timeout: TIMEOUT,
+                                                read_timeout: TIMEOUT) do |http|
         http.get(uri.request_uri, "User-Agent" => "shugoi-ruby/#{VERSION}")
       end
       JSON.parse(res.body)
@@ -85,27 +74,27 @@ module Shugoi
 
     def get_raw(path, params)
       uri = build_uri(path, params)
-      res = Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https", open_timeout: TIMEOUT, read_timeout: TIMEOUT) do |http|
+      res = Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https", open_timeout: TIMEOUT,
+                                                read_timeout: TIMEOUT) do |http|
         http.get(uri.request_uri, "User-Agent" => "shugoi-ruby/#{VERSION}")
       end
-      # Décodage UTF-8 avec remplacement (parité fetch().text() du module Node).
       res.body.to_s.force_encoding(Encoding::UTF_8).scrub("\uFFFD")
     end
 
     def post_json(path, payload)
       uri = build_uri(path)
-      req = Net::HTTP::Post.new(uri.request_uri, "Content-Type" => "application/json", "User-Agent" => "shugoi-ruby/#{VERSION}")
+      req = Net::HTTP::Post.new(uri.request_uri, "Content-Type" => "application/json",
+                                                 "User-Agent" => "shugoi-ruby/#{VERSION}")
       req.body = JSON.generate(payload)
-      res = Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https", open_timeout: TIMEOUT, read_timeout: TIMEOUT) do |http|
+      res = Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https", open_timeout: TIMEOUT,
+                                                read_timeout: TIMEOUT) do |http|
         http.request(req)
       end
       JSON.parse(res.body)
     end
 
     def build_uri(path, params = {})
-      # Concatène le path RELATIF à base_url. URI.join avec un path absolu (`/guard-detect`)
-      # remplacerait le préfixe /api/v1 → https://shugoi.com/guard-detect (catch-all SPA) !
-      uri = URI(@base_url.sub(%r{/\z}, "") + "/" + path.sub(%r{\A/}, ""))
+      uri = URI("#{@base_url.sub(%r{/\z}, '')}/#{path.sub(%r{\A/}, '')}")
       uri.query = URI.encode_www_form(params) unless params.empty?
       uri
     end

data/lib/shugoi/bot_policy.rb

@@ -0,0 +1,15 @@
+module Shugoi
+  class BotPolicy
+    def initialize(config)
+      @config = config
+      @verifier = config.verify_bots ? BotVerifier.new : nil
+    end
+
+    def trusted?(user_agent, ip)
+      return false unless @config.whitelisted_bot?(user_agent)
+      return true unless @verifier
+
+      @verifier.verify(user_agent, ip) == true
+    end
+  end
+end

data/lib/shugoi/bot_verifier.rb

@@ -1,10 +1,6 @@
-# frozen_string_literal: true
-
 require "resolv"
 
 module Shugoi
-  # Vérifie par DNS inverse + forward que les bots whitelistés viennent bien de leurs
-  # plages officielles — parité avec verify-bot.ts (module Node). Résultat mis en cache.
   class BotVerifier
     BOT_DOMAINS = [
       { pattern: /Googlebot|Google-InspectionTool|Storebot-Google/i, suffixes: [".googlebot.com", ".google.com"] },
@@ -23,9 +19,6 @@ module Shugoi
       @mutex = Mutex.new
     end
 
-    # @param ua [String] User-Agent
-    # @param ip [String] IP du client
-    # @return [true, false, nil] true/false si UA reconnu, nil sinon
     def verify(ua, ip)
       entry = BOT_DOMAINS.find { |b| b[:pattern].match?(ua) }
       return nil unless entry
@@ -48,21 +41,19 @@ module Shugoi
 
     private
 
-    # PTR de l'IP → nom se terminant par un suffixe attendu, puis forward vérifié.
     def reverse_forward_match?(ip, suffixes)
       names = Resolv.getnames(ip)
       name = names.find { |n| suffixes.any? { |s| n.downcase.end_with?(s) } }
       return false unless name
       addresses = Resolv.getaddresses(name)
       addresses.include?(ip)
-    rescue Resolv::ResolvError, StandardError
+    rescue StandardError
       false
     end
 
     def prune
-      if @cache.size >= MAX_ENTRIES
-        @cache.delete(@cache.keys.min_by { |k| @cache[k][:at] })
-      end
+      return unless @cache.size >= MAX_ENTRIES
+      @cache.delete(@cache.keys.min_by { |k| @cache[k][:at] })
     end
   end
 end

data/lib/shugoi/challenge_limiter.rb

@@ -0,0 +1,38 @@
+module Shugoi
+  class ChallengeLimiter
+    LIMIT = 60
+    WINDOW_MS = 60_000
+    MAX_BLOCK_MS = 15 * 60 * 1000
+
+    def initialize
+      @limits = {}
+      @mutex = Mutex.new
+    end
+
+    def allow?(ip)
+      return true if ip.to_s.empty? || ip == "unknown"
+
+      now = Utils.now_ms
+      @mutex.synchronize do
+        entry = @limits[ip]
+        if entry.nil? || now - entry[:window_start] >= WINDOW_MS
+          @limits[ip] = { count: 1, window_start: now, blocked_until: 0 }
+          next true
+        end
+
+        entry[:count] += 1
+        next false if entry[:blocked_until] > now
+
+        if entry[:count] > LIMIT
+          exponent = [entry[:count] - LIMIT, 10].min
+          backoff = [60_000 * (2**exponent), MAX_BLOCK_MS].min
+          entry[:blocked_until] = now + backoff
+          entry[:count] = 0
+          next false
+        end
+
+        true
+      end
+    end
+  end
+end

data/lib/shugoi/challenge_path.rb

@@ -0,0 +1,12 @@
+module Shugoi
+  class ChallengePath
+    def self.sanitize(path)
+      return '/' if path.to_s.empty?
+      return '/' unless path.start_with?('/')
+      return '/' if path.start_with?('//') || path.include?('\\')
+      return '/' if path.each_codepoint.any? { |codepoint| codepoint < 0x20 || codepoint == 0x7f }
+
+      path
+    end
+  end
+end

data/lib/shugoi/challenge_script.rb

@@ -0,0 +1,24 @@
+module Shugoi
+  class ChallengeScript
+    def self.render(difficulty)
+      <<~JS
+        (function(){
+        var P=new URLSearchParams(location.search);
+        var salt=P.get('salt')||'', ts=P.get('ts')||'', nonce=P.get('nonce')||'', diff=parseInt(P.get('diff')||'#{difficulty}',10), path=P.get('path')||'/';
+        if(path.charAt(0)!=='/'||path.charAt(1)==='/'||path.indexOf('\\\\')>=0)path='/';
+        var enc=new TextEncoder();
+        function bits(d){var l=0;for(var i=0;i<d.length;i++){var b=parseInt(d[i],16);if(b===0){l+=4;continue}l+=(b&8)?0:(b&4)?1:(b&2)?2:3;break}return l}
+        var n=0;
+        function step(){
+          crypto.subtle.digest('SHA-256',enc.encode(salt+':'+n.toString(16))).then(function(buf){
+            var h=Array.from(new Uint8Array(buf)).map(function(v){return v.toString(16).padStart(2,'0')}).join('');
+            if(bits(h)>=diff){var base=path;var q=(base.indexOf('?')>=0?'&':'?')+'sg_proof='+ts+':'+nonce+':'+n.toString(16);location.replace(base+q)}
+            else{n++;if(n<300000)step()}
+          }).catch(function(){location.reload()});
+        }
+        step();
+        })();
+      JS
+    end
+  end
+end

data/lib/shugoi/challenge_url.rb

@@ -0,0 +1,15 @@
+module Shugoi
+  class ChallengeUrl
+    def self.build(secret:, difficulty:, context:)
+      timestamp = Utils.now_sec
+      nonce = SecureRandom.hex(8)
+      salt = Utils.hmac_hex(secret, "#{timestamp}:#{nonce}")
+      prefix = context[:forwarded_prefix].to_s
+      prefix = "" if prefix == "/"
+      prefix = prefix.sub(%r{/\z}, "") unless prefix.empty?
+      path = ChallengePath.sanitize(context[:path].to_s)
+      query = "ts=#{timestamp}&salt=#{salt}&nonce=#{nonce}&diff=#{difficulty}"
+      "#{prefix}/__sg_challenge?#{query}&path=#{URI.encode_www_form_component(prefix + path)}"
+    end
+  end
+end

data/lib/shugoi/config.rb

@@ -1,7 +1,4 @@
-# frozen_string_literal: true
-
 module Shugoi
-  # Options du middleware — parité avec ShugoiCoreOptions (module Node).
   class Config
     DEFAULTS = {
       site_key: nil,
@@ -23,7 +20,6 @@ module Shugoi
       split_render: true,
       multi_process: false,
       verify_bots: true,
-      # Parité module Node : difficulté PoW 14 par défaut (configurable), TTL 60 s.
       pow_difficulty: 14,
       pow_ttl_ms: 60_000
     }.freeze
@@ -57,15 +53,15 @@ module Shugoi
     def pow_difficulty = @options[:pow_difficulty]
     def pow_ttl_ms = @options[:pow_ttl_ms]
 
-    def is_allowlisted?(path)
+    def allowlisted?(path)
       allowlist.any? { |p| path == p || path.start_with?("#{p}/") }
     end
 
-    def is_whitelisted_bot?(ua)
+    def whitelisted_bot?(ua)
       bot_whitelist.any? { |p| p.match?(ua) }
     end
 
-    def is_headless?(ua)
+    def headless?(ua)
       headless_patterns.any? { |p| p.match?(ua) }
     end
 

data/lib/shugoi/config_cache.rb

@@ -1,8 +1,4 @@
-# frozen_string_literal: true
-
 module Shugoi
-  # Cache mémoire de la config (whitelist + flags + skipPaths).
-  # Parité avec _configCache (render.ts) : TTL 30s + stale-refresh.
   class ConfigCache
     TTL_MS = 30_000
     STALE_MAX_MS = 600_000
@@ -15,16 +11,29 @@ module Shugoi
       @flags = {}
       @skip_paths = []
       @fetched_at = 0
+      @refreshing = false
     end
 
     def fetch(site_key)
+      now = Utils.now_ms
+      refresh_now = false
+      refresh_async = false
+
       @mutex.synchronize do
-        if @fetched_at.zero? || (Utils.now_ms - @fetched_at > STALE_MAX_MS)
-          refresh(site_key)
-        elsif Utils.now_ms - @fetched_at > TTL_MS
-          Thread.new { refresh(site_key) }.abort_on_exception = false
-        end
-        { whitelist: @whitelist, flags: @flags, skip_paths: @skip_paths }
+        age = now - @fetched_at
+        refresh_now = @fetched_at.zero? || age > STALE_MAX_MS
+        refresh_async = !refresh_now && age > TTL_MS && !@refreshing
+        @refreshing = true if refresh_now || refresh_async
+      end
+
+      if refresh_now
+        refresh(site_key)
+      elsif refresh_async
+        Thread.new { refresh(site_key) }.abort_on_exception = false
+      end
+
+      @mutex.synchronize do
+        { whitelist: @whitelist.dup, flags: @flags.dup, skip_paths: @skip_paths.dup }
       end
     end
 
@@ -32,10 +41,16 @@ module Shugoi
 
     def refresh(site_key)
       data = @api_client.fetch_whitelist(site_key, @signing_secret)
-      @whitelist = data[:whitelist]
-      @flags = data[:flags]
-      @skip_paths = data[:skip_paths]
-      @fetched_at = Utils.now_ms
+      @mutex.synchronize do
+        @whitelist = data[:whitelist]
+        @flags = data[:flags]
+        @skip_paths = data[:skip_paths]
+        @fetched_at = Utils.now_ms
+        @refreshing = false
+      end
+    rescue StandardError
+      @mutex.synchronize { @refreshing = false }
+      raise
     end
   end
 end

data/lib/shugoi/constants.rb

@@ -1,5 +1,3 @@
-# frozen_string_literal: true
-
 module Shugoi
   BLOCK_PAGE = [
     "+---------------------------------------------+",
@@ -13,10 +11,10 @@ module Shugoi
     "|                                             |",
     "|  - web: https://shugoi.com -                |",
     "+---------------------------------------------+"
-  ].join("\n") + "\n"
+  ].join("\n") << "\n"
 
   DEFAULT_HEADLESS_PATTERNS = [
-    /^curl/i, /^wget/i, /^python/i, /^Go-http-client/i, /^Java\//,
+    /^curl/i, /^wget/i, /^python/i, /^Go-http-client/i, %r{^Java/},
     /HTTPie/i, /^node-fetch/i, /axios/i, /^okhttp/i, /^scrapy/i,
     /PowerShell/i, /WinHttp/i
   ].freeze