bot_challenge_page 0.2.0 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 826b15d243c27b3003ad7c37650836ee67536319177d947b4a046eb200914df7
-  data.tar.gz: 4709e5302b7c4298fc06bb490646273cce5a916e3337ed7d788c5390f345777e
+  metadata.gz: 2e582326d0a139a9407ccd31a0bbb822d99d51a7040093d22427d563997dbea8
+  data.tar.gz: 68107c7fc6a9aa2d8e3e079e401708c740e3cf6ee840b7be6cc4f7c85e8b64df
 SHA512:
-  metadata.gz: e6e260de1875ebbb96a8b809907b50f8d92abe1a9e1d6aee711fd7f3f08567ba627325a035ac08b67f45e42468ad982a85b2f4c45c6709460abe186acbb71b6a
-  data.tar.gz: 3a11694e124de65ca11df02d6fc3aa38b7ee4d93881b2eba64f89a6c30924f3bee3d98297cc8ff9340aa029823bf5cef9e19077cad3b8acfc39aa9cc57e7fe29
+  metadata.gz: 3748c684eeca9e6e66b50fa5e0e5fe4b97c328862799a9dee8fc19632974df5c4489e076eede3d67f0ca8d6f5dc16476ae682d938be015606837c232866256ce
+  data.tar.gz: 2b0bf202ef9c4751a07d0acebeaedd722fd4de501b0a0ae94b004b70b03fb77682cd472aa7e1d3a34ff36810ce3b532662751f3cc79c5453c5fa673977afd0db

data/README.md

@@ -27,6 +27,8 @@ The motivating use case is fairly dumb (probably AI-related) crawlers, rather th
 
   * If you do not want to use rack-attack and want challenge on FIRST request, `rails g bot_challenge_page:install --no-rack-attack`
 
+  * By default challenge pages are "inline" at protected URL. To redirect to a separate challenge page URL instead, `--redirect-for-challenge`
+
 * If you are **not using rack-attack**, you need to add a before_action to the controller(s)
   you'd like to protect, eg:
 
@@ -59,10 +61,41 @@ To customize the layout or challenge page HTML more further, you can use configu
 ```ruby
 BotChallengePage::BotChallengePageController.bot_challenge_config.challenge_renderer = ()->  {
   render "my_local_view_folder/whatever", layout "another_layout"
-  render layout: "another_layout" # default html but change layout. etc.
 }
 ```
 
+## Logging
+
+By default we log when a challenge result is submitted to the back-end; you can find challenge passes or failures by searching your logs for `BotChallengePage`.
+
+We do not log when a challenge is issued -- experience shows challenge issues far outnumber challenge results, and can fill up the logs too fast.
+
+If you'd like to log or observe challenge issues, you can configure a proc that is executed
+in the context of the controller, and is called when a page is blocked by a challenge.
+
+```ruby
+BotChallengePage::BotChallengePageController.bot_challenge_config.after_blocked = (_bot_challenge_class)->  {
+  logger.info("page blocked by challenge: #{request.uri}")
+}
+```
+
+Or, here's how I managed to get it in [lograge](https://github.com/roidrage/lograge), so a page blocked results in a `bot_chlng=true` param in a lograge line.
+
+```ruby
+BotChallengePage::BotChallengePageController.bot_challenge_config.after_blocked =
+  ->(bot_detect_class) {
+    request.env["bot_detect.blocked_for_challenge"] = true
+  }
+
+
+# production.rb
+config.lograge.custom_payload do |controller|
+  {
+    bot_chlng: controller.request.env["bot_detect.blocked_for_challenge"]
+  }.compact
+end
+```
+
 ## Example possible Blacklight config
 
 Many of us in my professional community use [blacklight](https://github.com/projectblacklight/blacklight).  Here's a possible sample blacklight config to:
@@ -88,13 +121,12 @@ Rails.application.config.to_prepare do
   BotChallengePage::BotChallengePageController.bot_challenge_config.rate_limit_count = 3
 
   BotChallengePage::BotChallengePageController.allow_exempt = ->(controller) {
-    # Excempt any Catalog #facet action that looks like an ajax/fetch request, the redirect
-    # ain't gonna work there, we just exempt it.
+    # Excempt any Catalog #facet or #range_limit action that looks like an ajax/fetch request, the # challenge isn't going to work there, we just exempt it.
     #
     # sec-fetch-dest is set to 'empty' by browser on fetch requests, to limit us further;
     # sure an attacker could fake it, we don't mind if someone determined can avoid
     # bot challenge on this one action
-    ( controller.params[:action] == "facet" &&
+    ( controller.params[:action].in?(["facet", "range_limit"]) &&
       controller.request.headers["sec-fetch-dest"] == "empty" &&
       controller.kind_of?(CatalogController)
     )
@@ -117,6 +149,8 @@ Locally one way to test with a specific rails version appraisal is `bundle exec
 
 If you make any changes to `Gemfile` you may need to run `bundle exec appraisal install` and commit changes.
 
+**One reason tests are slow** is I think we're running system tests with real turnstile proof-of-work bot detection JS code? (Or is it, when we are are using a CF turnstile testing key that always passes?).  There aren't many tests so it's no big deal, but this is something that could be investigated/optmized more potentially.
+
 ## Possible future features?
 
 * allow regex in default location_matcher? Easy to do if you want it, just say so.
@@ -135,6 +169,8 @@ The gem is available as open source under the terms of the [MIT License](https:/
 
 * Joe's [similar plugin for drupal](https://drupal.org/project/turnstile_protect)
 
+* Joe's [similar plugin for traefik reverse-proxy](https://github.com/libops/captcha-protect)
+
 * [Similar feature built into PHP VuFind app](https://github.com/vufind-org/vufind/pull/4079)
 
 * [My own blog post about this approach](https://bibwild.wordpress.com/2025/01/16/using-cloudflare-turnstile-to-protect-certain-pages-on-a-rails-app/).

data/app/controllers/bot_challenge_page/bot_challenge_page_controller.rb

@@ -19,9 +19,6 @@ module BotChallengePage
     # different paths in your app if you like, is why config is with controller
     class_attribute :bot_challenge_config, default: ::BotChallengePage::Config.new
 
-    delegate :cf_turnstile_js_url, :cf_turnstile_sitekey, :still_around_delay_ms, to: :bot_challenge_config
-    helper_method :cf_turnstile_js_url, :cf_turnstile_sitekey, :still_around_delay_ms
-
     SESSION_DATETIME_KEY = "t"
     SESSION_IP_KEY = "i"
 
@@ -29,19 +26,22 @@ module BotChallengePage
     class_attribute :_track_notification_subscription, instance_accessor: false
 
 
+    # only used if config.redirect_for_challenge is true
     def challenge
-      # possible custom render to choose layouts or templates, but normally
-      # we just do default rails render and this proc is empty.
-      if self.bot_challenge_config.challenge_renderer
-        instance_exec &self.bot_challenge_config.challenge_renderer
-      end
+      # possible custom render to choose layouts or templates, but
+      # default is what would be default template for this action
+      #
+      # We put it in instancevar as a hacky way of passing to template that can be fulfilled
+      # both here and in arbitrary controllers for direct render.
+      @bot_challenge_config = bot_challenge_config
+      instance_exec &self.bot_challenge_config.challenge_renderer
     end
 
     def verify_challenge
       body = {
         secret: self.bot_challenge_config.cf_turnstile_secret_key,
         response: params["cf_turnstile_response"],
-        remoteip: request.remote_ip
+        remoteip: request.remote_ip,
       }
 
       http = HTTP.timeout(self.bot_challenge_config.cf_timeout)
@@ -64,7 +64,10 @@ module BotChallengePage
         Rails.logger.warn("#{self.class.name}: Cloudflare Turnstile validation failed (#{request.remote_ip}, #{request.user_agent}): #{result}: #{params["dest"]}")
       end
 
-      # let's just return the whole thing to client? Is there anything confidential there?
+      # add config needed by JS to result
+      result["redirect_for_challenge"] = self.bot_challenge_config.redirect_for_challenge
+
+      # and let's just return the whole thing to client? Is there anything confidential there?
       render json: result
     rescue HTTP::Error, JSON::ParserError => e
       # probably an http timeout? or something weird.

data/app/controllers/concerns/bot_challenge_page/enforce_filter.rb

@@ -25,9 +25,20 @@ module BotChallengePage
             return
           end
 
-          Rails.logger.info("#{self.name}: Cloudflare Turnstile challenge redirect: (#{controller.request.remote_ip}, #{controller.request.user_agent}): from #{controller.request.url}")
-          # status code temporary
-          controller.redirect_to controller.bot_detect_challenge_path(dest: controller.request.original_fullpath), status: 307
+          # Prevent caching of bot challenge page
+          controller.response.headers["Cache-Control"] = "no-store"
+
+          if self.bot_challenge_config.redirect_for_challenge
+            # status code temporary
+            controller.redirect_to controller.bot_detect_challenge_path(dest: controller.request.original_fullpath), status: 307
+          else
+            # hacky way to get config to view template in an arbitrary controller, good enough for now
+            controller.instance_variable_set("@bot_challenge_config", self.bot_challenge_config) unless controller.instance_variable_get("@bot_challenge_config")
+            controller.instance_exec &self.bot_challenge_config.challenge_renderer
+          end
+
+          # allow app to see and log if desired
+          controller.instance_exec(self, &self.bot_challenge_config.after_blocked)
         end
       end
 

data/app/models/bot_challenge_page/config.rb

@@ -27,6 +27,10 @@ module BotChallengePage
       end
     end
 
+    # Should we redirect to a challenge page (true) or just display it inline
+    # with a 403 status (false)
+    attribute :redirect_for_challenge, default: false
+
     attribute :enabled, default: false # Must set to true to turn on at all
 
     attribute :cf_turnstile_sitekey, default: "1x00000000000000000000AA" # a testing key that always passes
@@ -55,7 +59,11 @@ module BotChallengePage
     attribute :allow_exempt, default: ->(controller, config) { false }
 
     # replace with say `->() { render layout: 'something' }`, or `render "somedir/some_template"`
-    attribute :challenge_renderer, default: nil
+    attribute :challenge_renderer, default: ->() {
+      render "bot_challenge_page/bot_challenge_page/challenge", status: 403
+    }
+
+    attribute :after_blocked, default: ->(bot_detect_class) {}
 
 
     # rate limit per subnet, following lehigh's lead, although we use a smaller
@@ -63,9 +71,9 @@ module BotChallengePage
     # https://git.drupalcode.org/project/turnstile_protect/-/blob/0dae9f95d48f9d8cae5a8e61e767c69f64490983/src/EventSubscriber/Challenge.php#L140-151
     attribute :rate_limit_discriminator, default: (lambda do |req, config|
       if req.ip.index(":") # ipv6
-        IPAddr.new("#{req.ip}/24").to_string
-      else
         IPAddr.new("#{req.ip}/72").to_string
+      else
+        IPAddr.new("#{req.ip}/24").to_string
       end
     rescue IPAddr::InvalidAddressError
       req.ip

data/app/models/bot_challenge_page/simple_pow1.rb

@@ -0,0 +1,90 @@
+require 'digest'
+require "base64"
+
+module BotChallengePage
+  # A simple proof-of-work algorithm, that we can also do in javascript
+  #
+  # ## Algorithm
+  #
+  # We calculate a deterministic "challenge" based on a secret key (salt?), current time period,
+  # and the specific client request characteristics (prob just client IP).
+  #
+  # The client has to find a prefix than when prepended to the challenge yields a Sha256 hash
+  # that begins with a certain number of zeroes in the hex representtion. The number of zeroes is the "difficulty".
+  # Each zero in hex rep is 4 bits.
+  #
+  # They send the prefix back to us as a solution, and we confirm that when prefixed to
+  # our challenge, and hashed, it has the required number of leading zeroes.
+  #
+  # (TODO: Leading zeroes in a hex represnetation or what?)
+  class SimplePow1
+    # how long is a challenge good for, it will really be good for somewhere between this and 2x this,
+    # since we always try previous challenge to avoid race condition on switch
+    CHALLENGE_PERIOD = 6.minutes
+
+    # how many leading 0 *BITS* -- and time varies a LOT and expands RAPIDLY when we add, we dont' totally knokw what we're doing
+    DEFAULT_DIFFICULTY = 18
+
+    DEFAULT_SECRET = ActiveSupport::KeyGenerator.new(Rails.application.config.secret_key_base).generate_key("BotChallengePage::SimplePow1")
+
+    attr_reader :client_id, :difficulty
+
+    def initialize(client_id:, secret: DEFAULT_SECRET, difficulty: DEFAULT_DIFFICULTY)
+      @client_id = client_id # usually client ip
+      @difficulty = difficulty
+      @secret = secret
+    end
+
+    # challenge is determinsitic based on our secret, the current time, and the client_id
+    def challenge(for_time: Time.now.utc)
+      period_normalized_time = for_time - (for_time.to_i % CHALLENGE_PERIOD)
+
+      Digest::SHA256.hexdigest "#{period_normalized_time.to_s}_#{client_id.to_s}_#{@secret.to_s}"
+    end
+
+    def challenge_for_last_period
+      challenge(for_time: Time.now.utc - CHALLENGE_PERIOD)
+    end
+
+    def challenge_params
+      {
+        challenge: challenge,
+        difficulty: difficulty
+      }
+    end
+
+    # Check solution against current challenge, AND against the previous period's challenge,
+    # in case we just had a race condition, meaning our time goid is actually
+    # min CHALLENGE_PERIOD and max 2 * CHALLENGE_PERIOD
+    #
+    # @param solution [String] *Base64-encoded data*, that when prefixed to the challenge,
+    #   results in a sha256 digest with `difficulty` leading 0 bits.
+    #
+    def verify_solution(solution)
+      solution = Base64.decode64(solution)
+
+      verify_solution_for_challenge(solution, challenge) ||
+        verify_solution_for_challenge(solution, challenge(for_time: Time.now.utc - CHALLENGE_PERIOD))
+    end
+
+    # @param solution [String] actual data, **not** base64 encoded
+    #
+    def verify_solution_for_challenge(aSolution, aChallenge)
+      # there's prob a more efficient mathematical way to do this wihtout converting
+      # to hex string, but this is what we've got.
+      bindigest = Digest::SHA256.digest(aSolution + aChallenge)
+
+      # hopefully we are not going to have a problem with endian-ness here. :(
+
+      bytes_required = (difficulty / 8) + 1
+      prefix_bytes = bindigest.byteslice(0, bytes_required).bytes
+      prefix_bits = prefix_bytes.collect do |byte|
+        reversed_bits = byte.digits(2)
+        reversed_bits.fill(0, reversed_bits.length..7).reverse
+      end.compact.join.slice(0, difficulty)
+
+      prefix_bits == ("0" * difficulty)
+    end
+
+  end
+end

data/app/views/bot_challenge_page/_local_turnstile_script_tag.html.erb

@@ -1,6 +1,7 @@
+<%# locals: (bot_challenge_config:) -%>
+
 <%# we deliver our simple javascript as inline script to make deployment more
   reliable without having to deal with different asset pipelines, and it's really a fine choice anyway  %>
-
 <script type="text/javascript">
   async function turnstileCallback(token) {
     try {
@@ -31,12 +32,6 @@
 
       result = await response.json();
       if (result["success"] == true) {
-        const dest = new URLSearchParams(window.location.search).get("dest");
-        // For security make sure it only has path and on
-        if (!dest.startsWith("/") || dest.startsWith("//")) {
-          throw new Error("illegal non-local redirect: " + dest);
-        }
-
         // in case this page stays around, (say it was rediret to media asset), let's add a failsafe message after
         // a couple seconds.
         const delay = document.querySelector("#botChallengePageStillAroundTemplate")?.getAttribute("data-still-around-delay-ms") || 1200;
@@ -44,8 +39,19 @@
           _displayStillAroundNote()
         }, delay);
 
-        // replace the challenge page in history
-        window.location.replace(dest);
+        if (result["redirect_for_challenge"] == true) {
+          const dest = new URLSearchParams(window.location.search).get("dest");
+          // For security make sure it only has path and on
+          if (!dest.startsWith("/") || dest.startsWith("//")) {
+            throw new Error("illegal non-local redirect: " + dest);
+          }
+
+          // replace the challenge page in history
+          window.location.replace(dest);
+        } else {
+          // just need to reload and now we'll get through
+          window.location.reload();
+        }
       } else {
         console.error("Turnstile response reported as failure: " + JSON.stringify(result))
         _displayChallengeError();

data/app/views/bot_challenge_page/_turnstile_widget_placeholder.html.erb

@@ -1,5 +1,6 @@
+<%# locals: (bot_challenge_config:) -%>
 <div
   class="cf-turnstile"
-  data-sitekey="<%= cf_turnstile_sitekey %>"
+  data-sitekey="<%= bot_challenge_config.cf_turnstile_sitekey %>"
   data-callback="turnstileCallback"
 ></div>

data/app/views/bot_challenge_page/bot_challenge_page/challenge.html.erb

@@ -1,7 +1,7 @@
 <div class="bot_challenge_page">
   <h1 class="mb-4"><%= t('bot_challenge_page.title') %></h1>
 
-  <%= render "bot_challenge_page/turnstile_widget_placeholder" %>
+  <%= render "bot_challenge_page/turnstile_widget_placeholder", bot_challenge_config: @bot_challenge_config %>
 
   <noscript>
     <div class="alert alert-danger"><%= t('bot_challenge_page.noscript') %></div>
@@ -16,14 +16,14 @@
     </div>
   </template>
 
-  <template id="botChallengePageStillAroundTemplate" data-still_around_delay_ms="<%= still_around_delay_ms %>">
+  <template id="botChallengePageStillAroundTemplate" data-still_around_delay_ms="<%= @bot_challenge_config.still_around_delay_ms %>">
     <div class="alert alert-info" role="alert">
       <i class="fa fa-info-circle" aria-hidden="true"></i>
       <%= t('bot_challenge_page.still_around') %>
     </div>
   </template>
 
-  <script src="<%= cf_turnstile_js_url %>" async defer></script>
+  <script src="<%= @bot_challenge_config.cf_turnstile_js_url %>" async defer></script>
 
-  <%= render "bot_challenge_page/local_turnstile_script_tag" %>
+  <%= render "bot_challenge_page/local_turnstile_script_tag", bot_challenge_config: @bot_challenge_config %>
 </div>

data/lib/bot_challenge_page/version.rb

@@ -1,3 +1,3 @@
 module BotChallengePage
-  VERSION = "0.2.0"
+  VERSION = "0.3.1"
 end

data/lib/generators/bot_challenge_page/install_generator.rb

@@ -3,10 +3,14 @@ module BotChallengePage
     source_root File.expand_path("templates", __dir__)
 
     class_option :'rack_attack', type: :boolean, default: true, desc: "Support rate-limit allowance configuration"
+    class_option :redirect_for_challenge, type: :boolean, default: false, desc: "Redirect to separate challenge page instead of inline challenge"
 
     def generate_routes
-      route 'get "/challenge", to: "bot_challenge_page/bot_challenge_page#challenge", as: :bot_detect_challenge'
-      route 'post "/challenge", to: "bot_challenge_page/bot_challenge_page#verify_challenge"'
+      route 'post "/challenge", to: "bot_challenge_page/bot_challenge_page#verify_challenge", as: :bot_detect_challenge'
+
+      if options[:redirect_for_challenge]
+        route 'get "/challenge", to: "bot_challenge_page/bot_challenge_page#challenge"'
+      end
     end
 
     def add_before_filter_enforcement

data/lib/generators/bot_challenge_page/templates/initializer.rb.erb

@@ -4,9 +4,14 @@ Rails.application.config.to_prepare do
 
   # Get from CloudFlare Turnstile: https://www.cloudflare.com/application-services/products/turnstile/
   # Some testing keys are also available: https://developers.cloudflare.com/turnstile/troubleshooting/testing/
+  #
+  # Always pass testing sitekey: "1x00000000000000000000AA"
   BotChallengePage::BotChallengePageController.bot_challenge_config.cf_turnstile_sitekey = "MUST GET"
+  # Always pass testing secret_key: "1x0000000000000000000000000000000AA"
   BotChallengePage::BotChallengePageController.bot_challenge_config.cf_turnstile_secret_key = "MUST GET"
 
+  BotChallengePage::BotChallengePageController.bot_challenge_config.redirect_for_challenge = <%= options[:redirect_for_challenge] %>
+
   <%- if options[:rack_attack] %>
   # What paths do you want to protect?
   #
@@ -33,7 +38,7 @@ Rails.application.config.to_prepare do
   # BotChallengePage::BotChallengePageController.bot_challenge_config.session_passed_good_for = 36.hours
 
   # Exempt some requests from bot challenge protection
-  # BotChallengePage::BotChallengePageController.allow_exempt = ->(controller) {
+  # BotChallengePage::BotChallengePageController.bot_challenge_config.allow_exempt = ->(controller) {
   #   # controller.params
   #   # controller.request
   #   # controller.session

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bot_challenge_page
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.1
 platform: ruby
 authors:
 - Jonathan Rochkind
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2025-03-03 00:00:00.000000000 Z
+date: 2025-04-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: appraisal
@@ -156,6 +156,8 @@ files:
 - app/controllers/concerns/bot_challenge_page/enforce_filter.rb
 - app/controllers/concerns/bot_challenge_page/rack_attack_init.rb
 - app/models/bot_challenge_page/config.rb
+- app/models/bot_challenge_page/simple_pow1.rb
+- app/models/bot_challenge_page/test.html
 - app/views/bot_challenge_page/_local_turnstile_script_tag.html.erb
 - app/views/bot_challenge_page/_turnstile_widget_placeholder.html.erb
 - app/views/bot_challenge_page/bot_challenge_page/challenge.html.erb