jekyll-webmention_io 3.3.7 → 4.1.0

data/lib/jekyll/webmention_io.rb

@@ -15,16 +15,30 @@ require "json"
 require "net/http"
 require "uri"
 require "openssl"
-require "string_inflection"
+require "active_support"
 require "indieweb/endpoints"
 require "webmention"
 
 module Jekyll
   module WebmentionIO
+    module UriState
+      UNSUPPORTED = "unsupported"
+      ERROR = "error"
+      FAILURE = "failure"
+      SUCCESS = "success"
+    end
+
+    module UriPolicy
+      BAN = "ban"
+      IGNORE = "ignore"
+      RETRY = "retry"
+    end
+
     class << self
       # define simple getters and setters
       attr_reader :config, :jekyll_config, :cache_files, :cache_folder,
-                  :file_prefix, :types, :supported_templates, :js_handler
+                  :file_prefix, :types, :supported_templates, :js_handler,
+                  :uri_whitelist, :uri_blacklist
       attr_writer :api_suffix
     end
 
@@ -69,6 +83,24 @@ module Jekyll
       end
 
       @js_handler = WebmentionIO::JSHandler.new(site)
+
+      @uri_whitelist = @config
+        .fetch("bad_uri_policy", {})
+        .fetch("whitelist", [])
+        .clone
+        .insert(-1, "^https?://webmention.io/")
+        .map { |expr| Regexp.new(expr) }
+
+      @uri_blacklist = @config
+        .fetch("bad_uri_policy", {})
+        .fetch("blacklist", [])
+        .map { |expr| Regexp.new(expr) }
+
+      # Backward compatibility config for html_proofer setting
+
+      if @config['html_proofer'] == true
+        @config['html_proofer_ignore'] = "templates"
+      end
     end
 
     # Setter
@@ -81,6 +113,10 @@ module Jekyll
       Jekyll.sanitized_path(@cache_folder, "#{@file_prefix}#{filename}")
     end
 
+    def self.max_attempts()
+      @config.dig("max_attempts")
+    end
+
     def self.get_cache_file_path(key)
       @cache_files[key] || false
     end
@@ -213,11 +249,11 @@ module Jekyll
         endpoint = IndieWeb::Endpoints.get(uri)[:webmention]
         unless endpoint
           log("info", "Could not find a webmention endpoint at #{uri}")
-          uri_is_not_ok(uri)
+          update_uri_cache(uri, UriState::UNSUPPORTED)
         end
       rescue StandardError => e
         log "info", "Endpoint lookup failed for #{uri}: #{e.message}"
-        uri_is_not_ok(uri)
+        update_uri_cache(uri, UriState::FAILURE)
         endpoint = false
       end
       endpoint
@@ -231,10 +267,24 @@ module Jekyll
       case response.code
       when 200, 201, 202
         log "info", "Webmention successful!"
+        update_uri_cache(target, UriState::SUCCESS)
         response.body
       else
         log "info", response.inspect
         log "info", "Webmention failed, but will remain queued for next time"
+
+        if response.body
+          begin
+            body = JSON.parse(response.body)
+
+            if body.key? "error"
+              log "msg", "Endpoint returned error: #{body['error']}"
+            end
+          rescue
+          end
+        end
+
+        update_uri_cache(target, UriState::ERROR)
         false
       end
     end
@@ -260,7 +310,8 @@ module Jekyll
     end
 
     def self.html_templates
-      proofer = if @config['html_proofer'] == true
+      setting = @config['html_proofer_ignore']
+      proofer = if setting == "all" || setting == "templates"
                   ' data-proofer-ignore'
                 else
                   ''
@@ -291,12 +342,12 @@ module Jekyll
           redirect_to = redirect_to.relative? ? "#{original_uri.scheme}://#{original_uri.host}" + redirect_to.to_s : redirect_to.to_s
           return get_uri_source(redirect_to, redirect_limit - 1, original_uri)
         else
-          uri_is_not_ok(uri)
+          update_uri_cache(uri, UriState::FAILURE)
           return false
         end
       else
         log("warn", "too many redirects for #{original_uri}") if original_uri
-        uri_is_not_ok(uri)
+        update_uri_cache(uri, UriState::FAILURE)
         return false
       end
     end
@@ -347,37 +398,176 @@ module Jekyll
         return response
       rescue *EXCEPTIONS => e
         log "warn", "Got an error checking #{uri}: #{e}"
-        uri_is_not_ok(uri)
+        update_uri_cache(uri, UriState::FAILURE)
         return false
       end
     end
 
-    # Cache bad URLs for a bit
-    def self.uri_is_not_ok(uri)
+    # Given the provided state value (see UriState), retrieve the policy
+    # entry.  If no entry exists, return a new default entry that
+    # indicates unlimited retries.
+    def self.get_bad_uri_policy_entry(state)
+      settings = @config.fetch("bad_uri_policy", {})
+
+      default_policy = { "policy" => UriPolicy::RETRY }
+      policy_entry = nil
+
+      # Retrieve the policy entry, the default entry, or the canned default
+      policy_entry = settings.fetch(state) {
+        settings.fetch("default", default_policy)
+      }
+
+      # Convert shorthand entry to full policy record
+      if policy_entry.instance_of? String
+        policy_entry = { "policy" => policy_entry }
+      end
+
+      if policy_entry["policy"] == UriPolicy::RETRY and ! policy_entry.key? "retry_delay"
+        # If this is a retry policy and no delay is set, set up the default
+        # delay policy.  This inherits from the legacy cache_bad_uris_for
+        # setting to enable backward compatibility with older configurations.
+        #
+        # We do this here to make the rule enforcement logic a little tidier.
+
+        policy_entry["retry_delay"] = [ @config.fetch("cache_bad_uris_for", 1) * 24 ]
+      end
+
+      return policy_entry
+    end
+
+    # Retrieve the bad_uris cache entry for the given URI.  This method
+    # takes the cache and a URI instance (i.e. parsing must already be done).
+    #
+    # If the URI has no entry in the cache, returns nil and *not* a default
+    # entry.
+    def self.get_bad_uri_cache_entry(bad_uris, uri)
+      return nil if ! bad_uris.key? uri.host
+
+      entry = bad_uris[uri.host].clone
+
+      if entry.instance_of? String
+        # Older version of the bad URL cache, convert to new format with some
+        # "sensible" defaults.
+
+        entry = {
+          "state" => UriState::UNSUPPORTED,
+          "last_checked" => DateTime.parse(entry).to_time,
+          "attempts" => 1
+        }
+      else
+        # Otherwise, parse the check time into a real Time object before
+        # returning the entry.
+        #
+        # We convert to a Time object so we can do arithmetic on it later.
+
+        entry["last_checked"] = DateTime.parse(entry["last_checked"]).to_time
+      end
+
+      return entry
+    end
+
+    # Update the URI cache for this entry.
+    #
+    # If the state is UriState.SUCCESS or the URI is whitelisted or
+    # blacklisted, we delete any existing entries since no policy will
+    # apply.  This ensures we reset the policy state when a webmention
+    # succeeds.
+    #
+    # Otherwise, we either create or update an entry for the URI, recording
+    # the state and the current attempt counter.
+    def self.update_uri_cache(uri, state)
       uri = URI::Parser.new.parse(uri.to_s)
-      # Never cache webmention.io in here
-      return if uri.host == "webmention.io"
+      uri_str = uri.to_s
 
       cache_file = @cache_files["bad_uris"]
       bad_uris = load_yaml(cache_file)
-      bad_uris[uri.host] = Time.now.to_s
+
+      if state == UriState::SUCCESS or
+          @uri_whitelist.any? { |expr| expr.match uri_str } or
+          @uri_blacklist.any? { |expr| expr.match uri_str }
+
+        return if bad_uris.delete(uri.host).nil?
+      else
+        old_entry = get_bad_uri_cache_entry(bad_uris, uri) || {}
+
+        bad_uris[uri.host] = {
+          "state" => state,
+          "attempts" => old_entry.fetch("attempts", 0) + 1,
+          "last_checked" => Time.now.to_s
+        }
+      end
+
       dump_yaml(cache_file, bad_uris)
     end
 
+    # Check if we should attempt to send a webmention to the given URI based
+    # on the error handling policy and the last attempt.
     def self.uri_ok?(uri)
       uri = URI::Parser.new.parse(uri.to_s)
       now = Time.now.to_s
+      uri_str = uri.to_s
+
+      # If the URI is whitelisted, it's always ok!
+      return true if @uri_whitelist.any? { |expr| expr.match uri_str }
+
+      # If the URI is blacklisted, it's never ok!
+      return false if @uri_blacklist.any? { |expr| expr.match uri_str }
+
       bad_uris = load_yaml(@cache_files["bad_uris"])
-      if bad_uris.key? uri.host
-        last_checked = DateTime.parse(bad_uris[uri.host])
-        cache_bad_uris_for = @config["cache_bad_uris_for"] || 1 # in days
-        recheck_at = last_checked.next_day(cache_bad_uris_for).to_s
-        return false if recheck_at > now
+      entry = get_bad_uri_cache_entry(bad_uris, uri)
+
+      # If the entry isn't in our cache yet, then it's ok.
+      return true if entry.nil?
+
+      # Okay, the last time we tried to send a webmention to this URI it
+      # failed, so depending on what happened and the policy, we need to
+      # decide what to do.
+      #
+      # First pull the retry policy given the type of the last error for the URI
+      policy_entry = get_bad_uri_policy_entry(entry["state"])
+      policy = policy_entry["policy"]
+
+      if policy == UriPolicy::BAN
+        return false
+      elsif policy == UriPolicy::IGNORE
+        return true
+      elsif policy == UriPolicy::RETRY
+        now = Time.now
+
+        attempts = entry["attempts"]
+        max_attempts = policy_entry["max_attempts"]
+
+        if ! max_attempts.nil? and attempts >= max_attempts
+          # If there's a retry limit and we've hit it, URI is not ok.
+          log "msg", "Skipping #{uri}, attempted #{attempts} times and max is #{max_attempts}"
+
+          return false
+        end
+
+        retry_delay = policy_entry["retry_delay"]
+
+        # Sneaky trick.  By clamping to the array length, the last entry in
+        # the retry_delay list is used for all remaining retries.
+        delay = retry_delay[(attempts - 1).clamp(0, retry_delay.length - 1)]
+
+        recheck_at = (entry["last_checked"] + delay * 3600)
+
+        if recheck_at.to_r > now.to_r
+          log "msg", "Skipping #{uri}, next attempt will happen after #{recheck_at}"
+
+          return false
+        end
+      else
+        log "error", "Invalid bad URI policy type: #{policy}"
       end
+
       return true
     end
 
-    private_class_method :get_http_response, :uri_is_not_ok
+    private_class_method :get_http_response,
+                         :get_bad_uri_policy_entry,
+                         :get_bad_uri_cache_entry,
+                         :update_uri_cache
   end
 end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: jekyll-webmention_io
 version: !ruby/object:Gem::Version
-  version: 3.3.7
+  version: 4.1.0
 platform: ruby
 authors:
 - Aaron Gustafson
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-02-19 00:00:00.000000000 Z
+date: 2025-08-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: jekyll
@@ -65,19 +65,25 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '4.0'
 - !ruby/object:Gem::Dependency
-  name: string_inflection
+  name: activesupport
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.1'
+        version: '7.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 7.0.4.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.1'
+        version: '7.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 7.0.4.3
 - !ruby/object:Gem::Dependency
   name: htmlbeautifier
   requirement: !ruby/object:Gem::Requirement
@@ -120,6 +126,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '7.0'
+- !ruby/object:Gem::Dependency
+  name: jsonpath
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.0.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.0.1
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -154,14 +174,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12.0'
+        version: '13.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12.0'
+        version: '13.0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -297,7 +317,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.3
+rubygems_version: 3.4.22
 signing_key:
 specification_version: 4
 summary: A Jekyll plugin for sending & receiving webmentions via Webmention.io.