jekyll-webmention_io 4.1.0 → 4.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cc9d31a7e443e2a0b12a6d4b5ce65fff675e8302113f751eff6b5bbde3d090e3
-  data.tar.gz: 3c57fc8d14d3345def7d2bd432745fbcd9551d9931adc7ea870d7ae7ed638f6f
+  metadata.gz: eefc9862bc9a4e40c060cc0cbf7bd549e834e427b710b7340a8bb0b3bf6e20ad
+  data.tar.gz: 68506fdb42fb4e79639baedbfabc44c3aeefd2b2e9bbd5db7824a3948b7d04f6
 SHA512:
-  metadata.gz: a650de9ca6309c29e09f5f9533ac5ca62dddec88f8dca8302efb9aee4254fb7445c86f68f4bc06f070189e6cf47f04efcaa6942e9cdd523536e2bc319b70c531
-  data.tar.gz: afd4a141acc8cb97e78e27f0bfcac62f4d8d3d355a4362b7ef94789ba3e5710023f3813bb04a7a9d6de27af0e84167524440d88181aaceb3d18469362deae115
+  metadata.gz: 4badbaedab8e97b0d310586104c8179a8b0f8952214bc58de83c1df1ff97603ceba6321e18932be494eb5956b3457a197d76092dbe041c1d0ea674d50f6c3264
+  data.tar.gz: 505e06d78be628a3962a7885ac731c4f7aaab0b0079a4b40fc3a9c19ed77039c0460e468d8c5ceb3254c84f1b9eba6d30ca39e06743dd18db56fdca4b75de642

data/lib/jekyll/assets/webmention_loader.js

@@ -24,10 +24,12 @@
     redirects = false;
   }
 
-  // Load up any unpublished webmentions on load
+  // Load up any unpublished webmentions on load. The API base is injected at
+  // build time (see CompileJS#add_config); fall back to webmention.io.
   $script = document.createElement('script');
   $script.async = true;
-  $script.src = 'https://webmention.io/api/mentions?' +
+  $script.src = ( window.JekyllWebmentionIO.api_base || 'https://webmention.io/api' ) +
+                '/mentions?' +
                 'jsonp=window.JekyllWebmentionIO.processWebmentions&target[]=' +
                 targets.join( '&target[]=' );
   document.querySelector('head').appendChild( $script );

data/lib/jekyll/caches.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+require 'forwardable'
+require 'fileutils'
+
+module Jekyll
+  module WebmentionIO
+    # The Caches class is a utility service that provides access to the cache files used
+    # by this plugin.
+    #
+    # It is initialized with a config object and creates a folder in the configured cache folder
+    # to store the cache files.
+    #
+    # The class is a singleton and the instance is accessed via the WebmentionIO.caches method.
+    class Caches
+      def initialize(config)
+        @config = config
+
+        FileUtils.makedirs(@config.cache_folder)
+      end
+
+      def incoming_webmentions
+        @@incoming_webmentions ||= Cache.new(cache_file_path('incoming'))
+      end
+
+      def outgoing_webmentions
+        @@outgoing_webmentions ||= Cache.new(cache_file_path('outgoing'))
+      end
+
+      def bad_uris
+        @@bad_uris ||= Cache.new(cache_file_path('bad_uris'))
+      end
+
+      def site_lookups
+        @@site_lookups ||= Cache.new(cache_file_path('lookups'))
+      end
+
+      # Resets all singleton cache instances by clearing the class variables.
+      # This is useful for testing to ensure clean state between tests.
+      def self.reset
+        @@incoming_webmentions = nil
+        @@outgoing_webmentions = nil
+        @@bad_uris = nil
+        @@site_lookups = nil
+      end
+
+      private
+
+      def cache_file_path(name)
+        Jekyll.sanitized_path(@config.cache_folder, "webmention_io_#{name}.yml")
+      end
+
+      # A class that represents a single cache file. The initalizer takes a full path
+      # where the cache data will be stored and retrieved.
+      #
+      # Upon initialization the current contents of the cache are loaded. Writes are not
+      # saved until the `write` method is called.
+      class Cache
+        extend Forwardable
+
+        attr_reader :path
+
+        def_delegator :@data, :each
+        def_delegator :@data, :key?
+        def_delegator :@data, :delete
+        def_delegator :@data, :dig
+        def_delegator :@data, :[]
+        def_delegator :@data, :[]=
+        def_delegator :@data, :empty?
+        def_delegator :@data, :map
+
+        def initialize(path)
+          # NOTE: This is a deviation from the old code! Previously if the configured
+          # cache folder had the word 'webmention' in it, the 'webmention_io' prefix
+          # would be removed, but the extra complexity wasn't worth replicating.
+          @path = path
+
+          begin
+            @data = SafeYAML.load_file(path)
+          rescue StandardError
+            @data = {}
+          end
+        end
+
+        def write
+          File.open(@path, 'wb') { |f| f.puts YAML.dump(@data) }
+        end
+
+        def clear
+          @data = {}
+          FileUtils.rm_f(@path)
+        end
+      end
+
+      private_constant :Cache
+    end
+  end
+end

data/lib/jekyll/commands/webmention.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require "json"
+require 'json'
 
 module Jekyll
   module WebmentionIO
@@ -8,8 +8,8 @@ module Jekyll
       class WebmentionCommand < Command
         def self.init_with_program(prog)
           prog.command(:webmention) do |c|
-            c.syntax "webmention"
-            c.description "Sends queued webmentions"
+            c.syntax 'webmention'
+            c.description 'Sends queued webmentions'
 
             c.action { |args, options| process args, options }
           end
@@ -17,68 +17,69 @@ module Jekyll
 
         def self.process(_args = [], options = {})
           options = configuration_from_options(options)
-          WebmentionIO.bootstrap(Jekyll::Site.new(options))
+          site = Jekyll::Site.new(options)
 
-          if File.exist? WebmentionIO.cache_file("sent.yml")
-            WebmentionIO.log "error", "Your outgoing webmentions queue needs to be upgraded. Please re-build your project."
-          end
+          WebmentionIO.bootstrap(site)
+
+          send_webmentions
+        end
 
-          WebmentionIO.log "msg", "Getting ready to send webmentions (this may take a while)."
+        def self.send_webmentions
+          WebmentionIO.log 'msg', 'Getting ready to send webmentions (this may take a while).'
 
           count = 0
-          max_attempts = WebmentionIO.max_attempts()
-          cached_outgoing = WebmentionIO.get_cache_file_path "outgoing"
-          if File.exist?(cached_outgoing)
-            outgoing = WebmentionIO.load_yaml(cached_outgoing)
-            outgoing.each do |source, targets|
-              targets.each do |target, response|
-                # skip ones we’ve handled
-                next unless response == false or response.instance_of? Integer
-
-                # skip protocol-less links, we'll need to revisit this again later
-                next if target.index("//").zero?
-
-                # produce an escaped version of the target (in case of special
-                # characters, etc).
-                escaped = URI::Parser.new.escape(target);
-
-                # skip bad URLs
-                next unless WebmentionIO.uri_ok?(escaped)
-
-                # give up if we've attempted this too many times
-                response = (response || 0) + 1
-
-                if ! max_attempts.nil? and response > max_attempts
-                  outgoing[source][target] = ""
-                  WebmentionIO.log "msg", "Giving up sending from #{source} to #{target}."
-                  next
-                else
-                  outgoing[source][target] = response
-                end
-
-                # get the endpoint
-                endpoint = WebmentionIO.get_webmention_endpoint(escaped)
-                next unless endpoint
-
-                # get the response
-                response = WebmentionIO.webmention(source, target)
-                next unless response
-
-                # capture JSON responses in case site wants to do anything with them
-                begin
-                  response = JSON.parse response
-                rescue JSON::ParserError
-                  response = ""
-                end
+          max_attempts = WebmentionIO.config.max_attempts
+          outgoing = WebmentionIO.caches.outgoing_webmentions
+
+          return if outgoing.empty?
+
+          outgoing.each do |source, targets|
+            targets.each do |target, response|
+              # skip ones we’ve handled
+              next unless response == false || response.instance_of?(Integer)
+
+              # skip protocol-less links, we'll need to revisit this again later
+              idx = target.index('//')
+              next if idx.nil? || idx.zero?
+
+              # produce an escaped version of the target (in case of special
+              # characters, etc).
+              escaped = URI::Parser.new.escape(target);
+
+              # skip bad URLs
+              next unless WebmentionIO.policy.uri_ok?(escaped)
+
+              # give up if we've attempted this too many times
+              response = (response || 0) + 1
+
+              if !max_attempts.nil? && response > max_attempts
+                outgoing[source][target] = ''
+                WebmentionIO.log 'msg', "Giving up sending from #{source} to #{target}."
+                next
+              else
                 outgoing[source][target] = response
-                count += 1
               end
+
+              # get the response
+              response = WebmentionIO.webmentions.send_webmention(source, target)
+              next unless response
+
+              # capture JSON responses in case site wants to do anything with them
+              begin
+                response = JSON.parse response
+              rescue JSON::ParserError
+                response = ''
+              end
+
+              outgoing[source][target] = response
+              count += 1
             end
-            WebmentionIO.dump_yaml(cached_outgoing, outgoing)
-            WebmentionIO.log "msg", "#{count} webmentions sent."
-          end # file exists (outgoing)
-        end # def process
-      end # WebmentionCommand
-    end # Commands
-  end # WebmentionIO
-end # Jekyll
+          end
+
+          outgoing.write
+          WebmentionIO.log 'msg', "#{count} webmentions sent."
+        end
+      end
+    end
+  end
+end

data/lib/jekyll/config.rb

@@ -0,0 +1,336 @@
+# frozen_string_literal: true
+
+require 'uri'
+
+module Jekyll
+  module WebmentionIO
+    class Config
+      module HtmlProofer
+        NONE = 'none'
+        ALL = 'all'
+        TEMPLATES = 'templates'
+
+        def self.get_const(val)
+          constants.find { |sym| const_get(sym) == val }
+        end
+      end
+
+      module UriPolicy
+        BAN = 'ban'
+        IGNORE = 'ignore'
+        RETRY = 'retry'
+      end
+
+      TIMEFRAMES = {
+        'last_week' => 'weekly',
+        'last_month' => 'monthly',
+        'last_year' => 'yearly',
+      }.freeze
+
+      attr_accessor :html_proofer_ignore, :max_attempts,
+                    :templates, :bad_uri_policy, :throttle_lookups, :cache_folder,
+                    :legacy_domains, :pause_lookups, :site_url, :syndication, :js,
+                    :username, :debug, :api_url
+
+      # The scheme://host[:port] origin and bare host of the configured API,
+      # derived from api_url. Used to build the service links emitted into the
+      # page head (and the JS) so they track a custom endpoint instead of being
+      # hard-coded to webmention.io.
+      attr_reader :api_origin, :api_host
+
+      # The default base URL for the Webmention.io API. Exposed as a config key
+      # so the endpoint can be pointed elsewhere (e.g. a local stand-in during
+      # integration testing) instead of being hard-coded in the network layer.
+      DEFAULT_API_URL = 'https://webmention.io/api'
+
+      # Resolves a webmention API URL to a URI, falling back to the default
+      # endpoint when the configured value has no host.
+      def self.api_uri(url)
+        uri = URI.parse(url)
+        uri.host ? uri : URI.parse(DEFAULT_API_URL)
+      end
+
+      # The host of a parsed API URI, plus the port when it isn't the scheme's
+      # default (so custom/local endpoints still match the full URL).
+      def self.authority(uri)
+        uri.port == uri.default_port ? uri.host : "#{uri.host}:#{uri.port}"
+      end
+
+      def initialize(site = nil)
+        @site = site
+
+        if site.nil?
+          parse
+        else
+          parse(@site.config['webmentions'], @site.config['url'].to_s, @site.config['baseurl'].to_s)
+        end
+      end
+
+      def parse(config = nil, site_url = '', base_url = '')
+        config ||= {}
+
+        @site_url = site_url
+        @username = config['username']
+        @debug = config['debug']
+        @api_url = config['api_url'] || DEFAULT_API_URL
+        api_uri = self.class.api_uri(@api_url)
+        @api_host = api_uri.host
+        @api_origin = "#{api_uri.scheme}://#{self.class.authority(api_uri)}"
+
+        @pause_lookups =
+          if !@site.nil? && @site.config['serving']
+            WebmentionIO.log 'msg', 'Webmentions won’t be gathered when running `jekyll serve`.'
+
+            true
+          elsif !@site.nil? && @site_url.include?('localhost')
+            WebmentionIO.log 'msg', 'Webmentions won’t be gathered on localhost.'
+
+            true
+          else
+            config['pause_lookups']
+          end
+
+        @cache_folder = config['cache_folder'] || '.jekyll-cache'
+        @cache_folder = @site.in_source_dir(@cache_folder) if !@site.nil?
+
+        @pages = config['pages']
+        @collections = config['collections'] || {}
+        @templates = config['templates'] || {}
+
+        @js = JsConfig.new(base_url, config['js'] || false)
+
+        @html_proofer_ignore = HtmlProofer.get_const(
+          config['html_proofer_ignore'] ||
+          (config['html_proofer'] ? 'templates' : nil) ||
+          'none'
+        )
+
+        @max_attempts = config['max_attempts']
+
+        @bad_uri_policy = BadUriPolicy.new(config)
+
+        @throttle_lookups = config['throttle_lookups'] || {}
+
+        @legacy_domains = config['legacy_domains'] || []
+
+        @syndication = (config['syndication'] || {}).transform_values { |entry| SyndicationRule.new(entry) }
+      end
+
+      # The next lookup date has to be before this date to be allowed to
+      # request webmentions again.
+      def last_lookup_threshold(date)
+        age = get_timeframe_from_date(date)
+
+        throttle = @throttle_lookups[age]
+
+        throttle.nil? ? nil : get_date_from_string(throttle)
+      end
+
+      # Given a webmention endpoint, find the corresponding syndication rule
+      # Yes, this is a kind of reverse lookup so we can figure out of a given
+      # queued webmention was a result of a syndication rule.
+      def syndication_rule_for_uri(uri)
+        @syndication.values.detect { |rule| rule.endpoint == uri }
+      end
+
+      # Based on the specified configuration, return the list of documents for
+      # the site that should be processed.
+      def documents
+        documents = @site.posts.docs.clone
+
+        if @pages == true
+          WebmentionIO.log 'info', 'Including site pages.'
+          documents.concat @site.pages.clone
+        end
+
+        if @collections.empty?
+          WebmentionIO.log 'info', 'Adding collections.'
+
+          @site.collections.each do |name, collection|
+            # skip _posts
+            next if name == 'posts'
+
+            if collections.include?(name)
+              documents.concat collection.docs.clone
+            end
+          end
+        end
+
+        documents
+      end
+
+      def collections
+        @site.collections
+      end
+
+      class BadUriPolicy
+        BadUriPolicyEntry = Struct.new(:policy, :max_attempts, :retry_delay)
+
+        attr_reader :whitelist, :blacklist
+
+        def initialize(site_config)
+          @bad_uri_policy = site_config['bad_uri_policy'] || {}
+
+          @bad_uri_policy['whitelist'] ||= []
+          @bad_uri_policy['blacklist'] ||= []
+
+          # We always want to collect webmentions from the configured API host,
+          # so we explicitly whitelist it. This way a transient service outage
+          # won't get the endpoint banned by the bad-URI policy. Derived from the
+          # configured api_url (default webmention.io) so a custom endpoint gets
+          # the same protection.
+          @bad_uri_policy['whitelist'].insert(-1, api_host_pattern(site_config))
+
+          @whitelist = @bad_uri_policy['whitelist'].map { |expr| Regexp.new(expr) }
+          @blacklist = @bad_uri_policy['blacklist'].map { |expr| Regexp.new(expr) }
+        end
+
+        def set_policy(state, policy, max_attempts = nil, retry_delay = nil)
+          @bad_uri_policy[state] = {
+            'policy' => policy,
+            'max_attempts' => max_attempts,
+            'retry_delay' => retry_delay
+          }
+        end
+
+        # Given the provided state value (see WebmentionPolicy::State),
+        # retrieve the policy entry.  If no entry exists, return a new default
+        # entry that indicates unlimited retries.
+        def for_state(state)
+          default_policy = { 'policy' => UriPolicy::RETRY }
+
+          # Retrieve the policy entry, the default entry, or the canned default
+          policy_entry = @bad_uri_policy[state] || @bad_uri_policy['default'] || default_policy
+
+          # Convert shorthand entry to full policy record
+          if policy_entry.instance_of? String
+            policy_entry = { 'policy' => policy_entry }
+          end
+
+          if policy_entry['policy'] == UriPolicy::RETRY && !policy_entry.key?('retry_delay')
+            # If this is a retry policy and no delay is set, set up the default
+            # delay policy.  This inherits from the legacy cache_bad_uris_for
+            # setting to enable backward compatibility with older configurations.
+            #
+            # We do this here to make the rule enforcement logic a little tidier.
+
+            policy_entry['retry_delay'] = [(@bad_uri_policy['cache_bad_uris_for'] || 1) * 24]
+          end
+
+          # Now finally convert into a proper policy entry structure
+          BadUriPolicyEntry.new(
+            policy_entry['policy'],
+            policy_entry['max_attempts'],
+            policy_entry['retry_delay']
+          )
+        end
+
+        private
+
+        # Builds an anchored host pattern for the configured webmention API so it
+        # is always exempt from the bad-URI policy, mirroring the api_url read in
+        # Config#parse. A non-default port is included so custom/local endpoints
+        # still match the full URL the network layer checks.
+        def api_host_pattern(site_config)
+          uri = Config.api_uri(site_config['api_url'] || DEFAULT_API_URL)
+          "^https?://#{Regexp.escape(Config.authority(uri))}/"
+        end
+      end
+
+      class SyndicationRule
+        attr_reader :endpoint, :response_mapping, :shorturl, :fragment
+
+        def initialize(entry)
+          @endpoint = entry['endpoint']
+          @shorturl = entry['shorturl']
+          @fragment = entry['fragment']
+          @response_mapping = {}
+
+          return unless entry.key?('response_mapping')
+
+          entry['response_mapping'].each do |key, pattern|
+            @response_mapping[key] = JsonPath.new(pattern)
+          rescue StandardError => e
+            WebmentionIO.log 'error', "Ignoring invalid JsonPath expression #{pattern}: #{e}"
+          end
+        end
+      end
+
+      class JsConfig
+        attr_reader :destination, :resource_name, :resource_url
+
+        def initialize(base_url, js_config)
+          if js_config == false
+            @disabled = true
+            return
+          end
+
+          @disabled = false
+          @destination = js_config['destination'] || 'js'
+
+          # rubocop:disable Style/RedundantCondition
+          # Apparently this cop is broken...
+          @deploy = js_config['deploy'].nil? ? true : js_config['deploy']
+          @source = js_config['source'].nil? ? true : js_config['source']
+          @uglify = js_config['uglify'].nil? ? true : js_config['uglify']
+          # rubocop:enable Style/RedundantCondition
+
+          @resource_name = 'JekyllWebmentionIO.js'
+          @resource_url = File.join('', base_url, @destination, @resource_name)
+        end
+
+        def disabled?; @disabled; end
+
+        def source?; @source; end
+
+        def deploy?; @deploy; end
+
+        def uglify?; @uglify; end
+      end
+
+      private
+
+      def get_timeframe_from_date(time)
+        date = time.to_date
+
+        timeframe = nil
+
+        TIMEFRAMES.each do |key, value|
+          if date.to_date > get_date_from_string(value)
+            timeframe = key
+            break
+          end
+        end
+
+        timeframe ||= 'older'
+      end
+
+      def get_date_from_string(text)
+        today = Date.today
+        pattern = /every\s(?:(\d+)\s)?(day|week|month|year)s?/
+        matches = text.match(pattern)
+
+        unless matches
+          text = if text == 'daily'
+                   'every 1 day'
+                 else
+                   "every 1 #{text.sub('ly', '')}"
+                 end
+          matches = text.match(pattern)
+        end
+
+        n = matches[1] ? matches[1].to_i : 1
+        unit = matches[2]
+
+        # weeks aren't natively supported in Ruby
+        if unit == 'week'
+          n *= 7
+          unit = 'day'
+        end
+
+        # dynamic method call
+        today.send "prev_#{unit}", n
+      end
+    end
+  end
+end