html2rss 0.17.0 → 0.19.0

data/lib/html2rss/auto_source/scraper/wordpress_api/page_scope.rb

@@ -0,0 +1,311 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class AutoSource
+    module Scraper
+      class WordpressApi
+        ##
+        # Determines whether a WordPress page can safely be mapped to a posts query.
+        class PageScope
+          # Canonical path segment for category archives.
+          CATEGORY_SEGMENT = 'category'
+          # Canonical path segment for tag archives.
+          TAG_SEGMENT = 'tag'
+          # Canonical path segment for author archives.
+          AUTHOR_SEGMENT = 'author'
+          # Canonical path segment for paginated archives.
+          PAGE_SEGMENT = 'page'
+          # Canonical query key used for paginated archives.
+          PAGED_QUERY_KEY = 'paged'
+
+          ##
+          # @param parsed_body [Nokogiri::HTML::Document] parsed HTML document
+          # @param url [Html2rss::Url] canonical page URL
+          # @return [PageScope] derived page scope
+          def self.from(parsed_body:, url:)
+            Resolver.new(parsed_body:, url:).call
+          end
+
+          ##
+          # @param query [Hash{String => String}] scoped query params for the posts endpoint
+          # @param fetchable [Boolean] whether a posts follow-up is safe for this page
+          # @param reason [Symbol] classification of the resolved page scope
+          def initialize(query:, fetchable:, reason:)
+            @query = query.freeze
+            @fetchable = fetchable
+            @reason = reason
+            freeze
+          end
+
+          ##
+          # @return [Hash{String => String}] query params to apply to the posts request
+          attr_reader :query
+
+          ##
+          # @return [Boolean] whether the page may safely use the posts API follow-up
+          def fetchable?
+            @fetchable
+          end
+
+          ##
+          # @return [Symbol] classification of the resolved page scope
+          attr_reader :reason
+
+          ##
+          # Resolves the page scope from page markup and canonical URL signals.
+          class Resolver # rubocop:disable Metrics/ClassLength
+            ##
+            # @param parsed_body [Nokogiri::HTML::Document] parsed HTML document
+            # @param url [Html2rss::Url] canonical page URL
+            def initialize(parsed_body:, url:)
+              @parsed_body = parsed_body
+              @url = Html2rss::Url.from_absolute(url)
+            end
+
+            ##
+            # @return [PageScope] derived page scope
+            def call
+              scope = category_scope ||
+                      tag_scope ||
+                      author_scope ||
+                      date_scope ||
+                      fallback_scope
+
+              apply_pagination(scope)
+            end
+
+            private
+
+            attr_reader :parsed_body, :url
+
+            def category_scope
+              return unless category_archive?
+
+              scoped_scope('categories' => archive_id('category'))
+            end
+
+            def tag_scope
+              return unless tag_archive?
+
+              scoped_scope('tags' => archive_id('tag'))
+            end
+
+            def author_scope
+              return unless author_archive?
+
+              scoped_scope('author' => archive_id('author'))
+            end
+
+            def date_scope
+              return unless date_archive?
+
+              range = date_archive_range
+              return unknown_archive_scope unless range
+
+              PageScope.new(query: range, fetchable: true, reason: :archive)
+            end
+
+            def fallback_scope
+              return unknown_archive_scope if archive_like?
+              return non_archive_scope if singular_like?
+
+              PageScope.new(query: {}, fetchable: true, reason: :unscoped)
+            end
+
+            def apply_pagination(scope)
+              page = archive_page_number
+              return scope unless scope.fetchable? && page
+
+              PageScope.new(
+                query: scope.query.merge('page' => page.to_s),
+                fetchable: scope.fetchable?,
+                reason: scope.reason
+              )
+            end
+
+            def scoped_scope(query)
+              return unknown_archive_scope if query.values.any?(&:nil?)
+
+              PageScope.new(query:, fetchable: true, reason: :archive)
+            end
+
+            def unknown_archive_scope
+              PageScope.new(query: {}, fetchable: false, reason: :unsupported_archive)
+            end
+
+            def non_archive_scope
+              PageScope.new(query: {}, fetchable: false, reason: :non_archive)
+            end
+
+            def category_archive?
+              body_classes.include?('category') || leading_path_segment == CATEGORY_SEGMENT
+            end
+
+            def tag_archive?
+              body_classes.include?('tag') || leading_path_segment == TAG_SEGMENT
+            end
+
+            def author_archive?
+              body_classes.include?('author') || leading_path_segment == AUTHOR_SEGMENT
+            end
+
+            def date_archive?
+              body_classes.include?('date') || date_archive_path?
+            end
+
+            def archive_like?
+              category_archive? || tag_archive? || author_archive? || date_archive? || body_classes.include?('archive')
+            end
+
+            def singular_like?
+              body_classes.intersect?(%w[page single singular attachment]) ||
+                body_classes.any? { _1.match?(/\A(?:page-id|postid)-\d+\z/) }
+            end
+
+            def body_classes
+              @body_classes ||= parsed_body.at_css('body')&.[]('class').to_s.split
+            end
+
+            def archive_id(prefix)
+              body_classes.filter_map do |klass|
+                klass[Regexp.new("^#{Regexp.escape(prefix)}-(\\d+)$"), 1]
+              end.first
+            end
+
+            def canonical_or_current_url
+              href = parsed_body.at_css(WordpressApi::CANONICAL_LINK_SELECTOR)&.[]('href').to_s.strip
+              return url if href.empty?
+
+              canonical_url = Html2rss::Url.from_relative(href, url)
+              same_origin_url?(canonical_url, url) ? canonical_url : url
+            rescue ArgumentError
+              url
+            end
+
+            def path_segments
+              @path_segments ||= canonical_or_current_url.path_segments
+            end
+
+            def scoped_path_segments
+              @scoped_path_segments ||= paginated_path? ? path_segments[0...-2] : path_segments
+            end
+
+            def leading_path_segment
+              scoped_path_segments.first
+            end
+
+            def date_archive_path?
+              !date_archive_segments.nil?
+            end
+
+            def date_archive_range
+              components = date_archive_components
+              return unless components
+
+              start_date = Date.new(*components.fetch(:start_date_parts))
+              {
+                'after' => iso8601_start(start_date),
+                'before' => iso8601_start(next_archive_boundary(start_date, components.fetch(:precision)))
+              }
+            rescue Date::Error
+              nil
+            end
+
+            def date_archive_components
+              segments = date_archive_segments
+              return unless segments
+
+              year = segments.fetch(0).to_i
+              month = parse_archive_segment(segments[1], 1, 12)
+              day = parse_archive_segment(segments[2], 1, 31)
+
+              {
+                start_date_parts: [year, month || 1, day || 1],
+                precision: archive_precision(month:, day:)
+              }
+            end
+
+            def date_archive_segments
+              year_index = scoped_path_segments.find_index { _1.match?(/\A\d{4}\z/) }
+              return unless year_index
+
+              segments = scoped_path_segments.drop(year_index)
+              return unless segments.length.between?(1, 3)
+              return unless archive_segment_shape?(segments)
+
+              segments
+            end
+
+            def paginated_path?
+              return false if path_segments.length < 2
+              return false unless path_segments[-2] == PAGE_SEGMENT
+
+              !parse_positive_integer(path_segments[-1]).nil?
+            end
+
+            def archive_page_number
+              parse_positive_integer(canonical_or_current_url.query_values[PAGED_QUERY_KEY]) || path_page_number
+            end
+
+            def path_page_number
+              return unless paginated_path?
+
+              parse_positive_integer(path_segments[-1])
+            end
+
+            def archive_segment_shape?(segments)
+              month = segments[1]
+              day = segments[2]
+              return false if day && month.nil?
+              return false unless month.nil? || month.match?(/\A\d+\z/)
+              return false unless day.nil? || day.match?(/\A\d+\z/)
+
+              true
+            end
+
+            def same_origin_url?(left, right)
+              [left.scheme, left.host, left.port] == [right.scheme, right.host, right.port]
+            end
+
+            def archive_precision(month:, day:)
+              return :day if day
+              return :month if month
+
+              :year
+            end
+
+            def next_archive_boundary(start_date, precision)
+              {
+                year: start_date.next_year,
+                month: start_date.next_month,
+                day: start_date.next_day
+              }.fetch(precision)
+            end
+
+            def iso8601_start(date)
+              date.strftime('%Y-%m-%dT00:00:00Z')
+            end
+
+            def parse_archive_segment(value, minimum, maximum)
+              return nil unless value&.match?(/\A\d+\z/)
+
+              number = value.to_i
+              return nil if number < minimum || number > maximum
+
+              number
+            end
+
+            def parse_positive_integer(value)
+              return nil unless value.to_s.match?(/\A\d+\z/)
+
+              number = value.to_i
+              return nil if number < 1
+
+              number
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/auto_source/scraper/wordpress_api/posts_endpoint.rb

@@ -0,0 +1,135 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class AutoSource
+    module Scraper
+      class WordpressApi
+        ##
+        # Resolves the WordPress posts endpoint for a given page and scope.
+        class PostsEndpoint
+          # REST API collection path for posts resources.
+          POSTS_PATH = 'wp/v2/posts'
+
+          ##
+          # @param parsed_body [Nokogiri::HTML::Document] parsed HTML document
+          # @param page_url [Html2rss::Url] canonical page URL
+          # @param page_scope [Html2rss::AutoSource::Scraper::WordpressApi::PageScope] derived page scope
+          # @param posts_query [Hash{String => String}] query params for the posts request
+          # @param logger [Logger] logger used for operational warnings
+          # @return [Html2rss::Url, nil] resolved posts endpoint or nil when unavailable
+          def self.resolve(parsed_body:, page_url:, page_scope:, posts_query:, logger: Html2rss::Log)
+            new(parsed_body:, page_url:, page_scope:, posts_query:, logger:).call
+          end
+
+          ##
+          # @param parsed_body [Nokogiri::HTML::Document] parsed HTML document
+          # @param page_url [Html2rss::Url] canonical page URL
+          # @param page_scope [Html2rss::AutoSource::Scraper::WordpressApi::PageScope] derived page scope
+          # @param posts_query [Hash{String => String}] query params for the posts request
+          # @param logger [Logger] logger used for operational warnings
+          def initialize(parsed_body:, page_url:, page_scope:, posts_query:, logger:)
+            @parsed_body = parsed_body
+            @page_url = Html2rss::Url.from_absolute(page_url)
+            @page_scope = page_scope
+            @posts_query = posts_query
+            @logger = logger
+          end
+
+          ##
+          # @return [Html2rss::Url, nil] resolved posts endpoint or nil when unavailable
+          def call
+            api_root = api_root_url
+            return unless api_root
+            return unless fetchable_page_scope?
+
+            query_style_api_root?(api_root) ? query_root_posts_url(api_root) : posts_collection_url(api_root)
+          end
+
+          private
+
+          attr_reader :parsed_body, :page_url, :page_scope, :posts_query, :logger
+
+          def api_root_url
+            href = parsed_body.at_css(WordpressApi::API_LINK_SELECTOR)&.[]('href').to_s.strip
+            return log_missing_api_root if href.empty?
+
+            Html2rss::Url.from_relative(href, page_url)
+          rescue Addressable::URI::InvalidURIError, ArgumentError => error
+            logger.warn("#{WordpressApi}: invalid WordPress API endpoint #{href.inspect} (#{error.message})")
+            nil
+          end
+
+          def fetchable_page_scope?
+            return true if page_scope.fetchable?
+
+            if page_scope.reason == :non_archive
+              logger.debug(
+                "#{WordpressApi}: page advertised WordPress API support " \
+                'without a safe WordPress archive scope'
+              )
+              return false
+            end
+
+            logger.warn("#{WordpressApi}: unable to derive safe WordPress archive scope for #{page_url}")
+            false
+          end
+
+          def log_missing_api_root
+            logger.debug("#{WordpressApi}: page advertised WordPress API support without a usable API root")
+            nil
+          end
+
+          def query_style_api_root?(api_root)
+            api_root.query_values.key?('rest_route')
+          end
+
+          def query_root_posts_url(api_root)
+            query = api_root.query_values
+            route = normalized_rest_route(query.fetch('rest_route', '/'))
+            api_root.with_query_values(
+              query.merge(
+                'rest_route' => append_posts_route(route),
+                **posts_query
+              )
+            )
+          end
+
+          def posts_collection_url(api_root)
+            Html2rss::Url.from_relative(POSTS_PATH, normalized_api_root(api_root))
+                         .with_query_values(api_root.query_values.merge(posts_query))
+          end
+
+          def normalized_api_root(api_root)
+            api_root.with_path(normalized_api_path(api_root.path))
+          end
+
+          def normalized_api_path(path)
+            segments = path.to_s.split('/').reject(&:empty?)
+            normalized_path = "/#{segments.join('/')}"
+            normalized_path = '/' if normalized_path == '/'
+            normalized_path.end_with?('/') ? normalized_path : "#{normalized_path}/"
+          end
+
+          def normalized_rest_route(route)
+            value = route.to_s
+            value = '/' if value.empty?
+            value = "/#{value}" unless value.start_with?('/')
+            trim_trailing_slashes(value)
+          end
+
+          def trim_trailing_slashes(value)
+            end_index = value.length
+            end_index -= 1 while end_index > 1 && value.getbyte(end_index - 1) == 47
+            value[0, end_index]
+          end
+
+          def append_posts_route(route)
+            return '/wp/v2/posts' if route == '/'
+
+            "#{route}/wp/v2/posts"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/auto_source/scraper/wordpress_api.rb

@@ -0,0 +1,186 @@
+# frozen_string_literal: true
+
+require 'date'
+require 'nokogiri'
+
+module Html2rss
+  class AutoSource
+    module Scraper
+      # Scrapes WordPress sites through their REST API instead of parsing article HTML.
+      class WordpressApi # rubocop:disable Metrics/ClassLength
+        include Enumerable
+
+        # Selector for WordPress API discovery link tags.
+        API_LINK_SELECTOR = 'link[rel="https://api.w.org/"][href]'
+        # Selector for canonical link tags used for scope normalization.
+        CANONICAL_LINK_SELECTOR = 'link[rel="canonical"][href]'
+        # Fields requested from the WordPress posts endpoint.
+        POSTS_FIELDS = %w[id title excerpt content link date categories].freeze
+        # Baseline query sent to WordPress posts API follow-ups.
+        POSTS_QUERY_DEFAULTS = {
+          '_fields' => POSTS_FIELDS.join(','),
+          'per_page' => '100'
+        }.freeze
+        # @return [Symbol] scraper config key
+        def self.options_key = :wordpress_api
+
+        ##
+        # @param parsed_body [Nokogiri::HTML::Document, nil] parsed HTML document
+        # @return [Boolean] whether the page advertises a WordPress REST API endpoint
+        def self.articles?(parsed_body)
+          return false unless parsed_body
+
+          !parsed_body.at_css(API_LINK_SELECTOR).nil?
+        end
+
+        ##
+        # @param parsed_body [Nokogiri::HTML::Document] parsed HTML document
+        # @param url [String, Html2rss::Url] canonical page URL
+        # @param request_session [Html2rss::RequestSession, nil] shared request session for follow-up fetches
+        # @param _opts [Hash] unused scraper-specific options
+        # @option _opts [Object] :_reserved reserved for future scraper-specific options
+        # @return [void]
+        def initialize(parsed_body, url:, request_session: nil, **_opts)
+          @parsed_body = parsed_body
+          @url = Html2rss::Url.from_absolute(url)
+          @request_session = request_session
+          @page_scope = PageScope.from(parsed_body:, url: @url)
+        end
+
+        ##
+        # Yields article hashes from the WordPress posts API.
+        #
+        # @yieldparam article [Hash{Symbol => Object}] normalized article hash
+        # @return [Enumerator, void] enumerator when no block is given
+        def each
+          return enum_for(:each) unless block_given?
+          return unless (posts = fetch_posts)
+
+          posts.filter_map { article_from(_1) }.each { yield(_1) }
+        end
+
+        private
+
+        attr_reader :parsed_body, :url, :request_session, :page_scope
+
+        def fetch_posts
+          response = posts_response
+          return unless response
+
+          Array(response.parsed_body)
+        rescue RequestService::UnsupportedResponseContentType => error
+          Log.warn("#{self.class}: unsupported WordPress API posts content type (#{error.message})")
+          nil
+        rescue JSON::ParserError => error
+          Log.warn("#{self.class}: failed to parse WordPress API posts JSON (#{error.message})")
+          nil
+        end
+
+        def posts_response
+          return unless request_session
+          return unless (resolved_posts_url = posts_endpoint_url)
+
+          request_session.follow_up(
+            url: resolved_posts_url,
+            relation: :auto_source,
+            origin_url: url
+          )
+        rescue Html2rss::Error => error
+          Log.warn("#{self.class}: failed to fetch WordPress API posts (#{error.class}: #{error.message})")
+          nil
+        end
+
+        def article_from(post)
+          return unless post.is_a?(Hash)
+
+          article_url = article_url(post)
+          return unless article_url
+
+          article_attributes(post, article_url).compact
+        end
+
+        def article_url(post)
+          absolute_link(post[:link])
+        end
+
+        def article_id(_post, article_url)
+          root_path_query_id(article_url) || present_string(article_url.path) || article_url.to_s
+        end
+
+        def article_title(post)
+          rendered_text(post.dig(:title, :rendered))
+        end
+
+        def article_description(post)
+          rendered_html(post.dig(:content, :rendered)) || rendered_html(post.dig(:excerpt, :rendered))
+        end
+
+        def article_published_at(post)
+          present_string(post[:date])
+        end
+
+        def article_categories(post)
+          Array(post[:categories]).filter_map { |value| present_string(value) }
+        end
+
+        def article_attributes(post, article_url)
+          {
+            id: article_id(post, article_url),
+            title: article_title(post),
+            description: article_description(post),
+            url: article_url,
+            published_at: article_published_at(post),
+            categories: article_categories(post)
+          }
+        end
+
+        def absolute_link(link)
+          value = present_string(link)
+          return unless value
+
+          Html2rss::Url.from_relative(value, url)
+        rescue ArgumentError
+          nil
+        end
+
+        def rendered_text(value)
+          rendered_html(value)&.then { Nokogiri::HTML.fragment(_1).text.strip }
+        end
+
+        def rendered_html(value)
+          text = present_string(value)
+          text unless text.nil?
+        end
+
+        def present_string(value)
+          text = value.to_s.strip
+          text unless text.empty?
+        end
+
+        def root_path_query_id(article_url)
+          query = present_string(article_url.query)
+          return unless query
+
+          path = article_url.path.to_s
+          return unless path.empty? || path == '/'
+
+          "/?#{query}"
+        end
+
+        def posts_query
+          POSTS_QUERY_DEFAULTS.merge(page_scope.query).transform_values(&:to_s)
+        end
+
+        def posts_endpoint_url
+          PostsEndpoint.resolve(
+            parsed_body:,
+            page_url: url,
+            page_scope:,
+            posts_query:,
+            logger: Log
+          )
+        end
+      end
+    end
+  end
+end