html2rss 0.17.0 → 0.18.0

data/lib/html2rss/auto_source/scraper/wordpress_api/posts_endpoint.rb

@@ -0,0 +1,134 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class AutoSource
+    module Scraper
+      class WordpressApi
+        ##
+        # Resolves the WordPress posts endpoint for a given page and scope.
+        class PostsEndpoint
+          POSTS_PATH = 'wp/v2/posts'
+
+          ##
+          # @param parsed_body [Nokogiri::HTML::Document] parsed HTML document
+          # @param page_url [Html2rss::Url] canonical page URL
+          # @param page_scope [Html2rss::AutoSource::Scraper::WordpressApi::PageScope] derived page scope
+          # @param posts_query [Hash<String, String>] query params for the posts request
+          # @param logger [Logger] logger used for operational warnings
+          # @return [Html2rss::Url, nil] resolved posts endpoint or nil when unavailable
+          def self.resolve(parsed_body:, page_url:, page_scope:, posts_query:, logger: Html2rss::Log)
+            new(parsed_body:, page_url:, page_scope:, posts_query:, logger:).call
+          end
+
+          ##
+          # @param parsed_body [Nokogiri::HTML::Document] parsed HTML document
+          # @param page_url [Html2rss::Url] canonical page URL
+          # @param page_scope [Html2rss::AutoSource::Scraper::WordpressApi::PageScope] derived page scope
+          # @param posts_query [Hash<String, String>] query params for the posts request
+          # @param logger [Logger] logger used for operational warnings
+          def initialize(parsed_body:, page_url:, page_scope:, posts_query:, logger:)
+            @parsed_body = parsed_body
+            @page_url = Html2rss::Url.from_absolute(page_url)
+            @page_scope = page_scope
+            @posts_query = posts_query
+            @logger = logger
+          end
+
+          ##
+          # @return [Html2rss::Url, nil] resolved posts endpoint or nil when unavailable
+          def call
+            api_root = api_root_url
+            return unless api_root
+            return unless fetchable_page_scope?
+
+            query_style_api_root?(api_root) ? query_root_posts_url(api_root) : posts_collection_url(api_root)
+          end
+
+          private
+
+          attr_reader :parsed_body, :page_url, :page_scope, :posts_query, :logger
+
+          def api_root_url
+            href = parsed_body.at_css(WordpressApi::API_LINK_SELECTOR)&.[]('href').to_s.strip
+            return log_missing_api_root if href.empty?
+
+            Html2rss::Url.from_relative(href, page_url)
+          rescue Addressable::URI::InvalidURIError, ArgumentError => error
+            logger.warn("#{WordpressApi}: invalid WordPress API endpoint #{href.inspect} (#{error.message})")
+            nil
+          end
+
+          def fetchable_page_scope?
+            return true if page_scope.fetchable?
+
+            if page_scope.reason == :non_archive
+              logger.debug(
+                "#{WordpressApi}: page advertised WordPress API support " \
+                'without a safe WordPress archive scope'
+              )
+              return false
+            end
+
+            logger.warn("#{WordpressApi}: unable to derive safe WordPress archive scope for #{page_url}")
+            false
+          end
+
+          def log_missing_api_root
+            logger.debug("#{WordpressApi}: page advertised WordPress API support without a usable API root")
+            nil
+          end
+
+          def query_style_api_root?(api_root)
+            api_root.query_values.key?('rest_route')
+          end
+
+          def query_root_posts_url(api_root)
+            query = api_root.query_values
+            route = normalized_rest_route(query.fetch('rest_route', '/'))
+            api_root.with_query_values(
+              query.merge(
+                'rest_route' => append_posts_route(route),
+                **posts_query
+              )
+            )
+          end
+
+          def posts_collection_url(api_root)
+            Html2rss::Url.from_relative(POSTS_PATH, normalized_api_root(api_root))
+                         .with_query_values(api_root.query_values.merge(posts_query))
+          end
+
+          def normalized_api_root(api_root)
+            api_root.with_path(normalized_api_path(api_root.path))
+          end
+
+          def normalized_api_path(path)
+            segments = path.to_s.split('/').reject(&:empty?)
+            normalized_path = "/#{segments.join('/')}"
+            normalized_path = '/' if normalized_path == '/'
+            normalized_path.end_with?('/') ? normalized_path : "#{normalized_path}/"
+          end
+
+          def normalized_rest_route(route)
+            value = route.to_s
+            value = '/' if value.empty?
+            value = "/#{value}" unless value.start_with?('/')
+            trim_trailing_slashes(value)
+          end
+
+          def trim_trailing_slashes(value)
+            end_index = value.length
+            end_index -= 1 while end_index > 1 && value.getbyte(end_index - 1) == 47
+            value[0, end_index]
+          end
+
+          def append_posts_route(route)
+            return '/wp/v2/posts' if route == '/'
+
+            "#{route}/wp/v2/posts"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/auto_source/scraper/wordpress_api.rb

@@ -0,0 +1,179 @@
+# frozen_string_literal: true
+
+require 'date'
+require 'nokogiri'
+
+module Html2rss
+  class AutoSource
+    module Scraper
+      # Scrapes WordPress sites through their REST API instead of parsing article HTML.
+      class WordpressApi # rubocop:disable Metrics/ClassLength
+        include Enumerable
+
+        API_LINK_SELECTOR = 'link[rel="https://api.w.org/"][href]'
+        CANONICAL_LINK_SELECTOR = 'link[rel="canonical"][href]'
+        POSTS_FIELDS = %w[id title excerpt content link date categories].freeze
+        def self.options_key = :wordpress_api
+
+        ##
+        # @param parsed_body [Nokogiri::HTML::Document, nil] parsed HTML document
+        # @return [Boolean] whether the page advertises a WordPress REST API endpoint
+        def self.articles?(parsed_body)
+          return false unless parsed_body
+
+          !parsed_body.at_css(API_LINK_SELECTOR).nil?
+        end
+
+        ##
+        # @param parsed_body [Nokogiri::HTML::Document] parsed HTML document
+        # @param url [String, Html2rss::Url] canonical page URL
+        # @param request_session [Html2rss::RequestSession, nil] shared request session for follow-up fetches
+        # @param _opts [Hash] unused scraper-specific options
+        # @return [void]
+        def initialize(parsed_body, url:, request_session: nil, **_opts)
+          @parsed_body = parsed_body
+          @url = Html2rss::Url.from_absolute(url)
+          @request_session = request_session
+          @page_scope = PageScope.from(parsed_body:, url: @url)
+        end
+
+        ##
+        # Yields article hashes from the WordPress posts API.
+        #
+        # @yieldparam article [Hash<Symbol, Object>] normalized article hash
+        # @return [Enumerator, void] enumerator when no block is given
+        def each
+          return enum_for(:each) unless block_given?
+          return unless (posts = fetch_posts)
+
+          posts.filter_map { article_from(_1) }.each { yield(_1) }
+        end
+
+        private
+
+        attr_reader :parsed_body, :url, :request_session, :page_scope
+
+        def fetch_posts
+          response = posts_response
+          return unless response
+
+          Array(response.parsed_body)
+        rescue RequestService::UnsupportedResponseContentType => error
+          Log.warn("#{self.class}: unsupported WordPress API posts content type (#{error.message})")
+          nil
+        rescue JSON::ParserError => error
+          Log.warn("#{self.class}: failed to parse WordPress API posts JSON (#{error.message})")
+          nil
+        end
+
+        def posts_response
+          return unless request_session
+          return unless (resolved_posts_url = posts_endpoint_url)
+
+          request_session.follow_up(
+            url: resolved_posts_url,
+            relation: :auto_source,
+            origin_url: url
+          )
+        rescue Html2rss::Error => error
+          Log.warn("#{self.class}: failed to fetch WordPress API posts (#{error.class}: #{error.message})")
+          nil
+        end
+
+        def article_from(post)
+          return unless post.is_a?(Hash)
+
+          article_url = article_url(post)
+          return unless article_url
+
+          article_attributes(post, article_url).compact
+        end
+
+        def article_url(post)
+          absolute_link(post[:link])
+        end
+
+        def article_id(_post, article_url)
+          root_path_query_id(article_url) || string(article_url.path) || article_url.to_s
+        end
+
+        def article_title(post)
+          rendered_text(post.dig(:title, :rendered))
+        end
+
+        def article_description(post)
+          rendered_html(post.dig(:content, :rendered)) || rendered_html(post.dig(:excerpt, :rendered))
+        end
+
+        def article_published_at(post)
+          string(post[:date])
+        end
+
+        def article_categories(post)
+          Array(post[:categories]).filter_map { |value| string(value) }
+        end
+
+        def article_attributes(post, article_url)
+          {
+            id: article_id(post, article_url),
+            title: article_title(post),
+            description: article_description(post),
+            url: article_url,
+            published_at: article_published_at(post),
+            categories: article_categories(post)
+          }
+        end
+
+        def absolute_link(link)
+          value = string(link)
+          return unless value
+
+          Html2rss::Url.from_relative(value, url)
+        rescue ArgumentError
+          nil
+        end
+
+        def rendered_text(value)
+          rendered_html(value)&.then { Nokogiri::HTML.fragment(_1).text.strip }
+        end
+
+        def rendered_html(value)
+          text = string(value)
+          text unless text.nil?
+        end
+
+        def string(value)
+          text = value.to_s.strip
+          text unless text.empty?
+        end
+
+        def root_path_query_id(article_url)
+          query = string(article_url.query)
+          return unless query
+
+          path = article_url.path.to_s
+          return unless path.empty? || path == '/'
+
+          "/?#{query}"
+        end
+
+        def posts_query
+          {
+            '_fields' => POSTS_FIELDS.join(','),
+            'per_page' => '100'
+          }.merge(page_scope.query)
+        end
+
+        def posts_endpoint_url
+          PostsEndpoint.resolve(
+            parsed_body:,
+            page_url: url,
+            page_scope:,
+            posts_query:,
+            logger: Log
+          )
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/auto_source/scraper.rb

@@ -4,31 +4,165 @@ module Html2rss
   class AutoSource
     ##
     # The Scraper module contains all scrapers that can be used to extract articles.
-    # Each scraper should implement a `call` method that returns an array of article hashes.
+    # Each scraper should implement an `each` method that yields article hashes.
     # Each scraper should also implement an `articles?` method that returns true if the scraper
     # can potentially be used to extract articles from the given HTML.
     #
+    # Detection is intentionally shallow for most scrapers, but instance-based
+    # matching is available for scrapers that need to carry expensive selection
+    # state forward into extraction.
+    # Scrapers run in parallel threads, so implementations must avoid shared
+    # mutable state and degrade by returning no articles when a follow-up would
+    # be unsafe or unsupported.
+    #
     module Scraper
+      APP_SHELL_ROOT_SELECTORS = '#app, #root, #__next, [data-reactroot], [ng-app], [id*="app-shell"]'
+      APP_SHELL_MAX_ANCHORS = 2
+      APP_SHELL_MAX_VISIBLE_TEXT_LENGTH = 220
+
       SCRAPERS = [
-        Html,
+        WordpressApi,
         Schema,
-        SemanticHtml
+        Microdata,
+        JsonState,
+        SemanticHtml,
+        Html
       ].freeze
 
       ##
       # Error raised when no suitable scraper is found.
-      class NoScraperFound < Html2rss::Error; end
+      class NoScraperFound < Html2rss::Error
+        CATEGORY_MESSAGES = {
+          blocked_surface: 'No scrapers found: blocked surface likely (anti-bot or interstitial). ' \
+                           'Retry with --strategy browserless, try a more specific public listing URL, ' \
+                           'or run from an environment that can complete anti-bot checks.',
+          app_shell: 'No scrapers found: app-shell surface detected (client-rendered page with little or no ' \
+                     'server-rendered article HTML). Retry with --strategy browserless, or target a direct ' \
+                     'listing/update URL instead of a homepage or shell entrypoint.',
+          unsupported_surface: 'No scrapers found: unsupported extraction surface for auto mode. ' \
+                               'Try a direct listing/changelog/category URL, ' \
+                               'or use explicit selectors in a feed config.'
+        }.freeze
+
+        def initialize(message = nil, category: :unsupported_surface)
+          validate_category!(category)
+          @category = category
+          super(message || CATEGORY_MESSAGES.fetch(@category))
+        end
+
+        attr_reader :category
+
+        private
+
+        def validate_category!(category)
+          return if CATEGORY_MESSAGES.key?(category)
+
+          valid_categories = CATEGORY_MESSAGES.keys.join(', ')
+          raise ArgumentError, "Unknown category: #{category.inspect}. Valid categories are: #{valid_categories}"
+        end
+      end
 
       ##
-      # Returns an array of scrapers that claim to find articles in the parsed body.
+      # Returns an array of scraper classes that claim to find articles in the parsed body.
       # @param parsed_body [Nokogiri::HTML::Document] The parsed HTML body.
+      # @param opts [Hash] The options hash.
       # @return [Array<Class>] An array of scraper classes that can handle the parsed body.
-      def self.from(parsed_body)
-        scrapers = SCRAPERS.select { |scraper| scraper.articles?(parsed_body) }
-        raise NoScraperFound, 'No suitable scraper found for URL.' if scrapers.empty?
+      def self.from(parsed_body, opts = Html2rss::AutoSource::DEFAULT_CONFIG[:scraper])
+        scrapers = SCRAPERS.select { |scraper| opts.dig(scraper.options_key, :enabled) }
+        scrapers.select! { |scraper| scraper.articles?(parsed_body) }
+
+        raise no_scraper_found_for(parsed_body) if scrapers.empty?
 
         scrapers
       end
+
+      # Returns scraper instances ready for extraction.
+      # @param parsed_body [Nokogiri::HTML::Document] The parsed HTML body.
+      # @param url [String, Html2rss::Url] The page url.
+      # @param request_session [Html2rss::RequestSession, nil] Shared follow-up session.
+      # @param opts [Hash] The options hash.
+      # @return [Array<Object>] An array of scraper instances that can handle the parsed body.
+      #
+      # `instances_for` is the main entrypoint for extraction. It lets a scraper
+      # decide whether it matches using the same instance that will later yield
+      # article hashes, which keeps precomputed state close to the scraper that
+      # owns it.
+      def self.instances_for(parsed_body, url:, request_session: nil,
+                             opts: Html2rss::AutoSource::DEFAULT_CONFIG[:scraper])
+        instances = SCRAPERS.filter_map do |scraper|
+          next unless opts.dig(scraper.options_key, :enabled)
+
+          instance = scraper.new(parsed_body, url:, request_session:, **opts.fetch(scraper.options_key, {}))
+          next unless extractable_instance?(instance, parsed_body)
+
+          instance
+        end
+
+        raise no_scraper_found_for(parsed_body) if instances.empty?
+
+        instances
+      end
+
+      def self.extractable_instance?(instance, parsed_body)
+        return instance.extractable? if instance.respond_to?(:extractable?)
+
+        instance.class.articles?(parsed_body)
+      end
+      private_class_method :extractable_instance?
+
+      def self.no_scraper_found_for(parsed_body)
+        NoScraperFound.new(category: classify_no_scraper_surface(parsed_body))
+      end
+      private_class_method :no_scraper_found_for
+
+      def self.classify_no_scraper_surface(parsed_body)
+        return :blocked_surface if blocked_surface?(parsed_body)
+        return :app_shell if app_shell_surface?(parsed_body)
+
+        :unsupported_surface
+      end
+      private_class_method :classify_no_scraper_surface
+
+      def self.blocked_surface?(parsed_body)
+        Html2rss::BlockedSurface.interstitial?(parsed_body.to_html)
+      end
+      private_class_method :blocked_surface?
+
+      def self.app_shell_surface?(parsed_body)
+        root_marker = parsed_body.at_css(APP_SHELL_ROOT_SELECTORS)
+        return false unless root_marker
+
+        sparse_anchor_surface?(parsed_body) &&
+          no_article_markers?(parsed_body) &&
+          short_visible_text?(parsed_body)
+      end
+      private_class_method :app_shell_surface?
+
+      def self.sparse_anchor_surface?(parsed_body)
+        parsed_body.css('body a[href]').size <= APP_SHELL_MAX_ANCHORS
+      end
+      private_class_method :sparse_anchor_surface?
+
+      def self.no_article_markers?(parsed_body)
+        parsed_body.css(
+          'article, main article, [itemtype*="Article"], [itemprop="articleBody"]'
+        ).empty?
+      end
+      private_class_method :no_article_markers?
+
+      def self.short_visible_text?(parsed_body)
+        visible_text_length(parsed_body) <= APP_SHELL_MAX_VISIBLE_TEXT_LENGTH
+      end
+      private_class_method :short_visible_text?
+
+      def self.visible_text_length(parsed_body)
+        body = parsed_body.at_css('body')
+        return 0 unless body
+
+        text_nodes = body.xpath('.//text()[not(ancestor::script or ancestor::style or ancestor::noscript)]')
+        text_nodes.map(&:text).join(' ').gsub(/\s+/, ' ').strip.length
+      end
+      private_class_method :visible_text_length
     end
   end
 end

data/lib/html2rss/auto_source.rb

@@ -1,73 +1,145 @@
 # frozen_string_literal: true
 
-require 'nokogiri'
 require 'parallel'
-require 'addressable'
+require 'dry-validation'
 
 module Html2rss
   ##
-  # The AutoSource class is responsible for extracting channel and articles
-  # from a given URL.
-  # It uses a set of ArticleExtractors to extract articles, utilizing popular ways of
-  # marking articles, e.g. schema, microdata, open graph, etc.
+  # The AutoSource class automatically extracts articles from a given URL by
+  # utilizing a collection of Scrapers. These scrapers analyze and
+  # parse popular structured data formats—such as schema, microdata, and
+  # open graph—to identify and compile article elements into unified articles.
+  #
+  # Scrapers supporting plain HTML are also available for sites without structured data,
+  # though results may vary based on page markup.
+  #
+  # @see Html2rss::AutoSource::Scraper::Schema
+  # @see Html2rss::AutoSource::Scraper::SemanticHtml
+  # @see Html2rss::AutoSource::Scraper::Html
   class AutoSource
-    class NoArticlesFound < Html2rss::Error; end
-
-    ##
-    # @param url [Addressable::URI] The URL to extract articles from.
-    # @param body [String] The body of the response.
-    # @param headers [Hash] The headers of the response.
-    def initialize(url, body:, headers: {})
-      @url = url
-      @body = body
-      @headers = headers
-    end
-
-    def build
-      raise NoArticlesFound if articles.empty?
+    DEFAULT_CONFIG = {
+      scraper: {
+        wordpress_api: {
+          enabled: true
+        },
+        schema: {
+          enabled: true
+        },
+        microdata: {
+          enabled: true
+        },
+        json_state: {
+          enabled: true
+        },
+        semantic_html: {
+          enabled: true
+        },
+        html: {
+          enabled: true,
+          minimum_selector_frequency: Scraper::Html::DEFAULT_MINIMUM_SELECTOR_FREQUENCY,
+          use_top_selectors: Scraper::Html::DEFAULT_USE_TOP_SELECTORS
+        }
+      },
+      cleanup: Cleanup::DEFAULT_CONFIG
+    }.freeze
+
+    SCRAPER_CONFIG = proc do
+      optional(:wordpress_api).hash do
+        optional(:enabled).filled(:bool)
+      end
+      optional(:schema).hash do
+        optional(:enabled).filled(:bool)
+      end
+      optional(:microdata).hash do
+        optional(:enabled).filled(:bool)
+      end
+      optional(:json_state).hash do
+        optional(:enabled).filled(:bool)
+      end
+      optional(:semantic_html).hash do
+        optional(:enabled).filled(:bool)
+      end
+      optional(:html).hash do
+        optional(:enabled).filled(:bool)
+        optional(:minimum_selector_frequency).filled(:integer, gt?: 0)
+        optional(:use_top_selectors).filled(:integer, gt?: 0)
+      end
+    end.freeze
+    private_constant :SCRAPER_CONFIG
 
-      Reducer.call(articles, url:)
-      Cleanup.call(articles, url:, keep_different_domain: true)
+    Config = Dry::Schema.Params do
+      optional(:scraper).hash(&SCRAPER_CONFIG)
 
-      channel.articles = articles
+      optional(:cleanup).hash do
+        optional(:keep_different_domain).filled(:bool)
+        optional(:min_words_title).filled(:integer, gt?: 0)
+      end
+    end
 
-      Html2rss::AutoSource::RssBuilder.new(
-        channel:,
-        articles:
-      ).call
+    ##
+    # @param response [Html2rss::RequestService::Response] initial page response
+    # @param opts [Hash] validated auto-source options
+    # @param request_session [Html2rss::RequestSession, nil] shared request session for follow-up fetches
+    # @return [void]
+    def initialize(response, opts = DEFAULT_CONFIG, request_session: nil)
+      @parsed_body = response.parsed_body
+      @url = response.url
+      @opts = opts
+      @request_session = request_session
     end
 
+    ##
+    # Extracts article candidates by selecting every scraper that can explain the
+    # page shape, running those scrapers, and normalizing the resulting hashes
+    # into `RssBuilder::Article` objects.
+    #
+    # The contributor-facing flow is:
+    # 1. choose scraper instances that match the page
+    # 2. let each scraper collect its own candidates
+    # 3. clean and deduplicate the merged article list
+    #
+    # Scrapers with expensive precomputation, such as `SemanticHtml`, keep that
+    # state on the instance so detection and extraction can reuse the same work.
+    #
+    # @return [Array<Html2rss::RssBuilder::Article>] extracted articles
     def articles
-      @articles ||= Scraper.from(parsed_body).flat_map do |scraper|
-        instance = scraper.new(parsed_body, url:)
+      @articles ||= extract_articles
+    rescue Html2rss::AutoSource::Scraper::NoScraperFound => error
+      Log.warn "#{self.class}: no scraper matched #{url} (#{error.message})"
+      []
+    end
 
-        articles_in_thread = Parallel.map(instance.each) do |article_hash|
-          Log.debug "Scraper: #{scraper} in worker: #{Parallel.worker_number} [#{article_hash[:url]}]"
+    private
 
-          Article.new(**article_hash, scraper:)
-        end
+    attr_reader :url, :parsed_body, :request_session
 
-        Reducer.call(articles_in_thread, url:)
+    def extract_articles
+      scraper_instances = Scraper.instances_for(parsed_body, url:, request_session:, opts: @opts[:scraper])
+      return [] if scraper_instances.empty?
 
-        articles_in_thread
+      # Scrapers are instantiated and run in parallel threads. Implementations
+      # must avoid shared mutable state, treat request_session calls as
+      # concurrency-safe from the scraper side, and return no articles when a
+      # follow-up would be unsafe or unsupported.
+      articles = Parallel.flat_map(scraper_instances, in_threads: thread_count_for(scraper_instances)) do |instance|
+        run_scraper(instance)
       end
+      Cleanup.call(articles, url:, **cleanup_options)
     end
 
-    def channel
-      @channel ||= Channel.new(parsed_body, headers: @headers, url:)
+    def run_scraper(instance)
+      instance.each.map do |article_hash|
+        RssBuilder::Article.new(**article_hash, scraper: instance.class)
+      end
     end
 
-    private
-
-    attr_reader :url
+    def cleanup_options
+      @opts.fetch(:cleanup, {})
+    end
 
-    # @return [Nokogiri::HTML::Document]
-    def parsed_body
-      @parsed_body ||= Nokogiri.HTML(@body)
-                               .tap do |doc|
-        # Remove comments from the document
-        doc.xpath('//comment()').each(&:remove)
-      end.freeze
+    def thread_count_for(scrapers)
+      count = [scrapers.size, Parallel.processor_count].min
+      count.zero? ? 1 : count
     end
   end
 end