html2rss 0.17.0 → 0.19.0

data/lib/html2rss/auto_source/scraper.rb

@@ -4,31 +4,183 @@ module Html2rss
   class AutoSource
     ##
     # The Scraper module contains all scrapers that can be used to extract articles.
-    # Each scraper should implement a `call` method that returns an array of article hashes.
+    # Each scraper should implement an `each` method that yields article hashes.
     # Each scraper should also implement an `articles?` method that returns true if the scraper
     # can potentially be used to extract articles from the given HTML.
     #
+    # Detection is intentionally shallow for most scrapers, but instance-based
+    # matching is available for scrapers that need to carry expensive selection
+    # state forward into extraction.
+    # Scrapers run in parallel threads, so implementations must avoid shared
+    # mutable state and degrade by returning no articles when a follow-up would
+    # be unsafe or unsupported.
     module Scraper
+      # Root markers indicating likely app-shell/client-rendered surfaces.
+      APP_SHELL_ROOT_SELECTORS = '#app, #root, #__next, [data-reactroot], [ng-app], [id*="app-shell"]'
+      # Maximum anchors tolerated before app-shell detection is considered unlikely.
+      APP_SHELL_MAX_ANCHORS = 2
+      # Maximum visible text length tolerated for app-shell classification.
+      APP_SHELL_MAX_VISIBLE_TEXT_LENGTH = 220
+
+      # Ordered scraper classes considered during auto-source extraction.
       SCRAPERS = [
-        Html,
+        WordpressApi,
         Schema,
-        SemanticHtml
+        Microdata,
+        JsonState,
+        SemanticHtml,
+        Html
       ].freeze
 
       ##
       # Error raised when no suitable scraper is found.
-      class NoScraperFound < Html2rss::Error; end
+      class NoScraperFound < Html2rss::Error
+        # User-facing messages grouped by no-scraper surface category.
+        CATEGORY_MESSAGES = {
+          blocked_surface: 'No scrapers found: blocked surface likely (anti-bot or interstitial). ' \
+                           'Retry with --strategy browserless, try a more specific public listing URL, ' \
+                           'or run from an environment that can complete anti-bot checks.',
+          app_shell: 'No scrapers found: app-shell surface detected (client-rendered page with little or no ' \
+                     'server-rendered article HTML). Retry with --strategy browserless, or target a direct ' \
+                     'listing/update URL instead of a homepage or shell entrypoint.',
+          unsupported_surface: 'No scrapers found: unsupported extraction surface for auto mode. ' \
+                               'Try a direct listing/changelog/category URL, ' \
+                               'or use explicit selectors in a feed config.'
+        }.freeze
+
+        # @param message [String, nil] custom error message override
+        # @param category [Symbol] no-scraper classification
+        def initialize(message = nil, category: :unsupported_surface)
+          validate_category!(category)
+          @category = category
+          super(message || CATEGORY_MESSAGES.fetch(@category))
+        end
+
+        attr_reader :category
+
+        private
+
+        def validate_category!(category)
+          return if CATEGORY_MESSAGES.key?(category)
+
+          valid_categories = CATEGORY_MESSAGES.keys.join(', ')
+          raise ArgumentError, "Unknown category: #{category.inspect}. Valid categories are: #{valid_categories}"
+        end
+      end
 
       ##
-      # Returns an array of scrapers that claim to find articles in the parsed body.
+      # Returns an array of scraper classes that claim to find articles in the parsed body.
       # @param parsed_body [Nokogiri::HTML::Document] The parsed HTML body.
+      # @param opts [Hash] The options hash.
+      # @option opts [Hash] :wordpress_api scraper toggle and configuration
+      # @option opts [Hash] :schema scraper toggle and configuration
+      # @option opts [Hash] :microdata scraper toggle and configuration
+      # @option opts [Hash] :json_state scraper toggle and configuration
+      # @option opts [Hash] :semantic_html scraper toggle and configuration
+      # @option opts [Hash] :html scraper toggle and configuration
       # @return [Array<Class>] An array of scraper classes that can handle the parsed body.
-      def self.from(parsed_body)
-        scrapers = SCRAPERS.select { |scraper| scraper.articles?(parsed_body) }
-        raise NoScraperFound, 'No suitable scraper found for URL.' if scrapers.empty?
+      def self.from(parsed_body, opts = Html2rss::AutoSource::DEFAULT_CONFIG[:scraper])
+        scrapers = SCRAPERS.select { |scraper| opts.dig(scraper.options_key, :enabled) }
+        scrapers.select! { |scraper| scraper.articles?(parsed_body) }
+
+        raise no_scraper_found_for(parsed_body) if scrapers.empty?
 
         scrapers
       end
+
+      # Returns scraper instances ready for extraction.
+      # @param parsed_body [Nokogiri::HTML::Document] The parsed HTML body.
+      # @param url [String, Html2rss::Url] The page url.
+      # @param request_session [Html2rss::RequestSession, nil] Shared follow-up session.
+      # @param opts [Hash] The options hash.
+      # @option opts [Hash] :wordpress_api scraper toggle and configuration
+      # @option opts [Hash] :schema scraper toggle and configuration
+      # @option opts [Hash] :microdata scraper toggle and configuration
+      # @option opts [Hash] :json_state scraper toggle and configuration
+      # @option opts [Hash] :semantic_html scraper toggle and configuration
+      # @option opts [Hash] :html scraper toggle and configuration
+      # @return [Array<Object>] An array of scraper instances that can handle the parsed body.
+      #
+      # `instances_for` is the main entrypoint for extraction. It lets a scraper
+      # decide whether it matches using the same instance that will later yield
+      # article hashes, which keeps precomputed state close to the scraper that
+      # owns it.
+      def self.instances_for(parsed_body, url:, request_session: nil,
+                             opts: Html2rss::AutoSource::DEFAULT_CONFIG[:scraper])
+        instances = SCRAPERS.filter_map do |scraper|
+          next unless opts.dig(scraper.options_key, :enabled)
+
+          instance = scraper.new(parsed_body, url:, request_session:, **opts.fetch(scraper.options_key, {}))
+          next unless extractable_instance?(instance, parsed_body)
+
+          instance
+        end
+
+        raise no_scraper_found_for(parsed_body) if instances.empty?
+
+        instances
+      end
+
+      def self.extractable_instance?(instance, parsed_body)
+        return instance.extractable? if instance.respond_to?(:extractable?)
+
+        instance.class.articles?(parsed_body)
+      end
+      private_class_method :extractable_instance?
+
+      def self.no_scraper_found_for(parsed_body)
+        NoScraperFound.new(category: classify_no_scraper_surface(parsed_body))
+      end
+      private_class_method :no_scraper_found_for
+
+      def self.classify_no_scraper_surface(parsed_body)
+        return :blocked_surface if blocked_surface?(parsed_body)
+        return :app_shell if app_shell_surface?(parsed_body)
+
+        :unsupported_surface
+      end
+      private_class_method :classify_no_scraper_surface
+
+      def self.blocked_surface?(parsed_body)
+        Html2rss::BlockedSurface.interstitial?(parsed_body.to_html)
+      end
+      private_class_method :blocked_surface?
+
+      def self.app_shell_surface?(parsed_body)
+        root_marker = parsed_body.at_css(APP_SHELL_ROOT_SELECTORS)
+        return false unless root_marker
+
+        sparse_anchor_surface?(parsed_body) &&
+          no_article_markers?(parsed_body) &&
+          short_visible_text?(parsed_body)
+      end
+      private_class_method :app_shell_surface?
+
+      def self.sparse_anchor_surface?(parsed_body)
+        parsed_body.css('body a[href]').size <= APP_SHELL_MAX_ANCHORS
+      end
+      private_class_method :sparse_anchor_surface?
+
+      def self.no_article_markers?(parsed_body)
+        parsed_body.css(
+          'article, main article, [itemtype*="Article"], [itemprop="articleBody"]'
+        ).empty?
+      end
+      private_class_method :no_article_markers?
+
+      def self.short_visible_text?(parsed_body)
+        visible_text_length(parsed_body) <= APP_SHELL_MAX_VISIBLE_TEXT_LENGTH
+      end
+      private_class_method :short_visible_text?
+
+      def self.visible_text_length(parsed_body)
+        body = parsed_body.at_css('body')
+        return 0 unless body
+
+        text_nodes = body.xpath('.//text()[not(ancestor::script or ancestor::style or ancestor::noscript)]')
+        text_nodes.map(&:text).join(' ').gsub(/\s+/, ' ').strip.length
+      end
+      private_class_method :visible_text_length
     end
   end
 end

data/lib/html2rss/auto_source.rb

@@ -1,73 +1,149 @@
 # frozen_string_literal: true
 
-require 'nokogiri'
 require 'parallel'
-require 'addressable'
+require 'dry-validation'
 
 module Html2rss
   ##
-  # The AutoSource class is responsible for extracting channel and articles
-  # from a given URL.
-  # It uses a set of ArticleExtractors to extract articles, utilizing popular ways of
-  # marking articles, e.g. schema, microdata, open graph, etc.
+  # The AutoSource class automatically extracts articles from a given URL by
+  # utilizing a collection of Scrapers. These scrapers analyze and
+  # parse popular structured data formats—such as schema, microdata, and
+  # open graph—to identify and compile article elements into unified articles.
+  #
+  # Scrapers supporting plain HTML are also available for sites without structured data,
+  # though results may vary based on page markup.
+  #
+  # @see Html2rss::AutoSource::Scraper::Schema
+  # @see Html2rss::AutoSource::Scraper::SemanticHtml
+  # @see Html2rss::AutoSource::Scraper::Html
   class AutoSource
-    class NoArticlesFound < Html2rss::Error; end
-
-    ##
-    # @param url [Addressable::URI] The URL to extract articles from.
-    # @param body [String] The body of the response.
-    # @param headers [Hash] The headers of the response.
-    def initialize(url, body:, headers: {})
-      @url = url
-      @body = body
-      @headers = headers
-    end
-
-    def build
-      raise NoArticlesFound if articles.empty?
+    # Default auto-source configuration shipped for scraper and cleanup behavior.
+    DEFAULT_CONFIG = {
+      scraper: {
+        wordpress_api: {
+          enabled: true
+        },
+        schema: {
+          enabled: true
+        },
+        microdata: {
+          enabled: true
+        },
+        json_state: {
+          enabled: true
+        },
+        semantic_html: {
+          enabled: true
+        },
+        html: {
+          enabled: true,
+          minimum_selector_frequency: Scraper::Html::DEFAULT_MINIMUM_SELECTOR_FREQUENCY,
+          use_top_selectors: Scraper::Html::DEFAULT_USE_TOP_SELECTORS
+        }
+      },
+      cleanup: Cleanup::DEFAULT_CONFIG
+    }.freeze
+
+    SCRAPER_CONFIG = proc do
+      optional(:wordpress_api).hash do
+        optional(:enabled).filled(:bool)
+      end
+      optional(:schema).hash do
+        optional(:enabled).filled(:bool)
+      end
+      optional(:microdata).hash do
+        optional(:enabled).filled(:bool)
+      end
+      optional(:json_state).hash do
+        optional(:enabled).filled(:bool)
+      end
+      optional(:semantic_html).hash do
+        optional(:enabled).filled(:bool)
+      end
+      optional(:html).hash do
+        optional(:enabled).filled(:bool)
+        optional(:minimum_selector_frequency).filled(:integer, gt?: 0)
+        optional(:use_top_selectors).filled(:integer, gt?: 0)
+      end
+    end.freeze
+    private_constant :SCRAPER_CONFIG
 
-      Reducer.call(articles, url:)
-      Cleanup.call(articles, url:, keep_different_domain: true)
+    # Runtime schema used to validate auto-source config values.
+    Config = Dry::Schema.Params do
+      optional(:scraper).hash(&SCRAPER_CONFIG)
 
-      channel.articles = articles
+      optional(:cleanup).hash do
+        optional(:keep_different_domain).filled(:bool)
+        optional(:min_words_title).filled(:integer, gt?: 0)
+      end
+    end
 
-      Html2rss::AutoSource::RssBuilder.new(
-        channel:,
-        articles:
-      ).call
+    ##
+    # @param response [Html2rss::RequestService::Response] initial page response
+    # @param opts [Hash] validated auto-source options
+    # @param request_session [Html2rss::RequestSession, nil] shared request session for follow-up fetches
+    # @option opts [Hash] :scraper scraper configuration map
+    # @option opts [Hash] :cleanup cleanup configuration map
+    # @return [void]
+    def initialize(response, opts = DEFAULT_CONFIG, request_session: nil)
+      @parsed_body = response.parsed_body
+      @url = response.url
+      @opts = opts
+      @request_session = request_session
     end
 
+    ##
+    # Extracts article candidates by selecting every scraper that can explain the
+    # page shape, running those scrapers, and normalizing the resulting hashes
+    # into `RssBuilder::Article` objects.
+    #
+    # The contributor-facing flow is:
+    # 1. choose scraper instances that match the page
+    # 2. let each scraper collect its own candidates
+    # 3. clean and deduplicate the merged article list
+    #
+    # Scrapers with expensive precomputation, such as `SemanticHtml`, keep that
+    # state on the instance so detection and extraction can reuse the same work.
+    #
+    # @return [Array<Html2rss::RssBuilder::Article>] extracted articles
     def articles
-      @articles ||= Scraper.from(parsed_body).flat_map do |scraper|
-        instance = scraper.new(parsed_body, url:)
+      @articles ||= extract_articles
+    rescue Html2rss::AutoSource::Scraper::NoScraperFound => error
+      Log.warn "#{self.class}: no scraper matched #{url} (#{error.message})"
+      []
+    end
 
-        articles_in_thread = Parallel.map(instance.each) do |article_hash|
-          Log.debug "Scraper: #{scraper} in worker: #{Parallel.worker_number} [#{article_hash[:url]}]"
+    private
 
-          Article.new(**article_hash, scraper:)
-        end
+    attr_reader :url, :parsed_body, :request_session
 
-        Reducer.call(articles_in_thread, url:)
+    def extract_articles
+      scraper_instances = Scraper.instances_for(parsed_body, url:, request_session:, opts: @opts[:scraper])
+      return [] if scraper_instances.empty?
 
-        articles_in_thread
+      # Scrapers are instantiated and run in parallel threads. Implementations
+      # must avoid shared mutable state, treat request_session calls as
+      # concurrency-safe from the scraper side, and return no articles when a
+      # follow-up would be unsafe or unsupported.
+      articles = Parallel.flat_map(scraper_instances, in_threads: thread_count_for(scraper_instances)) do |instance|
+        run_scraper(instance)
       end
+      Cleanup.call(articles, url:, **cleanup_options)
     end
 
-    def channel
-      @channel ||= Channel.new(parsed_body, headers: @headers, url:)
+    def run_scraper(instance)
+      instance.each.map do |article_hash|
+        RssBuilder::Article.new(**article_hash, scraper: instance.class)
+      end
     end
 
-    private
-
-    attr_reader :url
+    def cleanup_options
+      @opts.fetch(:cleanup, {})
+    end
 
-    # @return [Nokogiri::HTML::Document]
-    def parsed_body
-      @parsed_body ||= Nokogiri.HTML(@body)
-                               .tap do |doc|
-        # Remove comments from the document
-        doc.xpath('//comment()').each(&:remove)
-      end.freeze
+    def thread_count_for(scrapers)
+      count = [scrapers.size, Parallel.processor_count].min
+      count.zero? ? 1 : count
     end
   end
 end

data/lib/html2rss/blocked_surface.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module Html2rss
+  ##
+  # Shared anti-bot/interstitial signatures used by request and auto-source flows.
+  #
+  # This module centralizes signature matching so request-time guards and
+  # auto-source surface classification stay consistent.
+  module BlockedSurface
+    # Known interstitial fingerprints used to detect blocked or anti-bot surfaces.
+    INTERSTITIAL_SIGNATURES = [
+      {
+        key: :cloudflare_interstitial,
+        min_matches: 2,
+        patterns: [
+          %r{<title>\s*just a moment\.\.\.\s*</title>}i,
+          /checking your browser before accessing/i,
+          /please (?:enable|turn on) javascript and cookies/i,
+          %r{cdn-cgi/challenge-platform}i,
+          /cloudflare ray id/i
+        ],
+        message: 'Blocked surface detected: Cloudflare anti-bot interstitial page. ' \
+                 'Retry with --strategy browserless, try a more specific public listing URL, ' \
+                 'or run from an environment that can complete anti-bot checks.'
+      }
+    ].freeze
+
+    ##
+    # Returns the first matching interstitial signature for the provided body.
+    #
+    # @param body [String, nil] response body candidate
+    # @return [Hash, nil] signature hash when matched, otherwise nil
+    def self.interstitial_signature_for(body)
+      normalized_body = normalize_body(body)
+      INTERSTITIAL_SIGNATURES.find { |signature| interstitial_signature_match?(normalized_body, signature) }
+    end
+
+    ##
+    # @param body [String, nil] response body candidate
+    # @return [Boolean] true when body matches a known interstitial signature
+    def self.interstitial?(body)
+      !interstitial_signature_for(body).nil?
+    end
+
+    def self.interstitial_signature_match?(body, signature)
+      min_matches = signature.fetch(:min_matches, 1)
+      matches = 0
+
+      signature.fetch(:patterns).each do |pattern|
+        matches += 1 if pattern.match?(body)
+        return true if matches >= min_matches
+      end
+
+      false
+    end
+    private_class_method :interstitial_signature_match?
+
+    def self.normalize_body(body)
+      body.to_s.encode(Encoding::UTF_8, invalid: :replace, undef: :replace, replace: '')
+    rescue Encoding::CompatibilityError, Encoding::UndefinedConversionError, Encoding::InvalidByteSequenceError
+      body.to_s.force_encoding(Encoding::UTF_8).scrub
+    end
+    private_class_method :normalize_body
+  end
+end

data/lib/html2rss/category_extractor.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+module Html2rss
+  ##
+  # CategoryExtractor is responsible for extracting categories from HTML elements
+  # by looking for CSS class names containing common category-related terms.
+  class CategoryExtractor
+    # Common category-related terms to look for in class names
+    CATEGORY_TERMS = %w[category tag topic section label theme subject].freeze
+
+    # CSS selectors to find elements with category-related class names
+    CATEGORY_SELECTORS = CATEGORY_TERMS.map { |term| "[class*=\"#{term}\"]" }.freeze
+
+    # Regex pattern for matching category-related attribute names
+    CATEGORY_ATTR_PATTERN = /#{CATEGORY_TERMS.join('|')}/i
+
+    ##
+    # Extracts categories from the given article tag by looking for elements
+    # with class names containing common category-related terms.
+    #
+    # @param article_tag [Nokogiri::XML::Element] The article element to extract categories from
+    # @return [Array<String>] Array of category strings, empty if none found
+    def self.call(article_tag)
+      return [] unless article_tag
+
+      # Single optimized traversal that extracts all category types
+      extract_all_categories(article_tag)
+        .map(&:strip)
+        .reject(&:empty?)
+    end
+
+    ##
+    # Optimized single DOM traversal that extracts all category types.
+    #
+    # @param article_tag [Nokogiri::XML::Element] The article element
+    # @return [Set<String>] Set of category strings
+    def self.extract_all_categories(article_tag)
+      Set.new.tap do |categories|
+        article_tag.css('*').each do |element|
+          # Extract text categories from elements with category-related class names
+          categories.merge(extract_text_categories(element)) if element['class']&.match?(CATEGORY_ATTR_PATTERN)
+
+          # Extract data categories from all elements
+          categories.merge(extract_element_data_categories(element))
+        end
+      end
+    end
+
+    ##
+    # Extracts categories from data attributes of a single element.
+    #
+    # @param element [Nokogiri::XML::Element] metadata element that may contain category links
+    # @return [Set<String>] Set of category strings
+    def self.extract_element_data_categories(element)
+      Set.new.tap do |categories|
+        element.attributes.each_value do |attr|
+          next unless attr.name.match?(CATEGORY_ATTR_PATTERN)
+
+          value = attr.value&.strip
+          categories.add(value) if value && !value.empty?
+        end
+      end
+    end
+
+    ##
+    # Extracts text-based categories from elements, splitting content into discrete values.
+    #
+    # @param element [Nokogiri::XML::Element] metadata element whose text may contain delimiters
+    # @return [Set<String>] Set of category strings
+    def self.extract_text_categories(element)
+      anchor_values = element.css('a').filter_map do |node|
+        HtmlExtractor.extract_visible_text(node)
+      end
+      return Set.new(anchor_values.reject(&:empty?)) if anchor_values.any?
+
+      text = HtmlExtractor.extract_visible_text(element)
+      return Set.new unless text
+
+      Set.new(text.split(/\n+/).map(&:strip).reject(&:empty?))
+    end
+  end
+end