html2rss 0.22.2 → 0.24.0

data/lib/html2rss/auto_source/scraper/semantic_html.rb

@@ -1,8 +1,5 @@
 # frozen_string_literal: true
 
-require_relative 'semantic_html/anchor_selector'
-require_relative 'semantic_html/deduplicator'
-
 module Html2rss
   class AutoSource
     module Scraper
@@ -18,21 +15,9 @@ module Html2rss
       # The result is lower recall on weak-signal blocks, but much better link
       # quality on modern teaser cards that mix headlines, utility links, and
       # duplicate image overlays.
-      class SemanticHtml # rubocop:disable Metrics/ClassLength
+      class SemanticHtml
         include Enumerable
 
-        # Regexp to match content-related tokens.
-        CONTENT_REGEXP = begin
-          words = LinkHeuristics::PathClassifier::SEGMENT_SETS.fetch(:content)
-          /(?:^|\s|[-_])(#{Regexp.union(words.to_a).source})(?:\s|[-_]|$)/i
-        end.freeze
-
-        # Regexp to match junk/utility-related tokens.
-        JUNK_REGEXP = begin
-          words = LinkHeuristics::PathClassifier::SEGMENT_SETS.fetch(:utility)
-          /(?:^|\s|[-_])(#{Regexp.union(words.to_a).source})(?:\s|[-_]|$)/i
-        end.freeze
-
         # Container plus selected anchor, scoring metadata, and extracted article.
         Entry = Data.define(
           :container,
@@ -61,14 +46,13 @@ module Html2rss
         # @param url [String, Html2rss::Url] base url
         # @param extractor [Class] extractor class used for article extraction
         # @param opts [Hash] scraper-specific options
-        # @option opts [Boolean] :fallback_anchorless whether to extract anchorless blocks
-        def initialize(parsed_body, url:, extractor: HtmlExtractor, **opts)
+        # @option opts [Boolean] :fallback_anchorless whether to keep containers without a primary anchor
+        def initialize(parsed_body, url:, extractor: Html2rss::Html::ArticleExtractor, **opts)
           @parsed_body = parsed_body
           @url = url
           @extractor = extractor
-          @fallback_anchorless = opts.fetch(:fallback_anchorless, false)
+          @permit_unanchored = opts.fetch(:fallback_anchorless, false)
           @link_heuristics = LinkHeuristics.new(url)
-          @anchor_selector = AnchorSelector.new(url)
         end
 
         attr_reader :parsed_body
@@ -98,48 +82,53 @@ module Html2rss
           extractable_entries.any?
         end
 
-        protected
-
+        # @return [Array<Nokogiri::XML::Node>]
         def candidate_containers
           @candidate_containers ||= collect_candidate_containers
         end
 
+        # @param container [Nokogiri::XML::Node]
+        # @return [Nokogiri::XML::Node, nil]
         def primary_anchor_for(container)
-          @anchor_selector.primary_anchor_for(container)
+          Discovery::SemanticAnchorCandidates.new(
+            container,
+            link_heuristics: @link_heuristics
+          ).to_a.max_by(&:score)&.anchor
         end
 
-        # rubocop:disable Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
+        # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
         def extractable_entries
           @extractable_entries ||= candidate_containers.filter_map do |container|
             selected_anchor = primary_anchor_for(container)
 
-            next unless selected_anchor || @fallback_anchorless
+            next unless selected_anchor || @permit_unanchored
 
             destination_facts = selected_anchor ? normalized_destination(selected_anchor) : nil
             next if selected_anchor && !destination_facts
-            next if hard_junk_entry?(container, selected_anchor, destination_facts)
+            # Cheap path-only reject before title/DOM hard-junk observations.
+            next if destination_facts&.high_confidence_junk_path
 
-            quality = quality_score(container, selected_anchor, destination_facts)
-            junk = junk_score(container, selected_anchor, destination_facts)
+            signals = @link_heuristics.assess_container(container, selected_anchor, destination_facts:)
+            next if signals.hard_junk?
 
             Entry.new(
               container:,
               selected_anchor:,
               destination_facts:,
-              quality_score: quality,
-              junk_score: junk,
-              final_score: quality - junk,
+              quality_score: signals.quality_score,
+              junk_score: signals.junk_score,
+              final_score: signals.final_score,
               position: document_position(container),
               article: nil
             )
           end
         end
-        # rubocop:enable Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
+        # rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
 
         # rubocop:disable Metrics/MethodLength
         def ranked_entries
           @ranked_entries ||= begin
-            deduplicator = Deduplicator.new(@url, @extractor)
+            deduplicator = EntryDeduplicator.new(@url, @extractor)
             entries = deduplicator.call(extractable_entries)
             entries = stable_rank(entries)
 
@@ -163,7 +152,7 @@ module Html2rss
         # rubocop:enable Metrics/MethodLength
 
         def collect_candidate_containers
-          HtmlExtractor::SemanticContainers.call(parsed_body)
+          Discovery::SemanticContainers.call(parsed_body)
         end
 
         private
@@ -172,147 +161,10 @@ module Html2rss
           (@document_positions ||= candidate_containers.each_with_index.to_h).fetch(container)
         end
 
-        def quality_score(container, selected_anchor, destination_facts) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
-          title = entry_title(container, selected_anchor)
-          words = word_count(title)
-          container_text = visible_text(container)
-          score = 0
-
-          score += 40 if words >= 3
-          score += 15 if words >= 7
-          score += 20 if destination_facts&.url&.path.to_s.length > 6
-          score += 15 if destination_facts&.content_path
-          score += 15 if publish_marker?(container)
-          score += 10 if descriptive_context?(container_text, title)
-          score += 10 if article_container?(container)
-          score += 10 if content_tokens?(container_tokens(container))
-          score
-        end
-
-        def junk_score(container, selected_anchor, destination_facts) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
-          title = entry_title(container, selected_anchor)
-          utility_text = @link_heuristics.utility_prefix_text?(title)
-          recommended_text = @link_heuristics.recommended_text?(title)
-          content_signal = destination_facts&.content_path
-          no_content_signal = !content_signal
-          non_content_utility_path =
-            destination_facts&.utility_path &&
-            no_content_signal &&
-            !destination_facts&.strong_post_suffix
-          publish_signal = publish_marker?(container)
-          descriptive_signal = descriptive_context?(visible_text(container), title)
-          weak_container = !publish_signal && !descriptive_signal
-          score = 0
-
-          score += 25 if non_content_utility_path
-          score += 15 if utility_text && word_count(title) <= 6
-          score += 10 if destination_facts&.shallow
-          score += 10 if weak_container
-          score += 10 if recommended_text && no_content_signal
-          score += 5 if destination_facts&.high_confidence_junk_path
-          score += 15 if junk_tokens?(container_tokens(container))
-          score
-        end
-
-        # rubocop:disable Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
-        def hard_junk_entry?(container, selected_anchor, destination_facts)
-          title = entry_title(container, selected_anchor)
-          publish_signal = publish_marker?(container)
-          descriptive_signal = descriptive_context?(visible_text(container), title)
-          content_signal = destination_facts&.content_path
-          weak_article_candidate = article_signal_count(
-            container,
-            publish_signal:,
-            descriptive_signal:,
-            content_signal:
-          ) < 2
-
-          destination_facts&.high_confidence_junk_path ||
-            (selected_anchor &&
-              @link_heuristics.recommended_text?(title) &&
-              destination_facts&.shallow &&
-              weak_article_candidate) ||
-            (selected_anchor && @link_heuristics.utility_prefix_text?(title) &&
-              destination_facts&.high_confidence_utility_destination &&
-              weak_article_candidate)
-        end
-        # rubocop:enable Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
-
-        ##
-        # @param container [Nokogiri::XML::Node]
-        # @return [Boolean]
-        def publish_marker?(container)
-          (@publish_markers ||= {}.compare_by_identity)[container] ||=
-            !!container.at_css('time, [datetime], [itemprop="datePublished"], [itemprop="dateModified"]')
-        end
-
-        ##
-        # @param container [Nokogiri::XML::Node]
-        # @param publish_signal [Boolean]
-        # @param descriptive_signal [Boolean]
-        # @param content_signal [Boolean]
-        # @return [Integer]
-        def article_signal_count(container, publish_signal:, descriptive_signal:, content_signal:)
-          [article_container?(container), publish_signal, descriptive_signal, content_signal].count(&:itself)
-        end
-
-        ##
-        # @param container [Nokogiri::XML::Node]
-        # @return [Boolean]
-        def article_container?(container) = container.name == 'article'
-
-        def descriptive_context?(container_text, title)
-          snippet = container_text.to_s.sub(/\A#{Regexp.escape(title.to_s)}/i, '')
-          # Only check for existence of enough words if snippet is long enough to have them
-          snippet.length > 30 && word_count(snippet) >= 8
-        end
-
-        ##
-        # @param container [Nokogiri::XML::Node]
-        # @return [Nokogiri::XML::Node, nil]
-        def heading_for(container)
-          (@headings ||= {}.compare_by_identity)[container] ||= container.at_css(AnchorSelector::HEADING_SELECTOR)
-        end
-
         def normalized_destination(anchor)
           (@normalized_destinations ||= {}.compare_by_identity)[anchor] ||= @link_heuristics.destination_facts(anchor)
         end
 
-        def visible_text(node)
-          return '' unless node
-
-          (@visible_texts ||= {}.compare_by_identity)[node] ||= HtmlExtractor.extract_visible_text(node).to_s.strip
-        end
-
-        ##
-        # @param container [Nokogiri::XML::Node]
-        # @param selected_anchor [Nokogiri::XML::Node]
-        # @return [String]
-        def entry_title(container, selected_anchor) = visible_text(heading_for(container) || selected_anchor)
-
-        ##
-        # @param text [String, #to_s]
-        # @return [Integer]
-        def word_count(text)
-          (@word_counts ||= {})[text] ||= begin
-            count = 0
-            text.to_s.scan(/\p{Alnum}+/) { count += 1 }
-            count
-          end
-        end
-
-        def container_tokens(container)
-          (@container_tokens ||= {}.compare_by_identity)[container] ||= "#{container['class']} #{container['id']}"
-        end
-
-        def content_tokens?(tokens)
-          tokens.match?(CONTENT_REGEXP)
-        end
-
-        def junk_tokens?(tokens)
-          tokens.match?(JUNK_REGEXP)
-        end
-
         def stable_rank(entries)
           entries.sort_by { |entry| [-entry.final_score, entry.position] }
         end

data/lib/html2rss/auto_source/scraper/sitemap.rb

@@ -0,0 +1,153 @@
+# frozen_string_literal: true
+
+require 'nokogiri'
+
+module Html2rss
+  class AutoSource
+    module Scraper
+      ##
+      # Scrapes articles from XML sitemaps explicitly linked in HTML or discovered via standard sitemap paths.
+      class Sitemap
+        include Enumerable
+
+        # Selector for sitemap link tags in HTML head
+        SITEMAP_LINK_SELECTOR = 'link[rel="sitemap"][href]'
+
+        # Maximum number of sub-sitemaps to fetch when fanning out through a sitemapindex.
+        MAX_SUB_SITEMAPS = 3
+
+        # @return [Symbol] scraper config key
+        def self.options_key = :sitemap
+
+        ##
+        # Returns the number of request slots needed for sitemap discovery and fan-out.
+        #
+        # @param _opts [Hash] unused options
+        # @return [Integer] number of follow-up requests needed
+        def self.request_slots(_opts = {})
+          1 + MAX_SUB_SITEMAPS
+        end
+
+        ##
+        # @param parsed_body [Nokogiri::HTML::Document, nil] parsed document
+        # @return [Boolean] whether the page links to a sitemap or is a sitemap
+        def self.articles?(parsed_body)
+          return false unless parsed_body
+
+          !parsed_body.at_css(SITEMAP_LINK_SELECTOR).nil? ||
+            !parsed_body.at_xpath('//*[local-name()="urlset" or local-name()="sitemapindex"]').nil?
+        end
+
+        ##
+        # @param parsed_body [Nokogiri::HTML::Document] parsed HTML document
+        # @param url [String, Html2rss::Url] canonical page URL
+        # @param request_session [Html2rss::RequestSession, nil] shared request session for follow-up fetches
+        # @option _opts [Float] :min_priority minimum priority score (default: 0.3)
+        # @option _opts [Integer] :max_age_days maximum entry age in days (default: 30)
+        def initialize(parsed_body, url:, request_session: nil, **opts)
+          @parsed_body = parsed_body
+          @url = Html2rss::Url.from_absolute(url)
+          @request_session = request_session
+          @min_priority = opts.fetch(:min_priority, Discovery::Sitemap::DEFAULT_MIN_PRIORITY)
+          @max_age_days = opts.fetch(:max_age_days, Discovery::Sitemap::DEFAULT_MAX_AGE_DAYS)
+        end
+
+        ##
+        # Yields article hashes from the sitemap.
+        #
+        # @yieldparam article [Hash{Symbol => Object}] normalized article hash
+        # @return [Enumerator, void] enumerator when no block is given
+        def each
+          return enum_for(:each) unless block_given?
+
+          entries = fetch_and_parse_entries
+          return if entries.empty?
+
+          entries.each do |entry|
+            article_hash = article_from_entry(entry)
+            yield article_hash if article_hash
+          end
+        end
+
+        private
+
+        attr_reader :parsed_body, :url, :request_session, :min_priority, :max_age_days
+
+        def fetch_and_parse_entries
+          entries = parse_direct_sitemap
+          return entries unless entries.empty?
+
+          xml_body = fetch_xml(target_sitemap_url)
+          return [] unless xml_body
+
+          parse_sitemap_xml(xml_body)
+        rescue Html2rss::RequestService::RequestBudgetExceeded
+          Log.warn("#{self.class}: sitemap fetch stopped — request budget exhausted")
+          []
+        end
+
+        def parse_sitemap_xml(xml_body)
+          result = Discovery::Sitemap.call(xml_body, min_priority:, max_age_days:)
+          return fetch_sub_sitemaps(result.sub_sitemap_urls) if result.sub_sitemap_urls.any?
+
+          result.entries
+        end
+
+        def parse_direct_sitemap
+          return [] unless parsed_body.at_xpath('//*[local-name()="urlset" or local-name()="sitemapindex"]')
+
+          Discovery::Sitemap.call(parsed_body.to_s, min_priority:, max_age_days:).entries
+        end
+
+        def fetch_sub_sitemaps(sub_urls)
+          entries = []
+          sub_urls.first(MAX_SUB_SITEMAPS).each do |sub_url|
+            sub_entries = fetch_sub_sitemap_entries(sub_url)
+            break unless sub_entries
+
+            entries.concat(sub_entries)
+          end
+          entries
+        end
+
+        def fetch_sub_sitemap_entries(sub_url)
+          xml = fetch_xml(sub_url)
+          return [] unless xml
+
+          Discovery::Sitemap.call(xml, min_priority:, max_age_days:).entries
+        rescue Html2rss::RequestService::RequestBudgetExceeded
+          Log.warn("#{self.class}: sitemap fan-out stopped — request budget exhausted")
+          nil
+        end
+
+        def fetch_xml(target_url)
+          return unless request_session && target_url
+
+          response = request_session.follow_up(url: target_url, relation: :auto_source, origin_url: url)
+          response&.body
+        rescue Html2rss::RequestService::RequestBudgetExceeded
+          raise
+        rescue Html2rss::Error => error
+          Log.warn("#{self.class}: failed to fetch sitemap (#{error.class}: #{error.message})")
+          nil
+        end
+
+        def target_sitemap_url
+          link_tag = parsed_body.at_css(SITEMAP_LINK_SELECTOR)
+          href = link_tag ? link_tag['href'] : '/sitemap.xml'
+          Html2rss::Url.from_relative(href, url)
+        end
+
+        def article_from_entry(entry)
+          return unless entry.url
+
+          {
+            url: entry.url,
+            title: entry.title,
+            published_at: entry.published_at
+          }.compact
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/auto_source/scraper/wordpress_api/page_scope/date_archive_range.rb

@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+
+require 'date'
+
+module Html2rss
+  class AutoSource
+    module Scraper
+      class WordpressApi
+        class PageScope
+          ##
+          # Maps scoped archive path segments to WordPress REST +after+/+before+ bounds.
+          #
+          # Owns date-path shape detection and calendar range math only. Body-class
+          # taxonomy/author signals and pagination stay on {Resolver}.
+          class DateArchiveRange
+            ##
+            # @param segments [Array<String>] path segments with pagination already stripped
+            # @return [Hash{String => String}, nil] +after+/+before+ ISO8601 bounds, or +nil+
+            def self.from_segments(segments)
+              components = components_from(segments)
+              return unless components
+
+              start_date = Date.new(*components.fetch(:start_date_parts))
+              {
+                'after' => iso8601_start(start_date),
+                'before' => iso8601_start(next_archive_boundary(start_date, components.fetch(:precision)))
+              }
+            rescue Date::Error
+              nil
+            end
+
+            ##
+            # @param segments [Array<String>] path segments with pagination already stripped
+            # @return [Boolean] whether segments match a year[/month[/day]] archive shape
+            def self.path?(segments)
+              !date_segments_from(segments).nil?
+            end
+
+            class << self
+              private
+
+              def components_from(segments)
+                date_segments = date_segments_from(segments)
+                return unless date_segments
+
+                year = date_segments.fetch(0).to_i
+                month = parse_archive_segment(date_segments[1], 1, 12)
+                day = parse_archive_segment(date_segments[2], 1, 31)
+
+                {
+                  start_date_parts: [year, month || 1, day || 1],
+                  precision: archive_precision(month:, day:)
+                }
+              end
+
+              def date_segments_from(segments)
+                year_index = segments.find_index { _1.match?(/\A\d{4}\z/) }
+                return unless year_index
+
+                date_segments = segments.drop(year_index)
+                return unless date_segments.length.between?(1, 3)
+                return unless archive_segment_shape?(date_segments)
+
+                date_segments
+              end
+
+              def archive_segment_shape?(segments)
+                month = segments[1]
+                day = segments[2]
+                return false if day && month.nil?
+                return false unless month.nil? || month.match?(/\A\d+\z/)
+                return false unless day.nil? || day.match?(/\A\d+\z/)
+
+                true
+              end
+
+              def archive_precision(month:, day:)
+                return :day if day
+                return :month if month
+
+                :year
+              end
+
+              def next_archive_boundary(start_date, precision)
+                {
+                  year: start_date.next_year,
+                  month: start_date.next_month,
+                  day: start_date.next_day
+                }.fetch(precision)
+              end
+
+              def iso8601_start(date)
+                date.strftime('%Y-%m-%dT00:00:00Z')
+              end
+
+              def parse_archive_segment(value, minimum, maximum)
+                return nil unless value&.match?(/\A\d+\z/)
+
+                number = value.to_i
+                return nil if number < minimum || number > maximum
+
+                number
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/auto_source/scraper/wordpress_api/page_scope.rb

@@ -99,7 +99,7 @@ module Html2rss
             def date_scope
               return unless date_archive?
 
-              range = date_archive_range
+              range = DateArchiveRange.from_segments(scoped_path_segments)
               return unknown_archive_scope unless range
 
               PageScope.new(query: range, fetchable: true, reason: :archive)
@@ -195,45 +195,7 @@ module Html2rss
             end
 
             def date_archive_path?
-              !date_archive_segments.nil?
-            end
-
-            def date_archive_range
-              components = date_archive_components
-              return unless components
-
-              start_date = Date.new(*components.fetch(:start_date_parts))
-              {
-                'after' => iso8601_start(start_date),
-                'before' => iso8601_start(next_archive_boundary(start_date, components.fetch(:precision)))
-              }
-            rescue Date::Error
-              nil
-            end
-
-            def date_archive_components
-              segments = date_archive_segments
-              return unless segments
-
-              year = segments.fetch(0).to_i
-              month = parse_archive_segment(segments[1], 1, 12)
-              day = parse_archive_segment(segments[2], 1, 31)
-
-              {
-                start_date_parts: [year, month || 1, day || 1],
-                precision: archive_precision(month:, day:)
-              }
-            end
-
-            def date_archive_segments
-              year_index = scoped_path_segments.find_index { _1.match?(/\A\d{4}\z/) }
-              return unless year_index
-
-              segments = scoped_path_segments.drop(year_index)
-              return unless segments.length.between?(1, 3)
-              return unless archive_segment_shape?(segments)
-
-              segments
+              DateArchiveRange.path?(scoped_path_segments)
             end
 
             def paginated_path?
@@ -253,48 +215,10 @@ module Html2rss
               parse_positive_integer(path_segments[-1])
             end
 
-            def archive_segment_shape?(segments)
-              month = segments[1]
-              day = segments[2]
-              return false if day && month.nil?
-              return false unless month.nil? || month.match?(/\A\d+\z/)
-              return false unless day.nil? || day.match?(/\A\d+\z/)
-
-              true
-            end
-
             def same_origin_url?(left, right)
               [left.scheme, left.host, left.port] == [right.scheme, right.host, right.port]
             end
 
-            def archive_precision(month:, day:)
-              return :day if day
-              return :month if month
-
-              :year
-            end
-
-            def next_archive_boundary(start_date, precision)
-              {
-                year: start_date.next_year,
-                month: start_date.next_month,
-                day: start_date.next_day
-              }.fetch(precision)
-            end
-
-            def iso8601_start(date)
-              date.strftime('%Y-%m-%dT00:00:00Z')
-            end
-
-            def parse_archive_segment(value, minimum, maximum)
-              return nil unless value&.match?(/\A\d+\z/)
-
-              number = value.to_i
-              return nil if number < minimum || number > maximum
-
-              number
-            end
-
             def parse_positive_integer(value)
               return nil unless value.to_s.match?(/\A\d+\z/)
 

data/lib/html2rss/auto_source/scraper/wordpress_api.rb

@@ -24,6 +24,15 @@ module Html2rss
         # @return [Symbol] scraper config key
         def self.options_key = :wordpress_api
 
+        ##
+        # Returns the number of request slots needed for WordPress API posts fetch.
+        #
+        # @param _opts [Hash] unused options
+        # @return [Integer] number of follow-up requests needed
+        def self.request_slots(_opts = {})
+          1
+        end
+
         ##
         # @param parsed_body [Nokogiri::HTML::Document, nil] parsed HTML document
         # @return [Boolean] whether the page advertises a WordPress REST API endpoint