html2rss 0.24.0 → 0.26.0

data/lib/html2rss/auto_source/scraper/semantic_html.rb

@@ -4,31 +4,12 @@ module Html2rss
   class AutoSource
     module Scraper
       ##
-      # Scrapes semantic containers by choosing one primary content link per
-      # block before extraction.
-      #
-      # This scraper is intentionally container-first:
-      # 1. collect candidate semantic containers once
-      # 2. select the strongest content-like anchor within each container
-      # 3. extract fields from the container while honoring that anchor choice
-      #
-      # The result is lower recall on weak-signal blocks, but much better link
-      # quality on modern teaser cards that mix headlines, utility links, and
-      # duplicate image overlays.
+      # Scrapes semantic containers via SST: Normalizer → Segmenter → Scoring → SstArticleExtractor.
       class SemanticHtml
         include Enumerable
 
-        # Container plus selected anchor, scoring metadata, and extracted article.
-        Entry = Data.define(
-          :container,
-          :selected_anchor,
-          :destination_facts,
-          :quality_score,
-          :junk_score,
-          :final_score,
-          :position,
-          :article
-        )
+        # Maximum articles materialized after ranking.
+        TOP_K = Scoring::Engine::TOP_K
 
         ##
         # @return [Symbol] config key used to enable or configure this scraper
@@ -40,133 +21,72 @@ module Html2rss
           return false unless parsed_body
 
           new(parsed_body, url: 'https://example.com').extractable?
+        rescue ArgumentError
+          false
         end
 
-        # @param parsed_body [Nokogiri::HTML::Document] parsed HTML document
+        # @param parsed_body [Nokogiri::HTML::Document, nil] parsed HTML (when +document:+ omitted)
         # @param url [String, Html2rss::Url] base url
-        # @param extractor [Class] extractor class used for article extraction
+        # @param document [SST::Document, nil] memoized SST document from AutoSource
+        # @param link_resolver [Scoring::LinkResolver, nil] shared page-scoped resolver
         # @param opts [Hash] scraper-specific options
         # @option opts [Boolean] :fallback_anchorless whether to keep containers without a primary anchor
-        def initialize(parsed_body, url:, extractor: Html2rss::Html::ArticleExtractor, **opts)
+        def initialize(parsed_body = nil, url:, document: nil, link_resolver: nil, **opts)
           @parsed_body = parsed_body
+          @provided_document = document
+          @provided_link_resolver = link_resolver
           @url = url
-          @extractor = extractor
           @permit_unanchored = opts.fetch(:fallback_anchorless, false)
-          @link_heuristics = LinkHeuristics.new(url)
         end
 
-        attr_reader :parsed_body
-
         ##
-        # Yields extracted article hashes for each semantic container that
-        # survives anchor selection.
-        #
-        # Detection and extraction share the same memoized entry list so this
-        # scraper does not rerun anchor ranking once a page has already been
-        # accepted as extractable.
-        #
-        # @yieldparam article_hash [Hash] extracted article hash
-        # @return [Enumerator<Hash>]
+        # @yieldparam article [Html2rss::Article]
+        # @return [Enumerator<Html2rss::Article>]
         def each
           return enum_for(:each) unless block_given?
 
-          ranked_entries.each { yield _1.article }
+          articles.each { yield _1 }
         end
 
         ##
-        # Reports whether the page contains at least one semantic container with
-        # a selectable primary anchor.
-        #
-        # @return [Boolean] true when at least one candidate container yields a primary anchor
+        # @return [Boolean]
         def extractable?
-          extractable_entries.any?
-        end
-
-        # @return [Array<Nokogiri::XML::Node>]
-        def candidate_containers
-          @candidate_containers ||= collect_candidate_containers
-        end
-
-        # @param container [Nokogiri::XML::Node]
-        # @return [Nokogiri::XML::Node, nil]
-        def primary_anchor_for(container)
-          Discovery::SemanticAnchorCandidates.new(
-            container,
-            link_heuristics: @link_heuristics
-          ).to_a.max_by(&:score)&.anchor
+          ranked_segments.any?
+        rescue ArgumentError
+          false
         end
 
-        # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
-        def extractable_entries
-          @extractable_entries ||= candidate_containers.filter_map do |container|
-            selected_anchor = primary_anchor_for(container)
-
-            next unless selected_anchor || @permit_unanchored
-
-            destination_facts = selected_anchor ? normalized_destination(selected_anchor) : nil
-            next if selected_anchor && !destination_facts
-            # Cheap path-only reject before title/DOM hard-junk observations.
-            next if destination_facts&.high_confidence_junk_path
-
-            signals = @link_heuristics.assess_container(container, selected_anchor, destination_facts:)
-            next if signals.hard_junk?
+        private
 
-            Entry.new(
-              container:,
-              selected_anchor:,
-              destination_facts:,
-              quality_score: signals.quality_score,
-              junk_score: signals.junk_score,
-              final_score: signals.final_score,
-              position: document_position(container),
-              article: nil
-            )
+        def articles
+          @articles ||= begin
+            deduplicator = EntryDeduplicator.new(@url, scraper: self.class, link_resolver:)
+            entries = deduplicator.call(ranked_segments)
+            entries.filter_map { |ranked| deduplicator.article_for(ranked) }
           end
+        rescue ArgumentError
+          []
         end
-        # rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
-
-        # rubocop:disable Metrics/MethodLength
-        def ranked_entries
-          @ranked_entries ||= begin
-            deduplicator = EntryDeduplicator.new(@url, @extractor)
-            entries = deduplicator.call(extractable_entries)
-            entries = stable_rank(entries)
-
-            entries.filter_map do |entry|
-              article = deduplicator.article_for(entry)
-              next unless article
 
-              Entry.new(
-                container: entry.container,
-                selected_anchor: entry.selected_anchor,
-                destination_facts: entry.destination_facts,
-                quality_score: entry.quality_score,
-                junk_score: entry.junk_score,
-                final_score: entry.final_score,
-                position: entry.position,
-                article:
-              )
-            end
+        def ranked_segments
+          @ranked_segments ||= begin
+            segments = Segmenter.call(
+              document,
+              base_url: @url,
+              strategy: :semantic,
+              permit_unanchored: @permit_unanchored,
+              link_resolver:
+            )
+            Scoring::Engine.new(link_resolver:).rank_top(segments, limit: TOP_K)
           end
         end
-        # rubocop:enable Metrics/MethodLength
-
-        def collect_candidate_containers
-          Discovery::SemanticContainers.call(parsed_body)
-        end
-
-        private
-
-        def document_position(container)
-          (@document_positions ||= candidate_containers.each_with_index.to_h).fetch(container)
-        end
 
-        def normalized_destination(anchor)
-          (@normalized_destinations ||= {}.compare_by_identity)[anchor] ||= @link_heuristics.destination_facts(anchor)
+        def link_resolver
+          @link_resolver ||= @provided_link_resolver || Scoring::LinkResolver.new(@url)
         end
 
-        def stable_rank(entries)
-          entries.sort_by { |entry| [-entry.final_score, entry.position] }
+        def document
+          @document ||= @provided_document || SST::Normalizer.call(@parsed_body)
         end
       end
     end

data/lib/html2rss/auto_source/scraper/sitemap/parser.rb

@@ -0,0 +1,161 @@
+# frozen_string_literal: true
+
+require 'date'
+require 'nokogiri'
+
+module Html2rss
+  class AutoSource
+    module Scraper
+      class Sitemap
+        ##
+        # Parses XML sitemap documents and extracts prioritized, recent article entries.
+        class Parser
+          # Default minimum priority threshold (0.0 to 1.0)
+          DEFAULT_MIN_PRIORITY = 0.3
+          # Default maximum age in days for sitemap entries
+          DEFAULT_MAX_AGE_DAYS = 30
+          # Google News sitemap XML namespace
+          NEWS_NS = 'http://www.google.com/schemas/sitemap-news/0.9'
+
+          # Result value object returned by .call
+          # @!attribute entries [Array<SitemapEntry>] parsed entries (empty for sitemapindex docs)
+          # @!attribute sub_sitemap_urls [Array<String>] child sitemap URLs (empty for urlset docs)
+          Result = Data.define(:entries, :sub_sitemap_urls)
+
+          # Immutable facts for one sitemap entry
+          SitemapEntry = Data.define(:url, :title, :published_at, :priority, :changefreq)
+
+          class << self
+            ##
+            # Parses sitemap XML body and returns a Result.
+            #
+            # @param xml_content [String] sitemap XML body string
+            # @param min_priority [Float] minimum priority score (0.0..1.0)
+            # @param max_age_days [Integer] maximum age in days for entries
+            # @return [Result] result with entries and/or sub_sitemap_urls
+            def call(xml_content, min_priority: DEFAULT_MIN_PRIORITY, max_age_days: DEFAULT_MAX_AGE_DAYS)
+              new(xml_content, min_priority:, max_age_days:).call
+            end
+          end
+
+          ##
+          # @param xml_content [String]
+          # @param min_priority [Float]
+          # @param max_age_days [Integer]
+          def initialize(xml_content, min_priority: DEFAULT_MIN_PRIORITY, max_age_days: DEFAULT_MAX_AGE_DAYS)
+            @xml_content = xml_content
+            @min_priority = min_priority
+            @max_age_days = max_age_days
+          end
+
+          ##
+          # @return [Result]
+          def call
+            document = Nokogiri::XML(xml_content)
+            return Result.new(entries: [], sub_sitemap_urls: []) if invalid_xml?(document)
+
+            if sitemapindex?(document)
+              Result.new(entries: [], sub_sitemap_urls: extract_sub_sitemap_urls(document))
+            else
+              Result.new(entries: extract_entries(document), sub_sitemap_urls: [])
+            end
+          end
+
+          private
+
+          attr_reader :xml_content, :min_priority, :max_age_days
+
+          def invalid_xml?(document)
+            document.errors.any? && document.at_css('urlset, sitemapindex').nil?
+          end
+
+          def sitemapindex?(document)
+            !document.at_xpath('//*[local-name()="sitemapindex"]').nil?
+          end
+
+          def extract_sub_sitemap_urls(document)
+            document.xpath('//*[local-name()="sitemap"]/*[local-name()="loc"]').filter_map do |node|
+              text = node.text.strip
+              text unless text.empty?
+            end
+          end
+
+          def extract_entries(document)
+            document.xpath('//*[local-name()="url"]').filter_map { build_entry(_1) }
+          end
+
+          def build_entry(url_node)
+            loc = parse_loc(url_node)
+            return unless loc
+            return if invalid_entry_signals?(url_node)
+
+            published_at = parse_date(url_node)
+            return if stale_date?(published_at)
+
+            build_entry_facts(url_node, loc, published_at)
+          end
+
+          def build_entry_facts(url_node, loc, published_at)
+            SitemapEntry.new(
+              url: loc,
+              title: parse_title(url_node),
+              published_at:,
+              priority: parse_priority(url_node),
+              changefreq: parse_changefreq(url_node)
+            )
+          end
+
+          def parse_loc(url_node)
+            node = url_node.at_xpath('./*[local-name()="loc"]')
+            text = node&.text&.strip
+            text unless text.to_s.empty?
+          end
+
+          def parse_changefreq(url_node)
+            node = url_node.at_xpath('./*[local-name()="changefreq"]')
+            node ? node.text.strip.downcase : nil
+          end
+
+          def invalid_entry_signals?(url_node)
+            parse_priority(url_node) < min_priority || parse_changefreq(url_node) == 'never'
+          end
+
+          def parse_priority(url_node)
+            node = url_node.at_xpath('./*[local-name()="priority"]')
+            text = node&.text&.strip
+            text ? text.to_f : 0.5
+          end
+
+          def parse_date(url_node)
+            raw_date = raw_date_from(url_node)
+            return unless raw_date
+
+            Time.parse(raw_date).utc.iso8601
+          rescue ArgumentError
+            nil
+          end
+
+          def raw_date_from(url_node)
+            news_node = url_node.at_xpath('.//news:publication_date', 'news' => NEWS_NS)
+            lastmod_node = url_node.at_xpath('./*[local-name()="lastmod"]')
+            (news_node || lastmod_node)&.text&.strip
+          end
+
+          def parse_title(url_node)
+            node = url_node.at_xpath('.//news:title', 'news' => NEWS_NS)
+            node&.text&.strip
+          end
+
+          def stale_date?(published_at)
+            return false unless published_at
+
+            time = Time.parse(published_at)
+            (Time.now.utc - time) > (max_age_days * 86_400)
+          rescue ArgumentError
+            false
+          end
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/auto_source/scraper/sitemap.rb

@@ -42,14 +42,16 @@ module Html2rss
         # @param parsed_body [Nokogiri::HTML::Document] parsed HTML document
         # @param url [String, Html2rss::Url] canonical page URL
         # @param request_session [Html2rss::RequestSession, nil] shared request session for follow-up fetches
+        # @param body [String, nil] raw response body for direct sitemap XML parsing
         # @option _opts [Float] :min_priority minimum priority score (default: 0.3)
         # @option _opts [Integer] :max_age_days maximum entry age in days (default: 30)
-        def initialize(parsed_body, url:, request_session: nil, **opts)
+        def initialize(parsed_body, url:, request_session: nil, body: nil, **opts)
           @parsed_body = parsed_body
+          @body = body
           @url = Html2rss::Url.from_absolute(url)
           @request_session = request_session
-          @min_priority = opts.fetch(:min_priority, Discovery::Sitemap::DEFAULT_MIN_PRIORITY)
-          @max_age_days = opts.fetch(:max_age_days, Discovery::Sitemap::DEFAULT_MAX_AGE_DAYS)
+          @min_priority = opts.fetch(:min_priority, Scraper::Sitemap::Parser::DEFAULT_MIN_PRIORITY)
+          @max_age_days = opts.fetch(:max_age_days, Scraper::Sitemap::Parser::DEFAULT_MAX_AGE_DAYS)
         end
 
         ##
@@ -71,7 +73,7 @@ module Html2rss
 
         private
 
-        attr_reader :parsed_body, :url, :request_session, :min_priority, :max_age_days
+        attr_reader :parsed_body, :body, :url, :request_session, :min_priority, :max_age_days
 
         def fetch_and_parse_entries
           entries = parse_direct_sitemap
@@ -87,16 +89,17 @@ module Html2rss
         end
 
         def parse_sitemap_xml(xml_body)
-          result = Discovery::Sitemap.call(xml_body, min_priority:, max_age_days:)
+          result = Scraper::Sitemap::Parser.call(xml_body, min_priority:, max_age_days:)
           return fetch_sub_sitemaps(result.sub_sitemap_urls) if result.sub_sitemap_urls.any?
 
           result.entries
         end
 
         def parse_direct_sitemap
+          return [] unless body
           return [] unless parsed_body.at_xpath('//*[local-name()="urlset" or local-name()="sitemapindex"]')
 
-          Discovery::Sitemap.call(parsed_body.to_s, min_priority:, max_age_days:).entries
+          Scraper::Sitemap::Parser.call(body, min_priority:, max_age_days:).entries
         end
 
         def fetch_sub_sitemaps(sub_urls)
@@ -114,7 +117,7 @@ module Html2rss
           xml = fetch_xml(sub_url)
           return [] unless xml
 
-          Discovery::Sitemap.call(xml, min_priority:, max_age_days:).entries
+          Scraper::Sitemap::Parser.call(xml, min_priority:, max_age_days:).entries
         rescue Html2rss::RequestService::RequestBudgetExceeded
           Log.warn("#{self.class}: sitemap fan-out stopped — request budget exhausted")
           nil

data/lib/html2rss/auto_source/scraper/wordpress_api.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
+require 'cgi'
 require 'date'
-require 'nokogiri'
 
 module Html2rss
   class AutoSource
@@ -153,7 +153,11 @@ module Html2rss
         end
 
         def rendered_text(value)
-          rendered_html(value)&.then { Nokogiri::HTML.fragment(_1).text.strip }
+          html = rendered_html(value)
+          return unless html
+
+          text = CGI.unescapeHTML(html.gsub(/<[^>]*>/m, ' ')).gsub(/\s+/, ' ').strip
+          text unless text.empty?
         end
 
         def rendered_html(value)

data/lib/html2rss/auto_source/scraper/xhr_articles.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module Html2rss
+  class AutoSource
+    module Scraper
+      # Extracts articles from JSON XHR/fetch responses captured during a
+      # browser-tier scrape (see RequestService::Response#captured_responses).
+      class XhrArticles
+        include Enumerable
+
+        # @return [Symbol] scraper config key
+        def self.options_key = :xhr_articles
+
+        # @param _opts [Hash] unused scraper options
+        # @return [Integer] zero — no additional HTTP requests
+        def self.request_slots(_opts = {}) = 0
+
+        # Not detectable from HTML alone; instance {#extractable?} uses captures.
+        #
+        # @param _parsed_body [Nokogiri::HTML::Document, nil]
+        # @return [Boolean]
+        def self.articles?(_parsed_body) = false
+
+        # @param _parsed_body [Nokogiri::HTML::Document, nil] unused HTML document
+        # @param url [String, Html2rss::Url] page URL used to resolve relative links
+        # @param captured_responses [Array<Hash>] JSON bodies from Response#captured_responses
+        # @param _opts [Hash] scraper-specific options
+        # @option _opts [Object] :_reserved reserved for future scraper-specific options
+        def initialize(_parsed_body, url:, captured_responses: [], **_opts)
+          @url = url
+          @captured_responses = captured_responses
+        end
+
+        # @return [Boolean] true when any captured body contains article-like arrays
+        def extractable?
+          parsed_bodies.any? { |doc| JsonState::CandidateDetector.candidate_array?(doc) }
+        end
+
+        # @yield [Hash{Symbol => Object}] normalized article hash
+        # @return [Enumerator, void] article enumerator when no block is given
+        def each
+          return enum_for(:each) unless block_given?
+
+          parsed_bodies.each do |doc|
+            JsonState.discover_articles(doc, base_url: @url) { |article| yield article if article }
+          end
+        end
+
+        private
+
+        def parsed_bodies
+          @parsed_bodies ||= @captured_responses.filter_map { |captured| parse(captured) }
+                                                .select { |doc| JsonState::CandidateDetector.candidate_array?(doc) }
+        end
+
+        def parse(captured)
+          body = captured[:body] || captured['body']
+          return unless body.is_a?(String)
+
+          JSON.parse(body, symbolize_names: true)
+        rescue JSON::ParserError
+          nil
+        end
+      end
+    end
+  end
+end