html2rss 0.22.2 → 0.24.0

data/lib/html2rss/auto_source/link_heuristics.rb

@@ -0,0 +1,139 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class AutoSource
+    ##
+    # Shared link eligibility and scoring policy for AutoSource scrapers.
+    #
+    # Scrapers collect DOM observations; this module owns junk/noise rules and
+    # numeric weights so eligibility policy stays in one place.
+    class LinkHeuristics
+      # @param base_url [String, Html2rss::Url] page URL used to resolve relative hrefs
+      def initialize(base_url)
+        @base_url = base_url
+        @text_classifier = TextClassifier.new
+        @container_assessor = ContainerAssessor.new(text_classifier: @text_classifier)
+      end
+
+      # Builds normalized destination facts for an anchor element or href string.
+      #
+      # @param anchor_or_href [Nokogiri::XML::Element, String, #to_s] anchor element or href-like value
+      # @return [DestinationFacts, nil] normalized destination facts, or nil for blank/invalid URLs
+      def destination_facts(anchor_or_href)
+        return node_facts[anchor_or_href] if node_facts.key?(anchor_or_href)
+
+        href = HrefExtractor.call(anchor_or_href)
+        return unless href
+
+        res = memoized_destination_facts(href)
+
+        node_facts[anchor_or_href] = res if anchor_or_href.is_a?(Nokogiri::XML::Node)
+        res
+      rescue ArgumentError
+        nil
+      end
+
+      # @param text [String, #to_s] visible anchor text
+      # @return [Boolean] true when text matches a utility label
+      def utility_text?(text) = @text_classifier.utility?(text)
+
+      # @param text [String, #to_s] visible anchor text
+      # @return [Boolean] true when text begins with a utility label
+      def utility_prefix_text?(text) = @text_classifier.utility_prefix?(text)
+
+      # @param text [String, #to_s] visible anchor text
+      # @return [Boolean] true when text identifies recommendation chrome
+      def recommended_text?(text) = @text_classifier.recommended?(text)
+
+      ##
+      # Whether an anchor is junk chrome rather than a content permalink.
+      #
+      # One eligibility home for Html and SemanticHtml: taxonomy/utility text
+      # rules plus optional DOM checks (icon-only, utility landmarks).
+      #
+      # @param text [String, #to_s] visible anchor text
+      # @param destination_facts [DestinationFacts, nil] route facts for the href
+      # @param anchor [Nokogiri::XML::Node, nil] anchor node for icon/landmark checks
+      # @param container [Nokogiri::XML::Node, nil] content container bounding landmark walks
+      # @param heading_anchor [Boolean] whether the anchor is the container heading link
+      # @return [Boolean] true when the anchor should be ignored
+      def noise_anchor?(text:, destination_facts:, anchor: nil, container: nil, heading_anchor: false) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+        return true unless destination_facts
+
+        destination_facts.taxonomy_path ||
+          short_utility_label?(text, destination_facts) ||
+          recommended_chrome?(text, destination_facts, heading_anchor:) ||
+          (utility_prefix_text?(text) && destination_facts.high_confidence_utility_destination) ||
+          (utility_text?(text) && destination_facts.vanity_path) ||
+          utility_text_chrome?(text, destination_facts, heading_anchor:) ||
+          icon_only_anchor?(anchor, text) ||
+          utility_landmark_ancestor?(anchor, container)
+      end
+
+      ##
+      # Observes a container and builds ranking signals, including hard-junk.
+      #
+      # Delegates DOM observation to ContainerAssessor so SemanticHtml only
+      # orchestrates candidates and extraction, while ContainerSignals keeps
+      # scoring policy.
+      #
+      # @param container [Nokogiri::XML::Node] semantic container node
+      # @param selected_anchor [Nokogiri::XML::Node, nil] primary anchor for the container
+      # @param destination_facts [DestinationFacts, nil] route facts for the selected anchor
+      # @return [ContainerSignals] observation + scoring signals for the container
+      def assess_container(container, selected_anchor, destination_facts:)
+        @container_assessor.call(container, selected_anchor, destination_facts:)
+      end
+
+      private
+
+      def short_utility_label?(text, destination_facts)
+        destination_facts.utility_path &&
+          !destination_facts.content_path &&
+          !destination_facts.strong_post_suffix &&
+          text.to_s.scan(/\p{Alnum}+/).size <= 3
+      end
+
+      def recommended_chrome?(text, destination_facts, heading_anchor:)
+        # Heading-linked titles may start with "Recommended …" while still being
+        # real posts; container hard_junk? owns that call with publish markers.
+        !heading_anchor && recommended_text?(text) && destination_facts.shallow
+      end
+
+      def utility_text_chrome?(text, destination_facts, heading_anchor:)
+        return false if destination_facts.content_path
+        return false unless utility_text?(text)
+
+        !heading_anchor && !destination_facts.strong_post_suffix
+      end
+
+      def icon_only_anchor?(anchor, text)
+        return false unless anchor
+
+        !text.to_s.match?(/\p{Alnum}/) && !anchor.at_css('img, svg').nil?
+      end
+
+      def utility_landmark_ancestor?(anchor, container)
+        return false unless anchor && container
+
+        condition = lambda { |node|
+          node == container || Html2rss::Html::Navigator::UTILITY_LANDMARK_TAGS.include?(node.name)
+        }
+        landmark = Html2rss::Html::Navigator.parent_until_condition(anchor.parent, condition)
+
+        !landmark.nil? && landmark != container
+      end
+
+      def node_facts
+        @node_facts ||= {}.compare_by_identity
+      end
+
+      def memoized_destination_facts(href)
+        (@destination_facts ||= {})[href] ||= begin
+          url = Html2rss::Url.from_relative(href, @base_url)
+          DestinationFacts.build(url)
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/auto_source/scraper/html.rb

@@ -48,7 +48,7 @@ module Html2rss
         # @param xpath [String] original XPath
         # @return [String] XPath without positional indexes
         def self.simplify_xpath(xpath)
-          HtmlExtractor::ListCandidates.simplify_xpath(xpath)
+          Discovery::ListCandidates.simplify_xpath(xpath)
         end
 
         # @param parsed_body [Nokogiri::HTML::Document] The parsed HTML document.
@@ -57,7 +57,7 @@ module Html2rss
         # @param opts [Hash] Additional options.
         # @option opts [Integer] :minimum_selector_frequency minimum count before a selector is considered stable
         # @option opts [Integer] :use_top_selectors number of top selectors to keep
-        def initialize(parsed_body, url:, extractor: HtmlExtractor, **opts)
+        def initialize(parsed_body, url:, extractor: Html2rss::Html::ArticleExtractor, **opts)
           @parsed_body = parsed_body
           @url = url
           @extractor = extractor
@@ -96,7 +96,7 @@ module Html2rss
         # @return [Boolean] true when the node is a good extraction boundary
         def article_tag_condition?(node)
           # Ignore tags that are below ignored DOM chrome.
-          return false if HtmlExtractor.ignored_container_path?(node, @ignored_cache)
+          return false if Html2rss::Html::Navigator.ignored_container_path?(node, @ignored_cache)
           return true if %w[body html].include?(node.name)
           return false unless (parent = node.parent)
 
@@ -117,7 +117,10 @@ module Html2rss
         end
 
         def find_anchorless_articles
-          ClassClustering.call(parsed_body, minimum_selector_frequency:).map do |node|
+          Discovery::DomClustering.call(
+            parsed_body,
+            minimum_selector_frequency:
+          ).map do |node|
             @extractor.new(node, base_url: @url, selected_anchor: nil, fallback_anchorless: true).call
           end
         end
@@ -144,7 +147,7 @@ module Html2rss
           destination_facts = @link_heuristics.destination_facts(node)
           return false unless destination_facts
 
-          !noise_anchor?(node, destination_facts)
+          !noise_anchor?(node, destination_facts:)
         end
 
         ##
@@ -168,52 +171,38 @@ module Html2rss
         def extract_article(article_tag, selected_anchor: nil)
           selected_anchor ||= preferred_anchor_for(article_tag)
           return unless selected_anchor
-          return if noise_anchor?(selected_anchor, @link_heuristics.destination_facts(selected_anchor))
+          return if noise_anchor?(
+            selected_anchor,
+            destination_facts: @link_heuristics.destination_facts(selected_anchor),
+            container: article_tag
+          )
 
-          @extractor.new(article_tag, base_url: @url, selected_anchor:).call
+          @extractor.call(article_tag, base_url: @url, selected_anchor:)
         end
 
         ##
         # @param anchor [Nokogiri::XML::Node]
-        # @param destination_facts [DestinationFacts]
+        # @param destination_facts [LinkHeuristics::DestinationFacts, nil]
         # @return [Boolean]
-        def noise_anchor?(anchor, destination_facts) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
-          return true unless destination_facts
-
+        def noise_anchor?(anchor, destination_facts:, container: nil)
           (@noise_anchors ||= {}.compare_by_identity)[anchor] ||= begin
-            text = HtmlExtractor.extract_visible_text(anchor).to_s.strip
-
-            destination_facts.taxonomy_path ||
-              short_utility_label?(text, destination_facts) ||
-              (@link_heuristics.recommended_text?(text) && destination_facts.shallow) ||
-              (@link_heuristics.utility_prefix_text?(text) && destination_facts.high_confidence_utility_destination) ||
-              (@link_heuristics.utility_text?(text) && destination_facts.vanity_path)
+            text = Html2rss::Html::Navigator.extract_visible_text(anchor).to_s.strip
+            @link_heuristics.noise_anchor?(text:, destination_facts:, anchor:, container:)
           end
         end
 
-        ##
-        # @param text [String]
-        # @param destination_facts [DestinationFacts]
-        # @return [Boolean]
-        def short_utility_label?(text, destination_facts)
-          destination_facts.utility_path &&
-            !destination_facts.content_path &&
-            !destination_facts.strong_post_suffix &&
-            text.scan(/\p{Alnum}+/).size <= 3
-        end
-
         ##
         # @param article_tag [Nokogiri::XML::Node]
         # @return [Nokogiri::XML::Node, nil]
         def preferred_anchor_for(article_tag)
-          article_tag.css(HtmlExtractor::MAIN_ANCHOR_SELECTOR).find { relevant_anchor?(_1) } ||
-            HtmlExtractor.main_anchor_for(article_tag)
+          article_tag.css(Html2rss::Html::Navigator::MAIN_ANCHOR_SELECTOR).find { relevant_anchor?(_1) } ||
+            Html2rss::Html::Navigator.main_anchor_for(article_tag)
         end
 
         ##
-        # @return [HtmlExtractor::ListCandidates]
+        # @return [Discovery::ListCandidates]
         def list_candidates
-          HtmlExtractor::ListCandidates.new(
+          Discovery::ListCandidates.new(
             parsed_body,
             minimum_selector_frequency:,
             use_top_selectors:

data/lib/html2rss/auto_source/scraper/json_state.rb

@@ -9,7 +9,7 @@ module Html2rss
       # Scrapes JSON state blobs embedded in script tags such as Next.js, Nuxt,
       # or custom window globals. The scraper searches `<script type="application/json">`
       # tags and well-known JavaScript globals for arrays of article-like hashes
-      # and normalises them to a structure compatible with HtmlExtractor.
+      # and normalises them to a structure compatible with Html2rss::Html::ArticleExtractor.
       class JsonState
         include Enumerable
 

data/lib/html2rss/auto_source/scraper/meta_oembed.rb

@@ -0,0 +1,163 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'nokogiri'
+
+module Html2rss
+  class AutoSource
+    module Scraper
+      ##
+      # Scrapes OpenGraph meta tags and oEmbed JSON descriptors from HTML pages.
+      class MetaOembed
+        include Enumerable
+
+        # Selector for OpenGraph meta tags.
+        OG_META_SELECTOR = 'meta[property^="og:"], meta[property^="article:"], meta[name^="twitter:"]'
+        # Selector for oEmbed JSON link tag.
+        OEMBED_LINK_SELECTOR = 'link[rel="alternate"][type="application/json+oembed"][href]'
+
+        # Mapping of meta property names to article attribute keys.
+        META_MAP = {
+          'og:title' => :title,
+          'twitter:title' => :title,
+          'og:url' => :url,
+          'og:description' => :description,
+          'twitter:description' => :description,
+          'og:image' => :image,
+          'twitter:image' => :image,
+          'article:published_time' => :published_at,
+          'article:author' => :author
+        }.freeze
+        private_constant :META_MAP
+
+        # @return [Symbol] scraper config key
+        def self.options_key = :meta_oembed
+
+        class << self
+          # @param parsed_body [Nokogiri::HTML::Document, nil] parsed HTML document
+          # @return [Boolean] whether OpenGraph tags or oEmbed link exist
+          def articles?(parsed_body)
+            return false unless parsed_body
+
+            !parsed_body.at_css('meta[property="og:title"]').nil? ||
+              !parsed_body.at_css(OEMBED_LINK_SELECTOR).nil?
+          end
+        end
+
+        # @param parsed_body [Nokogiri::HTML::Document] parsed HTML document
+        # @param url [String, Html2rss::Url] base page URL
+        # @param request_session [Html2rss::RequestSession, nil] shared request session for follow-up fetches
+        # @param _opts [Hash] unused scraper-specific options
+        # @option _opts [Object] :_reserved reserved for future scraper-specific options
+        # @return [void]
+        def initialize(parsed_body, url:, request_session: nil, **_opts)
+          @parsed_body = parsed_body
+          @url = Html2rss::Url.from_absolute(url)
+          @request_session = request_session
+        end
+
+        ##
+        # Yields normalized article hash extracted from OpenGraph meta tags and optional oEmbed data.
+        #
+        # @yieldparam article [Hash{Symbol => Object}] normalized article hash
+        # @return [Enumerator, void] enumerator when no block is given
+        def each
+          return enum_for(:each) unless block_given?
+
+          article = build_article
+          yield article if article
+        end
+
+        private
+
+        attr_reader :parsed_body, :url, :request_session
+
+        # @return [Hash{Symbol => Object}, nil] assembled article hash or nil
+        def build_article
+          meta_data = extract_meta_data
+          oembed_data = fetch_oembed_data
+
+          article_url = resolve_url(meta_data[:url]) || url
+          title = oembed_data[:title] || meta_data[:title]
+
+          return unless title
+
+          assemble_article(meta_data, oembed_data, article_url, title)
+        end
+
+        # @return [Hash{Symbol => Object}] extracted metadata hash
+        def extract_meta_data
+          {}.tap do |data|
+            parsed_body.css(OG_META_SELECTOR).each do |meta|
+              prop = meta['property'] || meta['name']
+              content = meta['content']
+              next if prop.nil? || content.nil? || content.empty?
+
+              assign_meta_prop(data, prop, content)
+            end
+          end
+        end
+
+        # @param data [Hash{Symbol => Object}] target metadata accumulator
+        # @param prop [String] meta tag property/name
+        # @param content [String] meta tag content
+        # @return [void]
+        def assign_meta_prop(data, prop, content)
+          key = META_MAP[prop]
+          data[key] ||= content if key
+        end
+
+        # @return [Hash{Symbol => Object}] oEmbed fields hash
+        def fetch_oembed_data
+          return {} unless request_session && (link_node = parsed_body.at_css(OEMBED_LINK_SELECTOR))
+          return {} unless (oembed_url = resolve_url(link_node['href']))
+
+          response = request_session.follow_up(url: oembed_url, relation: :auto_source, origin_url: url)
+          parse_oembed_response(response)
+        rescue Html2rss::Error, RequestService::UnsupportedResponseContentType, JSON::ParserError => error
+          Log.warn("#{self.class}: failed to fetch oEmbed payload (#{error.class}: #{error.message})")
+          {}
+        end
+
+        # @param response [Html2rss::RequestService::Response, nil] HTTP response
+        # @return [Hash{Symbol => Object}] parsed oEmbed fields
+        def parse_oembed_response(response)
+          return {} unless response
+
+          payload = response.parsed_body
+          return {} unless payload.is_a?(Hash)
+
+          thumbnail = payload['thumbnail_url'] || (payload['type'] == 'photo' ? payload['url'] : nil)
+
+          { title: payload['title'], author: payload['author_name'], thumbnail:, html: payload['html'] }.compact
+        end
+
+        # @param meta_data [Hash{Symbol => Object}] extracted OpenGraph meta tags
+        # @param oembed_data [Hash{Symbol => Object}] parsed oEmbed fields
+        # @param article_url [Html2rss::Url] resolved article URL
+        # @param title [String] resolved article title
+        # @return [Hash{Symbol => Object}] final article hash
+        def assemble_article(meta_data, oembed_data, article_url, title)
+          {
+            url: article_url,
+            title:,
+            description: meta_data[:description] || oembed_data[:html],
+            author: oembed_data[:author] || meta_data[:author],
+            published_at: meta_data[:published_at],
+            image: oembed_data[:thumbnail] || meta_data[:image]
+          }.compact
+        end
+
+        # @param raw_url [String, nil] raw relative or absolute URL string
+        # @return [Html2rss::Url, nil] resolved absolute URL or nil
+        def resolve_url(raw_url)
+          return if raw_url.nil? || raw_url.empty?
+
+          Html2rss::Url.from_relative(raw_url, url)
+        rescue Html2rss::Url::InvalidUrlError
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/auto_source/scraper/microformats2.rb

@@ -0,0 +1,146 @@
+# frozen_string_literal: true
+
+require 'nokogiri'
+
+module Html2rss
+  class AutoSource
+    module Scraper
+      ##
+      # Scrapes h-entry objects based on Microformats2 specifications.
+      #
+      # @see https://microformats.org/wiki/h-entry
+      class Microformats2
+        include Enumerable
+
+        # Selector for Microformats2 h-entry containers.
+        ENTRY_SELECTOR = '.h-entry, [class*="h-entry"]'
+
+        # @return [Symbol] scraper config key
+        def self.options_key = :microformats2
+
+        class << self
+          # @param parsed_body [Nokogiri::HTML::Document, nil] parsed HTML document
+          # @return [Boolean] whether Microformats2 h-entry elements exist
+          def articles?(parsed_body)
+            return false unless parsed_body
+
+            !parsed_body.at_css(ENTRY_SELECTOR).nil?
+          end
+        end
+
+        # @param parsed_body [Nokogiri::HTML::Document] parsed HTML document
+        # @param url [String, Html2rss::Url] base page URL
+        # @param _opts [Hash] unused scraper-specific options
+        # @option _opts [Object] :_reserved reserved for future options
+        # @return [void]
+        def initialize(parsed_body, url:, **_opts)
+          @parsed_body = parsed_body
+          @url = Html2rss::Url.from_absolute(url)
+        end
+
+        ##
+        # Yields normalized article hashes extracted from h-entry elements.
+        #
+        # @yieldparam article [Hash{Symbol => Object}] normalized article hash
+        # @return [Enumerator, void] enumerator when no block is given
+        def each
+          return enum_for(:each) unless block_given?
+
+          parsed_body.css(ENTRY_SELECTOR).each do |entry|
+            article = build_article(entry)
+            yield article if article
+          end
+        end
+
+        private
+
+        attr_reader :parsed_body, :url
+
+        # @param entry [Nokogiri::XML::Element] entry DOM node
+        # @return [Hash{Symbol => Object}, nil] article hash or nil
+        def build_article(entry)
+          title = extract_title(entry)
+          article_url = extract_url(entry)
+          return unless title || article_url
+
+          assemble_article(entry, title, article_url)
+        end
+
+        # @param entry [Nokogiri::XML::Element] entry DOM node
+        # @return [String, nil] title text
+        def extract_title(entry)
+          node = entry.at_css('.p-name')
+          text = node&.text&.strip
+          text unless text.to_s.empty?
+        end
+
+        # @param entry [Nokogiri::XML::Element] entry DOM node
+        # @return [Html2rss::Url, nil] resolved article URL
+        def extract_url(entry)
+          node = entry.at_css('.u-url')
+          raw_url = node&.attr('href')&.strip
+          resolve_url(raw_url)
+        end
+
+        # @param entry [Nokogiri::XML::Element] entry DOM node
+        # @param title [String, nil] article title
+        # @param article_url [Html2rss::Url, nil] resolved article URL
+        # @return [Hash{Symbol => Object}] assembled article hash
+        def assemble_article(entry, title, article_url)
+          { title:, url: article_url || url, description: extract_description(entry),
+            image: extract_image(entry), author: extract_author(entry),
+            published_at: extract_published_at(entry), categories: extract_categories(entry) }.compact
+        end
+
+        # @param entry [Nokogiri::XML::Element] entry DOM node
+        # @return [String, nil] description text or html
+        def extract_description(entry)
+          node = entry.at_css('.e-content, .p-summary')
+          content = node&.inner_html&.strip || node&.text&.strip
+          content unless content.to_s.empty?
+        end
+
+        # @param entry [Nokogiri::XML::Element] entry DOM node
+        # @return [String, nil] image URL
+        def extract_image(entry)
+          node = entry.at_css('.u-featured, .u-photo')
+          raw_src = node&.attr('src')&.strip || node&.attr('href')&.strip
+          resolve_url(raw_src)&.to_s
+        end
+
+        # @param entry [Nokogiri::XML::Element] entry DOM node
+        # @return [String, nil] author text
+        def extract_author(entry)
+          node = entry.at_css('.p-author .p-name, .p-author, .h-card .p-name')
+          text = node&.text&.strip
+          text unless text.to_s.empty?
+        end
+
+        # @param entry [Nokogiri::XML::Element] entry DOM node
+        # @return [String, nil] published timestamp
+        def extract_published_at(entry)
+          node = entry.at_css('.dt-published')
+          timestamp = node&.attr('datetime')&.strip || node&.text&.strip
+          timestamp unless timestamp.to_s.empty?
+        end
+
+        # @param entry [Nokogiri::XML::Element] entry DOM node
+        # @return [Array<String>, nil] categories array
+        def extract_categories(entry)
+          categories = entry.css('.p-category').map { _1.text.strip }.reject(&:empty?)
+          categories unless categories.empty?
+        end
+
+        # @param raw_url [String, nil] raw URL string
+        # @return [Html2rss::Url, nil] resolved absolute URL or nil
+        def resolve_url(raw_url)
+          return if raw_url.to_s.empty?
+
+          Html2rss::Url.from_relative(raw_url, url)
+        rescue Html2rss::Url::InvalidUrlError
+          nil
+        end
+      end
+    end
+  end
+end