html2rss 0.19.1 → 0.20.0

data/lib/html2rss/configuration.rb

@@ -0,0 +1,176 @@
+# frozen_string_literal: true
+
+module Html2rss
+  ##
+  # Global configuration defaults for the Html2rss gem.
+  class Configuration
+    # The valid symbol log levels.
+    VALID_LOG_LEVELS = %i[debug info warn error fatal unknown].freeze
+
+    # @return [Object] the logger
+    attr_reader :logger
+
+    # @return [Proc, nil] the logger formatter
+    attr_reader :logger_formatter
+
+    # @return [Symbol, Integer] the current log level
+    attr_reader :log_level
+
+    # @return [Hash, Proc, nil] the globally configured headers
+    attr_reader :headers
+
+    # @return [Symbol, nil] the default strategy name
+    attr_reader :default_strategy
+
+    # @return [Integer, nil] the minimum TTL in minutes
+    attr_reader :min_ttl
+
+    # @return [Array<Hash>] the globally configured stylesheets
+    attr_reader :stylesheets
+
+    ##
+    # Initializes a new Configuration instance with defaults.
+    def initialize
+      @logger_formatter = proc do |severity, datetime, _progname, msg|
+        "#{datetime} [#{severity}] #{msg}\n"
+      end
+      @logger = Logger.new($stdout)
+      @logger.formatter = @logger_formatter
+      self.log_level = ENV.fetch('LOG_LEVEL', :warn)
+      @headers = nil
+      @default_strategy = nil
+      @min_ttl = nil
+      @stylesheets = [].freeze
+    end
+
+    ##
+    # Sets the logger.
+    #
+    # @param logger [Object]
+    # @return [Object] the logger
+    def logger=(logger)
+      @logger = logger
+      @logger.level = @log_level if @logger.respond_to?(:level=)
+      @logger.formatter = @logger_formatter if @logger_formatter && @logger.respond_to?(:formatter=)
+    end
+
+    ##
+    # Sets the log level.
+    #
+    # @param level [Symbol, String, Integer] the new log level
+    # @return [Integer] the normalized log level
+    # @raise [ArgumentError] if the log level is invalid
+    def log_level=(level)
+      @log_level = normalize_log_level(level)
+      @logger.level = @log_level if @logger.respond_to?(:level=)
+    end
+
+    ##
+    # Sets the logger formatter.
+    #
+    # @param formatter [Proc, #call, nil] the new logger formatter
+    # @return [Proc, #call, nil] the new logger formatter
+    # @raise [ArgumentError] if formatter does not respond to #call
+    def logger_formatter=(formatter)
+      raise ArgumentError, 'formatter must respond to #call or be nil' if formatter && !formatter.respond_to?(:call)
+
+      @logger_formatter = formatter
+      @logger.formatter = @logger_formatter if @logger.respond_to?(:formatter=)
+    end
+
+    ##
+    # Sets the global request headers.
+    #
+    # @param headers [Hash, Proc, #call, nil] the HTTP request headers to globally apply
+    # @return [Hash, Proc, #call, nil] the assigned headers
+    # @raise [ArgumentError] if headers is not a Hash or callable
+    def headers=(headers)
+      if headers && !headers.is_a?(Hash) && !headers.respond_to?(:call)
+        raise ArgumentError, 'headers must be a Hash or respond to #call'
+      end
+
+      @headers = headers.is_a?(Hash) ? headers.dup.freeze : headers
+    end
+
+    ##
+    # Sets the default strategy.
+    #
+    # @param strategy [Symbol, String, nil] the strategy name
+    # @return [Symbol, nil] the normalized strategy name
+    # @raise [ArgumentError] if the strategy is not registered
+    def default_strategy=(strategy)
+      if strategy.nil?
+        @default_strategy = nil
+      else
+        unless strategy.is_a?(Symbol) || strategy.is_a?(String)
+          raise ArgumentError, 'strategy must be a Symbol or String'
+        end
+
+        normalized = strategy.to_sym
+        raise ArgumentError, "unknown strategy: #{strategy}" unless RequestService.strategy_registered?(normalized)
+
+        @default_strategy = normalized
+      end
+    end
+
+    ##
+    # Sets the minimum TTL in minutes.
+    #
+    # @param ttl [Integer, String, nil] the minimum TTL
+    # @return [Integer, nil] the normalized minimum TTL
+    # @raise [ArgumentError] if ttl is not a positive integer
+    def min_ttl=(ttl)
+      if ttl.nil?
+        @min_ttl = nil
+      else
+        val = Integer(ttl)
+        raise ArgumentError unless val.positive?
+
+        @min_ttl = val
+      end
+    rescue ArgumentError, TypeError
+      raise ArgumentError, "min_ttl must be a positive integer, got #{ttl.inspect}"
+    end
+
+    ##
+    # Sets the global stylesheets.
+    #
+    # @param stylesheets [Array<Hash>] the XML stylesheet processing instructions to include in the generated feed
+    # @return [Array<Hash>] the assigned stylesheets
+    # @raise [ArgumentError] if stylesheets is not an Array of hashes
+    def stylesheets=(stylesheets)
+      raise ArgumentError, 'stylesheets must be an Array' unless stylesheets.is_a?(Array)
+      raise ArgumentError, 'stylesheets must be an Array of Hashes' unless stylesheets.all?(Hash)
+
+      @stylesheets = stylesheets.map { |h| h.dup.freeze }.freeze
+    end
+
+    protected
+
+    ##
+    # Copy constructor for duplicating configuration.
+    #
+    # @param other [Html2rss::Configuration] the original configuration
+    # @return [void]
+    def initialize_copy(other)
+      super
+      @headers = @headers.dup if @headers.is_a?(Hash)
+      @stylesheets = @stylesheets.map(&:dup) if @stylesheets.is_a?(Array)
+    end
+
+    private
+
+    def normalize_log_level(level)
+      if level.is_a?(Integer)
+        raise ArgumentError, "invalid log level: #{level}" unless level.between?(0, 5)
+
+        level
+      else
+        sym = level.to_s.downcase.to_sym
+        raise ArgumentError, "invalid log level: #{level}" unless VALID_LOG_LEVELS.include?(sym)
+
+        Logger.const_get(sym.upcase)
+      end
+    end
+  end
+end

data/lib/html2rss/html_extractor/list_candidates.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class HtmlExtractor
+    ##
+    # Builds repeated-list article container candidates from generic HTML.
+    class ListCandidates
+      ##
+      # Simplify an XPath selector by removing index notation.
+      #
+      # @param xpath [String] original XPath
+      # @return [String] XPath without positional indexes
+      def self.simplify_xpath(xpath)
+        xpath.gsub(/\[\d+\]/, '')
+      end
+
+      # @param parsed_body [Nokogiri::HTML::Document] parsed document
+      # @param minimum_selector_frequency [Integer] minimum repeated anchor path count
+      # @param use_top_selectors [Integer] number of frequent anchor paths to inspect
+      def initialize(parsed_body, minimum_selector_frequency:, use_top_selectors:)
+        @parsed_body = parsed_body
+        @minimum_selector_frequency = minimum_selector_frequency
+        @use_top_selectors = use_top_selectors
+      end
+
+      ##
+      # @param anchor_filter [#call] predicate for scraper-specific anchor eligibility
+      # @param boundary_condition [#call] predicate for article container boundary
+      # @yieldparam article_tag [Nokogiri::XML::Node] candidate article container
+      # @yieldparam selected_anchor [Nokogiri::XML::Node] anchor that made the container eligible
+      # @return [Enumerator]
+      def each_article_tag(anchor_filter:, boundary_condition:)
+        return enum_for(:each_article_tag, anchor_filter:, boundary_condition:) unless block_given?
+
+        article_tags(anchor_filter:, boundary_condition:).each { yield _1[:article_tag], _1[:selected_anchor] }
+      end
+
+      private
+
+      attr_reader :parsed_body, :minimum_selector_frequency, :use_top_selectors
+
+      def article_tags(anchor_filter:, boundary_condition:)
+        selectors(anchor_filter:).flat_map do |selector|
+          article_tags_for_selector(selector, boundary_condition)
+        end
+      end
+
+      def article_tags_for_selector(selector, boundary_condition)
+        parsed_body.xpath(selector).filter_map do |selected_tag|
+          next if HtmlExtractor.ignored_container_path?(selected_tag)
+
+          article_tag = HtmlNavigator.parent_until_condition(selected_tag, boundary_condition)
+          next unless article_tag
+
+          { article_tag:, selected_anchor: selected_tag }
+        end
+      end
+
+      def selectors(anchor_filter:)
+        anchor_counts(anchor_filter:)
+          .select { |_selector, count| count >= minimum_selector_frequency }
+          .max_by(use_top_selectors, &:last)
+          .map(&:first)
+      end
+
+      def anchor_counts(anchor_filter:)
+        Hash.new(0).tap do |counts|
+          each_anchor(anchor_filter:) do |node|
+            path = self.class.simplify_xpath(node.path)
+            counts[path] += 1 unless HtmlExtractor.ignored_container_path?(path)
+          end
+        end
+      end
+
+      def each_anchor(anchor_filter:)
+        return enum_for(:each_anchor, anchor_filter:) unless block_given?
+
+        traversal_root&.traverse do |node|
+          yield node if relevant_anchor?(node, anchor_filter:)
+        end
+      end
+
+      def relevant_anchor?(node, anchor_filter:)
+        node.element? &&
+          node.matches?(HtmlExtractor::MAIN_ANCHOR_SELECTOR) &&
+          anchor_filter.call(node)
+      end
+
+      def traversal_root
+        parsed_body.at_css('body, html') || parsed_body.root
+      end
+    end
+  end
+end

data/lib/html2rss/html_extractor/semantic_anchor_candidates.rb

@@ -0,0 +1,257 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class HtmlExtractor
+    ##
+    # Builds ranked anchor facts for one semantic content container.
+    class SemanticAnchorCandidates
+      # Anchor candidate plus scoring signals used by semantic anchor ranking.
+      AnchorFacts = Data.define(
+        :anchor,
+        :text,
+        :url,
+        :destination,
+        :segments,
+        :meaningful_text,
+        :content_like_destination,
+        :heading_anchor,
+        :heading_text_match,
+        :score
+      ) do
+        # @param candidate [Candidate] eligible semantic anchor candidate
+        # @return [AnchorFacts] serializable facts used for ranking and dedupe
+        def self.from_candidate(candidate)
+          new(
+            **candidate.anchor_identity_attributes,
+            **candidate.anchor_signal_attributes,
+            score: Score.new(candidate).value
+          )
+        end
+      end
+
+      # Shared context for all anchors in one semantic container.
+      class Context
+        # Ancestor tags that usually indicate navigation/utility regions.
+        UTILITY_LANDMARK_TAGS = %w[nav aside footer menu].freeze
+
+        # @param container [Nokogiri::XML::Node] semantic container
+        # @param link_heuristics [Html2rss::AutoSource::Scraper::LinkHeuristics] destination/text heuristics
+        def initialize(container, link_heuristics:)
+          @container = container
+          @link_heuristics = link_heuristics
+        end
+
+        # @return [Nokogiri::XML::Node, nil] heading used to identify title anchors
+        def heading
+          @heading ||= @container.at_css(HtmlExtractor::HEADING_TAGS.join(','))
+        end
+
+        # @return [String] visible heading text
+        def heading_text
+          @heading_text ||= visible_text(heading)
+        end
+
+        # @param node [Nokogiri::XML::Node, nil] node to extract text from
+        # @return [String] visible text for the node
+        def visible_text(node)
+          return '' unless node
+
+          HtmlExtractor.extract_visible_text(node).to_s.strip
+        end
+
+        # @param anchor [Nokogiri::XML::Node] anchor candidate
+        # @return [Html2rss::AutoSource::Scraper::LinkHeuristics::DestinationFacts, nil] destination facts
+        def destination_facts(anchor)
+          @link_heuristics.destination_facts(anchor)
+        end
+
+        # @param text [String] visible anchor text
+        # @return [Boolean] true when text is utility chrome
+        def utility_text?(text)
+          @link_heuristics.utility_text?(text)
+        end
+
+        # @param ancestors [Array<Nokogiri::XML::Node>]
+        # @return [Boolean] true when the anchor lives inside navigation chrome
+        def utility_landmark?(ancestors)
+          ancestors.any? { |node| UTILITY_LANDMARK_TAGS.include?(node.name) }
+        end
+      end
+
+      # One anchor plus the facts needed to decide whether it represents content.
+      class Candidate
+        attr_reader :anchor
+
+        # @param anchor [Nokogiri::XML::Node] anchor candidate
+        # @param context [Context] semantic container context
+        def initialize(anchor, context)
+          @anchor = anchor
+          @context = context
+        end
+
+        # @return [AnchorFacts, nil] ranked anchor facts when the anchor is eligible
+        def facts
+          return unless destination_facts
+          return if utility_text_suppressed? || ineligible_anchor?
+          return unless representative_content_anchor?
+
+          AnchorFacts.from_candidate(self)
+        end
+
+        # @return [Html2rss::AutoSource::Scraper::LinkHeuristics::DestinationFacts, nil] destination facts
+        def destination_facts
+          @destination_facts ||= @context.destination_facts(@anchor)
+        end
+
+        # @return [String] visible anchor text
+        def text
+          @text ||= @context.visible_text(@anchor)
+        end
+
+        # @return [Hash] anchor identity attributes used to build AnchorFacts
+        def anchor_identity_attributes
+          {
+            anchor:,
+            text:,
+            url: destination_facts.url,
+            destination: destination_facts.destination,
+            segments: destination_facts.segments
+          }
+        end
+
+        # @return [Hash] anchor signal attributes used to build AnchorFacts
+        def anchor_signal_attributes
+          {
+            meaningful_text: meaningful_text?,
+            content_like_destination: content_like_destination?,
+            heading_anchor: heading_anchor?,
+            heading_text_match: heading_text_match?
+          }
+        end
+
+        # @return [Boolean] true when visible anchor text has words
+        def meaningful_text?
+          text.scan(/\p{Alnum}+/).any?
+        end
+
+        # @return [Boolean] true when the destination route has content signals
+        def content_like_destination?
+          destination_facts.content_path
+        end
+
+        # @return [Boolean] true when the anchor is inside the selected heading
+        def heading_anchor?
+          heading = @context.heading
+
+          heading && @anchor.ancestors.include?(heading)
+        end
+
+        # @return [Boolean] true when anchor text exactly matches heading text
+        def heading_text_match?
+          heading_text = @context.heading_text
+
+          meaningful_text? &&
+            heading_text.scan(/\p{Alnum}+/).any? &&
+            heading_text == text
+        end
+
+        private
+
+        def representative_content_anchor?
+          heading_anchor? || meaningful_text? || content_like_destination?
+        end
+
+        def utility_text_suppressed?
+          !content_like_destination? &&
+            @context.utility_text?(text) &&
+            (destination_facts.high_confidence_utility_destination || non_heading_weak_post?)
+        end
+
+        def non_heading_weak_post?
+          !heading_anchor? && !destination_facts.strong_post_suffix
+        end
+
+        def ineligible_anchor?
+          destination_facts.high_confidence_utility_destination ||
+            icon_only_anchor? ||
+            @context.utility_landmark?(@anchor.ancestors.to_a)
+        end
+
+        def icon_only_anchor?
+          !meaningful_text? && @anchor.at_css('img, svg')
+        end
+      end
+
+      # Scores an eligible semantic anchor candidate.
+      class Score
+        # Score weights keyed by candidate signal predicate.
+        RULES = {
+          heading_anchor?: 100,
+          heading_text_match?: 20,
+          meaningful_text?: 10,
+          content_like_destination?: 10
+        }.freeze
+
+        # @param candidate [Candidate] eligible semantic anchor candidate
+        def initialize(candidate)
+          @candidate = candidate
+        end
+
+        # @return [Integer] ranking score
+        def value
+          RULES.sum { |predicate, weight| @candidate.public_send(predicate) ? weight : 0 }
+        end
+      end
+
+      # Keeps the strongest semantic anchor fact for each destination.
+      class DestinationWinners
+        def initialize
+          @winners = {}
+        end
+
+        # @return [Array<AnchorFacts>] strongest candidate per destination
+        def to_a
+          @winners.values
+        end
+
+        # @param facts [AnchorFacts] candidate anchor facts
+        # @return [void]
+        def add(facts)
+          destination = facts.destination
+          @winners[destination] = stronger_fact(@winners[destination], facts)
+        end
+
+        private
+
+        def stronger_fact(current, candidate)
+          return candidate unless current
+
+          current.score >= candidate.score ? current : candidate
+        end
+      end
+
+      # @param container [Nokogiri::XML::Node] semantic container
+      # @param link_heuristics [Html2rss::AutoSource::Scraper::LinkHeuristics] destination/text heuristics
+      def initialize(container, link_heuristics:)
+        @container = container
+        @context = Context.new(container, link_heuristics:)
+      end
+
+      # @return [Array<AnchorFacts>] strongest candidate per destination
+      def to_a
+        @container.css(HtmlExtractor::MAIN_ANCHOR_SELECTOR)
+                  .each_with_object(DestinationWinners.new) { |anchor, winners| add_anchor(anchor, winners) }
+                  .to_a
+      end
+
+      private
+
+      def add_anchor(anchor, winners)
+        return if HtmlExtractor.ignored_container_path?(anchor)
+
+        facts = Candidate.new(anchor, @context).facts
+        winners.add(facts) if facts
+      end
+    end
+  end
+end

data/lib/html2rss/html_extractor/semantic_containers.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class HtmlExtractor
+    ##
+    # Collects semantic content containers from a parsed HTML document.
+    class SemanticContainers
+      # Candidate selectors used to locate extractable semantic content blocks.
+      SELECTORS = [
+        'article:not(:has(article))',
+        'section:not(:has(section))',
+        'li:not(:has(li))',
+        'tr:not(:has(tr))',
+        'div:not(:has(div))'
+      ].freeze
+
+      # @param parsed_body [Nokogiri::HTML::Document] parsed document
+      # @return [Array<Nokogiri::XML::Node>] candidate semantic containers
+      def self.call(parsed_body)
+        new(parsed_body).call
+      end
+
+      # @param parsed_body [Nokogiri::HTML::Document] parsed document
+      def initialize(parsed_body)
+        @parsed_body = parsed_body
+      end
+
+      # @return [Array<Nokogiri::XML::Node>] candidate semantic containers
+      def call
+        containers = SELECTORS.each_with_object([]) do |selector, memo|
+          collect_selector_containers(selector, memo)
+        end
+
+        containers.sort_by { document_order.fetch(_1) }
+      end
+
+      private
+
+      def document_order
+        @document_order ||= begin
+          order = {}
+          index = 0
+
+          @parsed_body.traverse do |node|
+            next unless node.element?
+
+            order[node] = index
+            index += 1
+          end
+
+          order.compare_by_identity
+        end
+      end
+
+      def collect_selector_containers(selector, containers)
+        @parsed_body.css(selector).each do |container|
+          next if HtmlExtractor.ignored_container_path?(container)
+          next if seen[container]
+
+          seen[container] = true
+          containers << container
+        end
+      end
+
+      def seen
+        @seen ||= {}.compare_by_identity
+      end
+    end
+  end
+end

data/lib/html2rss/html_extractor.rb

@@ -7,6 +7,8 @@ module Html2rss
   class HtmlExtractor
     # Tags ignored when extracting visible text content from article containers.
     INVISIBLE_CONTENT_TAGS = %w[svg script noscript style template].to_set.freeze
+    # Element path pattern ignored when traversing candidate article containers.
+    IGNORED_CONTAINER_PATH = /(nav|footer|header|svg|script|style)/i
     # Heading tags used to prioritize title extraction.
     HEADING_TAGS = %w[h1 h2 h3 h4 h5 h6].freeze
     # Selector used to derive non-headline description nodes.
@@ -87,6 +89,15 @@ module Html2rss
 
         article_tag.at_css(MAIN_ANCHOR_SELECTOR)
       end
+
+      ##
+      # @param node [Nokogiri::XML::Node, String] node or path to test
+      # @return [Boolean] true when the node belongs to ignored DOM chrome
+      def ignored_container_path?(node)
+        path = node.respond_to?(:path) ? node.path : node.to_s
+
+        path.match?(IGNORED_CONTAINER_PATH)
+      end
     end
 
     def extract_url

data/lib/html2rss/rss_builder/channel.rb

@@ -41,13 +41,16 @@ module Html2rss
 
       # @return [Integer] cache time-to-live in minutes
       def ttl
-        return overrides[:ttl] if overrides[:ttl]
-
-        if (ttl = headers['cache-control']&.match(/max-age=(\d+)/)&.[](1))
-          return ttl.to_i.fdiv(60).ceil
-        end
-
-        DEFAULT_TTL_IN_MINUTES
+        calculated = if overrides[:ttl]
+                       overrides[:ttl].to_i
+                     elsif (max_age = headers['cache-control']&.match(/max-age=(\d+)/)&.[](1))
+                       max_age.to_i.fdiv(60).ceil
+                     else
+                       DEFAULT_TTL_IN_MINUTES
+                     end
+
+        min_ttl = Html2rss.configuration.min_ttl
+        min_ttl ? [calculated, min_ttl].max : calculated
       end
 
       # @return [String, nil] ISO-like language code when available

data/lib/html2rss/url.rb

@@ -43,6 +43,8 @@ module Html2rss
       base_uri.path = '/' if base_uri.path.empty?
 
       new(base_uri.join(url).normalize)
+    rescue Addressable::URI::InvalidURIError
+      raise ArgumentError, 'URL could not be parsed'
     end
 
     ##

data/lib/html2rss/version.rb

@@ -4,6 +4,6 @@
 # The Html2rss namespace.
 module Html2rss
   # Current application version.
-  VERSION = '0.19.1'
+  VERSION = '0.20.0'
   public_constant :VERSION
 end