html2rss 0.17.0 → 0.18.0

data/exe/html2rss

@@ -1,6 +1,6 @@
 #!/usr/bin/env ruby
 # frozen_string_literal: true
 
-require 'html2rss/cli'
+require 'html2rss'
 
 Html2rss::CLI.start(ARGV)

data/html2rss.gemspec

@@ -26,15 +26,18 @@ Gem::Specification.new do |spec|
   end
 
   spec.files = `git ls-files -z`.split("\x0").select do |f|
-    f.match(%r{^(lib/|exe/|README.md|LICENSE|html2rss.gemspec)})
+    f.match(%r{^(lib/|exe/|schema/|README.md|LICENSE|html2rss.gemspec)})
   end
   spec.bindir        = 'exe'
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
 
   spec.add_dependency 'addressable', '~> 2.7'
+  spec.add_dependency 'brotli'
+  spec.add_dependency 'dry-validation'
   spec.add_dependency 'faraday', '> 2.0.1', '< 3.0'
   spec.add_dependency 'faraday-follow_redirects'
+  spec.add_dependency 'faraday-gzip', '~> 3'
   spec.add_dependency 'kramdown'
   spec.add_dependency 'mime-types', '> 3.0'
   spec.add_dependency 'nokogiri', '>= 1.10', '< 2.0'
@@ -43,7 +46,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'regexp_parser'
   spec.add_dependency 'reverse_markdown', '~> 3.0'
   spec.add_dependency 'rss'
-  spec.add_dependency 'sanitize', '~> 6.0'
+  spec.add_dependency 'sanitize'
   spec.add_dependency 'thor'
   spec.add_dependency 'tzinfo'
   spec.add_dependency 'zeitwerk'

data/lib/html2rss/articles/deduplicator.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require 'set' # rubocop:disable Lint/RedundantRequireStatement
+
+module Html2rss
+  module Articles
+    ##
+    # Deduplicates a list of articles while preserving their original order.
+    #
+    # The deduplicator prefers each article's URL (combined with its ID when
+    # available) to determine uniqueness. When no URL is present, it falls
+    # back to the article ID, then to the GUID enriched with title and
+    # description metadata. If none of these identifiers are available it
+    # defaults to the article object's hash to preserve the original entry.
+    class Deduplicator
+      ##
+      # @param articles [Array<Html2rss::RssBuilder::Article>]
+      # @raise [ArgumentError] if articles are not provided
+      def initialize(articles)
+        raise ArgumentError, 'articles must be provided' unless articles
+
+        @articles = articles
+      end
+
+      ##
+      # Returns the list of unique articles, preserving the order of the
+      # original collection and keeping the first occurrence of a duplicate.
+      # @return [Array<Html2rss::RssBuilder::Article>]
+      def call
+        seen = Set.new
+
+        articles.filter do |article|
+          fingerprint = deduplication_fingerprint_for(article) || article.hash
+          seen.add?(fingerprint)
+        end
+      end
+
+      private
+
+      attr_reader :articles
+
+      def deduplication_fingerprint_for(article)
+        return unless article.respond_to?(:deduplication_fingerprint)
+
+        article.deduplication_fingerprint
+      end
+    end
+  end
+end

data/lib/html2rss/auto_source/cleanup.rb

@@ -7,8 +7,15 @@ module Html2rss
     # :reek:MissingSafeMethod { enabled: false }
     # It applies various strategies to filter and refine the article list.
     class Cleanup
+      DEFAULT_CONFIG = {
+        keep_different_domain: false,
+        min_words_title: 3
+      }.freeze
+
+      VALID_SCHEMES = %w[http https].to_set.freeze
+
       class << self
-        def call(articles, url:, keep_different_domain: false)
+        def call(articles, url:, keep_different_domain:, min_words_title:)
           Log.debug "Cleanup: start with #{articles.size} articles"
 
           articles.select!(&:valid?)
@@ -17,13 +24,12 @@ module Html2rss
 
           keep_only_http_urls!(articles)
           reject_different_domain!(articles, url) unless keep_different_domain
+          keep_only_with_min_words_title!(articles, min_words_title:)
 
           Log.debug "Cleanup: end with #{articles.size} articles"
           articles
         end
 
-        private
-
         ##
         # Deduplicates articles by a given key.
         #
@@ -42,18 +48,40 @@ module Html2rss
         #
         # @param articles [Array<Article>] The list of articles to process.
         def keep_only_http_urls!(articles)
-          articles.select! { |article| %w[http https].include?(article.url&.scheme) }
+          articles.select! { |article| VALID_SCHEMES.include?(article.url&.scheme) }
         end
 
         ##
         # Rejects articles that have a URL not on the same domain as the source.
         #
         # @param articles [Array<Article>] The list of articles to process.
-        # @param base_url [Addressable::URI] The source URL to compare against.
+        # @param base_url [Html2rss::Url] The source URL to compare against.
         def reject_different_domain!(articles, base_url)
           base_host = base_url.host
           articles.select! { |article| article.url&.host == base_host }
         end
+
+        ##
+        # Keeps only articles with a title that is present and has at least `min_words_title` words.
+        #
+        # @param articles [Array<Article>] The list of articles to process.
+        # @param min_words_title [Integer] The minimum number of words in the title.
+        def keep_only_with_min_words_title!(articles, min_words_title:)
+          articles.select! do |article|
+            article.title ? word_count_at_least?(article.title, min_words_title) : true
+          end
+        end
+
+        private
+
+        def word_count_at_least?(str, min_words)
+          count = 0
+          str.to_s.scan(/\p{Alnum}+/) do
+            count += 1
+            return true if count >= min_words
+          end
+          false
+        end
       end
     end
   end

data/lib/html2rss/auto_source/scraper/html.rb

@@ -6,34 +6,58 @@ module Html2rss
   class AutoSource
     module Scraper
       ##
-      # Scrapes articles from HTML pages by
-      # finding similar structures around anchor tags in the parsed_body.
+      # Scrapes article-like blocks from plain HTML by looking for repeated link
+      # structures when richer structured data is unavailable.
+      #
+      # The approach is intentionally heuristic:
+      # 1. collect repeated anchor paths
+      # 2. walk upward to a shared container shape
+      # 3. extract the best anchor found inside each container
+      #
+      # This scraper is broader and noisier than `SemanticHtml`, so it acts as a
+      # fallback for pages without stronger semantic signals.
       class Html
         include Enumerable
 
-        TAGS_TO_IGNORE = /(nav|footer|header)/i
+        TAGS_TO_IGNORE = /(nav|footer|header|svg|script|style)/i
 
-        def self.articles?(parsed_body)
-          new(parsed_body, url: '').any?
-        end
+        DEFAULT_MINIMUM_SELECTOR_FREQUENCY = 2
+        DEFAULT_USE_TOP_SELECTORS = 5
 
-        def self.parent_until_condition(node, condition)
-          return nil if !node || node.document? || node.parent.name == 'html'
-          return node if condition.call(node)
+        ##
+        # @return [Symbol] config key used to enable or configure this scraper
+        def self.options_key = :html
 
-          parent_until_condition(node.parent, condition)
+        ##
+        # Probes whether the document appears to contain repeated anchor
+        # structures that this fallback scraper can cluster into article-like
+        # containers.
+        #
+        # @param parsed_body [Nokogiri::HTML::Document] parsed HTML document
+        # @return [Boolean] true when the scraper can likely extract articles
+        def self.articles?(parsed_body)
+          new(parsed_body, url: '').any?
         end
 
         ##
         # Simplify an XPath selector by removing the index notation.
+        # This keeps repeated anchor paths comparable across sibling blocks.
+        #
+        # @param xpath [String] original XPath
+        # @return [String] XPath without positional indexes
         def self.simplify_xpath(xpath)
           xpath.gsub(/\[\d+\]/, '')
         end
 
-        def initialize(parsed_body, url:)
+        # @param parsed_body [Nokogiri::HTML::Document] The parsed HTML document.
+        # @param url [String] The base URL.
+        # @param extractor [Class] The extractor class to handle article extraction.
+        # @param opts [Hash] Additional options.
+        def initialize(parsed_body, url:, extractor: HtmlExtractor, **opts)
           @parsed_body = parsed_body
           @url = url
-          @selectors = Hash.new(0)
+          @extractor = extractor
+          @opts = opts
         end
 
         attr_reader :parsed_body
@@ -44,51 +68,102 @@ module Html2rss
         def each
           return enum_for(:each) unless block_given?
 
-          return if frequent_selectors.empty?
+          each_article_tag do |article_tag|
+            article_hash = extract_article(article_tag)
+            yield article_hash if article_hash
+          end
+        end
 
-          frequent_selectors.each do |selector|
-            parsed_body.xpath(selector).each do |selected_tag|
-              article_tag = self.class.parent_until_condition(selected_tag, method(:article_condition))
+        ##
+        # Decides whether a traversed node has reached a useful article-like
+        # boundary for the generic HTML scraper.
+        #
+        # The predicate prefers containers that add surrounding link context,
+        # which helps the scraper move from a leaf anchor toward a repeated
+        # teaser/card wrapper.
+        #
+        # @param node [Nokogiri::XML::Node] candidate boundary node
+        # @return [Boolean] true when the node is a good extraction boundary
+        def article_tag_condition?(node)
+          # Ignore tags that are below a tag which is in TAGS_TO_IGNORE.
+          return false if node.path.match?(TAGS_TO_IGNORE)
+          return true if %w[body html].include?(node.name)
+          return false unless (parent = node.parent)
 
-              if article_tag && (article_hash = SemanticHtml::Extractor.new(article_tag, url: @url).call)
-                yield article_hash
-              end
-            end
+          anchor_count(parent) > anchor_count(node)
+        end
+
+        private
+
+        ##
+        # Find relevant anchors in root.
+        # @return [Set<String>] The set of XPath selectors
+        def selectors
+          @selectors ||= Hash.new(0).tap do |selectors|
+            each_relevant_anchor { |node| increment_selector_count(selectors, node) }
           end
         end
 
         ##
-        # Find all the anchors in root.
-        # @param root [Nokogiri::XML::Node] The root node to search for anchors
-        # @return [Set<String>] The set of XPath selectors which exist at least min_frequency times
-        def frequent_selectors(root = @parsed_body.at_css('body'), min_frequency: 2)
-          @frequent_selectors ||= begin
-            root.traverse do |node|
-              next if !node.element? || node.name != 'a'
-
-              @selectors[self.class.simplify_xpath(node.path)] += 1
-            end
+        # Filter the frequent selectors by the minimum_selector_frequency and use_top_selectors.
+        # @return [Array<String>] The filtered selectors
+        def filtered_selectors
+          selectors.select { |_selector, count| count >= minimum_selector_frequency }
+                   .max_by(use_top_selectors, &:last)
+                   .map(&:first)
+        end
+
+        def minimum_selector_frequency = @opts[:minimum_selector_frequency] || DEFAULT_MINIMUM_SELECTOR_FREQUENCY
+        def use_top_selectors = @opts[:use_top_selectors] || DEFAULT_USE_TOP_SELECTORS
+
+        def anchor_count(node)
+          @anchor_counts ||= {}
+          @anchor_counts[node.path] ||= node.name == 'a' ? 1 : node.css('a').size
+        end
+
+        def each_relevant_anchor
+          return enum_for(:each_relevant_anchor) unless block_given?
 
-            @selectors.keys
-                      .select { |selector| (@selectors[selector]).to_i >= min_frequency }
-                      .to_set
+          traversal_root&.traverse do |node|
+            yield node if relevant_anchor?(node)
           end
         end
 
-        def article_condition(node)
-          # Ignore tags that are below a tag which is in TAGS_TO_IGNORE.
-          return false if node.path.match?(TAGS_TO_IGNORE)
+        def relevant_anchor?(node)
+          node.element? && node.name == 'a' && !String(node['href']).empty?
+        end
 
-          # Ignore tags that are below a tag which has a class which matches TAGS_TO_IGNORE.
-          return false if self.class.parent_until_condition(node, proc do |current_node|
-            current_node.classes.any? { |klass| klass.match?(TAGS_TO_IGNORE) }
-          end)
+        def increment_selector_count(selectors, node)
+          path = self.class.simplify_xpath(node.path)
+          selectors[path] += 1 unless path.match?(TAGS_TO_IGNORE)
+        end
 
-          return true if %w[body html].include?(node.name)
+        def traversal_root
+          parsed_body.at_css('body, html') || parsed_body.root
+        end
+
+        def each_article_tag
+          return enum_for(:each_article_tag) unless block_given?
+
+          filtered_selectors.each do |selector|
+            parsed_body.xpath(selector).each do |selected_tag|
+              article_tag = article_tag_for(selected_tag)
+              yield article_tag if article_tag
+            end
+          end
+        end
+
+        def article_tag_for(selected_tag)
+          return if selected_tag.path.match?(Html::TAGS_TO_IGNORE)
+
+          HtmlNavigator.parent_until_condition(selected_tag, method(:article_tag_condition?))
+        end
 
-          return true if node.parent.css('a').size > 1
+        def extract_article(article_tag)
+          selected_anchor = HtmlExtractor.main_anchor_for(article_tag)
+          return unless selected_anchor
 
-          false
+          @extractor.new(article_tag, base_url: @url, selected_anchor:).call
         end
       end
     end