html2rss 0.21.0 → 0.22.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8168109d2cc60920d8a18b6b99970a5558e43163ad5cd11cb3d3f0d944d46943
-  data.tar.gz: 833a936f89f9ce31c0b4fb0036020c7962a4ac77e0dfa72f1134a0bae8bea4c4
+  metadata.gz: 750b7fb967b328cef2238b66729cafe122d1bae23bee05fd8504bb31e760b8a7
+  data.tar.gz: 327406de9c7c97ea13e90c89bec1c2653c962bbcaccfd29ddb78b282477f7578
 SHA512:
-  metadata.gz: 734f286a486d49c86ab7baf48d157cdee9d988fdc8b693ac7d79bf3c64c661fcd54538d5e94dc19bdc8a6f3021168c1ecac2d8e34417f56879392d71600c7340
-  data.tar.gz: f008a767b452557cff1b45b1abb0eccb26f38d839417e95d21b8cf74f4546f9143b067e2d11f9fc00f5955c3f145f8933a1b1d4912ad25756c890280d4bb1a37
+  metadata.gz: 5f41e00edfdd19ceb012900db7518f28236b417662dc4f11f45d9c498ede0720bdbbc0fb31443d80495eaf6076df646062fdf7f972b27bd71c50cc4e198b4540
+  data.tar.gz: 7a7eff85bd7f98cd872131041aa58faf9d3fba1aff47893a6d286378b6f52904ee11120f053e3ca8beca3060d86d1438cf2037e3e4e7e1586d3d3c4679b026f0

data/lib/html2rss/auto_source/scraper/html/class_clustering.rb

@@ -0,0 +1,196 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class AutoSource
+    module Scraper
+      class Html
+        ##
+        # ClassClustering clusters DOM elements on anchorless pages by class lists and scores
+        # candidate groups to find the best list of content cards/articles.
+        # rubocop:disable Metrics/ClassLength
+        class ClassClustering
+          # Node tags considered layout containers
+          LAYOUT_TAG_NAMES = Set['div', 'section', 'article', 'li', 'ul', 'ol'].freeze
+          # HTML/layout tags excluded from candidate nodes
+          EXCLUDED_TAGS = Set['html', 'body', 'nav', 'footer', 'header', 'svg', 'script', 'style'].freeze
+
+          class << self
+            ##
+            # Clusters elements in parsed_body and returns the best set of content card nodes.
+            #
+            # @param parsed_body [Nokogiri::HTML::Document] parsed HTML document
+            # @param minimum_selector_frequency [Integer] minimum frequency for class groups
+            # @return [Array<Nokogiri::XML::Node>] candidate nodes of the top-scoring class group
+            def call(parsed_body, minimum_selector_frequency:)
+              new(parsed_body, minimum_selector_frequency:).call
+            end
+          end
+
+          # @param parsed_body [Nokogiri::HTML::Document]
+          # @param minimum_selector_frequency [Integer]
+          def initialize(parsed_body, minimum_selector_frequency:)
+            @parsed_body = parsed_body
+            @minimum_frequency = minimum_selector_frequency
+            @text_words = {}.compare_by_identity
+            @has_date = {}.compare_by_identity
+          end
+
+          # @return [Array<Nokogiri::XML::Node>]
+          def call
+            candidate_groups = collect_candidate_groups
+            return [] if candidate_groups.empty?
+
+            non_containers = filter_containers(candidate_groups)
+            final_groups = filter_1_to_1_overlap(non_containers)
+
+            select_best_group(final_groups)
+          end
+
+          private
+
+          def collect_candidate_groups
+            class_groups = Hash.new { |h, k| h[k] = [] }
+            cache = {}.compare_by_identity
+
+            @parsed_body.css('[class]').each { |node| add_node_to_groups(node, class_groups, cache) }
+
+            class_groups.select { |_, nodes| nodes.size >= @minimum_frequency }
+          end
+
+          def add_node_to_groups(node, class_groups, cache)
+            return if EXCLUDED_TAGS.include?(node.name) || HtmlExtractor.ignored_container_path?(node, cache)
+
+            cls = normalize_class(node['class'])
+            class_groups[cls] << node unless cls.empty?
+          end
+
+          def normalize_class(class_attr)
+            class_str = class_attr.to_s.strip
+            return '' if class_str.empty?
+
+            # Bypass split/sort/join allocation for single-class lists
+            if class_str.include?(' ')
+              class_str.split(/\s+/).sort.join(' ')
+            else
+              class_str
+            end
+          end
+
+          # Discard group A if any node of A contains > 1 node of another group B
+          def filter_containers(groups)
+            groups.reject do |cls_a, nodes_a|
+              groups.any? { |cls_b, nodes_b| cls_a != cls_b && container_of?(nodes_a, nodes_b) }
+            end
+          end
+
+          # rubocop:disable Metrics/MethodLength
+          def container_of?(nodes_a, nodes_b)
+            return false unless LAYOUT_TAG_NAMES.include?(nodes_b.first.name)
+
+            nodes_a.any? do |node_a|
+              count = 0
+              nodes_b.each do |node_b|
+                next if node_a == node_b
+
+                if HtmlNavigator.descendant_of?(node_b, node_a)
+                  count += 1
+                  break if count > 1
+                end
+              end
+              count > 1
+            end
+          end
+          # rubocop:enable Metrics/MethodLength
+
+          # If group A contains group B, and they have the same size:
+          # - If B (the descendant) contains >= 80% of A's words, AND B's tag is div/section/article,
+          #   B is the actual content card. Discard A.
+          # - Otherwise, B is a sub-element (header, metadata line, button). Discard B.
+          def filter_1_to_1_overlap(groups)
+            discarded = Set.new
+            groups.each_key do |cls_a|
+              groups.each_key do |cls_b|
+                next if cls_a == cls_b || discarded.include?(cls_a) || discarded.include?(cls_b)
+
+                resolve_1_to_1_overlap(cls_a, cls_b, groups, discarded)
+              end
+            end
+
+            groups.except(*discarded)
+          end
+
+          def resolve_1_to_1_overlap(cls_a, cls_b, groups, discarded)
+            nodes_a = groups[cls_a]
+            nodes_b = groups[cls_b]
+            return if nodes_a.size != nodes_b.size
+            return unless nodes_a.zip(nodes_b).all? { |a, b| a != b && HtmlNavigator.descendant_of?(b, a) }
+
+            discarded << (keep_descendant?(nodes_a, nodes_b) ? cls_a : cls_b)
+          end
+
+          def keep_descendant?(nodes_a, nodes_b)
+            avg_words(nodes_b) >= 0.8 * avg_words(nodes_a) &&
+              LAYOUT_TAG_NAMES.include?(nodes_b.first.name)
+          end
+
+          def select_best_group(groups)
+            best_nodes = []
+            best_score = -1
+
+            groups.each_value do |nodes|
+              score = score_group(nodes)
+              next if score.negative?
+
+              (best_nodes = nodes) && (best_score = score) if score > best_score
+            end
+
+            best_nodes
+          end
+
+          def score_group(nodes)
+            avg_w = avg_words(nodes)
+            return -1 if avg_w < 5
+
+            score = nodes.size + (avg_w / 5.0)
+            score += 20 if nodes_heading?(nodes)
+            score += 20 if nodes_time?(nodes)
+            score += 40 if nodes_date?(nodes)
+            score
+          end
+
+          def nodes_heading?(nodes)
+            nodes.any? do |n|
+              n.at_css(HtmlExtractor::HEADING_TAGS.join(',')) ||
+                n.at_css('.font-bold, .font-semibold')
+            end
+          end
+
+          def nodes_time?(nodes)
+            nodes.any? { |n| n.at_css('time, [datetime]') }
+          end
+
+          def nodes_date?(nodes)
+            nodes.any? { |n| date?(n) }
+          end
+
+          def avg_words(nodes)
+            nodes.sum { |n| text_words(n) } / nodes.size.to_f
+          end
+
+          def text_words(node)
+            @text_words[node] ||= HtmlExtractor.extract_visible_text(node).to_s.scan(/\p{Alnum}+/).size
+          end
+
+          def date?(node)
+            @has_date[node] ||= begin
+              text = HtmlExtractor.extract_visible_text(node).to_s
+              text.match?(%r{\b\d{4}[-/]\d{2}[-/]\d{2}\b}) ||
+                text.match?(/\b(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\b/i)
+            end
+          end
+          # rubocop:enable Metrics/ClassLength
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/auto_source/scraper/html.rb

@@ -62,6 +62,7 @@ module Html2rss
           @url = url
           @extractor = extractor
           @opts = opts
+          @fallback_anchorless = opts.fetch(:fallback_anchorless, false)
           @link_heuristics = LinkHeuristics.new(url)
           @ignored_cache = {}.compare_by_identity
         end
@@ -105,8 +106,19 @@ module Html2rss
         private
 
         def articles
-          @articles ||= each_article_tag.filter_map do |article_tag, selected_anchor|
-            extract_article(article_tag, selected_anchor:)
+          @articles ||= begin
+            extracted = each_article_tag.filter_map do |article_tag, selected_anchor|
+              extract_article(article_tag, selected_anchor:)
+            end
+
+            extracted += find_anchorless_articles if @fallback_anchorless
+            extracted
+          end
+        end
+
+        def find_anchorless_articles
+          ClassClustering.call(parsed_body, minimum_selector_frequency:).map do |node|
+            @extractor.new(node, base_url: @url, selected_anchor: nil, fallback_anchorless: true).call
           end
         end
 

data/lib/html2rss/auto_source/scraper/json_state.rb

@@ -305,7 +305,7 @@ module Html2rss
           # rubocop:disable Metrics/MethodLength
           # @param entry [Hash] raw article entry candidate
           # @param base_url [String, Html2rss::Url] base URL for relative link resolution
-          # @return [Hash{Symbol => Object}, nil] normalized article hash for downstream extraction
+          # @return [Hash{Symbol => Object, nil}] normalized article hash for downstream extraction
           def normalise(entry, base_url:)
             return unless entry.is_a?(Hash)
 

data/lib/html2rss/auto_source/scraper/microdata.rb

@@ -55,7 +55,13 @@ module Html2rss
           def top_level_item?(node)
             return false if node.attribute('itemprop')
 
-            node.ancestors.none? { |ancestor| ancestor.attribute('itemscope') && ancestor.attribute('itemprop') }
+            curr = node.parent
+            while curr && !curr.document? && curr.name != 'html'
+              return false if curr.attribute('itemscope') && curr.attribute('itemprop')
+
+              curr = curr.parent
+            end
+            true
           end
         end
 
@@ -92,7 +98,7 @@ module Html2rss
         attr_reader :parsed_body, :url
 
         # @param root [Nokogiri::XML::Element] supported Microdata root node
-        # @return [Hash{Symbol => Object}, nil] normalized article hash
+        # @return [Hash{Symbol => Object, nil}] normalized article hash
         def article_from(root)
           schema_object = SchemaObjectBuilder.call(root)
           return unless schema_object
@@ -147,7 +153,13 @@ module Html2rss
           def direct_property?(root, node)
             return false if node == root
 
-            node.ancestors.take_while { _1 != root }.none? { |ancestor| ancestor.attribute('itemscope') }
+            curr = node.parent
+            while curr && curr != root
+              return false if curr.attribute('itemscope')
+
+              curr = curr.parent
+            end
+            true
           end
 
           # @param node [Nokogiri::XML::Element] itemprop node
@@ -378,7 +390,7 @@ module Html2rss
           extend ValueNormalizer
 
           # @param root [Nokogiri::XML::Element] supported microdata root node
-          # @return [Hash{Symbol => Object}, nil] compact schema-like object
+          # @return [Hash{Symbol => Object, nil}] compact schema-like object
           def call(root)
             type = Microdata.supported_type_name(root)
             return unless type

data/lib/html2rss/auto_source/scraper/semantic_html/deduplicator.rb

@@ -37,7 +37,7 @@ module Html2rss
 
             @article_cache.fetch(entry) do
               @article_cache[entry] = @extractor.new(
-                entry.container, base_url: @url, selected_anchor: entry.selected_anchor
+                entry.container, base_url: @url, selected_anchor: entry.selected_anchor, fallback_anchorless: true
               ).call
             end
           end

data/lib/html2rss/auto_source/scraper/semantic_html.rb

@@ -60,12 +60,13 @@ module Html2rss
         # @param parsed_body [Nokogiri::HTML::Document] parsed HTML document
         # @param url [String, Html2rss::Url] base url
         # @param extractor [Class] extractor class used for article extraction
-        # @param _opts [Hash] scraper-specific options
-        # @option _opts [Object] :_reserved reserved for future scraper-specific options
-        def initialize(parsed_body, url:, extractor: HtmlExtractor, **_opts)
+        # @param opts [Hash] scraper-specific options
+        # @option opts [Boolean] :fallback_anchorless whether to extract anchorless blocks
+        def initialize(parsed_body, url:, extractor: HtmlExtractor, **opts)
           @parsed_body = parsed_body
           @url = url
           @extractor = extractor
+          @fallback_anchorless = opts.fetch(:fallback_anchorless, false)
           @link_heuristics = LinkHeuristics.new(url)
           @anchor_selector = AnchorSelector.new(url)
         end
@@ -107,14 +108,15 @@ module Html2rss
           @anchor_selector.primary_anchor_for(container)
         end
 
-        def extractable_entries # rubocop:disable Metrics/MethodLength
+        # rubocop:disable Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
+        def extractable_entries
           @extractable_entries ||= candidate_containers.filter_map do |container|
             selected_anchor = primary_anchor_for(container)
 
-            next unless selected_anchor
+            next unless selected_anchor || @fallback_anchorless
 
-            destination_facts = normalized_destination(selected_anchor)
-            next unless destination_facts
+            destination_facts = selected_anchor ? normalized_destination(selected_anchor) : nil
+            next if selected_anchor && !destination_facts
             next if hard_junk_entry?(container, selected_anchor, destination_facts)
 
             quality = quality_score(container, selected_anchor, destination_facts)
@@ -132,6 +134,7 @@ module Html2rss
             )
           end
         end
+        # rubocop:enable Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
 
         # rubocop:disable Metrics/MethodLength
         def ranked_entries
@@ -177,8 +180,8 @@ module Html2rss
 
           score += 40 if words >= 3
           score += 15 if words >= 7
-          score += 20 if destination_facts.url.path.to_s.length > 6
-          score += 15 if destination_facts.content_path
+          score += 20 if destination_facts&.url&.path.to_s.length > 6
+          score += 15 if destination_facts&.content_path
           score += 15 if publish_marker?(container)
           score += 10 if descriptive_context?(container_text, title)
           score += 10 if article_container?(container)
@@ -190,12 +193,12 @@ module Html2rss
           title = entry_title(container, selected_anchor)
           utility_text = @link_heuristics.utility_prefix_text?(title)
           recommended_text = @link_heuristics.recommended_text?(title)
-          content_signal = destination_facts.content_path
+          content_signal = destination_facts&.content_path
           no_content_signal = !content_signal
           non_content_utility_path =
-            destination_facts.utility_path &&
+            destination_facts&.utility_path &&
             no_content_signal &&
-            !destination_facts.strong_post_suffix
+            !destination_facts&.strong_post_suffix
           publish_signal = publish_marker?(container)
           descriptive_signal = descriptive_context?(visible_text(container), title)
           weak_container = !publish_signal && !descriptive_signal
@@ -203,19 +206,20 @@ module Html2rss
 
           score += 25 if non_content_utility_path
           score += 15 if utility_text && word_count(title) <= 6
-          score += 10 if destination_facts.shallow
+          score += 10 if destination_facts&.shallow
           score += 10 if weak_container
           score += 10 if recommended_text && no_content_signal
-          score += 5 if destination_facts.high_confidence_junk_path
+          score += 5 if destination_facts&.high_confidence_junk_path
           score += 15 if junk_tokens?(container_tokens(container))
           score
         end
 
-        def hard_junk_entry?(container, selected_anchor, destination_facts) # rubocop:disable Metrics/MethodLength
+        # rubocop:disable Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
+        def hard_junk_entry?(container, selected_anchor, destination_facts)
           title = entry_title(container, selected_anchor)
           publish_signal = publish_marker?(container)
           descriptive_signal = descriptive_context?(visible_text(container), title)
-          content_signal = destination_facts.content_path
+          content_signal = destination_facts&.content_path
           weak_article_candidate = article_signal_count(
             container,
             publish_signal:,
@@ -223,12 +227,16 @@ module Html2rss
             content_signal:
           ) < 2
 
-          destination_facts.high_confidence_junk_path ||
-            (@link_heuristics.recommended_text?(title) && destination_facts.shallow && weak_article_candidate) ||
-            (@link_heuristics.utility_prefix_text?(title) &&
-              destination_facts.high_confidence_utility_destination &&
+          destination_facts&.high_confidence_junk_path ||
+            (selected_anchor &&
+              @link_heuristics.recommended_text?(title) &&
+              destination_facts&.shallow &&
+              weak_article_candidate) ||
+            (selected_anchor && @link_heuristics.utility_prefix_text?(title) &&
+              destination_facts&.high_confidence_utility_destination &&
               weak_article_candidate)
         end
+        # rubocop:enable Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
 
         ##
         # @param container [Nokogiri::XML::Node]

data/lib/html2rss/auto_source.rb

@@ -32,12 +32,14 @@ module Html2rss
           enabled: true
         },
         semantic_html: {
-          enabled: true
+          enabled: true,
+          fallback_anchorless: true
         },
         html: {
           enabled: true,
           minimum_selector_frequency: Scraper::Html::DEFAULT_MINIMUM_SELECTOR_FREQUENCY,
-          use_top_selectors: Scraper::Html::DEFAULT_USE_TOP_SELECTORS
+          use_top_selectors: Scraper::Html::DEFAULT_USE_TOP_SELECTORS,
+          fallback_anchorless: true
         }
       },
       cleanup: Cleanup::DEFAULT_CONFIG
@@ -58,11 +60,13 @@ module Html2rss
       end
       optional(:semantic_html).hash do
         optional(:enabled).filled(:bool)
+        optional(:fallback_anchorless).filled(:bool)
       end
       optional(:html).hash do
         optional(:enabled).filled(:bool)
         optional(:minimum_selector_frequency).filled(:integer, gt?: 0)
         optional(:use_top_selectors).filled(:integer, gt?: 0)
+        optional(:fallback_anchorless).filled(:bool)
       end
     end.freeze
     private_constant :SCRAPER_CONFIG

data/lib/html2rss/cli.rb

@@ -48,6 +48,9 @@ module Html2rss
     method_option :max_requests,
                   type: :numeric,
                   desc: 'Maximum requests to allow for this feed build'
+    method_option :input,
+                  type: :string,
+                  desc: 'Local HTML file path to read input from'
     # @param yaml_file [String] path to YAML config
     # @param feed_name [String, nil] optional named feed in multi-feed config
     # @return [void]
@@ -55,6 +58,7 @@ module Html2rss
       config = Html2rss.config_from_yaml_file(yaml_file, feed_name)
       config[:params] = options[:params] || {}
       apply_runtime_request_overrides!(config)
+      apply_local_file_input!(config, options[:input]) if options[:input]
 
       puts(execute_feed { Html2rss.feed(config) })
     end
@@ -76,20 +80,17 @@ module Html2rss
     method_option :max_requests,
                   type: :numeric,
                   desc: 'Maximum requests to allow for this feed build'
-    # @param url [String] source page URL for auto discovery
+    method_option :input,
+                  type: :string,
+                  desc: 'Local HTML file path to read input from'
+    # @param url [String, nil] source page URL for auto discovery
     # @return [void]
-    def auto(url) # rubocop:disable Metrics/MethodLength
+    def auto(url = nil)
       format = options.fetch(:format, 'rss')
-      source_method = format == 'jsonfeed' ? Html2rss.method(:auto_json_feed) : Html2rss.method(:auto_source)
+      strategy, local_file_path, url = prepare_auto_inputs(url, options[:input])
 
       result = execute_feed do
-        source_method.call(
-          url,
-          strategy: current_strategy,
-          items_selector: options[:items_selector],
-          max_redirects: options[:max_redirects],
-          max_requests: options[:max_requests]
-        )
+        source_call(url, strategy, local_file_path, format == 'jsonfeed')
       end
 
       puts(format == 'jsonfeed' ? JSON.pretty_generate(result) : result)
@@ -159,6 +160,33 @@ module Html2rss
       config.delete(:request) if request_config.empty?
     end
 
+    def apply_local_file_input!(config, input_path)
+      file_path = check_file_exists!(input_path)
+      config[:strategy] = :local_file
+      config[:request] = (config[:request] || {}).merge(local_file_path: file_path)
+
+      return unless config.dig(:channel, :url).to_s.empty?
+
+      config[:channel] = (config[:channel] || {}).merge(
+        url: detect_base_url!(file_path, 'Please specify a channel.url in the config.')
+      )
+    end
+
+    def prepare_auto_inputs(url, input_option)
+      if input_option.nil?
+        raise Thor::Error, 'A URL is required unless --input is specified' unless url
+
+        return [current_strategy, nil, url]
+      end
+
+      file_path = check_file_exists!(input_option)
+      detected_url = url || detect_base_url!(
+        file_path, 'Please specify a URL: html2rss auto [URL] --input <file>'
+      )
+
+      [:local_file, file_path, detected_url]
+    end
+
     def request_controls
       Html2rss::RequestControls.new(
         strategy: options[:strategy]&.to_sym,
@@ -213,5 +241,28 @@ module Html2rss
            Html2rss::NoFeedItemsExtracted => error
       raise Thor::Error, error.message
     end
+
+    def source_call(url, strategy, local_file_path, is_json)
+      method = is_json ? Html2rss.method(:auto_json_feed) : Html2rss.method(:auto_source)
+      method.call(
+        url,
+        strategy:,
+        items_selector: options[:items_selector],
+        max_redirects: options[:max_redirects],
+        max_requests: options[:max_requests],
+        local_file_path:
+      )
+    end
+
+    def check_file_exists!(path)
+      File.expand_path(path).tap do |file_path|
+        raise Thor::Error, "Input file does not exist: #{path}" unless File.exist?(file_path)
+      end
+    end
+
+    def detect_base_url!(file_path, error_hint)
+      Html2rss::Url.extract_from_html(File.read(file_path))&.to_s ||
+        raise(Thor::Error, "Could not auto-detect a base URL from HTML metadata. #{error_hint}")
+    end
   end
 end

data/lib/html2rss/config/class_methods.rb

@@ -29,7 +29,7 @@ module Html2rss
       # Validates a configuration hash with the runtime validator.
       #
       # @param config [Hash{Symbol => Object}] the configuration hash
-      # @param params [Hash{Symbol => Object}, Hash{String => Object}, nil] dynamic parameters for string formatting
+      # @param params [Hash{Symbol => Object, Hash{String => Object, nil}}] dynamic parameters for string formatting
       # @return [Dry::Validation::Result] validation result after defaults and deprecations are applied
       def validate(config, params: UNSET)
         prepared_config = prepare_for_validation(resolve_effective_config(config, params:))
@@ -56,7 +56,7 @@ module Html2rss
       # @param file [String] the YAML file to load
       # @param feed_name [String, nil] optional feed name for multi-feed files
       # @param multiple_feeds_key [Symbol] key under which multiple feeds are defined
-      # @param params [Hash{Symbol => Object}, Hash{String => Object}, nil] dynamic parameters for string formatting
+      # @param params [Hash{Symbol => Object, Hash{String => Object, nil}}] dynamic parameters for string formatting
       # @return [Dry::Validation::Result] validation result after defaults and deprecations are applied
       def validate_yaml(file, feed_name = nil, multiple_feeds_key: MultipleFeedsConfig::CONFIG_KEY_FEEDS, params: UNSET)
         validate(load_yaml(file, feed_name, multiple_feeds_key:), params:)
@@ -99,7 +99,7 @@ module Html2rss
       # and returns a new configuration object.
       #
       # @param config [Hash{Symbol => Object}] the configuration hash.
-      # @param params [Hash{Symbol => Object}, Hash{String => Object}, nil] dynamic parameters for string formatting.
+      # @param params [Hash{Symbol => Object, Hash{String => Object, nil}}] dynamic parameters for string formatting.
       # @return [Html2rss::Config] the configuration object.
       def from_hash(config, params: UNSET)
         new(resolve_effective_config(config, params:))

data/lib/html2rss/config/validator.rb

@@ -83,6 +83,7 @@ module Html2rss
         optional(:total_timeout_seconds).filled(:integer, gt?: 0)
         optional(:browserless).hash(BrowserlessRequestConfig)
         optional(:botasaurus).hash(BotasaurusRequestConfig)
+        optional(:local_file_path).filled(:string)
       end
 
       params do

data/lib/html2rss/config.rb

@@ -69,9 +69,9 @@ module Html2rss
     # @return [Hash{Symbol => Object}] request envelope configuration
     def request = config[:request]
 
-    # @return [Hash{Symbol => Object}, nil] selectors configuration
+    # @return [Hash{Symbol => Object, nil}] selectors configuration
     def selectors = config[:selectors]
-    # @return [Hash{Symbol => Object}, nil] auto-source configuration
+    # @return [Hash{Symbol => Object, nil}] auto-source configuration
     def auto_source = config[:auto_source]
 
     private

data/lib/html2rss/html_extractor/heading_extractor.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class HtmlExtractor
+    ##
+    # HeadingExtractor identifies and returns the best heading element within a container.
+    class HeadingExtractor
+      # Heading tags used to prioritize title extraction.
+      HEADING_TAGS = HtmlExtractor::HEADING_TAGS
+
+      class << self
+        ##
+        # @param article_tag [Nokogiri::XML::Element] container node
+        # @param fallback_anchorless [Boolean] whether to use fallback search
+        # @param selected_anchor [Nokogiri::XML::Node, nil] anchor element
+        # @return [Nokogiri::XML::Node, nil] the heading node, if found
+        def call(article_tag, fallback_anchorless:, selected_anchor:)
+          tags = article_tag.css(HEADING_TAGS.join(','))
+          if tags.any?
+            select_best_heading(tags)
+          elsif fallback_anchorless && selected_anchor.nil?
+            fallback_heading(article_tag)
+          end
+        end
+
+        private
+
+        def select_best_heading(tags)
+          min_tag_name = tags.map(&:name).min
+          best_tag = nil
+          max_size = -1
+
+          tags.each do |tag|
+            next if tag.name != min_tag_name
+
+            size = TextExtractor.call(tag)&.size.to_i
+            (best_tag = tag) && (max_size = size) if size > max_size
+          end
+
+          best_tag
+        end
+
+        def fallback_heading(article_tag)
+          fallback_tags = article_tag.css('strong, b, [class*="title"], [class*="font-bold"], [class*="font-semibold"]')
+          fallback_tags.find { |t| !TextExtractor.call(t).to_s.strip.empty? }
+        end
+      end
+    end
+  end
+end