html2rss 0.26.0 → 0.27.0

data/lib/html2rss/feed_pipeline/auto_fallback.rb

@@ -6,7 +6,7 @@ module Html2rss
     #
     # Owned by {FeedPipeline}; invoked only after {StrategyPlan} resolves +:auto+.
     # Hosted by the pipeline instance: session + extract call back into FeedPipeline.
-    class AutoFallback
+    class AutoFallback # rubocop:disable Metrics/ClassLength -- attempt state + logging stay co-located
       # Ordered list of concrete request strategies attempted by the :auto plan.
       CHAIN = %i[faraday botasaurus].freeze
 
@@ -15,7 +15,6 @@ module Html2rss
         RequestService::UnknownStrategy,
         RequestService::InvalidUrl,
         RequestService::UnsupportedUrlScheme,
-        RequestService::UnsupportedResponseContentType,
         RequestService::RequestBudgetExceeded,
         RequestService::PrivateNetworkDenied,
         RequestService::CrossOriginFollowUpDenied,
@@ -64,17 +63,22 @@ module Html2rss
         # @param dedup_dropped [Integer] articles removed by deduplication
         # @param selected_strategy [Symbol] concrete strategy that produced items
         # @param attempt_count [Integer] number of attempts recorded for this chain
+        # @param admission_drops [Hash{String => Integer}] Cleanup drop tallies
         # @return [void]
-        def succeed!(response:, articles:, dedup_dropped:, selected_strategy:, attempt_count:)
+        # rubocop:disable Metrics/ParameterLists -- PipelineOutcome kwargs stay co-located
+        def succeed!(response:, articles:, dedup_dropped:, selected_strategy:, attempt_count:,
+                     admission_drops: {})
           @result = PipelineOutcome.new(
             response:,
             articles:,
             dedup_dropped:,
             selected_strategy:,
             attempt_count:,
-            strategy_attempts: attempts
+            strategy_attempts: attempts,
+            admission_drops:
           )
         end
+        # rubocop:enable Metrics/ParameterLists
       end
 
       ##
@@ -120,6 +124,9 @@ module Html2rss
         return unless response
 
         process_response(response:, strategy:, next_strategy:, request_session:, state:)
+      rescue RequestService::UnsupportedResponseContentType => error
+        state.record_error(strategy:, error:)
+        log_fallback_error(strategy:, next_strategy:, error:, request_session:) if next_strategy
       end
 
       def fetch_response(request_session:, strategy:, next_strategy:, state:)
@@ -132,15 +139,18 @@ module Html2rss
         nil
       end
 
-      def process_response(response:, strategy:, next_strategy:, request_session:, state:)
+      def process_response(response:, strategy:, next_strategy:, request_session:, state:) # rubocop:disable Metrics/MethodLength -- extract + success path
+        articles, dedup_dropped, admission_drops = articles_for(response:, request_session:)
         state.remember_response(response)
-        articles, dedup_dropped = articles_for(response:, request_session:)
         items_count = articles.size
         state.record_items(strategy:, items_count:, transport_meta: response.transport_meta)
         Log.debug("#{self.class}: strategy=#{strategy} items=#{items_count} " \
                   "host=#{response.url.host} elapsed=#{format('%.3f', budget.elapsed_seconds)}s " \
                   "budget_remaining=#{budget_remaining_label}")
-        return record_success(response:, strategy:, articles:, dedup_dropped:, state:) if items_count.positive?
+        if items_count.positive?
+          return record_success(response:, strategy:, articles:, dedup_dropped:, admission_drops:,
+                                state:)
+        end
 
         log_info_fallback_zero_items(strategy:, next_strategy:, response:) if next_strategy
       end
@@ -149,15 +159,18 @@ module Html2rss
         pipeline.deduplicated_articles(config:, response:, request_session:)
       end
 
-      def record_success(response:, strategy:, articles:, dedup_dropped:, state:)
+      # rubocop:disable Metrics/ParameterLists -- success kwargs match PipelineOutcome
+      def record_success(response:, strategy:, articles:, dedup_dropped:, admission_drops:, state:)
         attempt_count = state.attempts.size
-        state.succeed!(response:, articles:, dedup_dropped:, selected_strategy: strategy, attempt_count:)
+        state.succeed!(response:, articles:, dedup_dropped:, selected_strategy: strategy,
+                       attempt_count:, admission_drops:)
         return unless attempt_count > 1
 
         Log.info("#{self.class}: auto selected strategy=#{strategy} after attempts=#{attempt_count} " \
                  "host=#{response.url.host} elapsed=#{format('%.3f', budget.elapsed_seconds)}s " \
                  "budget_remaining=#{budget_remaining_label}")
       end
+      # rubocop:enable Metrics/ParameterLists
 
       def finalize_failure(attempts:, response:)
         surface_category = response && AutoSource::Scraper.classify_no_scraper_surface(

data/lib/html2rss/feed_pipeline/strategy_plan.rb

@@ -41,6 +41,17 @@ module Html2rss
           Concrete.new(strategy: normalized)
         end
 
+        ##
+        # Cheap single-request diagnostic default: +:auto+ → +:faraday+.
+        # Does not run {AutoFallback}. Prefer {resolve} for scrape/capture feeds.
+        #
+        # @param name [Symbol, String, nil] plan name (+nil+ → +:auto+)
+        # @return [Symbol] concrete {RequestService} strategy
+        def concrete_for_diagnostic(name = AUTO_NAME)
+          plan = resolve(name || AUTO_NAME)
+          plan.is_a?(Auto) ? RequestService.default_strategy_name : plan.strategy
+        end
+
         ##
         # @param name [Symbol, String, nil] candidate plan name
         # @return [Boolean] whether +name+ is a valid feed-level strategy plan

data/lib/html2rss/feed_pipeline.rb

@@ -3,13 +3,16 @@
 module Html2rss
   ##
   # Builds feeds from validated config through request, extraction, and rendering stages.
-  class FeedPipeline
+  #
+  # {include:file:lib/html2rss/feed_pipeline/README.md}
+  class FeedPipeline # rubocop:disable Metrics/ClassLength -- outcome + collect seams stay co-located
     # Scrape-finished facts after request + extraction + dedup (before Channel/Status materialize).
     # selected_strategy: set on :auto success; nil otherwise.
     # attempt_count: auto attempts attempted; 0 outside :auto.
     # strategy_attempts: auto attempt hashes (with optional transport_meta); empty outside :auto.
     PipelineOutcome = Data.define(
-      :response, :articles, :dedup_dropped, :selected_strategy, :attempt_count, :strategy_attempts
+      :response, :articles, :dedup_dropped, :selected_strategy, :attempt_count, :strategy_attempts,
+      :admission_drops
     )
 
     ##
@@ -18,6 +21,16 @@ module Html2rss
       @raw_config = raw_config
     end
 
+    ##
+    # Runs the pipeline once and returns scrape-finished outcome (before Channel/Status).
+    # Used by {Capture} which needs the response body for SST selector derivation.
+    #
+    # @return [PipelineOutcome]
+    def to_outcome
+      config = Config.from_hash(raw_config, params: raw_config[:params])
+      pipeline_outcome_for(config)
+    end
+
     ##
     # Runs the pipeline once and returns an opaque, Marshal-cacheable result.
     #
@@ -32,7 +45,8 @@ module Html2rss
         dedup_dropped: outcome.dedup_dropped,
         selected_strategy: outcome.selected_strategy,
         attempt_count: outcome.attempt_count,
-        strategy_attempts: outcome.strategy_attempts
+        strategy_attempts: outcome.strategy_attempts,
+        admission_drops: outcome.admission_drops
       )
       FeedResult.new(channel:, articles: outcome.articles, status:, stylesheets: config.stylesheets)
     end
@@ -56,11 +70,11 @@ module Html2rss
     # @param config [Html2rss::Config]
     # @param response [Html2rss::RequestService::Response]
     # @param request_session [Html2rss::RequestSession]
-    # @return [Array(Array<Html2rss::Article>, Integer)] unique articles and drop count
+    # @return [Array(Array<Html2rss::Article>, Integer, Hash)] unique articles, dedup drops, admission drops
     def deduplicated_articles(config:, response:, request_session:)
-      collected = collect_articles(config:, response:, request_session:)
+      collected, admission_drops = collect_articles(config:, response:, request_session:)
       unique = Article::Deduplicator.new(collected).call
-      [unique, collected.size - unique.size]
+      [unique, collected.size - unique.size, admission_drops]
     end
 
     private
@@ -80,11 +94,12 @@ module Html2rss
     def run_pipeline_for_strategy(config, strategy:, resources:)
       request_session = request_session_for(config, strategy:, resources:)
       response = request_session.fetch_initial_response
-      articles, dedup_dropped = deduplicated_articles(config:, response:, request_session:)
+      articles, dedup_dropped, admission_drops = deduplicated_articles(config:, response:, request_session:)
       raise_empty_auto_source!(strategy:, response:) if config.auto_source && articles.empty?
 
       PipelineOutcome.new(
-        response:, articles:, dedup_dropped:, selected_strategy: nil, attempt_count: 0, strategy_attempts: []
+        response:, articles:, dedup_dropped:, selected_strategy: nil, attempt_count: 0,
+        strategy_attempts: [], admission_drops:
       )
     end
 
@@ -108,8 +123,9 @@ module Html2rss
     end
 
     def collect_articles(config:, response:, request_session:)
-      selector_articles(config:, response:, request_session:) +
-        auto_source_articles(config:, response:, request_session:)
+      selector = selector_articles(config:, response:, request_session:)
+      auto, admission_drops = auto_source_articles(config:, response:, request_session:)
+      [selector + auto, admission_drops]
     end
 
     # rubocop:disable Metrics/MethodLength
@@ -132,10 +148,12 @@ module Html2rss
     end
     # rubocop:enable Metrics/MethodLength
 
+    # @return [Array(Array<Html2rss::Article>, Hash{String => Integer})]
     def auto_source_articles(config:, response:, request_session:)
-      return [] unless (auto_source = config.auto_source)
+      return [[], {}] unless (auto_source = config.auto_source)
 
-      AutoSource.new(response, auto_source, request_session:).articles
+      source = AutoSource.new(response, auto_source, request_session:)
+      [source.articles, source.admission_drops]
     end
   end
 end

data/lib/html2rss/html/article_extractor/category_extractor.rb

@@ -9,24 +9,26 @@ module Html2rss
       class CategoryExtractor
         # Shared category vocabulary (owned by {ArticleRules::Category}).
         CATEGORY_TERMS = ArticleRules::Category::CATEGORY_TERMS
-        # Shared category attribute/class pattern (owned by {ArticleRules::Category}).
-        CATEGORY_ATTR_PATTERN = ArticleRules::Category::CATEGORY_ATTR_PATTERN
 
         # CSS selectors to find elements with category-related class names or data attributes
         CATEGORY_SELECTORS = CATEGORY_TERMS.flat_map do |term|
           ["[class*=\"#{term}\"]", "[data-#{term}]", "[#{term}]"]
         end.freeze
 
+        # Nested blocks that mean a "category" node is actually a content container.
+        CONTAINER_CHILD_SELECTOR = 'p, article, section, h1, h2, h3, h4, h5, h6'
+
         ##
         # Extracts categories from the given article tag by looking for elements
         # with class names containing common category-related terms.
         #
         # @param article_tag [Nokogiri::XML::Element] The article element to extract categories from
+        # @param title [String, nil] article title used to reject title-echo values
         # @return [Array<String>] Array of category strings, empty if none found
-        def self.call(article_tag)
+        def self.call(article_tag, title: nil)
           return [] unless article_tag
 
-          extract_all_categories(article_tag)
+          extract_all_categories(article_tag, title:)
             .map(&:strip)
             .reject(&:empty?)
         end
@@ -35,12 +37,18 @@ module Html2rss
         # Optimized single DOM traversal that extracts all category types.
         #
         # @param article_tag [Nokogiri::XML::Element] The article element
+        # @param title [String, nil]
         # @return [Set<String>] Set of category strings
-        def self.extract_all_categories(article_tag)
+        def self.extract_all_categories(article_tag, title: nil)
           Set.new.tap do |categories|
+            extract_element_data_categories!(categories, article_tag, title:)
             article_tag.css(CATEGORY_SELECTORS.join(',')).each do |element|
-              extract_text_categories!(categories, element) if ArticleRules::Category.class_match?(element['class'])
-              extract_element_data_categories!(categories, element)
+              next if element == article_tag
+
+              if ArticleRules::Category.class_match?(element['class'])
+                extract_text_categories!(categories, element, title:)
+              end
+              extract_element_data_categories!(categories, element, title:)
             end
           end
         end
@@ -50,12 +58,13 @@ module Html2rss
         #
         # @param categories [Set<String>] Accumulator set
         # @param element [Nokogiri::XML::Element] metadata element that may contain category links
+        # @param title [String, nil]
         # @return [void]
-        def self.extract_element_data_categories!(categories, element)
+        def self.extract_element_data_categories!(categories, element, title: nil)
           element.attributes.each_value do |attr|
             next unless ArticleRules::Category.attr_name_match?(attr.name)
 
-            ArticleRules::Category.add_text!(categories, attr.value)
+            ArticleRules::Category.add_text!(categories, attr.value, title:)
           end
         end
 
@@ -64,20 +73,20 @@ module Html2rss
         #
         # @param categories [Set<String>] Accumulator set
         # @param element [Nokogiri::XML::Element] metadata element whose text may contain delimiters
+        # @param title [String, nil]
         # @return [void]
-        def self.extract_text_categories!(categories, element)
-          if element.name == 'a'
-            add_text_to_categories!(categories, element)
-            return
-          end
-
-          anchors = element.css('a')
+        def self.extract_text_categories!(categories, element, title: nil)
+          return if category_container?(element)
 
+          anchors = element.name == 'a' ? [element] : element.css('a').to_a
           if anchors.any?
-            anchors.each { |node| add_text_to_categories!(categories, node) }
-          else
-            ArticleRules::Category.add_split_text!(categories, element.text)
+            anchors.each { |node| add_text_to_categories!(categories, node, title:) }
+            return
           end
+
+          ArticleRules::Category.add_split_text!(
+            categories, Navigator.extract_visible_text(element), title:
+          )
         end
 
         ##
@@ -85,12 +94,17 @@ module Html2rss
         #
         # @param categories [Set<String>] Accumulator set
         # @param element [Nokogiri::XML::Element] The element to extract text from
+        # @param title [String, nil]
         # @return [void]
-        def self.add_text_to_categories!(categories, element)
-          ArticleRules::Category.add_text!(categories, Navigator.extract_visible_text(element))
+        def self.add_text_to_categories!(categories, element, title: nil)
+          ArticleRules::Category.add_text!(categories, Navigator.extract_visible_text(element), title:)
+        end
+
+        def self.category_container?(element)
+          element.at_css(CONTAINER_CHILD_SELECTOR)
         end
 
-        private_class_method :add_text_to_categories!
+        private_class_method :add_text_to_categories!, :category_container?
       end
     end
   end

data/lib/html2rss/html/article_extractor/date_extractor.rb

@@ -6,10 +6,12 @@ module Html2rss
       # Extracts the earliest date from an article_tag.
       class DateExtractor
         # @param article_tag [Nokogiri::XML::Element] article container node
+        # @param leftover_lines [Array<String>] leftover visible lines (already split)
+        # @param time_zone [String] channel time zone for naive values
         # @return [DateTime, nil]
-        def self.call(article_tag)
-          datetimes = article_tag.css('[datetime]').map { |tag| tag['datetime'] }
-          ArticleRules::Date.earliest(datetimes)
+        def self.call(article_tag, leftover_lines: [], time_zone: 'UTC')
+          datetimes = article_tag.respond_to?(:css) ? article_tag.css('[datetime]').map { |tag| tag['datetime'] } : []
+          ArticleRules::Date.earliest(datetimes, leftover_lines:, time_zone:)
         end
       end
     end

data/lib/html2rss/html/article_extractor.rb

@@ -5,6 +5,7 @@ module Html2rss
     ##
     # ArticleExtractor is responsible for extracting details (headline, url, images, etc.)
     # from an article_tag DOM node. DOM chrome helpers live on {Navigator}.
+    # rubocop:disable Metrics/ClassLength -- leftover re-extract stays with field extractors
     class ArticleExtractor
       class << self
         ##
@@ -14,9 +15,10 @@ module Html2rss
         # @param base_url [String, Html2rss::Url] base url used to resolve relative links
         # @param selected_anchor [Nokogiri::XML::Node, nil] explicit primary anchor for the container
         # @param fallback_anchorless [Boolean] whether to fall back to anchorless extraction
+        # @param time_zone [String] channel time zone for naive leftover dates
         # @return [Hash{Symbol => Object}] extracted article attributes
-        def call(article_tag, base_url:, selected_anchor: nil, fallback_anchorless: false)
-          new(article_tag, base_url:, selected_anchor:, fallback_anchorless:).call
+        def call(article_tag, base_url:, selected_anchor: nil, fallback_anchorless: false, time_zone: 'UTC')
+          new(article_tag, base_url:, selected_anchor:, fallback_anchorless:, time_zone:).call
         end
       end
 
@@ -25,32 +27,38 @@ module Html2rss
       # @param base_url [String, Html2rss::Url] base url used to resolve relative links
       # @param selected_anchor [Nokogiri::XML::Node, nil] explicit primary anchor for the container
       # @param fallback_anchorless [Boolean] whether to fall back to anchorless extraction
-      def initialize(article_tag, base_url:, selected_anchor: nil, fallback_anchorless: false)
+      # @param time_zone [String] channel time zone for naive leftover dates
+      def initialize(article_tag, base_url:, selected_anchor: nil, fallback_anchorless: false, time_zone: 'UTC')
         raise ArgumentError, 'article_tag is required' unless article_tag
 
         @article_tag = article_tag
         @base_url = base_url
         @selected_anchor = selected_anchor
         @fallback_anchorless = fallback_anchorless
+        @time_zone = time_zone
       end
 
       # @return [Hash{Symbol => Object}] extracted article attributes
-      def call
+      def call # rubocop:disable Metrics/MethodLength
+        title = extract_title
+        lines = leftover_lines
+        published_at = extract_published_at(lines)
+        source, lines, published_at = parent_card_fields(title, lines, published_at)
         {
-          title: extract_title,
+          title:,
           url: extract_url,
           image: extract_image,
-          description: extract_description,
+          description: ArticleRules::Description.from_lines(lines, title:),
           id: generate_id,
-          published_at: extract_published_at,
+          published_at:,
           enclosures: extract_enclosures,
-          categories: extract_categories
+          categories: CategoryExtractor.call(source, title:)
         }
       end
 
       private
 
-      attr_reader :article_tag, :base_url, :selected_anchor
+      attr_reader :article_tag, :base_url, :selected_anchor, :time_zone
 
       def extract_url
         @extract_url ||= begin
@@ -106,13 +114,77 @@ module Html2rss
         end
       end
 
-      def extract_description
-        exclude = [heading, selected_anchor, kicker_node].compact
-        description = Navigator.extract_visible_text(article_tag, exclude_nodes: exclude)
-        return if description.nil?
+      def leftover_lines
+        leftover_lines_from(article_tag)
+      end
+
+      def leftover_lines_from(node)
+        ArticleRules::Description.lines_from(
+          Navigator.extract_visible_text(node, exclude_nodes: leftover_exclude_nodes_for(node))
+        )
+      end
+
+      def leftover_exclude_nodes_for(node)
+        times = node.respond_to?(:css) ? node.css('time') : []
+        [heading, selected_anchor, kicker_node, *times].compact
+      end
+
+      def heading_or_anchor_item?
+        heading_item? || article_tag.name.to_s == 'a'
+      end
+
+      def heading_item?
+        Navigator::HEADING_TAGS.include?(article_tag.name.to_s)
+      end
+
+      def heading_or_anchor_miss?(title, lines, published_at)
+        CardWalk.miss?(
+          heading_or_anchor_item: heading_or_anchor_item?,
+          published_at:,
+          description: ArticleRules::Description.from_lines(lines, title:)
+        )
+      end
+
+      def parent_card_fields(title, lines, published_at)
+        return article_tag, lines, published_at unless heading_or_anchor_miss?(title, lines, published_at)
+
+        parent = immediate_card_parent
+        if !parent || crowded_parent?(parent)
+          Log.debug { "parent-walk abort at #{article_tag.parent&.name}" }
+          return article_tag, lines, published_at
+        end
 
-        desc = description.strip
-        desc.empty? ? nil : desc
+        new_lines = leftover_lines_from(parent)
+        new_date = extract_published_at_from(parent, new_lines)
+        [parent, new_lines, new_date || published_at]
+      end
+
+      def immediate_card_parent
+        parent = article_tag.respond_to?(:parent) ? article_tag.parent : nil
+        Navigator.parent_until_condition(parent, method(:usable_walk_parent?))
+      end
+
+      def usable_walk_parent?(node)
+        Navigator.usable_card_parent?(node) && !thin_heading_wrapper?(node)
+      end
+
+      def thin_heading_wrapper?(node)
+        CardWalk.thin_wrapper?(
+          children: node.element_children,
+          item: article_tag,
+          descendant_of: ->(item, child) { Navigator.descendant_of?(item, child) }
+        )
+      end
+
+      def crowded_parent?(node)
+        CardWalk.crowded?(
+          heading_count: node.css(Navigator::HEADING_TAGS.join(',')).size,
+          distinct_main_hrefs: distinct_main_hrefs(node)
+        )
+      end
+
+      def distinct_main_hrefs(node)
+        node.css(Navigator::MAIN_ANCHOR_SELECTOR).map { |anchor| anchor['href'] }.uniq.size
       end
 
       def generate_id
@@ -126,9 +198,15 @@ module Html2rss
       end
 
       def extract_image = ImageExtractor.call(article_tag, base_url:)
-      def extract_published_at = DateExtractor.call(article_tag)
+
+      def extract_published_at(lines) = extract_published_at_from(article_tag, lines)
+
+      def extract_published_at_from(node, lines)
+        DateExtractor.call(node, leftover_lines: lines, time_zone:)
+      end
+
       def extract_enclosures = EnclosureExtractor.call(article_tag, base_url)
-      def extract_categories = CategoryExtractor.call(article_tag)
     end
+    # rubocop:enable Metrics/ClassLength
   end
 end

data/lib/html2rss/html/article_rules/category.rb

@@ -6,47 +6,64 @@ module Html2rss
       ##
       # Shared category term vocabulary and text accumulation (DOM walk stays in adapters).
       module Category
-        # Class/data attribute tokens that mark category metadata.
+        # Class/data attribute tokens that mark category metadata (not layout words).
         CATEGORY_TERMS = %w[
-          category categories tag tags topic topics section sections
-          label labels theme themes subject subjects
+          category categories tag tags topic topics
+          theme themes subject subjects
         ].freeze
 
-        # Regex matching category-related attribute or class names.
-        CATEGORY_ATTR_PATTERN = /#{CATEGORY_TERMS.join('|')}/i
+        # Frozen set for token membership checks.
+        TERM_SET = CATEGORY_TERMS.to_set.freeze
+
+        # Hyphen/underscore/BEM separators inside a single CSS class token.
+        CLASS_TOKEN_SPLIT = /[-_]+/
 
         class << self
           ##
           # @param name [String]
           # @return [Boolean]
           def attr_name_match?(name)
-            name.to_s.match?(CATEGORY_ATTR_PATTERN)
+            TERM_SET.include?(normalize_attr_name(name))
           end
 
           ##
           # @param class_attr [String, nil]
           # @return [Boolean]
           def class_match?(class_attr)
-            class_attr.to_s.match?(CATEGORY_ATTR_PATTERN)
+            class_attr.to_s.split(/\s+/).any? { |token| class_token_match?(token) }
           end
 
           ##
           # @param categories [Set<String>]
           # @param text [String, nil]
+          # @param title [String, nil]
           # @return [void]
-          def add_text!(categories, text)
+          def add_text!(categories, text, title: nil)
             value = text.to_s.strip
-            categories.add(value) unless value.empty?
+            return if value.empty? || !Description.keep?(value, title:, type_chips: false)
+
+            categories.add(value)
           end
 
           ##
           # @param categories [Set<String>]
           # @param text [String, nil]
+          # @param title [String, nil]
           # @return [void]
-          def add_split_text!(categories, text)
+          def add_split_text!(categories, text, title: nil)
             return unless text
 
-            text.split(/\n+/).each { |line| add_text!(categories, line) }
+            text.split(/\n+/).each { |line| add_text!(categories, line, title:) }
+          end
+
+          private
+
+          def class_token_match?(token)
+            token.downcase.split(CLASS_TOKEN_SPLIT).intersect?(CATEGORY_TERMS)
+          end
+
+          def normalize_attr_name(name)
+            name.to_s.downcase.delete_prefix('data-')
           end
         end
       end