html2rss 0.25.0 → 0.27.0

data/lib/html2rss/feed_pipeline.rb

@@ -3,13 +3,16 @@
 module Html2rss
   ##
   # Builds feeds from validated config through request, extraction, and rendering stages.
-  class FeedPipeline
+  #
+  # {include:file:lib/html2rss/feed_pipeline/README.md}
+  class FeedPipeline # rubocop:disable Metrics/ClassLength -- outcome + collect seams stay co-located
     # Scrape-finished facts after request + extraction + dedup (before Channel/Status materialize).
     # selected_strategy: set on :auto success; nil otherwise.
     # attempt_count: auto attempts attempted; 0 outside :auto.
     # strategy_attempts: auto attempt hashes (with optional transport_meta); empty outside :auto.
     PipelineOutcome = Data.define(
-      :response, :articles, :dedup_dropped, :selected_strategy, :attempt_count, :strategy_attempts
+      :response, :articles, :dedup_dropped, :selected_strategy, :attempt_count, :strategy_attempts,
+      :admission_drops
     )
 
     ##
@@ -18,6 +21,16 @@ module Html2rss
       @raw_config = raw_config
     end
 
+    ##
+    # Runs the pipeline once and returns scrape-finished outcome (before Channel/Status).
+    # Used by {Capture} which needs the response body for SST selector derivation.
+    #
+    # @return [PipelineOutcome]
+    def to_outcome
+      config = Config.from_hash(raw_config, params: raw_config[:params])
+      pipeline_outcome_for(config)
+    end
+
     ##
     # Runs the pipeline once and returns an opaque, Marshal-cacheable result.
     #
@@ -32,7 +45,8 @@ module Html2rss
         dedup_dropped: outcome.dedup_dropped,
         selected_strategy: outcome.selected_strategy,
         attempt_count: outcome.attempt_count,
-        strategy_attempts: outcome.strategy_attempts
+        strategy_attempts: outcome.strategy_attempts,
+        admission_drops: outcome.admission_drops
       )
       FeedResult.new(channel:, articles: outcome.articles, status:, stylesheets: config.stylesheets)
     end
@@ -56,11 +70,11 @@ module Html2rss
     # @param config [Html2rss::Config]
     # @param response [Html2rss::RequestService::Response]
     # @param request_session [Html2rss::RequestSession]
-    # @return [Array(Array<Html2rss::Article>, Integer)] unique articles and drop count
+    # @return [Array(Array<Html2rss::Article>, Integer, Hash)] unique articles, dedup drops, admission drops
     def deduplicated_articles(config:, response:, request_session:)
-      collected = collect_articles(config:, response:, request_session:)
+      collected, admission_drops = collect_articles(config:, response:, request_session:)
       unique = Article::Deduplicator.new(collected).call
-      [unique, collected.size - unique.size]
+      [unique, collected.size - unique.size, admission_drops]
     end
 
     private
@@ -80,13 +94,21 @@ module Html2rss
     def run_pipeline_for_strategy(config, strategy:, resources:)
       request_session = request_session_for(config, strategy:, resources:)
       response = request_session.fetch_initial_response
-      articles, dedup_dropped = deduplicated_articles(config:, response:, request_session:)
-      if config.auto_source && articles.empty?
-        raise NoFeedItemsExtracted.new(attempts: [{ strategy:, items_count: 0, error_class: nil }])
-      end
+      articles, dedup_dropped, admission_drops = deduplicated_articles(config:, response:, request_session:)
+      raise_empty_auto_source!(strategy:, response:) if config.auto_source && articles.empty?
 
       PipelineOutcome.new(
-        response:, articles:, dedup_dropped:, selected_strategy: nil, attempt_count: 0, strategy_attempts: []
+        response:, articles:, dedup_dropped:, selected_strategy: nil, attempt_count: 0,
+        strategy_attempts: [], admission_drops:
+      )
+    end
+
+    def raise_empty_auto_source!(strategy:, response:)
+      raise NoFeedItemsExtracted.new(
+        attempts: [{ strategy:, items_count: 0, error_class: nil }],
+        surface_category: AutoSource::Scraper.classify_no_scraper_surface(
+          response.parsed_body, body: response.body
+        )
       )
     end
 
@@ -101,8 +123,9 @@ module Html2rss
     end
 
     def collect_articles(config:, response:, request_session:)
-      selector_articles(config:, response:, request_session:) +
-        auto_source_articles(config:, response:, request_session:)
+      selector = selector_articles(config:, response:, request_session:)
+      auto, admission_drops = auto_source_articles(config:, response:, request_session:)
+      [selector + auto, admission_drops]
     end
 
     # rubocop:disable Metrics/MethodLength
@@ -125,10 +148,12 @@ module Html2rss
     end
     # rubocop:enable Metrics/MethodLength
 
+    # @return [Array(Array<Html2rss::Article>, Hash{String => Integer})]
     def auto_source_articles(config:, response:, request_session:)
-      return [] unless (auto_source = config.auto_source)
+      return [[], {}] unless (auto_source = config.auto_source)
 
-      AutoSource.new(response, auto_source, request_session:).articles
+      source = AutoSource.new(response, auto_source, request_session:)
+      [source.articles, source.admission_drops]
     end
   end
 end

data/lib/html2rss/hash_util.rb

@@ -54,6 +54,23 @@ module Html2rss
       end
     end
 
+    # Converts hash keys (and Symbol values) to strings recursively.
+    #
+    # @param object [Object] value to normalize
+    # @return [Object] normalized value with string keys
+    def deep_stringify_keys(object)
+      case object
+      when Hash
+        object.to_h { |key, value| [key.to_s, deep_stringify_keys(value)] }
+      when Array
+        object.map { deep_stringify_keys(_1) }
+      when Symbol
+        object.to_s
+      else
+        object
+      end
+    end
+
     # Validates that hash keys are symbols.
     #
     # @param value [Object] candidate hash container whose keys must be symbols

data/lib/html2rss/html/article_extractor/category_extractor.rb

@@ -9,24 +9,26 @@ module Html2rss
       class CategoryExtractor
         # Shared category vocabulary (owned by {ArticleRules::Category}).
         CATEGORY_TERMS = ArticleRules::Category::CATEGORY_TERMS
-        # Shared category attribute/class pattern (owned by {ArticleRules::Category}).
-        CATEGORY_ATTR_PATTERN = ArticleRules::Category::CATEGORY_ATTR_PATTERN
 
         # CSS selectors to find elements with category-related class names or data attributes
         CATEGORY_SELECTORS = CATEGORY_TERMS.flat_map do |term|
           ["[class*=\"#{term}\"]", "[data-#{term}]", "[#{term}]"]
         end.freeze
 
+        # Nested blocks that mean a "category" node is actually a content container.
+        CONTAINER_CHILD_SELECTOR = 'p, article, section, h1, h2, h3, h4, h5, h6'
+
         ##
         # Extracts categories from the given article tag by looking for elements
         # with class names containing common category-related terms.
         #
         # @param article_tag [Nokogiri::XML::Element] The article element to extract categories from
+        # @param title [String, nil] article title used to reject title-echo values
         # @return [Array<String>] Array of category strings, empty if none found
-        def self.call(article_tag)
+        def self.call(article_tag, title: nil)
           return [] unless article_tag
 
-          extract_all_categories(article_tag)
+          extract_all_categories(article_tag, title:)
             .map(&:strip)
             .reject(&:empty?)
         end
@@ -35,12 +37,18 @@ module Html2rss
         # Optimized single DOM traversal that extracts all category types.
         #
         # @param article_tag [Nokogiri::XML::Element] The article element
+        # @param title [String, nil]
         # @return [Set<String>] Set of category strings
-        def self.extract_all_categories(article_tag)
+        def self.extract_all_categories(article_tag, title: nil)
           Set.new.tap do |categories|
+            extract_element_data_categories!(categories, article_tag, title:)
             article_tag.css(CATEGORY_SELECTORS.join(',')).each do |element|
-              extract_text_categories!(categories, element) if ArticleRules::Category.class_match?(element['class'])
-              extract_element_data_categories!(categories, element)
+              next if element == article_tag
+
+              if ArticleRules::Category.class_match?(element['class'])
+                extract_text_categories!(categories, element, title:)
+              end
+              extract_element_data_categories!(categories, element, title:)
             end
           end
         end
@@ -50,12 +58,13 @@ module Html2rss
         #
         # @param categories [Set<String>] Accumulator set
         # @param element [Nokogiri::XML::Element] metadata element that may contain category links
+        # @param title [String, nil]
         # @return [void]
-        def self.extract_element_data_categories!(categories, element)
+        def self.extract_element_data_categories!(categories, element, title: nil)
           element.attributes.each_value do |attr|
             next unless ArticleRules::Category.attr_name_match?(attr.name)
 
-            ArticleRules::Category.add_text!(categories, attr.value)
+            ArticleRules::Category.add_text!(categories, attr.value, title:)
           end
         end
 
@@ -64,20 +73,20 @@ module Html2rss
         #
         # @param categories [Set<String>] Accumulator set
         # @param element [Nokogiri::XML::Element] metadata element whose text may contain delimiters
+        # @param title [String, nil]
         # @return [void]
-        def self.extract_text_categories!(categories, element)
-          if element.name == 'a'
-            add_text_to_categories!(categories, element)
-            return
-          end
-
-          anchors = element.css('a')
+        def self.extract_text_categories!(categories, element, title: nil)
+          return if category_container?(element)
 
+          anchors = element.name == 'a' ? [element] : element.css('a').to_a
           if anchors.any?
-            anchors.each { |node| add_text_to_categories!(categories, node) }
-          else
-            ArticleRules::Category.add_split_text!(categories, element.text)
+            anchors.each { |node| add_text_to_categories!(categories, node, title:) }
+            return
           end
+
+          ArticleRules::Category.add_split_text!(
+            categories, Navigator.extract_visible_text(element), title:
+          )
         end
 
         ##
@@ -85,12 +94,17 @@ module Html2rss
         #
         # @param categories [Set<String>] Accumulator set
         # @param element [Nokogiri::XML::Element] The element to extract text from
+        # @param title [String, nil]
         # @return [void]
-        def self.add_text_to_categories!(categories, element)
-          ArticleRules::Category.add_text!(categories, Navigator.extract_visible_text(element))
+        def self.add_text_to_categories!(categories, element, title: nil)
+          ArticleRules::Category.add_text!(categories, Navigator.extract_visible_text(element), title:)
+        end
+
+        def self.category_container?(element)
+          element.at_css(CONTAINER_CHILD_SELECTOR)
         end
 
-        private_class_method :add_text_to_categories!
+        private_class_method :add_text_to_categories!, :category_container?
       end
     end
   end

data/lib/html2rss/html/article_extractor/date_extractor.rb

@@ -6,10 +6,12 @@ module Html2rss
       # Extracts the earliest date from an article_tag.
       class DateExtractor
         # @param article_tag [Nokogiri::XML::Element] article container node
+        # @param leftover_lines [Array<String>] leftover visible lines (already split)
+        # @param time_zone [String] channel time zone for naive values
         # @return [DateTime, nil]
-        def self.call(article_tag)
-          datetimes = article_tag.css('[datetime]').map { |tag| tag['datetime'] }
-          ArticleRules::Date.earliest(datetimes)
+        def self.call(article_tag, leftover_lines: [], time_zone: 'UTC')
+          datetimes = article_tag.respond_to?(:css) ? article_tag.css('[datetime]').map { |tag| tag['datetime'] } : []
+          ArticleRules::Date.earliest(datetimes, leftover_lines:, time_zone:)
         end
       end
     end

data/lib/html2rss/html/article_extractor.rb

@@ -5,6 +5,7 @@ module Html2rss
     ##
     # ArticleExtractor is responsible for extracting details (headline, url, images, etc.)
     # from an article_tag DOM node. DOM chrome helpers live on {Navigator}.
+    # rubocop:disable Metrics/ClassLength -- leftover re-extract stays with field extractors
     class ArticleExtractor
       class << self
         ##
@@ -14,9 +15,10 @@ module Html2rss
         # @param base_url [String, Html2rss::Url] base url used to resolve relative links
         # @param selected_anchor [Nokogiri::XML::Node, nil] explicit primary anchor for the container
         # @param fallback_anchorless [Boolean] whether to fall back to anchorless extraction
+        # @param time_zone [String] channel time zone for naive leftover dates
         # @return [Hash{Symbol => Object}] extracted article attributes
-        def call(article_tag, base_url:, selected_anchor: nil, fallback_anchorless: false)
-          new(article_tag, base_url:, selected_anchor:, fallback_anchorless:).call
+        def call(article_tag, base_url:, selected_anchor: nil, fallback_anchorless: false, time_zone: 'UTC')
+          new(article_tag, base_url:, selected_anchor:, fallback_anchorless:, time_zone:).call
         end
       end
 
@@ -25,32 +27,38 @@ module Html2rss
       # @param base_url [String, Html2rss::Url] base url used to resolve relative links
       # @param selected_anchor [Nokogiri::XML::Node, nil] explicit primary anchor for the container
       # @param fallback_anchorless [Boolean] whether to fall back to anchorless extraction
-      def initialize(article_tag, base_url:, selected_anchor: nil, fallback_anchorless: false)
+      # @param time_zone [String] channel time zone for naive leftover dates
+      def initialize(article_tag, base_url:, selected_anchor: nil, fallback_anchorless: false, time_zone: 'UTC')
         raise ArgumentError, 'article_tag is required' unless article_tag
 
         @article_tag = article_tag
         @base_url = base_url
         @selected_anchor = selected_anchor
         @fallback_anchorless = fallback_anchorless
+        @time_zone = time_zone
       end
 
       # @return [Hash{Symbol => Object}] extracted article attributes
-      def call
+      def call # rubocop:disable Metrics/MethodLength
+        title = extract_title
+        lines = leftover_lines
+        published_at = extract_published_at(lines)
+        source, lines, published_at = parent_card_fields(title, lines, published_at)
         {
-          title: extract_title,
+          title:,
           url: extract_url,
           image: extract_image,
-          description: extract_description,
+          description: ArticleRules::Description.from_lines(lines, title:),
           id: generate_id,
-          published_at: extract_published_at,
+          published_at:,
           enclosures: extract_enclosures,
-          categories: extract_categories
+          categories: CategoryExtractor.call(source, title:)
         }
       end
 
       private
 
-      attr_reader :article_tag, :base_url, :selected_anchor
+      attr_reader :article_tag, :base_url, :selected_anchor, :time_zone
 
       def extract_url
         @extract_url ||= begin
@@ -106,13 +114,77 @@ module Html2rss
         end
       end
 
-      def extract_description
-        exclude = [heading, selected_anchor, kicker_node].compact
-        description = Navigator.extract_visible_text(article_tag, exclude_nodes: exclude)
-        return if description.nil?
+      def leftover_lines
+        leftover_lines_from(article_tag)
+      end
+
+      def leftover_lines_from(node)
+        ArticleRules::Description.lines_from(
+          Navigator.extract_visible_text(node, exclude_nodes: leftover_exclude_nodes_for(node))
+        )
+      end
+
+      def leftover_exclude_nodes_for(node)
+        times = node.respond_to?(:css) ? node.css('time') : []
+        [heading, selected_anchor, kicker_node, *times].compact
+      end
+
+      def heading_or_anchor_item?
+        heading_item? || article_tag.name.to_s == 'a'
+      end
+
+      def heading_item?
+        Navigator::HEADING_TAGS.include?(article_tag.name.to_s)
+      end
+
+      def heading_or_anchor_miss?(title, lines, published_at)
+        CardWalk.miss?(
+          heading_or_anchor_item: heading_or_anchor_item?,
+          published_at:,
+          description: ArticleRules::Description.from_lines(lines, title:)
+        )
+      end
+
+      def parent_card_fields(title, lines, published_at)
+        return article_tag, lines, published_at unless heading_or_anchor_miss?(title, lines, published_at)
+
+        parent = immediate_card_parent
+        if !parent || crowded_parent?(parent)
+          Log.debug { "parent-walk abort at #{article_tag.parent&.name}" }
+          return article_tag, lines, published_at
+        end
 
-        desc = description.strip
-        desc.empty? ? nil : desc
+        new_lines = leftover_lines_from(parent)
+        new_date = extract_published_at_from(parent, new_lines)
+        [parent, new_lines, new_date || published_at]
+      end
+
+      def immediate_card_parent
+        parent = article_tag.respond_to?(:parent) ? article_tag.parent : nil
+        Navigator.parent_until_condition(parent, method(:usable_walk_parent?))
+      end
+
+      def usable_walk_parent?(node)
+        Navigator.usable_card_parent?(node) && !thin_heading_wrapper?(node)
+      end
+
+      def thin_heading_wrapper?(node)
+        CardWalk.thin_wrapper?(
+          children: node.element_children,
+          item: article_tag,
+          descendant_of: ->(item, child) { Navigator.descendant_of?(item, child) }
+        )
+      end
+
+      def crowded_parent?(node)
+        CardWalk.crowded?(
+          heading_count: node.css(Navigator::HEADING_TAGS.join(',')).size,
+          distinct_main_hrefs: distinct_main_hrefs(node)
+        )
+      end
+
+      def distinct_main_hrefs(node)
+        node.css(Navigator::MAIN_ANCHOR_SELECTOR).map { |anchor| anchor['href'] }.uniq.size
       end
 
       def generate_id
@@ -126,9 +198,15 @@ module Html2rss
       end
 
       def extract_image = ImageExtractor.call(article_tag, base_url:)
-      def extract_published_at = DateExtractor.call(article_tag)
+
+      def extract_published_at(lines) = extract_published_at_from(article_tag, lines)
+
+      def extract_published_at_from(node, lines)
+        DateExtractor.call(node, leftover_lines: lines, time_zone:)
+      end
+
       def extract_enclosures = EnclosureExtractor.call(article_tag, base_url)
-      def extract_categories = CategoryExtractor.call(article_tag)
     end
+    # rubocop:enable Metrics/ClassLength
   end
 end

data/lib/html2rss/html/article_rules/category.rb

@@ -6,47 +6,64 @@ module Html2rss
       ##
       # Shared category term vocabulary and text accumulation (DOM walk stays in adapters).
       module Category
-        # Class/data attribute tokens that mark category metadata.
+        # Class/data attribute tokens that mark category metadata (not layout words).
         CATEGORY_TERMS = %w[
-          category categories tag tags topic topics section sections
-          label labels theme themes subject subjects
+          category categories tag tags topic topics
+          theme themes subject subjects
         ].freeze
 
-        # Regex matching category-related attribute or class names.
-        CATEGORY_ATTR_PATTERN = /#{CATEGORY_TERMS.join('|')}/i
+        # Frozen set for token membership checks.
+        TERM_SET = CATEGORY_TERMS.to_set.freeze
+
+        # Hyphen/underscore/BEM separators inside a single CSS class token.
+        CLASS_TOKEN_SPLIT = /[-_]+/
 
         class << self
           ##
           # @param name [String]
           # @return [Boolean]
           def attr_name_match?(name)
-            name.to_s.match?(CATEGORY_ATTR_PATTERN)
+            TERM_SET.include?(normalize_attr_name(name))
           end
 
           ##
           # @param class_attr [String, nil]
           # @return [Boolean]
           def class_match?(class_attr)
-            class_attr.to_s.match?(CATEGORY_ATTR_PATTERN)
+            class_attr.to_s.split(/\s+/).any? { |token| class_token_match?(token) }
           end
 
           ##
           # @param categories [Set<String>]
           # @param text [String, nil]
+          # @param title [String, nil]
           # @return [void]
-          def add_text!(categories, text)
+          def add_text!(categories, text, title: nil)
             value = text.to_s.strip
-            categories.add(value) unless value.empty?
+            return if value.empty? || !Description.keep?(value, title:, type_chips: false)
+
+            categories.add(value)
           end
 
           ##
           # @param categories [Set<String>]
           # @param text [String, nil]
+          # @param title [String, nil]
           # @return [void]
-          def add_split_text!(categories, text)
+          def add_split_text!(categories, text, title: nil)
             return unless text
 
-            text.split(/\n+/).each { |line| add_text!(categories, line) }
+            text.split(/\n+/).each { |line| add_text!(categories, line, title:) }
+          end
+
+          private
+
+          def class_token_match?(token)
+            token.downcase.split(CLASS_TOKEN_SPLIT).intersect?(CATEGORY_TERMS)
+          end
+
+          def normalize_attr_name(name)
+            name.to_s.downcase.delete_prefix('data-')
           end
         end
       end

data/lib/html2rss/html/article_rules/date.rb

@@ -1,22 +1,76 @@
 # frozen_string_literal: true
 
+require 'date'
+require 'tzinfo'
+
 module Html2rss
   module Html
     module ArticleRules
       ##
       # DOM-agnostic datetime aggregation for article published_at.
       module Date
+        # Skip DateTime.parse above this length (Ruby parse is not a validator).
+        MAX_DATE_CHARS = Description::MAX_DATE_CHARS
+        # Trailing offset / Zulu marker — present means the string already has a zone.
+        OFFSET = /(?:Z|[+-]\d{2}:?\d{2})\z/i
+        # Prefer standard time when a local clock time occurs twice (DST fall-back).
+        AMBIGUOUS_DST = false
+
         class << self
           ##
           # @param datetime_strings [Array<String, nil>] raw datetime attribute values
+          # @param leftover_lines [Array<String>] leftover visible lines (already split)
+          # @param time_zone [String] channel time zone for naive values
           # @return [DateTime, nil] earliest successfully parsed instant
-          def earliest(datetime_strings)
-            times = Array(datetime_strings).filter_map do |value|
-              DateTime.parse(value.to_s)
-            rescue ArgumentError, TypeError
-              nil
+          def earliest(datetime_strings, leftover_lines: [], time_zone: 'UTC')
+            values = Array(datetime_strings).filter_map { |value| parse_attr(value, time_zone:) }
+            Array(leftover_lines).each do |line|
+              parsed = parse_line(line, time_zone:)
+              values << parsed if parsed
             end
-            times.min
+            values.min
+          end
+
+          private
+
+          def parse_attr(value, time_zone:)
+            parse_datetime(value.to_s.strip, time_zone:)
+          end
+
+          def parse_line(line, time_zone:)
+            return unless Description.date_shaped?(line)
+
+            parse_datetime(Description.date_core(line), time_zone:)
+          end
+
+          def parse_datetime(string, time_zone:)
+            return if string.empty? || string.length > MAX_DATE_CHARS
+
+            parsed = DateTime.parse(string, limit: MAX_DATE_CHARS)
+            naive?(string) ? localize(parsed, time_zone) : parsed
+          rescue ArgumentError, TypeError
+            nil
+          end
+
+          def naive?(string) = !string.match?(OFFSET)
+
+          def localize(datetime, time_zone)
+            identifier = time_zone.to_s
+            identifier = 'UTC' if identifier.empty?
+            at_local(timezone_for(identifier), datetime)
+          rescue TZInfo::PeriodNotFound
+            at_local(timezone_for('UTC'), datetime)
+          end
+
+          def at_local(timezone, datetime)
+            timezone.local_datetime(datetime.year, datetime.month, datetime.day,
+                                    datetime.hour, datetime.min, datetime.sec, 0, AMBIGUOUS_DST, &:first)
+          end
+
+          def timezone_for(identifier)
+            TZInfo::Timezone.get(identifier)
+          rescue TZInfo::InvalidTimezoneIdentifier
+            TZInfo::Timezone.get('UTC')
           end
         end
       end