html2rss 0.25.0 → 0.27.0

data/lib/html2rss/html/article_rules/description.rb

@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Html
+    module ArticleRules
+      ##
+      # Leftover visible-text keep/drop for article descriptions.
+      # Split once on block newlines; Date and Category consume the same keepers.
+      module Description
+        # CTA lines dropped by whole-line equality (case-insensitive).
+        CTA = Set['read more', 'learn more'].freeze
+        # Type-chip lines dropped when the whole leftover line is one of these.
+        TYPE_CHIPS = Set['news article', 'press release', 'news'].freeze
+        # Listing section names that are chrome, not a dek.
+        SECTION_NAMES = Set['press releases'].freeze
+        # Skip date detection / DateTime.parse above this length.
+        MAX_DATE_CHARS = 128
+        # Optional type-chip / clock separators (ASCII/en/em dash, pipe, bullet).
+        CHIP_SEPARATORS = [' - ', ' – ', ' — ', ' | ', ' • '].freeze
+        # Trailing clock plus optional AM/PM and one timezone/offset token (IANA, GMT/UTC offset, or abbrev).
+        CLOCK_ZONE = %r{
+          \A
+          \d{1,2}:\d{2}(?::\d{2})?
+          (?:\s*(?:AM|PM))?
+          (?:\s+(?:[A-Za-z]+(?:/[A-Za-z_]+)+|(?:GMT|UTC)[+-]\d{1,2}(?::\d{2})?|[A-Za-z]{2,5}))?
+          \z
+        }ix
+        # Whole-line date shapes after normalize (ISO, numeric, day-month, month-day).
+        DATE_SHAPES = [
+          /\A\d{4}-\d{1,2}-\d{1,2}(?:[T ]\d{1,2}:\d{2}(?::\d{2})?(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?)?\z/i,
+          %r{\A\d{1,2}[./-]\d{1,2}[./-]\d{2,4}(?: \d{1,2}:\d{2}(?::\d{2})?)?\z},
+          /\A\d{1,2}\.? \p{L}{3,9}\.?\s*,?\s*\d{4}(?: \d{1,2}:\d{2}(?::\d{2})?)?\z/i,
+          /\A\p{L}{3,9}\.? \d{1,2}\s*,?\s*\d{4}(?: \d{1,2}:\d{2}(?::\d{2})?)?\z/i
+        ].freeze
+
+        class << self
+          ##
+          # @param text [String, nil]
+          # @return [Array<String>]
+          def lines_from(text)
+            text.to_s.split(/\n+/).map { |line| normalize(line) }.reject(&:empty?)
+          end
+
+          ##
+          # @param lines [Array<String>, nil]
+          # @param title [String, nil]
+          # @return [String, nil]
+          def from_lines(lines, title: nil)
+            kept = Array(lines).select { |line| keep?(line, title:) }
+            kept.empty? ? nil : kept.join("\n")
+          end
+
+          ##
+          # @param line [String, nil]
+          # @param title [String, nil]
+          # @param type_chips [Boolean] whether whole-line type chips are leftover chrome
+          # @return [Boolean]
+          def keep?(line, title: nil, type_chips: true)
+            normalized = normalize(line)
+            return false if normalized.empty?
+
+            !chrome?(normalized, title:, type_chips:)
+          end
+
+          ##
+          # @param line [String, nil]
+          # @return [Boolean]
+          def date_shaped?(line)
+            core = date_core(line)
+            return false if core.empty? || core.length > MAX_DATE_CHARS
+
+            DATE_SHAPES.any? { |shape| core.match?(shape) }
+          end
+
+          ##
+          # @param line [String, nil]
+          # @return [String]
+          def date_core(line)
+            normalized = normalize(line)
+            CHIP_SEPARATORS.each do |sep|
+              peeled = peel_separator(normalized, sep)
+              return peeled if peeled
+            end
+            normalized
+          end
+
+          private
+
+          def normalize(line) = line.to_s.gsub(/[[:space:]]+/, ' ').strip
+
+          def peel_separator(normalized, sep)
+            idx = normalized.rindex(sep)
+            return unless idx
+
+            left = normalized[0, idx]
+            right = normalized[(idx + sep.length)..]
+            return left if type_chip?(right) || clock_zone?(right)
+            return right if type_chip?(left)
+
+            nil
+          end
+
+          def type_chip?(text) = TYPE_CHIPS.include?(text.downcase)
+
+          def clock_zone?(text) = text.match?(CLOCK_ZONE)
+
+          def chrome?(normalized, title:, type_chips:)
+            key = normalized.downcase
+            CTA.include?(key) || (type_chips && TYPE_CHIPS.include?(key)) ||
+              SECTION_NAMES.include?(key) || normalized.end_with?(':') ||
+              date_shaped?(normalized) || title_echo?(key, title)
+          end
+
+          def title_echo?(key, title)
+            t = title.to_s.strip.downcase
+            !t.empty? && key == t
+          end
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/html/card_walk.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Html
+    ##
+    # Leftover parent-card walk policy shared by Nokogiri and SST extractors.
+    # Adapters traverse; this module decides miss / thin wrapper / crowded abort.
+    # Keep/drop stays on {ArticleRules::Description}. Capture's selector lift is a
+    # separate job (set-aware abort) and is not encoded here.
+    module CardWalk
+      class << self
+        ##
+        # @param heading_or_anchor_item [Boolean] true when the extract root is a heading or wrapping +a+
+        # @param published_at [Object, nil]
+        # @param description [String, nil] leftover description after Description keep/drop
+        # @return [Boolean] whether to climb to a parent card
+        def miss?(heading_or_anchor_item:, published_at:, description:)
+          heading_or_anchor_item && published_at.nil? && description.nil?
+        end
+
+        ##
+        # @param children [Enumerable] immediate children of the candidate parent
+        # @param item [Object] heading/anchor being extracted
+        # @param descendant_of [Proc] +(item, child)+ — whether +item+ sits inside +child+
+        # @return [Boolean] true when the parent only wraps the item (no sibling content)
+        def thin_wrapper?(children:, item:, descendant_of:)
+          children.none? do |child|
+            !child.equal?(item) && !descendant_of.call(item, child)
+          end
+        end
+
+        ##
+        # @param heading_count [Integer] headings inside the candidate parent
+        # @param distinct_main_hrefs [Integer] unique eligible main-article hrefs
+        # @return [Boolean] true when the parent looks like a listing, not one card
+        def crowded?(heading_count:, distinct_main_hrefs:)
+          heading_count > 1 || distinct_main_hrefs > 1
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/html/feed_link.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Html
+    # A native RSS/Atom hint from a document head +link[rel=alternate]+.
+    #
+    # Does not guess +/feed+ or +/rss.xml+ paths — that stays in configs +probe_rss+.
+    FeedLink = Data.define(:href, :mime_type) do
+      class << self
+        ##
+        # Collects RSS/Atom +rel=alternate+ links from +doc+'s +head+.
+        #
+        # @param doc [Nokogiri::XML::Node] parsed HTML document
+        # @return [Array<FeedLink>] native feed hints, omitting blanks and non-feed types
+        def from_document(doc)
+          doc.css('head link[rel~="alternate"][href]').filter_map { |node| from_node(node) }
+        end
+
+        private
+
+        def from_node(node)
+          href = node['href']&.strip
+          mime_type = feed_mime_type(node['type'])
+          new(href:, mime_type:) unless href.nil? || href.empty? || mime_type.nil?
+        end
+
+        def feed_mime_type(type)
+          media_type = type.to_s.split(';', 2).first.strip.downcase
+          media_type if media_type in 'application/rss+xml' | 'application/atom+xml'
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/html/navigator.rb

@@ -17,6 +17,12 @@ module Html2rss
       # Ancestor tags that usually indicate navigation/utility regions inside a content container.
       UTILITY_LANDMARK_TAGS = %w[nav aside footer menu].to_set.freeze
 
+      # Immediate parent walk stops here — not a usable article card.
+      CARD_WALK_STOP_TAGS = (UTILITY_LANDMARK_TAGS | IGNORED_CONTAINER_TAGS | %w[html body]).freeze
+
+      # Inner tags that mean a wrapping <a> is a card, not a span-styled list link.
+      WRAPPING_ANCHOR_CHILD_TAGS = (HEADING_TAGS + %w[p]).freeze
+
       # Anchor selector used to identify the canonical article link element.
       MAIN_ANCHOR_SELECTOR = begin
         buf = +'a[href]:not([href=""])'
@@ -94,6 +100,18 @@ module Html2rss
           end
         end
 
+        ##
+        # Immediate parent is a usable article card (not html/body/landmark chrome).
+        #
+        # @param node [Nokogiri::XML::Node, SST::Node, nil]
+        # @return [Boolean]
+        def usable_card_parent?(node)
+          return false unless node
+          return false if node.respond_to?(:document?) && node.document?
+
+          !CARD_WALK_STOP_TAGS.include?(node.name.to_s)
+        end
+
         ##
         # Think of it as `css_upwards` method.
         # It searches for the closest parent that matches the given selector.

data/lib/html2rss/html/sst_article_extractor.rb

@@ -13,6 +13,8 @@ module Html2rss
       KICKER_CLASS_PATTERN = /kicker|eyebrow|pre-title|pretitle|overline/i
       # Inline emphasis tags used as title fallbacks when no heading exists.
       FALLBACK_HEADING_NAMES = %i[strong b].freeze
+      # Nested blocks that mean a category node is actually a content container.
+      CATEGORY_CONTAINER_NAMES = %i[p article section].to_set.freeze
 
       class << self
         ##
@@ -20,9 +22,10 @@ module Html2rss
         # @param base_url [String, Html2rss::Url]
         # @param scraper [Class, nil]
         # @param fallback_anchorless [Boolean]
+        # @param time_zone [String] channel time zone for naive leftover dates
         # @return [Html2rss::Article, nil]
-        def call(ranked_or_segment, base_url:, scraper: nil, fallback_anchorless: false)
-          new(ranked_or_segment, base_url:, scraper:, fallback_anchorless:).call
+        def call(ranked_or_segment, base_url:, scraper: nil, fallback_anchorless: false, time_zone: 'UTC')
+          new(ranked_or_segment, base_url:, scraper:, fallback_anchorless:, time_zone:).call
         end
       end
 
@@ -30,7 +33,8 @@ module Html2rss
       # @param base_url [String, Html2rss::Url]
       # @param scraper [Class, nil]
       # @param fallback_anchorless [Boolean]
-      def initialize(ranked_or_segment, base_url:, scraper: nil, fallback_anchorless: false)
+      # @param time_zone [String] channel time zone for naive leftover dates
+      def initialize(ranked_or_segment, base_url:, scraper: nil, fallback_anchorless: false, time_zone: 'UTC')
         segment = ranked_or_segment.is_a?(Scoring::RankedSegment) ? ranked_or_segment.segment : ranked_or_segment
         raise ArgumentError, 'segment is required' unless segment.is_a?(AutoSource::Segment)
 
@@ -40,20 +44,25 @@ module Html2rss
         @base_url = base_url
         @scraper = scraper
         @fallback_anchorless = fallback_anchorless
+        @time_zone = time_zone
       end
 
       ##
       # @return [Html2rss::Article, nil]
       def call # rubocop:disable Metrics/MethodLength
+        title = extract_title
+        lines = leftover_lines
+        published_at = extract_published_at(lines)
+        root, lines, published_at = parent_card_fields(title, lines, published_at)
         attrs = {
-          title: extract_title,
+          title:,
           url: extract_url,
           image: extract_image,
-          description: extract_description,
+          description: ArticleRules::Description.from_lines(lines, title:),
           id: generate_id,
-          published_at: extract_published_at,
+          published_at:,
           enclosures: extract_enclosures,
-          categories: extract_categories,
+          categories: extract_categories(title, root:),
           scraper: @scraper
         }
         article = Article.new(**attrs)
@@ -80,19 +89,48 @@ module Html2rss
         Url.from_relative("##{id}", @base_url) if id
       end
 
-      def extract_title # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
-        source = heading || @selected_anchor
-        title_text = if source
-                       source.visible_text
-                     elsif @fallback_anchorless && @selected_anchor.nil?
-                       first_nonempty_own_text
-                     end
+      def extract_title
+        title_text = title_from_in_card_sources
         return unless title_text
 
         kicker = kicker_node&.visible_text.to_s.strip
         kicker && !kicker.empty? && !title_text.include?(kicker) ? "#{kicker}: #{title_text}" : title_text
       end
 
+      # Prefer a non-junk title. Credit-only cards may blank to nil (description can carry
+      # the item). Other junk shapes must stay present so Cleanup can hard-reject them —
+      # blanking slug/CMS/template text would admit titleless cards.
+      #
+      # @return [String, nil]
+      def title_from_in_card_sources
+        candidates = title_candidates
+        clean = candidates.find { |text| AutoSource::Cleanup.junk_reason(text).nil? }
+        return clean if clean
+
+        candidates.find { |text| AutoSource::Cleanup.junk_reason(text) != :credit }
+      end
+
+      # @return [Array<String>] ordered in-card title candidates (heading → anchor → fallback)
+      def title_candidates
+        [].tap do |candidates|
+          add_title_candidate!(candidates, heading&.visible_text)
+          add_title_candidate!(candidates, @selected_anchor&.visible_text)
+          next unless candidates.empty? && @fallback_anchorless && @selected_anchor.nil?
+
+          add_title_candidate!(candidates, first_nonempty_own_text)
+        end
+      end
+
+      # @param candidates [Array<String>]
+      # @param text [String, nil]
+      # @return [void]
+      def add_title_candidate!(candidates, text)
+        stripped = text.to_s.strip
+        return if stripped.empty? || candidates.include?(stripped)
+
+        candidates << stripped
+      end
+
       def heading
         @heading ||= begin
           tags = @root.find_all(&:heading?)
@@ -143,13 +181,73 @@ module Html2rss
         ancestor.descendants.any? { |d| d.equal?(child) }
       end
 
-      def extract_description
-        exclude = [heading, @selected_anchor, kicker_node].compact
-        description = @root.visible_text(exclude:)
-        return if description.nil?
+      def leftover_lines
+        leftover_lines_from(@root)
+      end
+
+      def leftover_lines_from(node)
+        times = node.find_all { |n| n.name == :time }
+        exclude = [heading, @selected_anchor, kicker_node, *times].compact
+        ArticleRules::Description.lines_from(node.visible_text(exclude:))
+      end
+
+      def heading_or_anchor_item?
+        @root.heading? || @root.name == :a
+      end
+
+      def heading_or_anchor_miss?(title, lines, published_at)
+        CardWalk.miss?(
+          heading_or_anchor_item: heading_or_anchor_item?,
+          published_at:,
+          description: ArticleRules::Description.from_lines(lines, title:)
+        )
+      end
+
+      def parent_card_fields(title, lines, published_at)
+        return @root, lines, published_at unless heading_or_anchor_miss?(title, lines, published_at)
+
+        parent = immediate_card_parent
+        if !parent || crowded_parent?(parent)
+          Log.debug { "parent-walk abort at #{sst_parent&.name}" }
+          return @root, lines, published_at
+        end
+
+        new_lines = leftover_lines_from(parent)
+        new_date = extract_published_at(new_lines, root: parent)
+        [parent, new_lines, new_date || published_at]
+      end
+
+      def immediate_card_parent
+        parent = sst_parent
+        index = SST::Index.for_node(@root)
+        index&.parent_until(parent, method(:usable_walk_parent?))
+      end
+
+      def usable_walk_parent?(node)
+        Navigator.usable_card_parent?(node) && !thin_heading_wrapper?(node)
+      end
+
+      def thin_heading_wrapper?(node)
+        CardWalk.thin_wrapper?(
+          children: node.children,
+          item: @root,
+          descendant_of: ->(item, child) { descendant_of?(item, child) }
+        )
+      end
+
+      def crowded_parent?(node)
+        CardWalk.crowded?(
+          heading_count: node.each_node.count(&:heading?),
+          distinct_main_hrefs: distinct_main_hrefs(node)
+        )
+      end
+
+      def distinct_main_hrefs(node)
+        node.each_node.filter_map { |candidate| candidate.attrs.href if candidate.link? }.uniq.size
+      end
 
-        desc = description.strip
-        desc.empty? ? nil : desc
+      def sst_parent
+        SST::Index.for_node(@root)&.parent_of(@root)
       end
 
       def generate_id # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
@@ -211,9 +309,9 @@ module Html2rss
         ArticleRules::Image.best_from_styles(styles)
       end
 
-      def extract_published_at
-        datetimes = @root.find_all { |n| n.attrs.datetime }.map { |n| n.attrs.datetime }
-        ArticleRules::Date.earliest(datetimes)
+      def extract_published_at(lines, root: @root)
+        datetimes = root.find_all { |n| n.attrs.datetime }.map { |n| n.attrs.datetime }
+        ArticleRules::Date.earliest(datetimes, leftover_lines: lines, time_zone: @time_zone)
       end
 
       def extract_enclosures
@@ -242,14 +340,14 @@ module Html2rss
         end
       end
 
-      def extract_categories
+      def extract_categories(title, root: @root)
         Set.new.tap do |categories|
-          @root.find_all { |n| category_candidate?(n) }.each do |element|
-            add_category_text!(categories, element) if ArticleRules::Category.class_match?(element.attrs.class_attr)
+          root.find_all { |n| category_candidate?(n) }.each do |element|
+            add_category_text!(categories, element, title:) if category_text_node?(element, root)
             element.attrs.raw.each do |name, value|
               next unless ArticleRules::Category.attr_name_match?(name)
 
-              ArticleRules::Category.add_text!(categories, value)
+              ArticleRules::Category.add_text!(categories, value, title:)
             end
           end
         end.to_a
@@ -260,17 +358,35 @@ module Html2rss
           node.attrs.raw.keys.any? { |k| ArticleRules::Category.attr_name_match?(k) }
       end
 
-      def add_category_text!(categories, element)
+      def category_text_node?(element, root)
+        !element.equal?(root) && ArticleRules::Category.class_match?(element.attrs.class_attr)
+      end
+
+      def add_category_text!(categories, element, title:)
+        return if category_text_container?(element)
+
         if element.link?
-          ArticleRules::Category.add_text!(categories, element.visible_text)
+          ArticleRules::Category.add_text!(categories, element.visible_text, title:)
           return
         end
 
+        add_descendant_or_visible_category!(categories, element, title:)
+      end
+
+      def add_descendant_or_visible_category!(categories, element, title:)
         anchors = element.find_all(&:link?)
         if anchors.any?
-          anchors.each { |a| ArticleRules::Category.add_text!(categories, a.visible_text) }
+          anchors.each { |a| ArticleRules::Category.add_text!(categories, a.visible_text, title:) }
         else
-          ArticleRules::Category.add_split_text!(categories, element.visible_text)
+          ArticleRules::Category.add_split_text!(categories, element.visible_text, title:)
+        end
+      end
+
+      def category_text_container?(element)
+        element.find do |node|
+          next if node.equal?(element)
+
+          node.heading? || CATEGORY_CONTAINER_NAMES.include?(node.name)
         end
       end
     end

data/lib/html2rss/link_destination/noise_policy.rb

@@ -9,10 +9,8 @@ module Html2rss
       ICON_NAMES = %i[img svg].freeze
 
       # @param link_resolver [LinkResolver]
-      # @param index [SST::Index]
-      def initialize(link_resolver:, index:)
+      def initialize(link_resolver:)
         @link_resolver = link_resolver
-        @index = index
       end
 
       ##
@@ -21,8 +19,11 @@ module Html2rss
       # @param anchor [SST::Node, nil]
       # @param container [SST::Node, nil]
       # @param heading_anchor [Boolean]
+      # @param utility_landmark_ancestor [Boolean] pre-computed by Segmenter tree walk
       # @return [Boolean]
-      def noise_anchor?(text:, destination_facts:, anchor: nil, container: nil, heading_anchor: false) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+      # rubocop:disable Metrics/CyclomaticComplexity, Metrics/ParameterLists, Metrics/PerceivedComplexity, Lint/UnusedMethodArgument
+      def noise_anchor?(text:, destination_facts:, anchor: nil, container: nil,
+                        heading_anchor: false, utility_landmark_ancestor: false)
         return true unless destination_facts
 
         destination_facts.taxonomy_path ||
@@ -32,8 +33,9 @@ module Html2rss
           (@link_resolver.utility_text?(text) && destination_facts.vanity_path) ||
           utility_text_chrome?(text, destination_facts, heading_anchor:) ||
           icon_only_anchor?(anchor, text) ||
-          utility_landmark_ancestor?(anchor, container)
+          utility_landmark_ancestor
       end
+      # rubocop:enable Metrics/CyclomaticComplexity, Metrics/ParameterLists, Metrics/PerceivedComplexity, Lint/UnusedMethodArgument
 
       private
 
@@ -61,19 +63,6 @@ module Html2rss
 
         !!anchor.find { |n| ICON_NAMES.include?(n.name) }
       end
-
-      def utility_landmark_ancestor?(anchor, container) # rubocop:disable Metrics/CyclomaticComplexity
-        return false unless anchor && container
-
-        curr = @index.parent_of(anchor)
-        while curr && curr.name != :html
-          return true if curr != container && curr.utility_landmark?
-          return false if curr.equal?(container)
-
-          curr = @index.parent_of(curr)
-        end
-        false
-      end
     end
   end
 end