html2rss 0.32.4 → 0.34.0

data/lib/html2rss/html/title_pieces/kicker.rb

@@ -0,0 +1,147 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Html
+    class TitlePieces
+      ##
+      # The kicker of one card, first match: the outermost kicker-marked element beside the headline
+      # ({KICKER_WORDS}, {KICKER_ATTRIBUTE}) holding no word besides its noise and kicker-marked parts; else
+      # a label leaf ({LABEL_WORDS}) opening the heading or anchor text; else the piece right before the
+      # headline piece. A paid-content label is never a kicker.
+      class Kicker
+        # Media element names of both node kinds: a piece holding one is a picture, not a kicker.
+        MEDIA_NAMES = Set['img', 'picture', 'figure', 'video', 'svg', :img, :picture, :figure, :video, :svg].freeze
+        # Paragraphs of both node kinds: running prose, never a kicker.
+        PARAGRAPH_NAMES = Set['p', :p].freeze
+        # Class tokens that may hold a {KICKER_WORDS} word: each word with any +-+ or +_+ between its letters,
+        # in any case. A cheap gate before the class words are split.
+        KICKER_TOKEN = Regexp.new(KICKER_WORDS.map { _1.chars.join('[-_]*') }.join('|'), Regexp::IGNORECASE)
+        private_constant :MEDIA_NAMES, :PARAGRAPH_NAMES, :KICKER_TOKEN
+
+        ##
+        # @param nodes [Nodes] reader of the card's nodes
+        # @param root [Nokogiri::XML::Node, SST::Node] card root
+        # @param source [Nokogiri::XML::Node, SST::Node, nil] headline source: the heading, else the anchor
+        # @yieldreturn [Nokogiri::XML::Node, SST::Node, nil] the source's headline piece, read only when
+        #   no marked or label kicker is found
+        def initialize(nodes, root, source, &)
+          @nodes = nodes
+          @root = root
+          @source = source
+          @element = marked || (label || preceding(yield) if source)
+        end
+
+        ##
+        # @return [Nokogiri::XML::Node, SST::Node, nil] the kicker element
+        attr_reader :element
+
+        ##
+        # @return [String, nil] kicker text without its noise pieces ({Article::Title.noise_piece?})
+        def text = element && text_without_noise(element)
+
+        private
+
+        # A kicker inside the heading or anchor wins over one beside it (a topline in the heading over a
+        # type badge above it).
+        def marked
+          candidates = @nodes.carrying(@root, KICKER_ATTRIBUTE).select { marked?(_1) && worded?(_1) && beside?(_1) }
+          kickers = candidates.select { |node| parts_only?(node, candidates) }
+          kickers.find { @nodes.inside?(_1, @source) } || kickers.first
+        end
+
+        def marked?(node)
+          @nodes.attribute(node, KICKER_ATTRIBUTE) ||
+            (@nodes.class_names(node).any?(KICKER_TOKEN) && @nodes.class_words(node).intersect?(KICKER_WORDS))
+        end
+
+        # A label leaf opening the source text, with headline text after it.
+        def label
+          unseen = @nodes.unseen(@source)
+          rest = @nodes.text(@source, unseen).to_s
+          @nodes.elements(@source).find do |element|
+            label_leaf?(element) && rest.start_with?(@nodes.text(element).to_s) &&
+              @nodes.text(@source, unseen + [element]).to_s.match?(WORD)
+          end
+        end
+
+        def label_leaf?(element) = @nodes.class_words(element).intersect?(LABEL_WORDS) && !@nodes.paid_label?(element)
+
+        # The nearest worded piece before the headline piece, when it is a kicker-shaped leaf.
+        def preceding(headline_piece)
+          candidate = piece_before(headline_piece)
+          return unless candidate && shaped?(candidate)
+
+          leaf = nearest_leaf(candidate)
+          leaf if shaped?(leaf)
+        end
+
+        # Walks from +piece+ up to the source, looking back over element siblings; direct text stops it.
+        def piece_before(piece)
+          current = piece
+          until current.nil? || current.equal?(@source)
+            before = @nodes.siblings_before(current)
+            return if before.nil?
+
+            worded = before.reverse.find { !skipped?(_1) }
+            return worded if worded
+
+            current = @nodes.parent(current)
+          end
+        end
+
+        # Descends into the last worded child of a piece made only of elements: the leaf next to the
+        # headline (a topic span after a badge in one box).
+        def nearest_leaf(node)
+          until @nodes.own_words?(node)
+            child = node.children.reverse.find { !skipped?(_1) }
+            return node unless child
+
+            node = child
+          end
+          node
+        end
+
+        def skipped?(node)
+          @nodes.concealed?(node) || @nodes.date_element?(node) || !text_without_noise(node).to_s.match?(WORD)
+        end
+
+        def shaped?(node)
+          !PARAGRAPH_NAMES.include?(node.name) && !@nodes.class_words(node).intersect?(LEAD_WORDS) &&
+            @nodes.elements(node).none? { MEDIA_NAMES.include?(_1.name) }
+        end
+
+        def worded?(node)
+          !@nodes.date_element?(node) && !@nodes.noise?(node) && !@nodes.concealed?(node) &&
+            text_without_noise(node).to_s.match?(WORD)
+        end
+
+        def text_without_noise(node) = @nodes.text(node, noise(node))
+
+        # Kicker pieces that are noise on their own: date, time and concealed elements, paid-content labels
+        # and {Article::Title.noise_piece?}.
+        def noise(node)
+          @nodes.elements(node).select do |piece|
+            @nodes.date_element?(piece) || @nodes.concealed?(piece) || @nodes.paid_label?(piece) ||
+              Article::Title.noise_piece?(@nodes.text(piece))
+          end
+        end
+
+        def beside?(node)
+          return true unless @source
+          return false if @nodes.inside?(@source, node)
+
+          !@nodes.inside?(node, @source) || @nodes.text(@source, [node]).to_s.match?(WORD)
+        end
+
+        # Whether +node+ holds no word outside its noise and its nested kicker candidates.
+        def parts_only?(node, candidates)
+          parts = candidates.select { |other| !other.equal?(node) && @nodes.inside?(other, node) }
+          return true if parts.empty?
+
+          !@nodes.text(node, noise(node) + parts).to_s.match?(WORD)
+        end
+      end
+      private_constant :Kicker
+    end
+  end
+end

data/lib/html2rss/html/title_pieces/nodes.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Html
+    class TitlePieces
+      ##
+      # Reads the nodes of one card for {TitlePieces}, SST and Nokogiri alike: element descendants, class
+      # words, visible text and the element facts both title rules read (concealed, date and noise
+      # elements), each read once per node. {SstNodes} and {NokogiriNodes} supply the reads that differ by
+      # node kind; the headline rule stays on {TitlePieces}, the kicker rule on {Kicker}.
+      class Nodes
+        # Text that holds a word.
+        WORD = /[[:alnum:]]/
+        # Splits a class token into words.
+        CLASS_WORD_BREAK = /[-_]+|(?<=[[:lower:][:digit:]])(?=[[:upper:]])/
+        # +time+ element names of both node kinds.
+        TIME_NAMES = Set['time', :time].freeze
+        private_constant :CLASS_WORD_BREAK, :TIME_NAMES
+
+        ##
+        # @param root [Nokogiri::XML::Node, SST::Node] card root
+        # @return [Nodes] the reader for the node kind of +root+
+        def self.for(root) = root.instance_of?(SST::Node) ? SstNodes.new : NokogiriNodes.new
+
+        def initialize
+          @words = {}.compare_by_identity
+          @elements = {}.compare_by_identity
+          @texts = {}.compare_by_identity
+        end
+
+        ##
+        # @param node [Nokogiri::XML::Node, SST::Node]
+        # @return [Array] element descendants of +node+, read once per node
+        def elements(node) = @elements[node] ||= descendants(node)
+
+        ##
+        # Lowercase words of all class tokens, plus each pair of neighbours joined (+region+ +Name+ gives
+        # +regionname+, +sub+ +title+ gives +subtitle+).
+        #
+        # @param node [Nokogiri::XML::Node, SST::Node]
+        # @return [Set<String>]
+        def class_words(node)
+          @words[node] ||= class_names(node).each_with_object(Set.new) do |token, words|
+            parts = token.split(CLASS_WORD_BREAK).map(&:downcase)
+            words.merge(parts).merge(parts.each_cons(2).map(&:join))
+          end
+        end
+
+        ##
+        # @param node [Nokogiri::XML::Node, SST::Node]
+        # @return [Boolean] whether a sighted reader does not see +node+ ({Visibility.concealed?})
+        def concealed?(node)
+          Visibility.concealed?(class_names(node), style: style(node), hidden: !attribute(node, 'hidden').nil?)
+        end
+
+        ##
+        # @param node [Nokogiri::XML::Node, SST::Node]
+        # @param exclude [Array, nil] elements left out
+        # @return [String, nil] visible text of +node+; text read without exclusions is read once per node
+        def text(node, exclude = nil)
+          return visible_text(node, exclude) if exclude&.any?
+
+          @texts[node] ||= visible_text(node, nil)
+        end
+
+        ##
+        # A date chip set apart from the headline words: a date element ({#date_element?}) with no direct text
+        # holding a word right before or right after it (+resume on <time>Friday</time>+ is running text).
+        #
+        # @param node [Nokogiri::XML::Node, SST::Node]
+        # @return [Boolean]
+        def dropped_date?(node)
+          return false unless date_element?(node)
+
+          before, after = texts_beside(node)
+          !before.to_s.match?(WORD) && !after.to_s.match?(WORD)
+        end
+
+        ##
+        # @param node [Nokogiri::XML::Node, SST::Node]
+        # @return [Boolean] whether +node+ is a +time+ element or its class marks a date, time or duration
+        #   ({DATE_WORDS})
+        def date_element?(node) = TIME_NAMES.include?(node.name) || class_words(node).intersect?(DATE_WORDS)
+
+        ##
+        # @param node [Nokogiri::XML::Node, SST::Node]
+        # @return [Array] elements of +node+ a reader does not see as its text: concealed and dropped date ones
+        def unseen(node) = elements(node).select { concealed?(_1) || dropped_date?(_1) }
+
+        ##
+        # @param node [Nokogiri::XML::Node, SST::Node]
+        # @return [Boolean] whether the text of +node+ is a paid-content label
+        #   ({LinkDestination::Sponsorship.paid_text?})
+        def paid_label?(node) = LinkDestination::Sponsorship.paid_text?(text(node))
+
+        ##
+        # @param node [Nokogiri::XML::Node, SST::Node]
+        # @return [Boolean] whether the text of +node+ is headline noise ({Article::Title.headline_noise_piece?})
+        #   or a paid-content label
+        def noise?(node)
+          Article::Title.headline_noise_piece?(text(node)) || LinkDestination::Sponsorship.paid_text?(text(node))
+        end
+      end
+      private_constant :Nodes
+    end
+  end
+end

data/lib/html2rss/html/title_pieces/nokogiri_nodes.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Html
+    class TitlePieces
+      ##
+      # {Nodes} reads of Nokogiri nodes: direct text is the text node siblings of an element.
+      class NokogiriNodes < Nodes
+        ##
+        # @param root [Nokogiri::XML::Node]
+        # @param attribute [String]
+        # @return [Array<Nokogiri::XML::Element>] +root+ and its descendants carrying a class or +attribute+
+        def carrying(root, attribute) = root.xpath("descendant-or-self::*[@class or @#{attribute}]")
+
+        ##
+        # @param node [Nokogiri::XML::Node]
+        # @return [Array<String>] class tokens
+        def class_names(node) = node['class'].to_s.split
+
+        ##
+        # @param node [Nokogiri::XML::Node]
+        # @param name [String]
+        # @return [String, nil] the attribute value
+        def attribute(node, name) = node[name]
+
+        ##
+        # @param node [Nokogiri::XML::Node]
+        # @return [Nokogiri::XML::Node, nil]
+        def parent(node) = node.parent
+
+        ##
+        # @param node [Nokogiri::XML::Node]
+        # @param ancestor [Nokogiri::XML::Node]
+        # @return [Boolean] whether +node+ is +ancestor+ or inside it
+        def inside?(node, ancestor) = node == ancestor || Navigator.descendant_of?(node, ancestor)
+
+        ##
+        # @param node [Nokogiri::XML::Node]
+        # @return [Boolean] whether +node+'s direct text (outside its child elements) holds a word
+        def own_words?(node) = node.children.any? { _1.text? && _1.text.match?(WORD) }
+
+        ##
+        # @param node [Nokogiri::XML::Node]
+        # @return [Array<Nokogiri::XML::Node>, nil] siblings before +node+, or nil when direct text holding a
+        #   word sits between them and +node+
+        def siblings_before(node)
+          before = []
+          sibling = node.previous_sibling
+          while sibling
+            return if sibling.text? && sibling.text.match?(WORD)
+
+            before.unshift(sibling)
+            sibling = sibling.previous_sibling
+          end
+          before
+        end
+
+        private
+
+        def descendants(node) = node.css('*')
+        def style(node) = node['style']
+        def visible_text(node, exclude) = Navigator.extract_visible_text(node, exclude_nodes: exclude)
+
+        # The direct text right before and right after +node+.
+        def texts_beside(node) = [node.previous_sibling, node.next_sibling].map { _1.text if _1&.text? }
+      end
+      private_constant :NokogiriNodes
+    end
+  end
+end

data/lib/html2rss/html/title_pieces/sst_nodes.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Html
+    class TitlePieces
+      ##
+      # {Nodes} reads of SST nodes: direct text is the node's lead text and the tail after each child.
+      class SstNodes < Nodes
+        ##
+        # @param root [SST::Node]
+        # @param attribute [String]
+        # @return [Array<SST::Node>] +root+ and its descendants carrying a class or +attribute+
+        def carrying(root, attribute) = root.find_all { _1.attrs.class_names.any? || _1.attrs.raw.key?(attribute) }
+
+        ##
+        # @param node [SST::Node]
+        # @return [Array<String>] class tokens
+        def class_names(node) = node.attrs.class_names
+
+        ##
+        # @param node [SST::Node]
+        # @param name [String]
+        # @return [String, nil] the attribute value
+        def attribute(node, name) = node.attrs.raw[name]
+
+        ##
+        # @param node [SST::Node]
+        # @return [SST::Node, nil] the parent of an indexed node
+        def parent(node) = SST::Index.for_node(node)&.parent_of(node)
+
+        ##
+        # @param node [SST::Node]
+        # @param ancestor [SST::Node]
+        # @return [Boolean] whether +node+ is +ancestor+ or inside it
+        def inside?(node, ancestor) = SST::Index.descendant_or_self?(node, ancestor)
+
+        ##
+        # @param node [SST::Node]
+        # @return [Boolean] whether +node+'s direct text (outside its child elements) holds a word
+        def own_words?(node) = node.own_text.match?(WORD)
+
+        ##
+        # @param node [SST::Node]
+        # @return [Array<SST::Node>, nil] siblings before +node+, or nil when direct text holding a word
+        #   sits between them and +node+
+        def siblings_before(node)
+          up = parent(node)
+          return unless up
+
+          index = up.children.index { _1.equal?(node) }
+          up.children.take(index) unless [up.lead_text, *up.tails][..index].any? { _1.match?(WORD) }
+        end
+
+        private
+
+        def descendants(node) = node.descendants
+        def style(node) = node.attrs.style
+        def visible_text(node, exclude) = node.visible_text(exclude:)
+
+        # The direct text right before and right after +node+.
+        def texts_beside(node)
+          up = parent(node)
+          return [] unless up
+
+          [up.lead_text, *up.tails][up.children.index { _1.equal?(node) }, 2]
+        end
+      end
+      private_constant :SstNodes
+    end
+  end
+end

data/lib/html2rss/html/title_pieces.rb

@@ -0,0 +1,169 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Html
+    ##
+    # Picks the kicker and the headline text of one teaser card: the one rule shared by
+    # {ArticleExtractor} (Nokogiri) and {SstArticleExtractor} (SST), so both give one title for the same
+    # markup. The noise vocabulary and composition stay on {Article::Title}; rules: CONTEXT.md § Article title.
+    #
+    # Everything here reads elements, never headline wording:
+    #
+    # - Headline: a +data-title+ attribute on the heading or anchor wins. A heading or anchor made only of
+    #   element pieces (no word in its own direct text) is read from its headline piece: its one outermost
+    #   headline-marked descendant ({HEADLINE_WORDS}; never a sub-title or a label-marked one, {LABEL_WORDS}),
+    #   else the selected anchor when it is the only link inside the heading. Pieces beside it (tags, dates,
+    #   lead paragraphs) are no part of the title.
+    # - Headline text leaves out the kicker, concealed elements ({Visibility.concealed?}; they stay when they
+    #   hold the only words, the accessible name of an otherwise empty link), date and time
+    #   elements ({Nodes#dropped_date?}) unless direct text holding a word touches them, lead pieces
+    #   ({LEAD_WORDS}), tag pieces closing the text ({TAG_WORDS}), clock-only or label-only elements before
+    #   its first word, and date chips closing it ({#date_chip?}).
+    # - A paid-content label ({LinkDestination::Sponsorship.paid_text?}) is item metadata, never title text:
+    #   it is no kicker and leaves the headline text where it opens it, so a kept sponsored item (cleanup
+    #   +keep_sponsored+) is titled by its headline alone. An ad word ("Anzeige", "Werbung") also names a
+    #   topic and stays a kicker.
+    # - Kicker: {Kicker}.
+    class TitlePieces
+      # Class words (tokens split on +-+, +_+ and camelCase, neighbours joined) marking kicker / eyebrow
+      # text anywhere in the card.
+      KICKER_WORDS = Set['kicker', 'eyebrow', 'pretitle', 'overline', 'topline'].freeze
+      # Attribute marking kicker text.
+      KICKER_ATTRIBUTE = 'data-tb-kicker'
+      # Attribute holding the publisher's clean title on the heading or anchor.
+      TITLE_ATTRIBUTE = 'data-title'
+      # Class words (tokens split on +-+, +_+ and camelCase) marking the headline piece. +subTitle+ and
+      # +sub-title+ are not one.
+      HEADLINE_WORDS = Set['title', 'headline', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6'].freeze
+      # Class words of a label leaf that is a kicker when it opens the heading or anchor text.
+      LABEL_WORDS = (KICKER_WORDS | Set['prefix', 'label', 'alert', 'alerte', 'live', 'regionname']).freeze
+      # Class words of date, time and duration elements.
+      DATE_WORDS = Set['date', 'time', 'duration', 'timestamp', 'datetime'].freeze
+      # Class words of a teaser lead or summary, never part of the title.
+      LEAD_WORDS = Set['lead', 'leadin', 'summary', 'excerpt', 'standfirst', 'dek', 'abstract'].freeze
+      # Class words of tag or subtitle pieces, dropped when they close the headline text.
+      TAG_WORDS = Set['tag', 'tags', 'subtitle'].freeze
+
+      # Text that holds a word.
+      WORD = Nodes::WORD
+      # Link element names of both node kinds.
+      LINK_NAMES = Set['a', :a].freeze
+      private_constant :WORD, :LINK_NAMES
+
+      ##
+      # @param root [Nokogiri::XML::Node, SST::Node] card root
+      # @param source [Nokogiri::XML::Node, SST::Node, nil] headline source: the heading, else the anchor
+      # @param anchor [Nokogiri::XML::Node, SST::Node, nil] the card's selected anchor
+      def initialize(root, source, anchor: nil)
+        @anchor = anchor
+        @nodes = Nodes.for(root)
+        @kicker = Kicker.new(@nodes, root, source) { headline_piece(source) }
+      end
+
+      ##
+      # @return [Nokogiri::XML::Node, SST::Node, nil] the kicker element ({Kicker})
+      def kicker = @kicker.element
+
+      ##
+      # @return [String, nil] kicker text without its noise pieces ({Article::Title.noise_piece?})
+      def kicker_text = @kicker.text
+
+      ##
+      # @param node [Nokogiri::XML::Node, SST::Node, nil] heading or anchor
+      # @return [String, nil] its headline text (see the class notes)
+      def headline_text(node)
+        return unless node
+        return @nodes.attribute(node, TITLE_ATTRIBUTE) if @nodes.attribute(node, TITLE_ATTRIBUTE).to_s.match?(WORD)
+
+        node = headline_piece(node) || node
+        @nodes.text(node, headline_drops(node))
+      end
+
+      private
+
+      # --- Headline ------------------------------------------------------------------------------------
+
+      # The piece of +node+ holding the headline when +node+ is made only of element pieces.
+      def headline_piece(node)
+        return if @nodes.own_words?(node)
+
+        marked = @nodes.elements(node).select { headline_marked?(_1) }
+        outermost = marked.reject { |el| marked.any? { |other| !other.equal?(el) && @nodes.inside?(el, other) } }
+        return outermost.first if outermost.one?
+
+        @anchor if sole_anchor_inside?(node)
+      end
+
+      # Whether the selected anchor is the only link inside +node+ (a headline split over several links
+      # stays whole).
+      def sole_anchor_inside?(node)
+        links = @nodes.elements(node).select { LINK_NAMES.include?(_1.name) }
+        links.one? && links.first.equal?(@anchor)
+      end
+
+      def headline_marked?(node)
+        words = @nodes.class_words(node)
+        words.intersect?(HEADLINE_WORDS) && !words.include?('subtitle') && !words.intersect?(LABEL_WORDS)
+      end
+
+      # Elements left out of the headline text of +node+, in rule order.
+      def headline_drops(node)
+        dropped = drop_concealed!(node, [kicker, *@nodes.elements(node).select { @nodes.dropped_date?(_1) }].compact)
+        drop_pieces!(node, dropped, LEAD_WORDS) { |rest, _| rest.match?(WORD) }
+        drop_pieces!(node, dropped, TAG_WORDS) { |rest, piece| rest.match?(WORD) && closing?(node, dropped, piece) }
+        drop_opening_noise!(node, dropped)
+        drop_closing_dates!(node, dropped)
+      end
+
+      # Concealed elements leave unless they hold the only words (the name of an otherwise empty link).
+      def drop_concealed!(node, dropped)
+        concealed = @nodes.elements(node).select { @nodes.concealed?(_1) }
+        @nodes.text(node, dropped + concealed).to_s.match?(WORD) ? dropped.concat(concealed) : dropped
+      end
+
+      # Drops each +words+-marked piece of +node+ the block admits, given the text left without it.
+      def drop_pieces!(node, dropped, words)
+        @nodes.elements(node).each do |element|
+          next unless @nodes.class_words(element).intersect?(words)
+
+          dropped << element if yield(@nodes.text(node, dropped + [element]).to_s, element)
+        end
+      end
+
+      def drop_opening_noise!(node, dropped)
+        @nodes.elements(node).each do |element|
+          next if within?(element, dropped) || !@nodes.noise?(element)
+          break unless opening?(node, dropped, element)
+
+          dropped << element
+        end
+      end
+
+      # Date chips ({#date_chip?}) closing the text.
+      def drop_closing_dates!(node, dropped)
+        @nodes.elements(node).reverse_each do |element|
+          next if within?(element, dropped) || !date_chip?(element)
+          break unless closing?(node, dropped, element)
+
+          dropped << element
+        end
+        dropped
+      end
+
+      # The date chip of a teaser card: a date-only element ({Article::Title.date_piece?}), or a date element
+      # holding only a clock time (+late <time>22:10</time>+, while +at <b>20:15</b>+ is headline text).
+      def date_chip?(element)
+        Article::Title.date_piece?(@nodes.text(element)) || (@nodes.date_element?(element) && Article::Title.clock_piece?(@nodes.text(element)))
+      end
+
+      # Whether +element+ opens the text of +node+ left after +dropped+.
+      def opening?(node, dropped, element) = @nodes.text(node, dropped).to_s.start_with?(@nodes.text(element))
+
+      # Whether +element+ closes the text of +node+ left after +dropped+.
+      def closing?(node, dropped, element) = @nodes.text(node, dropped).end_with?(@nodes.text(element).to_s)
+
+      # Whether +element+ is or sits inside a dropped element.
+      def within?(element, dropped) = dropped.any? { @nodes.inside?(element, _1) }
+    end
+  end
+end

data/lib/html2rss/html/visibility.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Html
+    ##
+    # The one vocabulary of elements a sighted reader does not see, in two kinds:
+    #
+    # - unrendered: a +hidden+ attribute, an inline +display: none+, or a hiding utility class
+    #   ({HIDDEN_CLASSES}). Neither seen nor spoken.
+    # - screen-reader-only: an off-screen or visually-hidden class ({SCREEN_READER_CLASS}: +sr-only+,
+    #   +visually-hidden+, +h-offscreen+, +_screen-reader-only+, +screen-reader-text+, ...). Spoken, not seen.
+    #
+    # +aria-hidden="true"+ is neither: the element is seen, only not spoken. Callers pick the kind they
+    # need: a sponsorship label still counts when spoken ({LinkDestination::Sponsorship}), while titles
+    # and item anchors keep only what is seen ({.concealed?}). Inputs are plain attribute values, so
+    # SST and Nokogiri callers share it.
+    module Visibility
+      # Inline style that hides an element.
+      HIDDEN_STYLE = /display\s*:\s*none/i
+      # Utility class tokens that keep an element from rendering.
+      HIDDEN_CLASSES = Set['hidden', 'd-none', 'is-hidden', 'hide'].freeze
+      # Off-screen / visually-hidden class token, with an optional utility or BEM prefix (+u-+, +h-+,
+      # +card__+, a leading +_+) and a +-focusable+ variant that shows only on keyboard focus.
+      SCREEN_READER_CLASS = /
+        \A_?(?:[a-z0-9]+(?:-|__))?
+        (?:sr-only|visually-?hidden|screen-?reader-(?:only|text)|off-?screen|element-invisible)
+        (?:-focusable)?\z
+      /ix
+
+      class << self
+        ##
+        # @param class_names [Array<String>] class tokens
+        # @param style [String, nil] inline style
+        # @param hidden [Boolean] whether the element carries a +hidden+ attribute
+        # @return [Boolean] whether the element is not rendered at all
+        def unrendered?(class_names, style:, hidden:)
+          hidden || style.to_s.match?(HIDDEN_STYLE) || class_names.any? { HIDDEN_CLASSES.include?(_1) }
+        end
+
+        ##
+        # @param class_names [Array<String>] class tokens
+        # @return [Boolean] whether a class keeps the element off screen for screen readers only
+        def screen_reader_only?(class_names) = class_names.any? { _1.match?(SCREEN_READER_CLASS) }
+
+        ##
+        # @param class_names [Array<String>] class tokens
+        # @param style [String, nil] inline style
+        # @param hidden [Boolean] whether the element carries a +hidden+ attribute
+        # @return [Boolean] whether a sighted reader does not see the element
+        def concealed?(class_names, style:, hidden:)
+          unrendered?(class_names, style:, hidden:) || screen_reader_only?(class_names)
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/html.rb

@@ -4,5 +4,7 @@ module Html2rss
   ##
   # DOM helpers for navigating and extracting from HTML trees.
   module Html
+    # Raised when an extractor is built without the node or segment it extracts from.
+    class InvalidError < Html2rss::ContractError; end
   end
 end

data/lib/html2rss/link_destination/destination_facts.rb

@@ -21,7 +21,7 @@ module Html2rss
       # @param url [Html2rss::Url] normalized destination URL
       # @return [DestinationFacts] route facts for downstream link scoring
       def self.build(url) # rubocop:disable Metrics/MethodLength
-        classifier = PathClassifier.new(url.path_segments)
+        classifier = classifier_for(url)
 
         new(
           url:,
@@ -37,6 +37,22 @@ module Html2rss
           high_confidence_utility_destination: classifier.utility_destination?
         )
       end
+
+      # @param url [Html2rss::Url]
+      # @return [PathClassifier] path classifier that also sees the first host label (an obituary subdomain)
+      def self.classifier_for(url)
+        PathClassifier.new(url.path_segments.map { decode_segment(_1) },
+                           host_label: url.host.to_s.split('.').first, query: !url.query.nil?)
+      end
+      private_class_method :classifier_for
+
+      # Url keeps path segments percent-encoded; classification words are plain NFC text.
+      # @param segment [String] percent-encoded path segment
+      # @return [String] decoded, NFC-normalized segment
+      def self.decode_segment(segment)
+        Addressable::URI.unencode_component(segment).force_encoding(Encoding::UTF_8).scrub.unicode_normalize(:nfc)
+      end
+      private_class_method :decode_segment
     end
   end
 end