html2rss 0.32.4 → 0.34.0

data/lib/html2rss/link_destination/noise_policy.rb

@@ -33,6 +33,7 @@ module Html2rss
           (@link_resolver.utility_text?(text) && destination_facts.vanity_path) ||
           utility_text_chrome?(text, destination_facts, heading_anchor:) ||
           icon_only_anchor?(anchor, text) ||
+          concealed_anchor?(anchor) ||
           utility_landmark_ancestor
       end
 
@@ -42,16 +43,7 @@ module Html2rss
         destination_facts.utility_path &&
           !destination_facts.content_path &&
           !destination_facts.strong_post_suffix &&
-          word_count_at_most?(text, 3)
-      end
-
-      def word_count_at_most?(text, max)
-        count = 0
-        text.to_s.scan(/\p{Alnum}+/) do
-          count += 1
-          return false if count > max
-        end
-        true
+          !WordCount.at_least?(text, 4)
       end
 
       def recommended_chrome?(text, destination_facts, heading_anchor:)
@@ -65,6 +57,15 @@ module Html2rss
         !heading_anchor && !destination_facts.strong_post_suffix
       end
 
+      # An anchor a sighted reader does not see, such as the off-screen accessibility twin of a teaser
+      # link ({Html::Visibility.concealed?}), is never the item.
+      def concealed_anchor?(anchor)
+        return false unless anchor
+
+        attrs = anchor.attrs
+        Html::Visibility.concealed?(attrs.class_names, style: attrs.style, hidden: attrs.raw.key?('hidden'))
+      end
+
       def icon_only_anchor?(anchor, text)
         return false unless anchor
         return false if text.to_s.match?(/\p{Alnum}/)

data/lib/html2rss/link_destination/path_classifier.rb

@@ -15,13 +15,48 @@ module Html2rss
       # Segment groups used to classify article, taxonomy, utility, and vanity routes.
       # Utility = taxonomy ∪ chrome ∪ soft utility (do not re-list taxonomy tokens).
       SEGMENT_SETS = begin
-        content = Set[
-          'article', 'articles', 'blog', 'blogs', 'changelog', 'changelogs', 'insight', 'insights',
-          'launch', 'launches', 'news', 'post', 'posts', 'release', 'releases', 'story', 'stories',
-          'update', 'updates', 'artikel', 'beitrag', 'beitraege', 'nachrichten', 'neuigkeiten',
-          'aktuelles', 'articulo', 'articulos', 'noticia', 'noticias', 'entrada', 'entradas',
-          'publicacion', 'publicaciones', 'actualite', 'actualites', 'nouvelle', 'nouvelles',
-          'teaser', 'teasers', 'card', 'cards'
+        # Listing words: a path ending in one is a section index, not an article.
+        section = Set[
+          'articles', 'blog', 'blogs', 'changelogs', 'insights', 'launches', 'news', 'posts', 'releases',
+          'stories', 'updates', 'nachrichten', 'neuigkeiten', 'aktuelles', 'beitraege', 'articulos', 'noticias',
+          'entradas', 'publicaciones', 'actualites', 'nouvelles', 'teasers', 'cards',
+          'aktualnosci', 'aktualności', 'wiadomosci', 'wiadomości', 'artykuly', 'artykuły', 'wpisy', 'notizie',
+          'articoli', 'novita', 'novità', 'aggiornamenti', 'nieuws', 'nieuwsberichten', 'artikelen', 'berichten',
+          'nyheter', 'artiklar', 'ニュース', '記事', 'お知らせ', 'ブログ'
+        ].freeze
+        content = (section | Set[
+          'article', 'changelog', 'insight', 'launch', 'post', 'release', 'story', 'update', 'artikel', 'beitrag',
+          'articulo', 'noticia', 'entrada', 'publicacion', 'actualite', 'nouvelle', 'teaser', 'card',
+          'artykul', 'artykuł', 'wpis', 'notizia', 'articolo', 'bericht', 'nyhet', 'inlagg', 'inlägg'
+        ]).freeze
+        # People pages. `team` and `people` are left out on purpose: they name sports teams and
+        # newsmakers in article paths. The words below are unambiguous byline/staff words.
+        profile = Set[
+          'author', 'authors', 'staff', 'journalist', 'journalists', 'players',
+          'autor', 'autoren', 'redakteure', 'autores', 'periodista',
+          'periodistas', 'auteur', 'auteurs', 'journaliste', 'journalistes',
+          'autore', 'autori', 'giornalista', 'giornalisti', 'journalisten',
+          'autorzy', 'dziennikarz', 'dziennikarze', 'skribent', 'skribenter', 'forfattare',
+          'författare', '著者', '執筆者', '記者', 'ライター'
+        ].freeze
+        # Editorial-office and contributor words: their child pages are often editorial notes with a
+        # three-word slug, so only a one- or two-word name slug marks a profile.
+        office = Set[
+          'contributor', 'contributors', 'redaktion', 'redaccion', 'redacción', 'redaction', 'rédaction',
+          'redazione', 'redactie', 'redakcja', '編集部'
+        ].freeze
+        # Destinations that are never articles: games, horoscopes, quizzes, obituaries, livestreams, players.
+        non_article = Set[
+          'games', 'spiele', 'juegos', 'jeux', 'giochi', 'spellen', 'spel', 'gry', 'ゲーム',
+          'sudoku', 'crossword', 'crosswords', 'kreuzwortraetsel', 'kreuzworträtsel', 'crucigrama', 'crucigramas',
+          'quiz', 'quizzes', 'quizze', 'quizzen', 'kviz', 'クイズ',
+          'horoscope', 'horoscopes', 'horoskop', 'horoskope', 'horoscopo', 'horoscopos', 'oroscopo', 'oroscopi',
+          'horoscoop', 'horoskopy', '星座占い', '占い',
+          'obituary', 'obituaries', 'nachruf', 'nachrufe', 'traueranzeigen', 'todesanzeigen', 'necrologie',
+          'nécrologie', 'necrologio', 'necrologi', 'esquelas', 'obituarios', 'overlijdensberichten', 'nekrolog',
+          'nekrologi', 'dodsannonser', 'dödsannonser', '訃報', 'おくやみ',
+          'livestream', 'livestreams', 'endirect', 'envivo', 'indiretta', 'nazywo', 'ライブ配信',
+          'player'
         ].freeze
         taxonomy = Set[
           'category', 'categories', 'tag', 'tags', 'topic', 'topics',
@@ -52,15 +87,20 @@ module Html2rss
           'career', 'careers', 'deals', 'deal', 'shopping', 'shop', 'trading', 'broker',
           'versicherung', 'tierversicherung', 'insurance', 'vergleich', 'comparison',
           'partnerboerse', 'singleboerse', 'krypto', 'crypto', 'casinos', 'casino',
-          'kreditkarten', 'kreditkarte', 'kredit', 'echtgeld', 'vpn', 'games', 'kaufberater',
+          'kreditkarten', 'kreditkarte', 'kredit', 'echtgeld', 'vpn', 'kaufberater',
           'leasing'
         ].freeze
         utility = (taxonomy | chrome | SOFT_UTILITY).freeze
         {
           content:,
+          section:,
           utility:,
           high_confidence_junk: (utility - SOFT_UTILITY).freeze,
           taxonomy:,
+          non_article:,
+          profile:,
+          office:,
+          people_index: (profile | office).freeze,
           vanity:,
           deep_post_context: Set['press', 'newsroom', 'presse', 'pressemitteilungen', 'prensa'].freeze
         }.freeze
@@ -75,9 +115,23 @@ module Html2rss
       # Common file-extension suffixes excluded from host-shaped junk matching.
       FILE_EXTENSION_SEGMENT = /\.(?:pdf|jpe?g|png|gif|svg|webp|css|js|mjs|html?|xml|json|mp4|webm|zip|gz)\z/i
 
+      # A single person-name slug: letters only (dated and id slugs carry digits), up to three words.
+      NAME_SLUG = /\A\p{L}+(?:-\p{L}+){0,2}\z/
+      # Name slug of at most two words, for office/contributor sections whose notes carry longer slugs.
+      SHORT_NAME_SLUG = /\A\p{L}+(?:-\p{L}+)?\z/
+      # Name-slug shape that marks a profile page, by the word set of its parent segment.
+      NAME_SLUG_BY_OWNER = { profile: NAME_SLUG, office: SHORT_NAME_SLUG }.freeze
+
+      # Pop-up / embedded player segments such as `popup-audio-player` (hyphens removed before matching).
+      PLAYER_SEGMENT = /\A(?:(?:pop(?:up|out)|embed|audio|video|media)\w*)?player\z/
+
       # @param segments [Array<String>] normalized URL path segments
-      def initialize(segments)
+      # @param host_label [String, nil] first host label (e.g. an obituary subdomain)
+      # @param query [Boolean] whether the URL carries a query string (`/blog?p=12` selects one post)
+      def initialize(segments, host_label: nil, query: false)
         @segments = segments
+        @query = query
+        @host_label = host_label.to_s.downcase
         @lexicon = segments.map { ::Html2rss::Html::Probe.fold(_1) }.freeze
       end
 
@@ -106,13 +160,13 @@ module Html2rss
       def shallow?
         segment_count = segments.size
 
-        segment_count <= 1 || (segment_count == 2 && high_confidence_junk_at?(segment_count - 1))
+        segment_count <= 1 || (segment_count == 2 && high_confidence_junk_at?(1))
       end
 
       # @return [Boolean] true when the final path segment looks like a post slug
       def strong_post_suffix?
         @strong_post_suffix ||= segments.any? &&
-                                included_last_segment? &&
+                                slug_last_segment? &&
                                 trusted_post_context?(segments.size - 1)
       end
 
@@ -121,13 +175,9 @@ module Html2rss
         (0...segments.size).all? { |i| high_confidence_junk_at?(i) }
       end
 
-      # @return [Boolean] true when the route is shallow and contains high-confidence noise
-      def shallow_high_confidence_route?
-        vanity_segments = SEGMENT_SETS.fetch(:vanity)
-
-        shallow? && (0...segments.size).any? do |i|
-          high_confidence_junk_at?(i) || vanity_segments.include?(@lexicon[i])
-        end
+      # @return [Boolean] true when the route is shallow and names a soft vanity segment
+      def shallow_vanity_route?
+        shallow? && @lexicon.any? { |folded| SEGMENT_SETS.fetch(:vanity).include?(folded) }
       end
 
       # @return [Boolean] true when the leading segments are all utility chrome
@@ -135,19 +185,25 @@ module Html2rss
         all_junk?(segments.size - 1)
       end
 
+      # @return [Boolean] true when the host or a path segment names a game, horoscope, quiz,
+      #   obituary, livestream or player destination, or the path is a bare section index
+      def non_article_path?
+        return @non_article_path if defined?(@non_article_path)
+
+        @non_article_path = section_index? || profile_route? || (!strong_post_suffix? && non_article_word_present?)
+      end
+
       # @return [Boolean] true when the route is shallow and contains high-confidence noise
       def junk_path?
+        return true if non_article_path?
         return false if excluded_content_route?
 
-        any_high_confidence_junk_segment? ||
-          taxonomy_path? ||
-          utility_only_route? ||
-          deep_utility_context_route? ||
-          shallow_high_confidence_route?
+        any_high_confidence_junk_segment? || shallow_vanity_route?
       end
 
       # @return [Boolean] true when the route points at conversion or account chrome
       def utility_destination?
+        return true if non_article_path?
         return false if excluded_content_route?
 
         vanity_path? || utility_route?
@@ -155,6 +211,33 @@ module Html2rss
 
       private
 
+      def non_article_word_present?
+        ([@host_label] + @lexicon).any? { non_article_word?(_1) }
+      end
+
+      def non_article_word?(word)
+        joined = word.delete('-_')
+        SEGMENT_SETS[:non_article].include?(joined) || joined.match?(PLAYER_SEGMENT)
+      end
+
+      # A people word that ends the path (an index) or is followed by one name slug (a profile page).
+      def profile_route?
+        return false if segments.empty?
+
+        SEGMENT_SETS[:people_index].include?(@lexicon.last) || profile_page?
+      end
+
+      def profile_page?
+        return false if segments.size < 2
+
+        NAME_SLUG_BY_OWNER.any? { |set, slug| SEGMENT_SETS[set].include?(@lexicon[-2]) && segments.last.match?(slug) }
+      end
+
+      # A path ending in a section word (no slug) lists articles instead of being one.
+      def section_index?
+        !@query && segments.any? && SEGMENT_SETS[:section].include?(@lexicon.last)
+      end
+
       def leading_high_confidence_junk?
         segments.any? && high_confidence_junk_at?(0)
       end
@@ -164,8 +247,7 @@ module Html2rss
       end
 
       def yearish_content_context?
-        segments.any? { |segment| segment.match?(YEARISH_SEGMENT) } &&
-          (strong_post_suffix? || trusted_post_context?(segments.size - 1))
+        segments.any? { |segment| segment.match?(YEARISH_SEGMENT) } && trusted_post_context?(segments.size - 1)
       end
 
       def excluded_content_route?
@@ -199,28 +281,15 @@ module Html2rss
       end
 
       def trusted_post_context?(limit)
-        return false if limit <= 0
         return false if leading_high_confidence_junk?
 
-        content_segments = SEGMENT_SETS.fetch(:content)
-        context_segments = SEGMENT_SETS.fetch(:deep_post_context)
-
-        (0...limit).any? do |i|
-          content_segments.include?(@lexicon[i]) ||
-            segments[i].match?(PathClassifier::YEARISH_SEGMENT) ||
-            context_segments.include?(@lexicon[i])
+        content = SEGMENT_SETS.fetch(:content)
+        context = SEGMENT_SETS.fetch(:deep_post_context)
+        segments.zip(@lexicon).first(limit.clamp(0..)).any? do |segment, folded|
+          content.include?(folded) || segment.match?(YEARISH_SEGMENT) || context.include?(folded)
         end
       end
 
-      def included_last_segment?
-        !excluded_last_segment? && slug_last_segment?
-      end
-
-      def excluded_last_segment?
-        last_i = segments.size - 1
-        high_confidence_junk_at?(last_i) || SEGMENT_SETS[:vanity].include?(@lexicon[last_i])
-      end
-
       def slug_last_segment?
         last = segments.last
         last.match?(YEARISH_SEGMENT) || last.match?(POST_SLUG_SEGMENT)

data/lib/html2rss/link_destination/sponsorship/label_text.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module LinkDestination
+    module Sponsorship
+      ##
+      # The closed multilingual vocabulary of sponsorship label text (en, de, fr, es, it, nl, pl, sv, ja), in
+      # two kinds. A paid phrase states paid content; an ad word names an ad and also something editorial
+      # (the advertising trade; "Promoted" a team or a person moving up). "Provided by" attribution ("提供")
+      # is in neither: it also credits wire and content partners. Only whole texts match; {Sponsorship}
+      # decides where a text is read.
+      module LabelText
+        # Letters with an accented variant are alternations, not classes: with +/i+, Ruby on Linux does not
+        # fold an uppercase accented letter against a class that also holds its ASCII base. Texts are read in
+        # NFC, so a decomposed accent matches too.
+        #
+        # A whole label, optionally wrapped in punctuation or separators ("Anzeige |", "[Sponsored]",
+        # "Publicité:", "【広告】"), in two groups. +paid+ states paid content; +ad+ is an ad word, which also
+        # names the advertising trade. Paid attribution ("Sponsored by …") may name the sponsor in up to five
+        # words.
+        LABEL_PATTERN = /
+          \A[\s\u00A0\p{P}|]*(?:(?<paid>
+            # English
+            sponsored(?:\s+(?:content|post|story))?|advertorial|(?:paid|partner)\s+content|paid\s+post|
+            sponsored\s+by(?:\s+\S+){0,5}|
+            # German
+            gesponsert(?:\s+von(?:\s+\S+){0,5})?|
+            # French
+            (?:contenu\s+)?sponsoris(?:e|é)(?:\s+par(?:\s+\S+){0,5})?|publireportage|contenus?\s+partenaires?|
+            # Spanish
+            (?:contenido\s+)?patrocinado(?:\s+por(?:\s+\S+){0,5})?|publirreportaje|
+            # Italian
+            (?:contenuto\s+)?sponsorizzat[oa](?:\s+(?:da|a\s+cura\s+di)(?:\s+\S+){0,5})?|
+            informazione\s+pubblicitaria|pubbliredazionale|
+            # Dutch
+            gesponsord(?:e\s+(?:inhoud|content))?(?:\s+door(?:\s+\S+){0,5})?|
+            # Polish
+            sponsorowane|materia(?:ł|l)\s+(?:sponsorowany|partnera|partnerski|promocyjny|reklamowy)|
+            materia(?:ł|l)y\s+(?:sponsorowane|partnera|partnerskie|promocyjne|reklamowe)|artyku(?:ł|l)\s+sponsorowany|
+            # Swedish
+            sponsrad|sponsrat(?:\s+inneh(?:å|a)ll)?|annonssamarbete|
+            # Japanese
+            タイアップ(?:広告|企画)?|スポンサー(?:リンク|コンテンツ)|広告企画|PR企画
+          )|(?<ad>
+            ad|advertisement|promoted|
+            anzeige|werbung|publicit(?:e|é)|publicidad|pubblicit(?:à|a)|advertentie|reclame|reklama|annons|
+            reklam|広告|スポンサー
+          ))[\s\u00A0\p{P}|]*\z
+        /ix
+        # "PR" as a whole uppercase label element (a Japanese paid-content badge). Only element text counts:
+        # never an ARIA label, a heading or a topic chip, and never "Pr" or "pr".
+        ELEMENT_LABEL = /\A[\s\u00A0\p{P}|]*PR[\s\u00A0\p{P}|]*\z/
+
+        class << self
+          ##
+          # @param text [String, nil] a whole text
+          # @param element [Boolean] whether +text+ is an element's text, the only place "PR" counts
+          # @return [Symbol, nil] +:paid_label+ for a paid phrase, +:label+ for an ad word, else nil
+          def kind(text, element: true)
+            return if text.nil?
+
+            text = text.unicode_normalize
+            return :label if element && text.match?(ELEMENT_LABEL)
+
+            match = LABEL_PATTERN.match(text)
+            match && (match[:paid] ? :paid_label : :label)
+          end
+
+          ##
+          # Cheap filter for the direct text of every node: whether it may hold a label at all.
+          #
+          # @param text [String]
+          # @return [Boolean]
+          def candidate?(text)
+            text = text.unicode_normalize
+            text.match?(LABEL_PATTERN) || text.include?('PR')
+          end
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/link_destination/sponsorship.rb

@@ -0,0 +1,184 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module LinkDestination
+    ##
+    # Explicit sponsorship markers on SST nodes, in three kinds:
+    #
+    # - +:container+: a +rel+ containing +sponsored+, a truthy +data-sponsored+ or
+    #   +data-advertorial+, a sponsored, advertorial or native-advertising class token (BEM variants
+    #   included), or an ARIA label that is an ad label. The publisher states that the element is paid content.
+    # - +:label+: an ad-label class token (+ad-label+, +ad-badge__item+), or a standalone element whose whole
+    #   text is an ad word ({LabelText} ad words: "Anzeige", "Publicité", "Promoted", "Reklama", "広告", or
+    #   "PR" as an uppercase element text). An ad word also names something editorial (the advertising trade,
+    #   a promotion), so it counts only on a plain element: never on a heading or a topic chip (category, tag
+    #   or kicker class), and it never heads a section.
+    # - +:paid_label+: a standalone element whose whole text states paid content ({LabelText} paid
+    #   phrases: "Sponsored", "Contenus partenaires", "Materiał partnera", "タイアップ"; paid attribution may
+    #   name the sponsor in up to five words). It also counts on a heading or a topic chip, and
+    #   {AutoSource::PageLinks} lets it head a section.
+    #
+    # Both text vocabularies are one closed multilingual set (en, de, fr, es, it, nl, pl, sv, ja).
+    # Attribution alone ("Presented by", "In Kooperation mit", "proposé par", "In collaborazione con", "提供")
+    # states no payment and is no label, nor is "Partners" (an editorial word).
+    #
+    # A class or text label counts only on a rendered element: neither it nor an ancestor inside its
+    # card ({CARD_NAMES}) is unrendered ({Html::Visibility.unrendered?}). +aria-hidden="true"+ and
+    # screen-reader-only classes still count: one is a visual label, the other a spoken one. A class
+    # marker also needs content (an empty template slot is not a label). A text label never counts
+    # inside running text: anchor text is never read, nor are words inside a heading or an inline element
+    # of text-bearing prose (or of an inline topic chip, for an ad word). A label inside an anchor counts
+    # (a linked label); a headline about advertising is not a marker.
+    #
+    # Markers only; {AutoSource::PageLinks} scopes a marker to the card that holds it.
+    module Sponsorship
+      # +rel+ token a publisher sets on paid links.
+      REL_TOKEN = 'sponsored'
+      # Attribute names that mark a sponsored element when set to a value in {ON_VALUES}.
+      ATTRIBUTE_NAMES = %w[data-sponsored data-advertorial].freeze
+      # Attribute values (case-insensitive) that switch a marker attribute on. Anything else is
+      # off: an empty value (template output or a bare attribute; HTML parsing cannot tell them
+      # apart), +no+, +null+, +false+, +0+, +off+.
+      ON_VALUES = Set['true', '1', 'yes'].freeze
+      # Sponsored class token: whole, behind a BEM block (+card--sponsored+, +teaser__advertorial+)
+      # or an +is-+ state prefix, with an optional content suffix (+sponsored-content+).
+      # +unsponsored+, +non-sponsored+ and +has-sponsored+ (a wrapper that contains one) do not match.
+      SPONSORED_CLASS = /
+        \A(?:[a-z0-9_-]*(?:--|__))?(?:is-)?(?:sponsored|advertorial)
+        (?:[-_](?:content|post|story|article|card|item|teaser))?\z
+      /x
+      # Native-advertising class token: a whole +native+, behind a BEM block (+teaser--native+,
+      # +card__native+), or a native-ad block (+native-ad+, +adv-native+, +ad-native+) with optional suffix
+      # words (+adv-native-strip+). +native-input+, +nativeApp+, +native-teaser+, +bt-langnative+ and a
+      # blockless +--native+ modifier do not match.
+      NATIVE_AD_CLASS = /
+        \A(?:[a-z0-9][a-z0-9_-]*?(?:--|__))?
+        (?:native|(?:native-ads?|adv-native|ad-native)(?:-[a-z0-9]+)*)\z
+      /x
+      # Ad-label class token family, with an optional BEM element or modifier: +ad-label+,
+      # +ad-labels__item+, +ad-badge+, +ad-marker--small+. Ad slots (+ad-container+) are not labels.
+      AD_LABEL_CLASS = /\Aad-(?:labels?|badge|marker)(?:(?:__|--)[a-z0-9-]+)?\z/
+      # ARIA attributes read as labels.
+      ARIA_NAMES = %w[aria-label aria-roledescription].freeze
+      # Elements that delimit one card (+main+ is a page region, never a card). A hidden ancestor hides
+      # a marker only inside its card: a hidden tab panel or responsive wrapper around whole cards
+      # hides none of their labels.
+      CARD_NAMES = Set[:article, :section, :aside, :li].freeze
+      # Class-token words (split on +-+/+_+) of topic chips, whose text names a topic, not a sale.
+      CHIP_WORDS = Set['category', 'categories', 'tag', 'tags', 'kicker', 'topic', 'topics', 'rubric', 'rubrik',
+                       'eyebrow'].freeze
+      # Inline elements: a label inside one belongs to the surrounding text.
+      PHRASING_NAMES = Set[:span, :em, :strong, :b, :i, :q, :small, :mark, :u, :s, :cite, :abbr, :time, :sup,
+                           :sub, :label, :font].freeze
+      class << self
+        ##
+        # @param node [SST::Node]
+        # @param ancestors [Array<SST::Node>] +node+'s ancestors, root first
+        # @return [Symbol, nil] +:container+, +:label+ or +:paid_label+ when +node+ itself carries an
+        #   explicit sponsorship marker, else nil
+        def marker(node, ancestors)
+          return :container if raw_marker?(node.attrs.raw)
+
+          kind = class_marker(node.attrs.class_names)
+          return (kind if visible?(node, ancestors)) if kind
+
+          text_label(node, ancestors)
+        end
+
+        ##
+        # Whether +text+, as a whole, states paid content ({LabelText}). For readers
+        # of plain text such as title pieces; an ad word is left out, as it also names a topic.
+        #
+        # @param text [String, nil]
+        # @return [Boolean]
+        def paid_text?(text) = LabelText.kind(text) == :paid_label
+
+        private
+
+        # Runs on every node of the page: literal checks gate each regex.
+        def class_marker(class_names)
+          kind = nil
+          class_names.each do |token|
+            return :container if sponsored_class?(token) || native_ad_class?(token)
+
+            kind ||= :label if token.start_with?('ad-') && token.match?(AD_LABEL_CLASS)
+          end
+          kind
+        end
+
+        def sponsored_class?(token)
+          (token.include?('sponsored') || token.include?('advertorial')) && token.match?(SPONSORED_CLASS)
+        end
+
+        def native_ad_class?(token) = token.include?('native') && token.match?(NATIVE_AD_CLASS)
+
+        def visible?(node, ancestors)
+          (node.children.any? || node.own_text.match?(/\S/)) && !unrendered?(node, ancestors)
+        end
+
+        # Whether +node+, or an ancestor up to and including its card, is hidden.
+        def unrendered?(node, ancestors)
+          return true if hidden?(node)
+
+          ancestors.reverse_each do |ancestor|
+            return true if hidden?(ancestor)
+            return false if CARD_NAMES.include?(ancestor.name)
+          end
+          false
+        end
+
+        def hidden?(node)
+          attrs = node.attrs
+          Html::Visibility.unrendered?(attrs.class_names, style: attrs.style, hidden: attrs.raw.key?('hidden'))
+        end
+
+        # Most nodes carry no raw attributes, so the empty check skips every lookup.
+        def raw_marker?(raw)
+          return false if raw.empty?
+
+          sponsored_rel?(raw['rel']) ||
+            ATTRIBUTE_NAMES.any? { ON_VALUES.include?(raw[_1]&.strip&.downcase) } ||
+            ARIA_NAMES.any? { LabelText.kind(raw[_1], element: false) }
+        end
+
+        def sponsored_rel?(rel)
+          return false unless rel
+
+          rel.downcase.split.include?(REL_TOKEN)
+        end
+
+        # The own-text check runs first: it is the cheap filter for every node of the page.
+        def text_label(node, ancestors)
+          return unless LabelText.candidate?(node.own_text)
+
+          kind = LabelText.kind(node.visible_text)
+          kind if kind && readable?(node, ancestors, paid: kind == :paid_label)
+        end
+
+        # Anchor text is never read; an ad word is read only on a plain element.
+        def readable?(node, ancestors, paid:)
+          node.name != :a && (paid || plain?(node)) && ancestors.none?(&:heading?) &&
+            !in_running_text?(ancestors, paid:) && !unrendered?(node, ancestors)
+        end
+
+        def plain?(node) = !node.heading? && !chip?(node)
+
+        def chip?(node)
+          node.attrs.class_names.any? { |token| token.downcase.split(/[-_]+/).any? { CHIP_WORDS.include?(_1) } }
+        end
+
+        # Climbs inline ancestors up to the first block one: prose in any of them, or (for an ad word) a
+        # chip class on an inline one, makes the label part of the surrounding text. Block classes are not
+        # read (post classes such as +category-news+ sit on whole cards).
+        def in_running_text?(ancestors, paid:)
+          ancestors.reverse_each do |ancestor|
+            return true if ancestor.own_text.match?(/\p{Alnum}/)
+            return false unless PHRASING_NAMES.include?(ancestor.name)
+            return true if !paid && chip?(ancestor)
+          end
+          false
+        end
+      end
+    end
+  end
+end