html2rss 0.33.0 → 0.34.0

data/lib/html2rss/link_destination/noise_policy.rb

@@ -33,6 +33,7 @@ module Html2rss
           (@link_resolver.utility_text?(text) && destination_facts.vanity_path) ||
           utility_text_chrome?(text, destination_facts, heading_anchor:) ||
           icon_only_anchor?(anchor, text) ||
+          concealed_anchor?(anchor) ||
           utility_landmark_ancestor
       end
 
@@ -42,16 +43,7 @@ module Html2rss
         destination_facts.utility_path &&
           !destination_facts.content_path &&
           !destination_facts.strong_post_suffix &&
-          word_count_at_most?(text, 3)
-      end
-
-      def word_count_at_most?(text, max)
-        count = 0
-        text.to_s.scan(/\p{Alnum}+/) do
-          count += 1
-          return false if count > max
-        end
-        true
+          !WordCount.at_least?(text, 4)
       end
 
       def recommended_chrome?(text, destination_facts, heading_anchor:)
@@ -65,6 +57,15 @@ module Html2rss
         !heading_anchor && !destination_facts.strong_post_suffix
       end
 
+      # An anchor a sighted reader does not see, such as the off-screen accessibility twin of a teaser
+      # link ({Html::Visibility.concealed?}), is never the item.
+      def concealed_anchor?(anchor)
+        return false unless anchor
+
+        attrs = anchor.attrs
+        Html::Visibility.concealed?(attrs.class_names, style: attrs.style, hidden: attrs.raw.key?('hidden'))
+      end
+
       def icon_only_anchor?(anchor, text)
         return false unless anchor
         return false if text.to_s.match?(/\p{Alnum}/)

data/lib/html2rss/link_destination/path_classifier.rb

@@ -15,13 +15,48 @@ module Html2rss
       # Segment groups used to classify article, taxonomy, utility, and vanity routes.
       # Utility = taxonomy ∪ chrome ∪ soft utility (do not re-list taxonomy tokens).
       SEGMENT_SETS = begin
-        content = Set[
-          'article', 'articles', 'blog', 'blogs', 'changelog', 'changelogs', 'insight', 'insights',
-          'launch', 'launches', 'news', 'post', 'posts', 'release', 'releases', 'story', 'stories',
-          'update', 'updates', 'artikel', 'beitrag', 'beitraege', 'nachrichten', 'neuigkeiten',
-          'aktuelles', 'articulo', 'articulos', 'noticia', 'noticias', 'entrada', 'entradas',
-          'publicacion', 'publicaciones', 'actualite', 'actualites', 'nouvelle', 'nouvelles',
-          'teaser', 'teasers', 'card', 'cards'
+        # Listing words: a path ending in one is a section index, not an article.
+        section = Set[
+          'articles', 'blog', 'blogs', 'changelogs', 'insights', 'launches', 'news', 'posts', 'releases',
+          'stories', 'updates', 'nachrichten', 'neuigkeiten', 'aktuelles', 'beitraege', 'articulos', 'noticias',
+          'entradas', 'publicaciones', 'actualites', 'nouvelles', 'teasers', 'cards',
+          'aktualnosci', 'aktualności', 'wiadomosci', 'wiadomości', 'artykuly', 'artykuły', 'wpisy', 'notizie',
+          'articoli', 'novita', 'novità', 'aggiornamenti', 'nieuws', 'nieuwsberichten', 'artikelen', 'berichten',
+          'nyheter', 'artiklar', 'ニュース', '記事', 'お知らせ', 'ブログ'
+        ].freeze
+        content = (section | Set[
+          'article', 'changelog', 'insight', 'launch', 'post', 'release', 'story', 'update', 'artikel', 'beitrag',
+          'articulo', 'noticia', 'entrada', 'publicacion', 'actualite', 'nouvelle', 'teaser', 'card',
+          'artykul', 'artykuł', 'wpis', 'notizia', 'articolo', 'bericht', 'nyhet', 'inlagg', 'inlägg'
+        ]).freeze
+        # People pages. `team` and `people` are left out on purpose: they name sports teams and
+        # newsmakers in article paths. The words below are unambiguous byline/staff words.
+        profile = Set[
+          'author', 'authors', 'staff', 'journalist', 'journalists', 'players',
+          'autor', 'autoren', 'redakteure', 'autores', 'periodista',
+          'periodistas', 'auteur', 'auteurs', 'journaliste', 'journalistes',
+          'autore', 'autori', 'giornalista', 'giornalisti', 'journalisten',
+          'autorzy', 'dziennikarz', 'dziennikarze', 'skribent', 'skribenter', 'forfattare',
+          'författare', '著者', '執筆者', '記者', 'ライター'
+        ].freeze
+        # Editorial-office and contributor words: their child pages are often editorial notes with a
+        # three-word slug, so only a one- or two-word name slug marks a profile.
+        office = Set[
+          'contributor', 'contributors', 'redaktion', 'redaccion', 'redacción', 'redaction', 'rédaction',
+          'redazione', 'redactie', 'redakcja', '編集部'
+        ].freeze
+        # Destinations that are never articles: games, horoscopes, quizzes, obituaries, livestreams, players.
+        non_article = Set[
+          'games', 'spiele', 'juegos', 'jeux', 'giochi', 'spellen', 'spel', 'gry', 'ゲーム',
+          'sudoku', 'crossword', 'crosswords', 'kreuzwortraetsel', 'kreuzworträtsel', 'crucigrama', 'crucigramas',
+          'quiz', 'quizzes', 'quizze', 'quizzen', 'kviz', 'クイズ',
+          'horoscope', 'horoscopes', 'horoskop', 'horoskope', 'horoscopo', 'horoscopos', 'oroscopo', 'oroscopi',
+          'horoscoop', 'horoskopy', '星座占い', '占い',
+          'obituary', 'obituaries', 'nachruf', 'nachrufe', 'traueranzeigen', 'todesanzeigen', 'necrologie',
+          'nécrologie', 'necrologio', 'necrologi', 'esquelas', 'obituarios', 'overlijdensberichten', 'nekrolog',
+          'nekrologi', 'dodsannonser', 'dödsannonser', '訃報', 'おくやみ',
+          'livestream', 'livestreams', 'endirect', 'envivo', 'indiretta', 'nazywo', 'ライブ配信',
+          'player'
         ].freeze
         taxonomy = Set[
           'category', 'categories', 'tag', 'tags', 'topic', 'topics',
@@ -52,15 +87,20 @@ module Html2rss
           'career', 'careers', 'deals', 'deal', 'shopping', 'shop', 'trading', 'broker',
           'versicherung', 'tierversicherung', 'insurance', 'vergleich', 'comparison',
           'partnerboerse', 'singleboerse', 'krypto', 'crypto', 'casinos', 'casino',
-          'kreditkarten', 'kreditkarte', 'kredit', 'echtgeld', 'vpn', 'games', 'kaufberater',
+          'kreditkarten', 'kreditkarte', 'kredit', 'echtgeld', 'vpn', 'kaufberater',
           'leasing'
         ].freeze
         utility = (taxonomy | chrome | SOFT_UTILITY).freeze
         {
           content:,
+          section:,
           utility:,
           high_confidence_junk: (utility - SOFT_UTILITY).freeze,
           taxonomy:,
+          non_article:,
+          profile:,
+          office:,
+          people_index: (profile | office).freeze,
           vanity:,
           deep_post_context: Set['press', 'newsroom', 'presse', 'pressemitteilungen', 'prensa'].freeze
         }.freeze
@@ -75,9 +115,23 @@ module Html2rss
       # Common file-extension suffixes excluded from host-shaped junk matching.
       FILE_EXTENSION_SEGMENT = /\.(?:pdf|jpe?g|png|gif|svg|webp|css|js|mjs|html?|xml|json|mp4|webm|zip|gz)\z/i
 
+      # A single person-name slug: letters only (dated and id slugs carry digits), up to three words.
+      NAME_SLUG = /\A\p{L}+(?:-\p{L}+){0,2}\z/
+      # Name slug of at most two words, for office/contributor sections whose notes carry longer slugs.
+      SHORT_NAME_SLUG = /\A\p{L}+(?:-\p{L}+)?\z/
+      # Name-slug shape that marks a profile page, by the word set of its parent segment.
+      NAME_SLUG_BY_OWNER = { profile: NAME_SLUG, office: SHORT_NAME_SLUG }.freeze
+
+      # Pop-up / embedded player segments such as `popup-audio-player` (hyphens removed before matching).
+      PLAYER_SEGMENT = /\A(?:(?:pop(?:up|out)|embed|audio|video|media)\w*)?player\z/
+
       # @param segments [Array<String>] normalized URL path segments
-      def initialize(segments)
+      # @param host_label [String, nil] first host label (e.g. an obituary subdomain)
+      # @param query [Boolean] whether the URL carries a query string (`/blog?p=12` selects one post)
+      def initialize(segments, host_label: nil, query: false)
         @segments = segments
+        @query = query
+        @host_label = host_label.to_s.downcase
         @lexicon = segments.map { ::Html2rss::Html::Probe.fold(_1) }.freeze
       end
 
@@ -131,8 +185,17 @@ module Html2rss
         all_junk?(segments.size - 1)
       end
 
+      # @return [Boolean] true when the host or a path segment names a game, horoscope, quiz,
+      #   obituary, livestream or player destination, or the path is a bare section index
+      def non_article_path?
+        return @non_article_path if defined?(@non_article_path)
+
+        @non_article_path = section_index? || profile_route? || (!strong_post_suffix? && non_article_word_present?)
+      end
+
       # @return [Boolean] true when the route is shallow and contains high-confidence noise
       def junk_path?
+        return true if non_article_path?
         return false if excluded_content_route?
 
         any_high_confidence_junk_segment? || shallow_vanity_route?
@@ -140,6 +203,7 @@ module Html2rss
 
       # @return [Boolean] true when the route points at conversion or account chrome
       def utility_destination?
+        return true if non_article_path?
         return false if excluded_content_route?
 
         vanity_path? || utility_route?
@@ -147,6 +211,33 @@ module Html2rss
 
       private
 
+      def non_article_word_present?
+        ([@host_label] + @lexicon).any? { non_article_word?(_1) }
+      end
+
+      def non_article_word?(word)
+        joined = word.delete('-_')
+        SEGMENT_SETS[:non_article].include?(joined) || joined.match?(PLAYER_SEGMENT)
+      end
+
+      # A people word that ends the path (an index) or is followed by one name slug (a profile page).
+      def profile_route?
+        return false if segments.empty?
+
+        SEGMENT_SETS[:people_index].include?(@lexicon.last) || profile_page?
+      end
+
+      def profile_page?
+        return false if segments.size < 2
+
+        NAME_SLUG_BY_OWNER.any? { |set, slug| SEGMENT_SETS[set].include?(@lexicon[-2]) && segments.last.match?(slug) }
+      end
+
+      # A path ending in a section word (no slug) lists articles instead of being one.
+      def section_index?
+        !@query && segments.any? && SEGMENT_SETS[:section].include?(@lexicon.last)
+      end
+
       def leading_high_confidence_junk?
         segments.any? && high_confidence_junk_at?(0)
       end

data/lib/html2rss/link_destination/sponsorship/label_text.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module LinkDestination
+    module Sponsorship
+      ##
+      # The closed multilingual vocabulary of sponsorship label text (en, de, fr, es, it, nl, pl, sv, ja), in
+      # two kinds. A paid phrase states paid content; an ad word names an ad and also something editorial
+      # (the advertising trade; "Promoted" a team or a person moving up). "Provided by" attribution ("提供")
+      # is in neither: it also credits wire and content partners. Only whole texts match; {Sponsorship}
+      # decides where a text is read.
+      module LabelText
+        # Letters with an accented variant are alternations, not classes: with +/i+, Ruby on Linux does not
+        # fold an uppercase accented letter against a class that also holds its ASCII base. Texts are read in
+        # NFC, so a decomposed accent matches too.
+        #
+        # A whole label, optionally wrapped in punctuation or separators ("Anzeige |", "[Sponsored]",
+        # "Publicité:", "【広告】"), in two groups. +paid+ states paid content; +ad+ is an ad word, which also
+        # names the advertising trade. Paid attribution ("Sponsored by …") may name the sponsor in up to five
+        # words.
+        LABEL_PATTERN = /
+          \A[\s\u00A0\p{P}|]*(?:(?<paid>
+            # English
+            sponsored(?:\s+(?:content|post|story))?|advertorial|(?:paid|partner)\s+content|paid\s+post|
+            sponsored\s+by(?:\s+\S+){0,5}|
+            # German
+            gesponsert(?:\s+von(?:\s+\S+){0,5})?|
+            # French
+            (?:contenu\s+)?sponsoris(?:e|é)(?:\s+par(?:\s+\S+){0,5})?|publireportage|contenus?\s+partenaires?|
+            # Spanish
+            (?:contenido\s+)?patrocinado(?:\s+por(?:\s+\S+){0,5})?|publirreportaje|
+            # Italian
+            (?:contenuto\s+)?sponsorizzat[oa](?:\s+(?:da|a\s+cura\s+di)(?:\s+\S+){0,5})?|
+            informazione\s+pubblicitaria|pubbliredazionale|
+            # Dutch
+            gesponsord(?:e\s+(?:inhoud|content))?(?:\s+door(?:\s+\S+){0,5})?|
+            # Polish
+            sponsorowane|materia(?:ł|l)\s+(?:sponsorowany|partnera|partnerski|promocyjny|reklamowy)|
+            materia(?:ł|l)y\s+(?:sponsorowane|partnera|partnerskie|promocyjne|reklamowe)|artyku(?:ł|l)\s+sponsorowany|
+            # Swedish
+            sponsrad|sponsrat(?:\s+inneh(?:å|a)ll)?|annonssamarbete|
+            # Japanese
+            タイアップ(?:広告|企画)?|スポンサー(?:リンク|コンテンツ)|広告企画|PR企画
+          )|(?<ad>
+            ad|advertisement|promoted|
+            anzeige|werbung|publicit(?:e|é)|publicidad|pubblicit(?:à|a)|advertentie|reclame|reklama|annons|
+            reklam|広告|スポンサー
+          ))[\s\u00A0\p{P}|]*\z
+        /ix
+        # "PR" as a whole uppercase label element (a Japanese paid-content badge). Only element text counts:
+        # never an ARIA label, a heading or a topic chip, and never "Pr" or "pr".
+        ELEMENT_LABEL = /\A[\s\u00A0\p{P}|]*PR[\s\u00A0\p{P}|]*\z/
+
+        class << self
+          ##
+          # @param text [String, nil] a whole text
+          # @param element [Boolean] whether +text+ is an element's text, the only place "PR" counts
+          # @return [Symbol, nil] +:paid_label+ for a paid phrase, +:label+ for an ad word, else nil
+          def kind(text, element: true)
+            return if text.nil?
+
+            text = text.unicode_normalize
+            return :label if element && text.match?(ELEMENT_LABEL)
+
+            match = LABEL_PATTERN.match(text)
+            match && (match[:paid] ? :paid_label : :label)
+          end
+
+          ##
+          # Cheap filter for the direct text of every node: whether it may hold a label at all.
+          #
+          # @param text [String]
+          # @return [Boolean]
+          def candidate?(text)
+            text = text.unicode_normalize
+            text.match?(LABEL_PATTERN) || text.include?('PR')
+          end
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/link_destination/sponsorship.rb

@@ -0,0 +1,184 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module LinkDestination
+    ##
+    # Explicit sponsorship markers on SST nodes, in three kinds:
+    #
+    # - +:container+: a +rel+ containing +sponsored+, a truthy +data-sponsored+ or
+    #   +data-advertorial+, a sponsored, advertorial or native-advertising class token (BEM variants
+    #   included), or an ARIA label that is an ad label. The publisher states that the element is paid content.
+    # - +:label+: an ad-label class token (+ad-label+, +ad-badge__item+), or a standalone element whose whole
+    #   text is an ad word ({LabelText} ad words: "Anzeige", "Publicité", "Promoted", "Reklama", "広告", or
+    #   "PR" as an uppercase element text). An ad word also names something editorial (the advertising trade,
+    #   a promotion), so it counts only on a plain element: never on a heading or a topic chip (category, tag
+    #   or kicker class), and it never heads a section.
+    # - +:paid_label+: a standalone element whose whole text states paid content ({LabelText} paid
+    #   phrases: "Sponsored", "Contenus partenaires", "Materiał partnera", "タイアップ"; paid attribution may
+    #   name the sponsor in up to five words). It also counts on a heading or a topic chip, and
+    #   {AutoSource::PageLinks} lets it head a section.
+    #
+    # Both text vocabularies are one closed multilingual set (en, de, fr, es, it, nl, pl, sv, ja).
+    # Attribution alone ("Presented by", "In Kooperation mit", "proposé par", "In collaborazione con", "提供")
+    # states no payment and is no label, nor is "Partners" (an editorial word).
+    #
+    # A class or text label counts only on a rendered element: neither it nor an ancestor inside its
+    # card ({CARD_NAMES}) is unrendered ({Html::Visibility.unrendered?}). +aria-hidden="true"+ and
+    # screen-reader-only classes still count: one is a visual label, the other a spoken one. A class
+    # marker also needs content (an empty template slot is not a label). A text label never counts
+    # inside running text: anchor text is never read, nor are words inside a heading or an inline element
+    # of text-bearing prose (or of an inline topic chip, for an ad word). A label inside an anchor counts
+    # (a linked label); a headline about advertising is not a marker.
+    #
+    # Markers only; {AutoSource::PageLinks} scopes a marker to the card that holds it.
+    module Sponsorship
+      # +rel+ token a publisher sets on paid links.
+      REL_TOKEN = 'sponsored'
+      # Attribute names that mark a sponsored element when set to a value in {ON_VALUES}.
+      ATTRIBUTE_NAMES = %w[data-sponsored data-advertorial].freeze
+      # Attribute values (case-insensitive) that switch a marker attribute on. Anything else is
+      # off: an empty value (template output or a bare attribute; HTML parsing cannot tell them
+      # apart), +no+, +null+, +false+, +0+, +off+.
+      ON_VALUES = Set['true', '1', 'yes'].freeze
+      # Sponsored class token: whole, behind a BEM block (+card--sponsored+, +teaser__advertorial+)
+      # or an +is-+ state prefix, with an optional content suffix (+sponsored-content+).
+      # +unsponsored+, +non-sponsored+ and +has-sponsored+ (a wrapper that contains one) do not match.
+      SPONSORED_CLASS = /
+        \A(?:[a-z0-9_-]*(?:--|__))?(?:is-)?(?:sponsored|advertorial)
+        (?:[-_](?:content|post|story|article|card|item|teaser))?\z
+      /x
+      # Native-advertising class token: a whole +native+, behind a BEM block (+teaser--native+,
+      # +card__native+), or a native-ad block (+native-ad+, +adv-native+, +ad-native+) with optional suffix
+      # words (+adv-native-strip+). +native-input+, +nativeApp+, +native-teaser+, +bt-langnative+ and a
+      # blockless +--native+ modifier do not match.
+      NATIVE_AD_CLASS = /
+        \A(?:[a-z0-9][a-z0-9_-]*?(?:--|__))?
+        (?:native|(?:native-ads?|adv-native|ad-native)(?:-[a-z0-9]+)*)\z
+      /x
+      # Ad-label class token family, with an optional BEM element or modifier: +ad-label+,
+      # +ad-labels__item+, +ad-badge+, +ad-marker--small+. Ad slots (+ad-container+) are not labels.
+      AD_LABEL_CLASS = /\Aad-(?:labels?|badge|marker)(?:(?:__|--)[a-z0-9-]+)?\z/
+      # ARIA attributes read as labels.
+      ARIA_NAMES = %w[aria-label aria-roledescription].freeze
+      # Elements that delimit one card (+main+ is a page region, never a card). A hidden ancestor hides
+      # a marker only inside its card: a hidden tab panel or responsive wrapper around whole cards
+      # hides none of their labels.
+      CARD_NAMES = Set[:article, :section, :aside, :li].freeze
+      # Class-token words (split on +-+/+_+) of topic chips, whose text names a topic, not a sale.
+      CHIP_WORDS = Set['category', 'categories', 'tag', 'tags', 'kicker', 'topic', 'topics', 'rubric', 'rubrik',
+                       'eyebrow'].freeze
+      # Inline elements: a label inside one belongs to the surrounding text.
+      PHRASING_NAMES = Set[:span, :em, :strong, :b, :i, :q, :small, :mark, :u, :s, :cite, :abbr, :time, :sup,
+                           :sub, :label, :font].freeze
+      class << self
+        ##
+        # @param node [SST::Node]
+        # @param ancestors [Array<SST::Node>] +node+'s ancestors, root first
+        # @return [Symbol, nil] +:container+, +:label+ or +:paid_label+ when +node+ itself carries an
+        #   explicit sponsorship marker, else nil
+        def marker(node, ancestors)
+          return :container if raw_marker?(node.attrs.raw)
+
+          kind = class_marker(node.attrs.class_names)
+          return (kind if visible?(node, ancestors)) if kind
+
+          text_label(node, ancestors)
+        end
+
+        ##
+        # Whether +text+, as a whole, states paid content ({LabelText}). For readers
+        # of plain text such as title pieces; an ad word is left out, as it also names a topic.
+        #
+        # @param text [String, nil]
+        # @return [Boolean]
+        def paid_text?(text) = LabelText.kind(text) == :paid_label
+
+        private
+
+        # Runs on every node of the page: literal checks gate each regex.
+        def class_marker(class_names)
+          kind = nil
+          class_names.each do |token|
+            return :container if sponsored_class?(token) || native_ad_class?(token)
+
+            kind ||= :label if token.start_with?('ad-') && token.match?(AD_LABEL_CLASS)
+          end
+          kind
+        end
+
+        def sponsored_class?(token)
+          (token.include?('sponsored') || token.include?('advertorial')) && token.match?(SPONSORED_CLASS)
+        end
+
+        def native_ad_class?(token) = token.include?('native') && token.match?(NATIVE_AD_CLASS)
+
+        def visible?(node, ancestors)
+          (node.children.any? || node.own_text.match?(/\S/)) && !unrendered?(node, ancestors)
+        end
+
+        # Whether +node+, or an ancestor up to and including its card, is hidden.
+        def unrendered?(node, ancestors)
+          return true if hidden?(node)
+
+          ancestors.reverse_each do |ancestor|
+            return true if hidden?(ancestor)
+            return false if CARD_NAMES.include?(ancestor.name)
+          end
+          false
+        end
+
+        def hidden?(node)
+          attrs = node.attrs
+          Html::Visibility.unrendered?(attrs.class_names, style: attrs.style, hidden: attrs.raw.key?('hidden'))
+        end
+
+        # Most nodes carry no raw attributes, so the empty check skips every lookup.
+        def raw_marker?(raw)
+          return false if raw.empty?
+
+          sponsored_rel?(raw['rel']) ||
+            ATTRIBUTE_NAMES.any? { ON_VALUES.include?(raw[_1]&.strip&.downcase) } ||
+            ARIA_NAMES.any? { LabelText.kind(raw[_1], element: false) }
+        end
+
+        def sponsored_rel?(rel)
+          return false unless rel
+
+          rel.downcase.split.include?(REL_TOKEN)
+        end
+
+        # The own-text check runs first: it is the cheap filter for every node of the page.
+        def text_label(node, ancestors)
+          return unless LabelText.candidate?(node.own_text)
+
+          kind = LabelText.kind(node.visible_text)
+          kind if kind && readable?(node, ancestors, paid: kind == :paid_label)
+        end
+
+        # Anchor text is never read; an ad word is read only on a plain element.
+        def readable?(node, ancestors, paid:)
+          node.name != :a && (paid || plain?(node)) && ancestors.none?(&:heading?) &&
+            !in_running_text?(ancestors, paid:) && !unrendered?(node, ancestors)
+        end
+
+        def plain?(node) = !node.heading? && !chip?(node)
+
+        def chip?(node)
+          node.attrs.class_names.any? { |token| token.downcase.split(/[-_]+/).any? { CHIP_WORDS.include?(_1) } }
+        end
+
+        # Climbs inline ancestors up to the first block one: prose in any of them, or (for an ad word) a
+        # chip class on an inline one, makes the label part of the surrounding text. Block classes are not
+        # read (post classes such as +category-news+ sit on whole cards).
+        def in_running_text?(ancestors, paid:)
+          ancestors.reverse_each do |ancestor|
+            return true if ancestor.own_text.match?(/\p{Alnum}/)
+            return false unless PHRASING_NAMES.include?(ancestor.name)
+            return true if !paid && chip?(ancestor)
+          end
+          false
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/link_destination/text_classifier.rb

@@ -2,8 +2,10 @@
 
 module Html2rss
   module LinkDestination
-    # Classifies visible anchor text for utility and recommendation chrome.
-    class TextClassifier
+    # Classifies visible anchor text for utility, recommendation and read-more chrome in en, de, fr, es, it,
+    # nl, pl, sv and ja. Letters with an accented variant are alternations, not classes: with +/i+, Ruby on
+    # Linux does not fold an uppercase accented letter against a class that also holds its ASCII base.
+    class TextClassifier # rubocop:disable Metrics/ClassLength -- nine-language vocabulary tables
       # Prefix labels that usually identify navigation or subscription links.
       UTILITY_PREFIX_PATTERN = /
         \A\s*(
@@ -14,7 +16,17 @@ module Html2rss
           # Spanish
           ver\s+todos|ver\s+todo|todas\s+las\s+noticias|suscribirse|bolet(i|í)n|comentarios\s+feed|unirse|
           # French
-          voir\s+tout|voir\s+tous|toutes\s+les\s+nouvelles|s['’]abonner|flux\s+de\s+commentaires|rejoindre
+          voir\s+tout|voir\s+tous|toutes\s+les\s+nouvelles|s['’]abonner|flux\s+de\s+commentaires|rejoindre|
+          # Italian
+          vedi\s+tutto|vedi\s+tutti|tutte\s+le\s+notizie|abbonati|newsletter|iscriviti|
+          # Dutch
+          bekijk\s+alles|alles\s+bekijken|al\s+het\s+nieuws|abonneren|nieuwsbrief|lid\s+worden|
+          # Polish
+          zobacz\s+wszystko|wszystkie\s+wiadomo(ś|s)ci|subskrybuj|newsletter|do(łą|la)cz|
+          # Swedish
+          visa\s+alla|se\s+alla|alla\s+nyheter|prenumerera|nyhetsbrev|bli\s+medlem|
+          # Japanese
+          すべて見る|全て見る|ニュースレター|メルマガ|会員登録|購読
         )\b
       /ix
       # Short labels that usually identify non-article navigation links.
@@ -35,7 +47,22 @@ module Html2rss
           # French
           (a|à)\s+propos|(a|à)propos|contact|commentaires?|rejoindre|
           se\s+connecter|login|membre|abonnement|recommand(e|é)(\s+pour\s+vous)?|
-          voir\s+tout|partager|s['’]inscrire|s['’]abonner|newsletter
+          voir\s+tout|partager|s['’]inscrire|s['’]abonner|newsletter|
+          # Italian
+          chi\s+siamo|contatti|commenti|accedi|iscriviti|condividi|abbonati|registrati|vedi\s+tutto|
+          consigliato(\s+per\s+te)?|abbonamento|
+          # Dutch
+          over\s+ons|contact|reacties|inloggen|aanmelden|delen|abonneren|registreren|bekijk\s+alles|
+          aanbevolen(\s+voor\s+jou)?|nieuwsbrief|lid\s+worden|lidmaatschap|
+          # Polish
+          o\s+nas|kontakt|komentarze|zaloguj\s+si(ę|e)|zarejestruj\s+si(ę|e)|udost(ę|e)pnij|subskrybuj|
+          zobacz\s+wszystko|polecane(\s+dla\s+ciebie)?|do(łą|la)cz|
+          # Swedish
+          om\s+oss|kontakt|kommentarer|logga\s+in|bli\s+medlem|dela|prenumerera|registrera\s+dig|visa\s+alla|
+          se\s+alla|rekommenderat(\s+f(ö|o)r\s+dig)?|nyhetsbrev|medlemskap|
+          # Japanese
+          会社概要|お問い合わせ|コメント|ログイン|会員登録|新規登録|シェア|共有|購読|すべて見る|全て見る|
+          もっと見る|おすすめ|ニュースレター|メルマガ
         )\b
       /ix
       # Labels for recommendation chrome rather than source articles.
@@ -44,21 +71,90 @@ module Html2rss
           recommended(\s+for\s+you)?|
           empfohlen(\s+f(ue|ü)r\s+dich)?|
           recomendado(\s+para\s+ti)?|
-          recommand(e|é)(\s+pour\s+vous)?
+          recommand(e|é)(\s+pour\s+vous)?|
+          consigliato(\s+per\s+te)?|
+          aanbevolen(\s+voor\s+jou)?|
+          polecane(\s+dla\s+ciebie)?|
+          rekommenderat(\s+f(ö|o)r\s+dig)?|
+          おすすめ
         )\b
       /ix
 
+      # Whole read-more labels that stand alone ("Continue reading", "Lire la suite", "続きを読む").
+      READ_MORE_PHRASE = /
+        read\s+more|read\s+on|continue\s+reading|keep\s+reading|learn\s+more|find\s+out\s+more|
+        read\s+(?:the\s+)?(?:full\s+|whole\s+)?(?:story|article)|read\s+the\s+rest|
+        weiterlesen|weiter\s+lesen|mehr\s+lesen|mehr\s+erfahren|mehr\s+dazu|mehr\s+zum\s+thema|
+        lesen\s+sie\s+(?:mehr|weiter)|(?:den\s+)?(?:ganzen|gesamten|vollst(?:ä|a)ndigen)\s+artikel\s+lesen|
+        lire\s+la\s+suite|lire\s+plus|lire\s+l['’]article(?:\s+complet)?|en\s+savoir\s+plus|
+        pour\s+en\s+savoir\s+plus|plus\s+d['’]infos?|plus\s+d['’]informations?|
+        leer\s+m(?:á|a)s|seguir\s+leyendo|continuar\s+leyendo|saber\s+m(?:á|a)s|
+        m(?:á|a)s\s+informaci(?:ó|o)n|leer\s+(?:todo|el\s+art(?:í|i)culo\s+completo)|
+        leggi\s+tutto|leggi\s+di\s+pi(?:ù|u)|leggi\s+l['’]articolo|continua\s+a\s+leggere|scopri\s+di\s+pi(?:ù|u)|
+        maggiori\s+informazioni|ulteriori\s+informazioni|per\s+saperne\s+di\s+pi(?:ù|u)|approfondisci|
+        lees\s+meer|lees\s+verder|verder\s+lezen|meer\s+lezen|meer\s+informatie|meer\s+weten|
+        lees\s+(?:het\s+)?(?:hele|volledige)\s+artikel|
+        czytaj\s+wi(?:ę|e)cej|czytaj\s+dalej|dowiedz\s+si(?:ę|e)\s+wi(?:ę|e)cej|wi(?:ę|e)cej\s+informacji|
+        czytaj\s+ca(?:ł|l)o(?:ś|s)(?:ć|c)|zobacz\s+wi(?:ę|e)cej|
+        l(?:ä|a)s\s+mer|l(?:ä|a)s\s+vidare|forts(?:ä|a)tt\s+l(?:ä|a)sa|mer\s+information|
+        l(?:ä|a)s\s+hela\s+artikeln|
+        続きを読む|もっと読む|続きはこちら|詳しくはこちら|詳細を見る|詳細はこちら|全文を読む|続きを見る
+      /ix
+      # "More about X" lead-ins: a read-more phrase or a bare "more", then a topic preposition.
+      READ_MORE_TOPIC_LEAD = /
+        (?:#{READ_MORE_PHRASE.source}|more|mehr|plus\s+d['’]infos?|m(?:á|a)s|altro|di\s+pi(?:ù|u)|meer|wi(?:ę|e)cej|mer)
+        \s+(?:about|on|from|zu|zum\s+thema|zum|zur|(?:ü|ue)ber|von|sur|
+        (?:à|a)\s+propos\s+de|de|sobre|su|riguardo|over|van|
+        o|na\s+temat|om|av)
+      /ix
+      # A read-more label as a whole text, optionally wrapped in punctuation or arrows. The topic after a
+      # lead-in counts only when quoted ("More about «X»", „X“, "X", 'X', 「X」), so a headline that merely
+      # opens with one ("Mehr über die Zukunft der Rente") stays a headline; Japanese topic forms put the
+      # topic first.
+      READ_MORE_PATTERN = /
+        \A[\s\p{P}\p{S}]*(?:
+          (?:#{READ_MORE_PHRASE.source})|
+          (?:#{READ_MORE_TOPIC_LEAD.source})\s*[«„"“”'‘’‹「].+[»"“”'’›」]|
+          \S{1,40}(?:の続きを読む|をもっと読む|について詳しく(?:はこちら)?|の詳細(?:を見る|はこちら))
+        )[\s\p{P}\p{S}]*\z
+      /ix
+
+      # Visible new-window notice that trails a link label, in parentheses or brackets.
+      NEW_WINDOW_NOTICE = /
+        \s*[(\[（【]\s*(?:
+          opens\s+in\s+(?:a\s+)?new\s+(?:window|tab)|
+          (?:ö|oe)ffnet\s+(?:sich\s+)?in\s+(?:einem\s+neuen|neuem)\s+(?:fenster|tab)|
+          ouvre\s+dans\s+(?:une\s+)?nouvel(?:le)?\s+(?:fen(?:ê|e)tre|onglet)|
+          se\s+abre\s+en\s+(?:una\s+)?nueva\s+(?:ventana|pesta(?:ñ|n)a)|
+          si\s+apre\s+in\s+(?:una\s+)?nuova\s+(?:finestra|scheda)|
+          opent\s+in\s+(?:een\s+)?nieuw(?:e)?\s+(?:venster|tabblad)|
+          otwiera\s+si(?:ę|e)\s+w\s+nowym\s+(?:oknie|karcie)|
+          (?:ö|o)ppnas\s+i\s+(?:ett\s+)?nytt\s+(?:f(?:ö|o)nster|flik)|
+          新しい(?:ウィンドウ|ウインドウ|タブ)で開く
+        )\s*[)\]）】]\s*\z
+      /ix
+
       # @param text [String, #to_s] visible anchor text
       # @return [Boolean] true when text matches a utility label
-      def utility?(text) = text.to_s.match?(UTILITY_PATTERN)
+      def utility?(text) = normalize(text).match?(UTILITY_PATTERN)
 
       # @param text [String, #to_s] visible anchor text
       # @return [Boolean] true when text begins with a utility label
-      def utility_prefix?(text) = text.to_s.match?(UTILITY_PREFIX_PATTERN)
+      def utility_prefix?(text) = normalize(text).match?(UTILITY_PREFIX_PATTERN)
 
       # @param text [String, #to_s] visible anchor text
       # @return [Boolean] true when text identifies recommendation chrome
-      def recommended?(text) = text.to_s.match?(RECOMMENDED_PATTERN)
+      def recommended?(text) = normalize(text).match?(RECOMMENDED_PATTERN)
+
+      # @param text [String, #to_s] visible anchor text or extracted title
+      # @return [Boolean] true when the whole text is a read-more label, bare or with its topic
+      def read_more?(text) = normalize(text).match?(READ_MORE_PATTERN)
+
+      private
+
+      # Every label is read in NFC, so a decomposed accent matches the precomposed vocabulary, and without
+      # its trailing new-window notice ("(opens in new tab)").
+      def normalize(text) = text.to_s.unicode_normalize(:nfc).sub(NEW_WINDOW_NOTICE, '')
     end
   end
 end

data/lib/html2rss/mcp/README.md

@@ -39,6 +39,7 @@ Every tool result is one JSON object (text body and `structuredContent`):
 | `next_step` | Bare verb name or `done` / `read_runtime`                    |
 | `guidance`  | Agent instruction (from `Outcome::Playbook`)                 |
 | `payload`   | Tool-specific data (`yaml`, `rss`, `items`, recon fields, …) |
+| `cache`     | Optional: `{ hits, max_age_seconds }` when cached pages fed the result |
 
 `Contract.response` builds the wire body; `Outcome` owns `next_step` policy.
 
@@ -64,7 +65,7 @@ Do not duplicate playbook prose in `server.rb`.
 | `html2rss://schema`     | Full JSON Schema for feed configurations                   |
 | `html2rss://extractors` | Registered extractor names                                 |
 | `html2rss://strategies` | Published MCP strategies (`auto`, `default`, `httpx`, `botasaurus`) |
-| `html2rss://runtime`    | `version`, `mcp_contract_version`, `catalog_fingerprint`, `tools`, `botasaurus_configured` (never the scraper URL). Fingerprint covers tool names, required keys, and `oneOf` branches; bump `mcp_contract_version` for envelope semantics. |
+| `html2rss://runtime`    | `version`, `mcp_contract_version`, `catalog_fingerprint`, `tools`, `botasaurus_configured`, `response_cache_ttl` (never the scraper URL). Fingerprint covers tool names, required keys, and `oneOf` branches; bump `mcp_contract_version` for envelope semantics. |
 
 ## Prompts
 
@@ -73,6 +74,17 @@ Do not duplicate playbook prose in `server.rb`.
 | `scrape-webpage`      | One `scrape` call; `inspect` only if weak or recon needed    |
 | `capture-feed-config` | Capture YAML → test → apply; catalog rewrite; strive enhance |
 
+## Response cache
+
+The MCP process is long-lived, so an agent iterating `inspect → capture → validate → test` on one URL should not refetch it. `RequestService::ResponseCache` is installed at server start, **on by default with a 300s TTL**. Set `HTML2RSS_RESPONSE_CACHE_TTL=<seconds>` to change it, or `0` / `off` / `false` to disable; an unusable value logs a warning and uses the default.
+
+- Only initial 2xx fetches are cached, keyed by a digest of URL, strategy, headers, request options, and policy limits. Follow-ups and block pages always hit the network.
+- Memory is bounded: at most 32 entries and 32 MiB of bodies, oldest evicted first.
+- When a result was built from cached pages, the envelope carries `cache: { hits, max_age_seconds }`; absent means everything was fetched live. Wait out the TTL or restart with the cache off to force a refetch.
+- `html2rss://runtime` publishes `response_cache_ttl` (`null` when off).
+
+The one-shot CLI has no use for it; use `capture` or `--input -` for offline iteration.
+
 ## Strategy note
 
 `scrape` / `capture` with `strategy: "auto"` run the full AutoFallback chain. `inspect` maps `auto` to default (HTTPX) for cheap diagnostics; pin `botasaurus` when you need browser rendering for inspect.