html2rss 0.32.4 → 0.34.0

data/lib/html2rss/sst/index.rb

@@ -14,18 +14,32 @@ module Html2rss
       # @return [Index, nil]
       def self.for_node(node) = NODE_BINDINGS[node]
 
+      ##
+      # Whether +node+ is +ancestor+ or inside it: parent pointers when +node+ is indexed, else a
+      # scan of +ancestor+'s subtree.
+      #
+      # @param node [Node]
+      # @param ancestor [Node]
+      # @return [Boolean]
+      def self.descendant_or_self?(node, ancestor)
+        return true if node.equal?(ancestor)
+
+        index = for_node(node)
+        index ? index.descendant_of?(node, ancestor) : !ancestor.find { _1.equal?(node) }.nil?
+      end
+
       ##
       # @param root [Node]
       # @param parents [Hash{Node => Node, nil}]
       # @param depths [Hash{Node => Integer}]
-      # @param ignored_chrome [Hash{Node => Boolean}]
-      def initialize(root:, parents:, depths:, ignored_chrome:)
+      # @param scopes [Hash{Node => Symbol}] {Tags.chrome_scope} of each node
+      def initialize(root:, parents:, depths:, scopes:)
         @root = root
         @parents = parents
         @depths = depths
-        @ignored_chrome = ignored_chrome
+        @scopes = scopes
         @visible_text = {}.compare_by_identity
-        @word_count = {}.compare_by_identity
+        @run_count = {}.compare_by_identity
         bind_tree(root)
       end
 
@@ -44,7 +58,14 @@ module Html2rss
       ##
       # @param node [Node]
       # @return [Boolean]
-      def ignored_chrome?(node) = @ignored_chrome.fetch(node, false)
+      def ignored_chrome?(node) = @scopes[node] == :chrome
+
+      ##
+      # Inside a utility landmark that is not chrome (aside, in-content nav, sectioned footer).
+      #
+      # @param node [Node]
+      # @return [Boolean]
+      def peripheral?(node) = @scopes[node] == :peripheral
 
       ##
       # Memoized default visible text (separator space, no excludes).
@@ -59,13 +80,11 @@ module Html2rss
 
       ##
       # @param node [Node]
-      # @return [Integer]
-      def memo_word_count(node)
-        return @word_count[node] if @word_count.key?(node)
+      # @return [Integer] memoized letter/digit runs of the default visible text ({WordCount.runs})
+      def memo_run_count(node)
+        return @run_count[node] if @run_count.key?(node)
 
-        count = 0
-        memo_visible_text(node).to_s.scan(/\p{Alnum}+/) { count += 1 }
-        @word_count[node] = count
+        @run_count[node] = WordCount.runs(memo_visible_text(node))
       end
 
       ##

data/lib/html2rss/sst/node.rb

@@ -2,7 +2,7 @@
 
 module Html2rss
   module SST
-    Node = Data.define(:name, :attrs, :own_text, :children, :tag_path)
+    Node = Data.define(:name, :attrs, :own_text, :tails, :children, :tag_path)
 
     ##
     # Immutable simplified semantic tree node. Predicates and traversal live on
@@ -11,12 +11,13 @@ module Html2rss
       ##
       # @param name [Symbol, String]
       # @param attrs [Attrs, nil]
-      # @param own_text [String, nil]
+      # @param own_text [String, nil] all direct text, in document order
+      # @param tails [Array<String>, nil] direct text following each child, parallel to +children+
       # @param children [Array<Node>, nil]
       # @param tag_path [String, nil] index-free ancestry path for list clustering
       # @return [Node]
-      # @raise [ArgumentError] on invalid name/attrs/children
-      def self.build(name:, attrs: nil, own_text: nil, children: nil, tag_path: nil)
+      # @raise [InvalidError] on invalid name/attrs/children
+      def self.build(name:, attrs: nil, own_text: nil, children: nil, tag_path: nil, tails: nil) # rubocop:disable Metrics/ParameterLists
         attrs_value = coerce_attrs(attrs)
         kids = coerce_children(children)
 
@@ -24,13 +25,14 @@ module Html2rss
           name: coerce_name(name),
           attrs: attrs_value,
           own_text: own_text.to_s.freeze,
+          tails: Array(tails).map { _1.to_s.freeze }.freeze,
           children: kids,
           tag_path: tag_path.to_s.freeze
         )
       end
 
       def self.coerce_name(name)
-        raise ArgumentError, 'name is required' if name.nil? || name.to_s.strip.empty?
+        raise InvalidError, 'name is required' if name.nil? || name.to_s.strip.empty?
 
         name.to_sym
       end
@@ -38,7 +40,7 @@ module Html2rss
 
       def self.coerce_attrs(attrs)
         attrs_value = attrs.nil? ? Attrs.empty : attrs
-        raise ArgumentError, 'attrs must be SST::Attrs' unless attrs_value.is_a?(Attrs)
+        raise InvalidError, 'attrs must be SST::Attrs' unless attrs_value.is_a?(Attrs)
 
         attrs_value
       end
@@ -46,12 +48,16 @@ module Html2rss
 
       def self.coerce_children(children)
         kids = Array(children).freeze
-        raise ArgumentError, 'children must be SST::Node instances' unless kids.all?(Node)
+        raise InvalidError, 'children must be SST::Node instances' unless kids.all?(Node)
 
         kids
       end
       private_class_method :coerce_children
 
+      ##
+      # @return [String] direct text before the first child
+      def lead_text = own_text.delete_suffix(tails.join)
+
       ##
       # @return [Boolean]
       def link?
@@ -68,11 +74,7 @@ module Html2rss
 
       ##
       # @return [Boolean]
-      def utility_landmark? = Tags::UTILITY_LANDMARK_NAMES.include?(name)
-
-      ##
-      # @return [Boolean]
-      def ignored_container_name? = Tags::IGNORED_CONTAINER_NAMES.include?(name)
+      def utility_landmark? = Tags.utility_landmark?(name.name, attrs.raw['role'])
 
       ##
       # @return [Boolean]
@@ -117,15 +119,23 @@ module Html2rss
       end
 
       ##
-      # @return [Array<Node>]
+      # @yieldparam node [Node]
+      # @return [Array<Node>] self and descendants (depth-first) for which the block is truthy
       def find_all(&)
-        each_node.select(&)
+        collect_matching([], &)
       end
 
       ##
-      # @return [Node, nil]
-      def find(&)
-        each_node.find(&)
+      # @yieldparam node [Node]
+      # @return [Node, nil] first self-or-descendant (depth-first) for which the block is truthy
+      def find(&block)
+        return self if yield(self)
+
+        children.each do |child|
+          hit = child.find(&block)
+          return hit if hit
+        end
+        nil
       end
 
       ##
@@ -143,23 +153,26 @@ module Html2rss
       end
 
       ##
-      # @return [Integer] alphanumeric word count of visible text
-      def word_count
-        if (index = Index.for_node(self))
-          return index.memo_word_count(self)
-        end
-
-        count = 0
-        visible_text.to_s.scan(/\p{Alnum}+/) { count += 1 }
-        count
+      # @return [Integer] letter/digit runs of visible text ({WordCount.runs})
+      def run_count
+        index = Index.for_node(self)
+        index ? index.memo_run_count(self) : WordCount.runs(visible_text)
       end
 
       ##
-      # @return [Float] words per descendant link
+      # @return [Float] letter/digit runs per descendant link
       def text_density
-        words = word_count
+        runs = run_count
         links = count_descendants(&:link?)
-        links.zero? ? words.to_f : words.to_f / links
+        links.zero? ? runs.to_f : runs.to_f / links
+      end
+
+      protected
+
+      def collect_matching(found, &block)
+        found << self if yield(self)
+        children.each { |child| child.collect_matching(found, &block) }
+        found
       end
     end
   end

data/lib/html2rss/sst/normalizer.rb

@@ -8,13 +8,10 @@ module Html2rss
     # Sole Nokogiri consumer on the heuristic auto-source path. Builds an
     # immutable SST::Document with parent/depth/chrome indices.
     class Normalizer # rubocop:disable Metrics/ClassLength
-      # Raised when no SST nodes survive normalization for the chosen root.
-      class EmptyTree < ArgumentError; end
-
       # Hard ceiling for SST node allocations; beyond this we degrade.
       MAX_NODES = 5_000
 
-      # Tags stripped entirely from the SST.
+      # Tags stripped entirely from the SST; a +noscript+ keeps only its file links (see {#child_nodes}).
       STRIPPED_TAGS = Set['script', 'style', 'noscript', 'iframe', 'svg', 'template'].freeze
 
       # Kept when MAX_NODES forces semantic-tag-only degrade.
@@ -32,9 +29,12 @@ module Html2rss
           href |
           src |
           srcset |
+          poster\z |
           style |
           datetime |
           itemprop |
+          rel\z |
+          hidden\z |
           role |
           type |
           category |
@@ -57,17 +57,14 @@ module Html2rss
       # Typed Attrs fields excluded from the leftover raw hash.
       TYPED_ATTR_NAMES = Set['href', 'src', 'id', 'class', 'datetime', 'itemprop', 'style', 'srcset', 'type'].freeze
 
-      # String form of Tags::IGNORED_CONTAINER_NAMES for chrome checks without to_sym.
-      IGNORED_CONTAINER_TAGS = Tags::IGNORED_CONTAINER_NAMES.to_set(&:to_s).freeze
-
       class << self
         ##
         # @param input [String, Nokogiri::HTML::Document, Nokogiri::XML::Node]
         #   HTML string or already-parsed node (String is parsed once here)
-        # @return [Document]
-        # @raise [ArgumentError] when input is nil or unsupported
+        # @return [Document, nil] +nil+ when no SST nodes survive normalization
+        # @raise [InvalidError] when input is nil or unsupported
         def call(input)
-          raise ArgumentError, 'input is required' if input.nil?
+          raise InvalidError, 'input is required' if input.nil?
 
           new(coerce(input)).call
         end
@@ -81,7 +78,7 @@ module Html2rss
           when String then Nokogiri::HTML(input)
           when Nokogiri::XML::Node then input
           else
-            raise ArgumentError, "expected String or Nokogiri::XML::Node, got #{input.class}"
+            raise InvalidError, "expected String or Nokogiri::XML::Node, got #{input.class}"
           end
         end
       end
@@ -93,17 +90,17 @@ module Html2rss
         @degraded = false
         @parents = {}.compare_by_identity
         @depths = {}.compare_by_identity
-        @ignored_chrome = {}.compare_by_identity
+        @scopes = {}.compare_by_identity
       end
 
       ##
-      # @return [Document]
+      # @return [Document, nil] +nil+ when no SST nodes survive normalization
       def call
         root_nk = resolve_root_nk
-        root = normalize_element(root_nk, parent: nil, depth: 0, path: '', chrome: false)
-        raise EmptyTree, 'SST Normalizer produced an empty tree' unless root
+        root = normalize_element(root_nk, depth: 0, path: '', scope: :page)
+        return unless root
 
-        index = Index.new(root:, parents: @parents, depths: @depths, ignored_chrome: @ignored_chrome)
+        index = Index.new(root:, parents: @parents, depths: @depths, scopes: @scopes)
         Document.build(root:, index:, degraded: @degraded, node_count: @node_count)
       end
 
@@ -149,16 +146,16 @@ module Html2rss
         node.element? && !STRIPPED_TAGS.include?(Html2rss::Html::Probe.tag(node))
       end
 
-      # rubocop:disable-next Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
-      def normalize_element(nk_node, parent:, depth:, path:, chrome:)
+      # rubocop:disable-next Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength
+      def normalize_element(nk_node, depth:, path:, scope:)
         return unless nk_node.element?
 
-        tag = Html2rss::Html::Probe.tag(nk_node)
-        return if STRIPPED_TAGS.include?(tag)
+        tag = Html::Probe.tag(nk_node)
+        return unless (nk_children = child_nodes(nk_node, tag))
 
         if @node_count >= MAX_NODES && !@degraded
           @degraded = true
-          Html2rss::Log.warn({
+          Log.warn({
             event_name: 'sst.normalizer.max_nodes',
             max_nodes: MAX_NODES,
             message: "sst.normalizer MAX_NODES=#{MAX_NODES} breached; degrading to semantic-tag-only"
@@ -168,26 +165,32 @@ module Html2rss
         return if @degraded && !SEMANTIC_DEGRADE_TAGS.include?(tag)
 
         @node_count += 1
-        tag_path = path.empty? ? "/#{tag}" : "#{path}/#{tag}"
+        tag_path = "#{path}/#{tag}"
         attrs = extract_attrs(nk_node)
-        own_text = direct_text(nk_node)
-        chrome_here = chrome || IGNORED_CONTAINER_TAGS.include?(tag)
+        scope_here = Tags.chrome_scope(nk_node, scope)
 
-        children = nk_node.element_children.filter_map do |child|
-          normalize_element(child, parent: :pending, depth: depth + 1, path: tag_path, chrome: chrome_here)
-        end.freeze
+        children, lead, tails = normalize_children(nk_children, depth:, path: tag_path, scope: scope_here)
 
-        node = Node.build(name: tag, attrs:, own_text:, children:, tag_path:)
-        @parents[node] = parent == :pending ? nil : parent
+        node = Node.build(name: tag, attrs:, own_text: [lead, *tails].join, children:, tag_path:, tails:)
         @depths[node] = depth
-        @ignored_chrome[node] = chrome_here
-
-        # Fix parent pointers now that node exists.
+        @scopes[node] = scope_here
         children.each { |child| @parents[child] = node }
 
         node
       end
 
+      # The Nokogiri children to normalize, or nil for a stripped tag. A +noscript+ is kept only for the
+      # file links inside it ({Html::ArticleRules::Enclosure.file_link?}), as its only children, so a
+      # card's no-script download link still names its enclosure; its text stays invisible
+      # ({Tags::INVISIBLE_NAMES}) and a +noscript+ without one is stripped.
+      def child_nodes(nk_node, tag)
+        return nk_node.children unless STRIPPED_TAGS.include?(tag)
+        return unless tag == 'noscript'
+
+        links = nk_node.xpath('.//a[@href]').select { Html::ArticleRules::Enclosure.file_link?(_1['href']) }
+        links unless links.empty?
+      end
+
       def extract_attrs(nk_node) # rubocop:disable Metrics/MethodLength
         Attrs.build(
           href: nk_node['href'],
@@ -213,8 +216,18 @@ module Html2rss
         end
       end
 
-      def direct_text(nk_node)
-        nk_node.children.select(&:text?).map(&:text).join
+      # Walks child nodes once, keeping direct text in the slot of the preceding kept child
+      # (slot 0 is the lead text) so text order survives the element-only tree.
+      def normalize_children(nk_children, depth:, path:, scope:)
+        children = []
+        slots = [+'']
+        nk_children.each do |child|
+          next slots.last << child.text if child.text?
+
+          node = normalize_element(child, depth: depth + 1, path:, scope:)
+          (children << node) && (slots << +'') if node
+        end
+        [children, slots.first, slots.drop(1)]
       end
     end
   end

data/lib/html2rss/sst/tags.rb

@@ -11,16 +11,127 @@ module Html2rss
       BLOCK_NAMES = Set[:p, :div, :li, :ul, :ol, :h1, :h2, :h3, :h4, :h5, :h6, :tr, :br].freeze
       # Names skipped when collecting visible text.
       INVISIBLE_NAMES = Set[:svg, :script, :noscript, :style, :template].freeze
-      # Landmark names treated as utility chrome.
+      # Utility landmark names: peripheral unless chrome (see {.chrome_scope}).
       UTILITY_LANDMARK_NAMES = Set[:nav, :aside, :footer, :menu].freeze
       # Container names ignored as article roots.
-      IGNORED_CONTAINER_NAMES = Set[:nav, :footer, :header, :svg, :script, :style].freeze
+      IGNORED_CONTAINER_NAMES = Set[:nav, :footer, :header, :dialog, :svg, :script, :style].freeze
+      # Ignored containers that are chrome only at page level (ARIA banner/contentinfo scoping).
+      PAGE_CHROME_NAMES = Set[:header, :footer].freeze
+      # Ancestors that scope a header/footer to their own section instead of the page.
+      SECTIONING_NAMES = Set[:article, :section, :aside, :main].freeze
       # Names excluded from class/structure clustering.
       CLUSTER_EXCLUDED_NAMES = Set[:html, :body, :nav, :footer, :header, :svg, :script, :style].freeze
       # Href prefixes that never count as content destinations.
       SKIP_HREF_PREFIXES = ['#', 'javascript:', 'mailto:', 'tel:', 'file://', 'sms:', 'data:'].freeze
       # Layout-ish names used when resolving nested wrapper groups.
       LAYOUT_NAMES = Set[:div, :section, :article, :li, :ul, :ol].freeze
+
+      # header/footer/nav: chrome at page level, their parent's scope inside sectioning content.
+      PAGE_CHROME = PAGE_CHROME_NAMES.to_set(&:to_s).freeze
+      # +noscript+ too: the SST keeps one only for its file links (enclosures), never as content anchors.
+      ALWAYS_CHROME = ((IGNORED_CONTAINER_NAMES.to_set(&:to_s) - PAGE_CHROME - Set['nav']) | Set['noscript']).freeze
+      UTILITY_LANDMARK = UTILITY_LANDMARK_NAMES.to_set(&:to_s).freeze
+      SECTIONING = SECTIONING_NAMES.to_set(&:to_s).freeze
+      # ARIA roles whose subtree is chrome anywhere (+search+ is not: it may wrap the result list).
+      CHROME_ROLES = Set['navigation', 'menu', 'menubar', 'dialog', 'alertdialog'].freeze
+      # ARIA landmark roles that, like header/footer, are chrome only at page level.
+      PAGE_CHROME_ROLES = Set['banner', 'contentinfo'].freeze
+      # Utility landmark roles: the chrome roles plus +complementary+ (the aside role).
+      UTILITY_ROLES = (CHROME_ROLES | Set['complementary']).freeze
+      # A nav or list labelled (aria-label) or classed as pagination is chrome in any scope.
+      PAGER_NAMES = Set['nav', 'ul', 'ol'].freeze
+      # Pagination nouns ("Results pagination", "paginación"), never +paginated+.
+      PAGER_LABEL = /\bpagina(?:tion|tie|ción|cion|zione|cja|ção|cao)(?!\p{L})|\bpaginering\b|\bpaginierung\b|
+                     \bpager\b|\bpaging\b|\bpage\snavigation\b/ix
+      # nav: a class-token part (+pagination+, +gsc-pagination+, +pager__list+), not +paginated-list+.
+      PAGER_CLASS = /(?:\A|\s|[-_])(?:pagination|pager|paging)(?:\z|\s|[-_])/i
+      # ul/ol: only a whole class token (+infinite-pager+ or +js-pagination-container+ is a feed list).
+      PAGER_TOKENS = Set['pagination', 'pager', 'paging'].freeze
+      PAGER_RELS = Set['next', 'prev', 'previous'].freeze
+      # A nav linking rel=next/prev is a pager.
+      PAGER_LINK_XPATH = PAGER_RELS.map { "contains(concat(' ', normalize-space(@rel), ' '), ' #{_1} ')" }
+                                   .join(' or ').then { ".//a[#{_1}]" }.freeze
+      # Page-navigation link shapes: a page number, an arrow, prev/next wording, or a page query/path.
+      PAGE_LINK_TEXT = /\A(?:\d{1,4}|[«‹›»←→<>]+|\W*(?:prev(?:ious)?|next|older|newer|first|last)\b.*)\z/im
+      PAGE_LINK_HREF = %r{[?&](?:page|paged|pg|p)=\d+|/page/\d+}i
+      private_constant :PAGE_CHROME, :ALWAYS_CHROME, :UTILITY_LANDMARK, :SECTIONING, :CHROME_ROLES,
+                       :PAGE_CHROME_ROLES, :UTILITY_ROLES, :PAGER_NAMES, :PAGER_LABEL, :PAGER_CLASS,
+                       :PAGER_TOKENS, :PAGER_RELS, :PAGER_LINK_XPATH, :PAGE_LINK_TEXT, :PAGE_LINK_HREF
+
+      ##
+      # The single chrome-scope rule, folded down the tree by both the SST normalizer
+      # and the Nokogiri ancestor walk. +:chrome+ sticks: dialog, noscript and the ARIA navigation,
+      # menu and dialog roles are chrome; header/footer/nav and the banner/contentinfo roles
+      # are chrome only at +:page+ level; a pager (a nav labelled or classed as pagination or
+      # linking rel=next/prev, or a list labelled or whole-token classed as pagination whose
+      # links mostly look like page navigation) is chrome in any scope. Otherwise a
+      # utility landmark ({.utility_landmark?}) opens a sticky +:peripheral+ scope, and
+      # article/section/main (or role=main) open +:sectioned+, so card headers and in-content
+      # nav lists stay out of chrome.
+      #
+      # @param element [Nokogiri::XML::Element] element being entered
+      # @param scope [Symbol] parent scope: +:page+, +:sectioned+, +:peripheral+, or +:chrome+
+      # @return [Symbol] scope the element's subtree inherits
+      def self.chrome_scope(element, scope)
+        tag = Html::Probe.tag(element)
+        roles = role_tokens(element['role'])
+        return :chrome if scope == :chrome || chrome?(element, tag, roles, scope)
+        return :peripheral if scope == :peripheral || utility?(tag, roles)
+
+        SECTIONING.include?(tag) || roles.include?('main') ? :sectioned : scope
+      end
+
+      ##
+      # Utility landmark: nav/aside/footer/menu, or a navigation, menu, dialog or
+      # complementary role. Peripheral content when not chrome; never an article card.
+      #
+      # @param tag [String] folded tag name
+      # @param role [String, nil] raw +role+ attribute
+      # @return [Boolean]
+      def self.utility_landmark?(tag, role) = utility?(tag, role_tokens(role))
+
+      ##
+      # Chrome only at page level: header/footer, or the banner/contentinfo role.
+      #
+      # @param tag [String] folded tag name
+      # @param role [String, nil] raw +role+ attribute
+      # @return [Boolean]
+      def self.page_level_chrome?(tag, role) = page_level?(tag, role_tokens(role))
+
+      def self.role_tokens(role) = role ? role.downcase.split : []
+
+      def self.utility?(tag, roles) = UTILITY_LANDMARK.include?(tag) || roles.any? { UTILITY_ROLES.include?(_1) }
+
+      def self.page_level?(tag, roles) = PAGE_CHROME.include?(tag) || roles.any? { PAGE_CHROME_ROLES.include?(_1) }
+
+      def self.chrome?(element, tag, roles, scope)
+        return true if ALWAYS_CHROME.include?(tag) || roles.any? { CHROME_ROLES.include?(_1) }
+
+        (scope == :page && (tag == 'nav' || page_level?(tag, roles))) || pager?(element, tag)
+      end
+
+      def self.pager?(element, tag)
+        return false unless PAGER_NAMES.include?(tag)
+        return list_pager?(element) unless tag == 'nav'
+        return true if element['aria-label'].to_s.match?(PAGER_LABEL) || element['class'].to_s.match?(PAGER_CLASS)
+
+        !element.at_xpath(PAGER_LINK_XPATH).nil?
+      end
+
+      # A list is a pager only when labelled or whole-token classed as one and at least
+      # half of its links look like page navigation.
+      def self.list_pager?(element)
+        named = element['aria-label'].to_s.match?(PAGER_LABEL) ||
+                element['class'].to_s.downcase.split.any? { PAGER_TOKENS.include?(_1) }
+        links = named ? element.css('a') : []
+        !links.empty? && links.count { page_link?(_1) } * 2 >= links.size
+      end
+
+      def self.page_link?(link)
+        link['rel'].to_s.downcase.split.any? { PAGER_RELS.include?(_1) } ||
+          link['href'].to_s.match?(PAGE_LINK_HREF) || link.text.strip.match?(PAGE_LINK_TEXT)
+      end
+      private_class_method :role_tokens, :utility?, :page_level?, :chrome?, :pager?, :list_pager?, :page_link?
     end
   end
 end