html2rss 0.32.4 → 0.34.0

data/lib/html2rss/auto_source/page_links/card_units.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class AutoSource
+    class PageLinks
+      ##
+      # The card units of one SST subtree: its largest one-card subtrees ({Html::CardWalk.crowded?}: one
+      # destination, at most one heading) that hold a destination and are rooted outside a heading (a
+      # heading holding a link is a headline, not a card). Headings outside every card unit are stray.
+      # Bare +div+ cards count like +article+ or +li+ ones: the unit is read from structure, not tag names.
+      class CardUnits
+        Unit = Data.define(:keys, :headings, :cards, :peers, :stray)
+        ##
+        # Destinations and headings of one subtree, and the card units and stray headings its children hold.
+        class Unit
+          # The empty unit (a left-out subtree).
+          NONE = new(keys: [].freeze, headings: 0, cards: 0, peers: false, stray: false)
+
+          # @return [Boolean] whether the unit holds more than one card
+          def crowded? = Html::CardWalk.crowded?(heading_count: headings, distinct_main_hrefs: keys.size)
+        end
+        private_constant :Unit
+
+        ##
+        # @param root [SST::Node] subtree to read
+        # @param skip [SST::Node, nil] a subtree left out (a section's own label)
+        # @param claims [AnchorClaims] resolves an anchor's destination
+        def initialize(root, skip:, claims:)
+          @skip = skip
+          @claims = claims
+          @unit = survey(root)
+          freeze
+        end
+
+        # @return [Integer] card units below the root
+        def cards = @unit.cards
+
+        # @return [Boolean] whether a card unit is a child of the root
+        def peers? = @unit.peers
+
+        # @return [Boolean] whether the root itself is one card unit holding a heading
+        def card? = !@unit.crowded? && @unit.keys.any? && @unit.headings == 1
+
+        # @return [Boolean] whether a heading below the root sits outside every card unit
+        def stray? = @unit.stray
+
+        private
+
+        def survey(node)
+          return Unit::NONE if node.equal?(@skip)
+
+          units = node.children.map { survey(_1) }
+          cards, peers, stray = split(node.children, units)
+          Unit.new(keys: keys(node, units), headings: headings(node, units), cards:, peers:, stray:)
+        end
+
+        def keys(node, units)
+          keys = units.flat_map(&:keys)
+          keys << @claims.key_of(node) if node.link?
+          keys.compact.uniq
+        end
+
+        def headings(node, units) = units.sum(&:headings) + (node.heading? ? 1 : 0)
+
+        def split(children, units)
+          children.zip(units).reduce([0, false, false]) do |(cards, peers, stray), (child, unit)|
+            next [cards + unit.cards, peers, stray || unit.stray] if unit.crowded?
+            next [cards + 1, true, stray] if unit.keys.any? && !child.heading?
+
+            [cards, peers, stray || unit.headings.positive?]
+          end
+        end
+      end
+      private_constant :CardUnits
+    end
+  end
+end

data/lib/html2rss/auto_source/page_links/label_binding.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class AutoSource
+    class PageLinks
+      ##
+      # Whether a +:paid_label+ outside every anchor marks the frame where it binds: its first ancestor
+      # holding a destination after it ({SponsoredCards}). The label marks that frame when it is
+      #
+      # - the label of one card: the frame is one card unit with a heading ({CardUnits#card?}) and the label
+      #   sits in it alone ({#alone?}), not in a box beside the card that holds more (an ad slot); or
+      # - the header of a section ({#header?}): the frame is no page region and holds no heading outside its
+      #   card units.
+      #
+      # A label in plain text before cards (an ad caption over an empty or stripped slot) heads nothing.
+      class LabelBinding
+        # Elements that hold a page, never a section: a section label stops below them.
+        PAGE_REGIONS = Set[:html, :body, :main].freeze
+        # Class-token words (split on +-+/+_+) of a header box.
+        HEADER_WORDS = Set['header', 'head', 'heading'].freeze
+        # Text that holds a word.
+        WORD = /[[:alnum:]]/
+        private_constant :HEADER_WORDS, :WORD
+
+        ##
+        # @param node [SST::Node] the frame where the label binds
+        # @param label [SST::Node] the label element
+        # @param claims [AnchorClaims] resolves the frame's destinations
+        def initialize(node, label, claims)
+          @node = node
+          @label = label
+          @claims = claims
+        end
+
+        ##
+        # @return [Boolean] whether the label marks every destination of the frame
+        def marks?
+          units = CardUnits.new(@node, skip: @label, claims: @claims)
+          (units.card? && alone?) || section?(units)
+        end
+
+        private
+
+        def section?(units)
+          return false if @node.find { PAGE_REGIONS.include?(_1.name) }
+
+          !units.stray? && header?
+        end
+
+        # Whether the label heads the frame: it sits in the frame's first child holding words, and it or a
+        # wrapper of it there is a heading, a +header+ element or a header box ({HEADER_WORDS}).
+        def header?
+          first = @node.children.find { _1.visible_text.to_s.match?(WORD) }
+          path.last.equal?(first) && path.any? { _1.heading? || _1.name == :header || header_box?(_1) }
+        end
+
+        def header_box?(node)
+          node.attrs.class_names.any? { |token| token.downcase.split(/[-_]+/).any? { HEADER_WORDS.include?(_1) } }
+        end
+
+        # Whether the frame's child around the label holds nothing but the label.
+        def alone?
+          path.drop(1).all? { _1.children.one? && !_1.own_text.match?(WORD) }
+        end
+
+        # The label and its wrappers up to the frame's child holding it, innermost first.
+        def path
+          @path ||= begin
+            index = SST::Index.for_node(@label)
+            nodes = [@label]
+            nodes << index.parent_of(nodes.last) until index.parent_of(nodes.last).equal?(@node)
+            nodes
+          end
+        end
+      end
+      private_constant :LabelBinding
+    end
+  end
+end

data/lib/html2rss/auto_source/page_links/sponsored_cards.rb

@@ -0,0 +1,147 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class AutoSource
+    class PageLinks
+      ##
+      # Article identities of sponsored cards, from one marker walk ({LinkDestination::Sponsorship.marker}).
+      # Anchors are resolved only once a marker binds; a page without markers resolves nothing.
+      #
+      # Each marker binds in one scope:
+      #
+      # - A +:container+ marker holding anchors marks every destination inside it.
+      # - A marker inside an anchor (a linked label) marks that anchor's destination across its card unit: the
+      #   largest unit around the anchor that is still one card ({Html::CardWalk.crowded?}: one destination,
+      #   at most one heading), and only when that unit holds a heading.
+      # - Any other label (or an anchorless container marker) marks the card it sits in: its nearest
+      #   {LinkDestination::Sponsorship::CARD_NAMES} element, itself included. That card marks its dominant
+      #   destination: its only one, else its only headline destination (an anchor in or around a heading);
+      #   a card with several headlines marks nothing, and so does a label outside every card.
+      # - A +:paid_label+ outside every anchor binds at its first ancestor holding a destination after it
+      #   (none before it) and climbs no further. There {LabelBinding} decides whether it marks every
+      #   destination of that frame: the one card it labels alone, or the section it heads.
+      #
+      # Marking is per card: a URL is sponsored only when every anchor to it sits in a marked card,
+      # so a promo slot never hides the editorial card linking the same story.
+      class SponsoredCards
+        LinkedLabel = Data.define(:range, :headings)
+        ##
+        # A label inside an anchor, climbing to its card unit: the anchor range and heading count of the
+        # largest one-card unit around it so far. It starts empty and takes the anchor's frame first.
+        class LinkedLabel
+          # A unit not yet measured.
+          OPEN = new(range: (0...0), headings: 0)
+        end
+
+        # A +:paid_label+ outside every anchor, climbing to the section it may head: the label node and the
+        # anchor count before and after it.
+        SectionLabel = Data.define(:label, :before, :after)
+
+        private_constant :LinkedLabel, :SectionLabel
+
+        # @param document [SST::Document, nil]
+        # @param link_resolver [Scoring::LinkResolver, nil]
+        def initialize(document, link_resolver)
+          @claims = AnchorClaims.new(link_resolver)
+          @linked = Set.new
+          @ancestors, @labelled, @links = Array.new(3) { [] }
+          @headings = 0
+          # Linked labels still climbing at the root bind there.
+          walk(document.root)&.each { bind_link(_1) if _1.is_a?(LinkedLabel) } if document
+          @keys = @claims.sponsored_keys
+          freeze
+        end
+
+        # @param key [Array] article identity
+        # @return [Boolean]
+        def include?(key) = @keys.include?(key)
+
+        private
+
+        # Returns the labels still climbing out of +node+ (nil when none). +@labelled+ holds one flag per
+        # open card, innermost last; +@links+ the anchor index of each open anchor.
+        def walk(node)
+          start = @claims.size
+          headings = @headings
+          card = LinkDestination::Sponsorship::CARD_NAMES.include?(node.name)
+          @labelled << false if card
+          pending = mark(node, start, descend(node))
+          pending = append(pending, [LinkedLabel::OPEN]) if node.link? && @linked.delete?(start)
+          bound = bind(start) if card && @labelled.pop
+          pending && settle(node, start, @headings - headings, pending, bound:)
+        end
+
+        def descend(node)
+          link = node.link?
+          record(node) if link
+          @headings += 1 if node.heading?
+          @ancestors << node
+          pending = nil
+          node.children.each { pending = append(pending, walk(_1)) }
+          @ancestors.pop
+          @links.pop if link
+          pending
+        end
+
+        def record(anchor) = @links << @claims.record(anchor, in_heading: @ancestors.any?(&:heading?))
+
+        def mark(node, start, pending)
+          marker = LinkDestination::Sponsorship.marker(node, @ancestors)
+          return pending unless marker
+          return pending.tap { @claims.claim(start...@claims.size) } if marker == :container && @claims.size > start
+          return pending.tap { @linked << @links.last } unless @links.empty?
+
+          @labelled[-1] = true unless @labelled.empty?
+          return pending unless marker == :paid_label
+
+          append(pending, [SectionLabel.new(label: node, before: start, after: @claims.size)])
+        end
+
+        def append(pending, labels) = pending && labels ? pending.concat(labels) : pending || labels
+
+        # Moves each climbing label to +node+'s frame; returns the labels that climb on. A section label
+        # stops at a card or page region: {#section?} would reject anything above, so it saves the survey.
+        def settle(node, start, headings, pending, bound:)
+          range = start...@claims.size
+          boundary = LinkDestination::Sponsorship::CARD_NAMES.include?(node.name) || LabelBinding::PAGE_REGIONS.include?(node.name)
+          climbing = pending.filter_map do |label|
+            case label
+            in LinkedLabel then climb(label, range, headings)
+            in SectionLabel then head(label, node, range, boundary:, bound:)
+            end
+          end
+          climbing unless climbing.empty?
+        end
+
+        # Grows a linked label's unit while it stays one card; binds it at the first crowded frame.
+        def climb(label, range, headings)
+          crowded = Html::CardWalk.crowded?(heading_count: headings, distinct_main_hrefs: @claims.keys_in(range).size)
+          crowded ? bind_link(label) : label.with(range:, headings:)
+        end
+
+        # Claims the unit (one destination at most) when it holds a heading; returns nil.
+        def bind_link(label)
+          @claims.claim(label.range) if label.headings == 1
+          nil
+        end
+
+        # A section label stops at a bound card, at an anchor before it, and at a card or page region holding
+        # no destination after it. At its first frame holding one it marks that card or section, then stops.
+        def head(label, node, range, boundary:, bound:)
+          return if bound || range.begin < label.before
+          return (label unless boundary) unless range.end > label.after
+
+          @claims.claim(range) if LabelBinding.new(node, label.label, @claims).marks?
+          nil
+        end
+
+        def bind(start)
+          range = start...@claims.size
+          key = @claims.dominant_key(range)
+          @claims.claim(range, [key]) if key
+        end
+      end
+      private_constant :SponsoredCards
+    end
+  end
+end

data/lib/html2rss/auto_source/page_links.rb

@@ -0,0 +1,164 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class AutoSource
+    ##
+    # Where each URL is rendered on the page, and which URLs the page marks as sponsored.
+    #
+    # Keys are {Html2rss::Url#article_identity}, so scheme, tracking query params, trailing
+    # slashes and fragments do not split a URL; other query params stay significant. Chrome and
+    # peripheral both come from the one SST scope rule ({SST::Tags.chrome_scope}, read through
+    # {SST::Index#ignored_chrome?} and {SST::Index#peripheral?}): utility landmarks that are not
+    # chrome (+aside+, +menu+, role=complementary, in-content +nav+, sectioned +footer+) are peripheral.
+    # A URL rendered several times keeps its strongest placement.
+    #
+    # Each {LinkDestination::Sponsorship} marker is scoped by {SponsoredCards}: a container marks every
+    # destination inside it, a linked label its anchor's card unit, a label the dominant destination of
+    # the +article+, +section+, +aside+ or +li+ it sits in, and a paid label opening a section every card
+    # of that section. PageLinks owns no keep or drop rule; {Corroboration} and {Cleanup} read it.
+    #
+    # Both facts are walked lazily, once, on first use: a sponsorship check walks markers and
+    # resolves only the anchors of marked cards; placement, canonical and sparse checks resolve
+    # every anchor.
+    class PageLinks
+      # Placement strength, strongest first; a URL missing from the page is +:absent+.
+      RANKS = { content: 0, peripheral: 1, chrome: 2 }.freeze
+      # A page with at most this many distinct content URLs is too sparse to corroborate.
+      SPARSE_MAX_CONTENT_LINKS = 2
+
+      ##
+      # @param document [SST::Document, nil] page SST; nil when there is no rendered page
+      # @param link_resolver [Scoring::LinkResolver, nil] page resolver (shared memo with heuristics);
+      #   required when +document+ is present
+      # @return [PageLinks]
+      def self.build(document, link_resolver:)
+        new(document, link_resolver)
+      end
+      private_class_method :new
+
+      # @param document [SST::Document, nil]
+      # @param link_resolver [Scoring::LinkResolver, nil]
+      def initialize(document, link_resolver)
+        @document = document
+        @link_resolver = link_resolver
+        @walks = {}
+        freeze
+      end
+
+      ##
+      # @param url [Html2rss::Url, nil] candidate item URL
+      # @return [Symbol] +:content+, +:peripheral+, +:chrome+, or +:absent+
+      def placement(url)
+        return :absent unless url
+
+        placements.placement(url.article_identity)
+      end
+
+      ##
+      # The rendered content anchor's URL for the article +url+ names, minus tracking params,
+      # so a tracked or +http+ variant takes one stable form (the feed item guid keys on it).
+      # An +https+ item never drops to an +http+ anchor.
+      #
+      # @param url [Html2rss::Url, nil] candidate item URL
+      # @return [Html2rss::Url, nil] nil unless +url+ is placed as +:content+
+      def canonical_url(url)
+        anchor = url && placements.content_url(url.article_identity)&.without_tracking_params
+        return anchor unless anchor && url.scheme == 'https'
+
+        anchor.with_scheme('https')
+      end
+
+      ##
+      # True when absence cannot be told from an unrendered list: no page, a degraded SST,
+      # or at most {SPARSE_MAX_CONTENT_LINKS} distinct content URLs.
+      #
+      # @return [Boolean]
+      def sparse? = placements.sparse?
+
+      ##
+      # Document position of the first content anchor to the article +url+ names, among the
+      # page's distinct content URLs (from the placement walk; no second walk).
+      #
+      # @param url [Html2rss::Url, nil] candidate item URL
+      # @return [Integer, nil] nil unless +url+ is placed as +:content+
+      def content_position(url)
+        url && content_positions[url.article_identity]
+      end
+
+      ##
+      # True when +url+ is rendered inside a card carrying a sponsorship marker.
+      #
+      # @param url [Html2rss::Url, nil] candidate item URL
+      # @return [Boolean]
+      def sponsored?(url)
+        return false unless url
+
+        sponsored_cards.include?(url.article_identity)
+      end
+
+      private
+
+      # The +@walks+ memo is the explicit lazy holder: the frozen PageLinks fills each walk once.
+      def placements = @walks[:placements] ||= Placements.new(@document, @link_resolver)
+
+      def sponsored_cards = @walks[:sponsored_cards] ||= SponsoredCards.new(@document, @link_resolver)
+
+      def content_positions = @walks[:content_positions] ||= placements.content_keys.each_with_index.to_h
+
+      ##
+      # Strongest placement and content anchor URL per article identity, from one walk that
+      # resolves every SST anchor.
+      class Placements
+        # @param document [SST::Document, nil]
+        # @param link_resolver [Scoring::LinkResolver, nil]
+        def initialize(document, link_resolver)
+          @by_key = {}
+          @content_urls = {}
+          @index = document&.index
+          @link_resolver = link_resolver
+          @index&.each_node { record(_1) if _1.link? }
+          @sparse = document.nil? || document.degraded || content_link_count <= SPARSE_MAX_CONTENT_LINKS
+          freeze
+        end
+
+        # @param key [String] article identity
+        # @return [Symbol] placement, or +:absent+
+        def placement(key) = @by_key.fetch(key, :absent)
+
+        # @param key [String] article identity
+        # @return [Html2rss::Url, nil] the content anchor URL for +key+
+        def content_url(key) = @content_urls[key]
+
+        # @return [Boolean]
+        def sparse? = @sparse
+
+        # @return [Array<String>] article identities placed as +:content+, in document order
+        def content_keys = @content_urls.keys
+
+        private
+
+        def record(anchor)
+          url = @link_resolver.destination_facts(anchor)&.url
+          return unless url
+
+          key = url.article_identity
+          placement = anchor_placement(anchor)
+          current = @by_key[key]
+          return unless current.nil? || RANKS.fetch(placement) < RANKS.fetch(current)
+
+          @by_key[key] = placement
+          @content_urls[key] = url if placement == :content
+        end
+
+        def anchor_placement(anchor)
+          return :chrome if @index.ignored_chrome?(anchor)
+
+          @index.peripheral?(anchor) ? :peripheral : :content
+        end
+
+        def content_link_count = @by_key.count { |_key, placement| placement == :content }
+      end
+      private_constant :Placements
+    end
+  end
+end

data/lib/html2rss/auto_source/scraper/html.rb

@@ -4,7 +4,7 @@ module Html2rss
   class AutoSource
     module Scraper
       ##
-      # Fallback HTML list/cluster scraper via SST pipeline
+      # Fallback HTML list scraper via SST pipeline
       # (Normalizer → Segmenter → Scoring::Engine → Html::SstArticleExtractor).
       class Html
         include Enumerable
@@ -34,8 +34,6 @@ module Html2rss
           return false unless parsed_body
 
           new(parsed_body, url: DETECTION_BASE_URL).extractable?
-        rescue ArgumentError
-          false
         end
 
         # @param parsed_body [Nokogiri::HTML::Document, nil] parsed HTML (when +document:+ omitted)
@@ -43,7 +41,6 @@ module Html2rss
         # @param document [SST::Document, nil] memoized SST document from AutoSource
         # @param link_resolver [Scoring::LinkResolver, nil] shared page-scoped resolver
         # @param opts [Hash]
-        # @option opts [Boolean] :fallback_anchorless keep anchorless cluster cards
         # @option opts [Integer] :minimum_selector_frequency list frequency floor
         # @option opts [Integer] :use_top_selectors list selector budget
         def initialize(parsed_body = nil, url:, document: nil, link_resolver: nil, **opts)
@@ -52,7 +49,6 @@ module Html2rss
           @provided_link_resolver = link_resolver
           @url = url
           @opts = opts
-          @fallback_anchorless = opts.fetch(:fallback_anchorless, false)
         end
 
         ##
@@ -68,57 +64,27 @@ module Html2rss
         # @return [Boolean]
         def extractable?
           ranked_segments.any?
-        rescue ArgumentError
-          false
         end
 
         private
 
         def articles
-          @articles ||= begin
-            ranked = list_ranked
-            if ranked.empty? && @fallback_anchorless
-              materialize(cluster_ranked, fallback_anchorless: true)
-            else
-              materialize(ranked)
-            end
-          end
-        rescue ArgumentError
-          []
+          @articles ||= materialize(ranked_segments)
         end
 
         def ranked_segments
-          @ranked_segments ||= begin
-            ranked = list_ranked
-            ranked = cluster_ranked if @fallback_anchorless && ranked.empty?
-            ranked
-          end
-        end
-
-        def list_ranked
-          @list_ranked ||= rank_strategy(:list, permit_unanchored: false)
-        end
-
-        def cluster_ranked
-          @cluster_ranked ||= rank_strategy(:cluster, permit_unanchored: true)
-        end
-
-        def rank_strategy(strategy, permit_unanchored:)
-          segments = Segmenter.call(
-            document,
-            base_url: @url,
-            strategy:,
-            permit_unanchored:,
-            minimum_selector_frequency:,
-            use_top_selectors:,
-            link_resolver:
-          )
-          engine.select_eligible(segments, limit: TOP_K)
+          @ranked_segments ||= if document
+                                 segments = Segmenter.call(document, base_url: @url, strategy: :list, link_resolver:,
+                                                                     minimum_selector_frequency:, use_top_selectors:)
+                                 engine.select_eligible(segments, limit: TOP_K)
+                               else
+                                 []
+                               end
         end
 
-        def materialize(ranked, fallback_anchorless: false)
+        def materialize(ranked)
           ranked.filter_map do |entry|
-            ::Html2rss::Html::SstArticleExtractor.call(entry, base_url: @url, scraper: self.class, fallback_anchorless:)
+            ::Html2rss::Html::SstArticleExtractor.call(entry, base_url: @url, scraper: self.class)
           end
         end
 

data/lib/html2rss/auto_source/scraper/json_document.rb

@@ -31,9 +31,7 @@ module Html2rss
 
         # @param parsed_body [Hash, Array, Object] parsed JSON document (or HTML when misrouted)
         # @param url [String, Html2rss::Url] base URL for relative link resolution
-        # @param _opts [Hash] scraper-specific options
-        # @option _opts [Object] :_reserved reserved for future scraper-specific options
-        def initialize(parsed_body, url:, **_opts)
+        def initialize(parsed_body, url:)
           @parsed_body = parsed_body
           @url = url
         end
@@ -47,7 +45,6 @@ module Html2rss
         # @return [Enumerator, void] article enumerator when no block is given
         def each(&block)
           return enum_for(:each) unless block
-          return unless extractable?
 
           if json_feed?
             each_json_feed_item(&block)
@@ -103,7 +100,7 @@ module Html2rss
           return unless str
 
           Url.from_relative(str, @url)
-        rescue ArgumentError
+        rescue Html2rss::Url::InvalidUrlError
           nil
         end
 
@@ -112,7 +109,7 @@ module Html2rss
           return unless str
 
           Url.from_absolute(str)
-        rescue ArgumentError
+        rescue Html2rss::Url::InvalidUrlError
           nil
         end