html2rss 0.24.0 → 0.26.0

data/lib/html2rss/html/article_rules/image.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Html
+    module ArticleRules
+      ##
+      # DOM-agnostic image URL selection (srcset sizing, CSS background urls).
+      #
+      # Uses string scans instead of Regexp: CodeQL rb/polynomial-redos flags the
+      # prior patterns on uncontrolled HTML attributes, and length guards alone
+      # do not clear the query.
+      module Image
+        # Skip absurdly large attributes before scanning.
+        MAX_ATTR_CHARS = 4_096
+        # Optional quoting around a CSS `url(...)` value.
+        QUOTES = ['"', "'"].freeze
+        # Srcset width/height descriptor suffixes.
+        UNITS = %w[w h].freeze
+        # HTML/CSS whitespace treated as srcset token separators.
+        WS = [' ', "\t", "\n", "\r", "\f"].freeze
+
+        class << self
+          ##
+          # @param srcset_strings [Array<String>] raw srcset attribute values
+          # @return [String, nil] URL of the largest width/height candidate
+          def largest_from_srcsets(srcset_strings)
+            by_size = {}
+            Array(srcset_strings).each { |srcset| merge_srcset!(by_size, srcset) }
+            by_size[by_size.keys.max]
+          end
+
+          ##
+          # @param style_strings [Array<String>] CSS style attribute values
+          # @return [String, nil] longest non-data background url
+          def best_from_styles(style_strings)
+            Array(style_strings)
+              .filter_map { |style| style_url(style) }
+              .reject { |src| src.start_with?('data:') }
+              .max_by(&:size)
+          end
+
+          private
+
+          def merge_srcset!(by_size, srcset)
+            s = srcset.to_s
+            return if s.length > MAX_ATTR_CHARS
+
+            each_srcset_pair(s) do |url, size|
+              next if url.start_with?('data:')
+
+              by_size[size] = url
+            end
+          end
+
+          # URL token + `Nw`/`Nh` pairs. Commas may appear inside URLs; a candidate
+          # boundary is whitespace before a width/height descriptor.
+          def each_srcset_pair(string) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+            i = 0
+            len = string.length
+            while i < len
+              i += 1 while i < len && sep?(string[i])
+              break if i >= len
+
+              start = i
+              i += 1 while i < len && !ws?(string[i])
+              url = string[start, i - start]
+
+              i += 1 while i < len && ws?(string[i])
+              size_start = i
+              i += 1 while i < len && string[i].between?('0', '9')
+              next unless i > size_start && i < len && UNITS.include?(string[i])
+
+              yield url, string[size_start, i - size_start].to_i
+              i += 1
+            end
+          end
+
+          def style_url(style)
+            s = style.to_s
+            return if s.length > MAX_ATTR_CHARS
+
+            pos = s.index('url(')
+            return unless pos
+
+            extract_url_body(s, pos + 4)
+          end
+
+          def extract_url_body(string, index)
+            quote = string[index]
+            if QUOTES.include?(quote)
+              slice_until(string, index + 1, quote)
+            else
+              slice_until(string, index, ')')
+            end
+          end
+
+          def slice_until(string, start, closer)
+            close = string.index(closer, start)
+            string[start, close - start] if close
+          end
+
+          def sep?(char) = ws?(char) || char == ','
+
+          def ws?(char) = WS.include?(char)
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/html/article_rules.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Html
+    ##
+    # DOM-agnostic article field rules shared by Nokogiri and SST adapters.
+    module ArticleRules
+    end
+  end
+end

data/lib/html2rss/html/rendering/audio_renderer.rb

@@ -1,12 +1,12 @@
 # frozen_string_literal: true
 
-require 'cgi'
-
 module Html2rss
   module Html
     module Rendering
       # Renders an HTML <audio> tag from a URL and type.
       class AudioRenderer
+        include EscapedAttributes
+
         # @param url [String, Html2rss::Url] media URL for the audio source
         # @param type [String] MIME type for the audio source
         def initialize(url:, type:)
@@ -22,16 +22,6 @@ module Html2rss
             '</audio>'
           ].join
         end
-
-        private
-
-        def escaped_url
-          CGI.escapeHTML(@url.to_s)
-        end
-
-        def escaped_type
-          CGI.escapeHTML(@type.to_s)
-        end
       end
     end
   end

data/lib/html2rss/html/rendering/escaped_attributes.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require 'cgi'
+
+module Html2rss
+  module Html
+    module Rendering
+      ##
+      # Shared HTML attribute escaping for media renderers.
+      module EscapedAttributes
+        private
+
+        def escaped_url
+          CGI.escapeHTML(@url.to_s)
+        end
+
+        def escaped_type
+          CGI.escapeHTML(@type.to_s)
+        end
+
+        def escaped_title
+          CGI.escapeHTML(@title.to_s)
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/html/rendering/image_renderer.rb

@@ -1,12 +1,12 @@
 # frozen_string_literal: true
 
-require 'cgi'
-
 module Html2rss
   module Html
     module Rendering
       # Renders an HTML <img> tag from a URL and title.
       class ImageRenderer
+        include EscapedAttributes
+
         # @param url [String, Html2rss::Url] image URL for the src attribute
         # @param title [String, nil] title/alt text for the image
         def initialize(url:, title:)
@@ -27,16 +27,6 @@ module Html2rss
           ]
           "<img #{attributes.join(' ')}>"
         end
-
-        private
-
-        def escaped_url
-          CGI.escapeHTML(@url.to_s)
-        end
-
-        def escaped_title
-          CGI.escapeHTML(@title.to_s)
-        end
       end
     end
   end

data/lib/html2rss/html/rendering/pdf_renderer.rb

@@ -1,12 +1,12 @@
 # frozen_string_literal: true
 
-require 'cgi'
-
 module Html2rss
   module Html
     module Rendering
       # Renders an HTML <iframe> for PDF documents.
       class PdfRenderer
+        include EscapedAttributes
+
         # @param url [String, Html2rss::Url] PDF URL rendered in the iframe
         def initialize(url:)
           @url = url
@@ -24,12 +24,6 @@ module Html2rss
           ]
           "<iframe #{attributes.join(' ')}></iframe>"
         end
-
-        private
-
-        def escaped_url
-          CGI.escapeHTML(@url.to_s)
-        end
       end
     end
   end

data/lib/html2rss/html/rendering/video_renderer.rb

@@ -1,12 +1,12 @@
 # frozen_string_literal: true
 
-require 'cgi'
-
 module Html2rss
   module Html
     module Rendering
       # Renders an HTML <video> tag from a URL and type.
       class VideoRenderer
+        include EscapedAttributes
+
         # @param url [String, Html2rss::Url] media URL for the video source
         # @param type [String] MIME type for the video source
         def initialize(url:, type:)
@@ -22,16 +22,6 @@ module Html2rss
             '</video>'
           ].join
         end
-
-        private
-
-        def escaped_url
-          CGI.escapeHTML(@url.to_s)
-        end
-
-        def escaped_type
-          CGI.escapeHTML(@type.to_s)
-        end
       end
     end
   end

data/lib/html2rss/html/sst_article_extractor.rb

@@ -0,0 +1,308 @@
+# frozen_string_literal: true
+
+require 'zlib'
+
+module Html2rss
+  module Html
+    ##
+    # Builds an {Html2rss::Article} from an SST segment / ranked segment.
+    # Port of Html::ArticleExtractor field logic onto SST::Node.
+    # rubocop:disable Metrics/ClassLength -- SST field extractors colocated for parity with ArticleExtractor
+    class SstArticleExtractor
+      # CSS class tokens that mark kicker / eyebrow text (excluded from titles).
+      KICKER_CLASS_PATTERN = /kicker|eyebrow|pre-title|pretitle|overline/i
+      # Inline emphasis tags used as title fallbacks when no heading exists.
+      FALLBACK_HEADING_NAMES = %i[strong b].freeze
+
+      class << self
+        ##
+        # @param ranked_or_segment [Scoring::RankedSegment, AutoSource::Segment]
+        # @param base_url [String, Html2rss::Url]
+        # @param scraper [Class, nil]
+        # @param fallback_anchorless [Boolean]
+        # @return [Html2rss::Article, nil]
+        def call(ranked_or_segment, base_url:, scraper: nil, fallback_anchorless: false)
+          new(ranked_or_segment, base_url:, scraper:, fallback_anchorless:).call
+        end
+      end
+
+      # @param ranked_or_segment [Scoring::RankedSegment, AutoSource::Segment]
+      # @param base_url [String, Html2rss::Url]
+      # @param scraper [Class, nil]
+      # @param fallback_anchorless [Boolean]
+      def initialize(ranked_or_segment, base_url:, scraper: nil, fallback_anchorless: false)
+        segment = ranked_or_segment.is_a?(Scoring::RankedSegment) ? ranked_or_segment.segment : ranked_or_segment
+        raise ArgumentError, 'segment is required' unless segment.is_a?(AutoSource::Segment)
+
+        @segment = segment
+        @root = segment.root_node
+        @selected_anchor = segment.primary_link
+        @base_url = base_url
+        @scraper = scraper
+        @fallback_anchorless = fallback_anchorless
+      end
+
+      ##
+      # @return [Html2rss::Article, nil]
+      def call # rubocop:disable Metrics/MethodLength
+        attrs = {
+          title: extract_title,
+          url: extract_url,
+          image: extract_image,
+          description: extract_description,
+          id: generate_id,
+          published_at: extract_published_at,
+          enclosures: extract_enclosures,
+          categories: extract_categories,
+          scraper: @scraper
+        }
+        article = Article.new(**attrs)
+        article.valid? ? article : nil
+      end
+
+      private
+
+      def extract_url
+        @extract_url ||= begin
+          href = @selected_anchor&.attrs&.href.to_s
+          if href.empty?
+            anchorless_url_fallback
+          else
+            Url.from_relative(href.split('#').first.strip, @base_url)
+          end
+        end
+      end
+
+      def anchorless_url_fallback
+        return unless @fallback_anchorless
+
+        id = generate_id
+        Url.from_relative("##{id}", @base_url) if id
+      end
+
+      def extract_title
+        title_text = title_from_in_card_sources
+        return unless title_text
+
+        kicker = kicker_node&.visible_text.to_s.strip
+        kicker && !kicker.empty? && !title_text.include?(kicker) ? "#{kicker}: #{title_text}" : title_text
+      end
+
+      # Prefer heading over credit-shaped visible text; never emit agency-only titles.
+      #
+      # @return [String, nil]
+      def title_from_in_card_sources
+        title_candidates.find { |text| !credit_shaped_title?(text) }
+      end
+
+      # @return [Array<String>] ordered in-card title candidates (heading → anchor → fallback)
+      def title_candidates
+        [].tap do |candidates|
+          add_title_candidate!(candidates, heading&.visible_text)
+          add_title_candidate!(candidates, @selected_anchor&.visible_text)
+          next unless candidates.empty? && @fallback_anchorless && @selected_anchor.nil?
+
+          add_title_candidate!(candidates, first_nonempty_own_text)
+        end
+      end
+
+      # @param candidates [Array<String>]
+      # @param text [String, nil]
+      # @return [void]
+      def add_title_candidate!(candidates, text)
+        stripped = text.to_s.strip
+        return if stripped.empty? || candidates.include?(stripped)
+
+        candidates << stripped
+      end
+
+      # @param text [String]
+      # @return [Boolean]
+      def credit_shaped_title?(text)
+        AutoSource::Cleanup::CREDIT_TITLE.match?(text)
+      end
+
+      def heading
+        @heading ||= begin
+          tags = @root.find_all(&:heading?)
+          if tags.any?
+            select_best_heading(tags)
+          elsif @fallback_anchorless && @selected_anchor.nil?
+            fallback_heading
+          end
+        end
+      end
+
+      def select_best_heading(tags)
+        min_name = tags.map { |t| t.name.to_s }.min
+        best = nil
+        max_size = -1
+        tags.each do |tag|
+          next unless tag.name.to_s == min_name
+
+          size = tag.visible_text.to_s.size
+          (best = tag) && (max_size = size) if size > max_size
+        end
+        best
+      end
+
+      def fallback_heading
+        @root.find do |n|
+          next true if FALLBACK_HEADING_NAMES.include?(n.name)
+
+          n.attrs.class_names.any? { |c| c.match?(/title|font-bold|font-semibold/) } &&
+            !n.visible_text.to_s.strip.empty?
+        end
+      end
+
+      def kicker_node # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+        @kicker_node ||= begin
+          node = @root.find do |n|
+            n.attrs.raw.key?('data-tb-kicker') ||
+              n.attrs.class_names.any? { |c| c.match?(KICKER_CLASS_PATTERN) }
+          end
+          node && heading && (node.equal?(heading) || descendant_of?(node, heading)) ? nil : node
+        end
+      end
+
+      def descendant_of?(child, ancestor)
+        # Walk via root children since Extractor may not hold Index; use tag walk on tree.
+        return true if child.equal?(ancestor)
+
+        ancestor.descendants.any? { |d| d.equal?(child) }
+      end
+
+      def extract_description
+        exclude = [heading, @selected_anchor, kicker_node].compact
+        description = @root.visible_text(exclude:)
+        return if description.nil?
+
+        desc = description.strip
+        desc.empty? ? nil : desc
+      end
+
+      def generate_id # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
+        @generate_id ||= begin
+          id_from_dom = parse_id_from_dom
+          if id_from_dom
+            id_from_dom
+          else
+            heading_text = heading&.visible_text
+            heading_text = first_nonempty_own_text if heading_text.to_s.strip.empty? && @fallback_anchorless
+            if heading_text && !heading_text.strip.empty?
+              generate_slug(heading_text)
+            elsif @fallback_anchorless
+              text = @root.visible_text.to_s.strip
+              Zlib.crc32(text).to_s(36) unless text.empty?
+            end
+          end
+        end
+      end
+
+      def parse_id_from_dom
+        candidates = [@root.attrs.id]
+        nested = @root.find { |n| n.attrs.id }
+        candidates << nested.attrs.id if nested
+        if @selected_anchor
+          url = extract_url
+          candidates += [url&.path, url&.query]
+        end
+        candidates.compact.reject(&:empty?).first
+      end
+
+      def generate_slug(text)
+        slug = text.downcase.gsub(/[^a-z0-9]+/, '-')
+        slug = slug[1..] if slug.start_with?('-')
+        slug = slug[0..-2] if slug.end_with?('-')
+        slug unless slug.empty?
+      end
+
+      def first_nonempty_own_text
+        @root.find { |n| !n.own_text.to_s.strip.empty? }&.own_text&.strip
+      end
+
+      def extract_image
+        img_src = image_from_srcset || image_from_img || image_from_style
+        Url.from_relative(img_src, @base_url) if img_src
+      end
+
+      def image_from_img
+        @root.find { |n| n.image? && n.attrs.src && !n.attrs.src.start_with?('data:') }&.attrs&.src
+      end
+
+      def image_from_srcset
+        srcsets = @root.find_all { |n| n.attrs.srcset }.map { |n| n.attrs.srcset }
+        ArticleRules::Image.largest_from_srcsets(srcsets)
+      end
+
+      def image_from_style
+        styles = @root.find_all { |n| n.attrs.style&.include?('url') }.map { |n| n.attrs.style }
+        ArticleRules::Image.best_from_styles(styles)
+      end
+
+      def extract_published_at
+        datetimes = @root.find_all { |n| n.attrs.datetime }.map { |n| n.attrs.datetime }
+        ArticleRules::Date.earliest(datetimes)
+      end
+
+      def extract_enclosures
+        @root.find_all { |n| enclosure_node?(n) }.filter_map { |element| enclosure_from(element) }
+      end
+
+      def enclosure_node?(node)
+        case node.name
+        when :img then node.attrs.src && !node.attrs.src.start_with?('data:')
+        when :source, :audio, :video, :iframe then node.attrs.src
+        when :a then ArticleRules::Enclosure.archive_href?(node.attrs.href)
+        else false
+        end
+      end
+
+      def enclosure_from(element)
+        case element.name
+        when :img
+          ArticleRules::Enclosure.from_image(element.attrs.src, @base_url)
+        when :video, :audio, :source
+          ArticleRules::Enclosure.from_media(element.attrs.src, element.attrs.type, @base_url)
+        when :iframe
+          ArticleRules::Enclosure.from_iframe(element.attrs.src, @base_url)
+        when :a
+          ArticleRules::Enclosure.from_anchor(element.attrs.href, @base_url)
+        end
+      end
+
+      def extract_categories
+        Set.new.tap do |categories|
+          @root.find_all { |n| category_candidate?(n) }.each do |element|
+            add_category_text!(categories, element) if ArticleRules::Category.class_match?(element.attrs.class_attr)
+            element.attrs.raw.each do |name, value|
+              next unless ArticleRules::Category.attr_name_match?(name)
+
+              ArticleRules::Category.add_text!(categories, value)
+            end
+          end
+        end.to_a
+      end
+
+      def category_candidate?(node)
+        ArticleRules::Category.class_match?(node.attrs.class_attr) ||
+          node.attrs.raw.keys.any? { |k| ArticleRules::Category.attr_name_match?(k) }
+      end
+
+      def add_category_text!(categories, element)
+        if element.link?
+          ArticleRules::Category.add_text!(categories, element.visible_text)
+          return
+        end
+
+        anchors = element.find_all(&:link?)
+        if anchors.any?
+          anchors.each { |a| ArticleRules::Category.add_text!(categories, a.visible_text) }
+        else
+          ArticleRules::Category.add_split_text!(categories, element.visible_text)
+        end
+      end
+    end
+    # rubocop:enable Metrics/ClassLength
+  end
+end

data/lib/html2rss/link_destination/destination_facts.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module LinkDestination
+    # Normalized URL plus reusable route-classification facts for one link.
+    DestinationFacts = Data.define(
+      :url,
+      :destination,
+      :segments,
+      :content_path,
+      :utility_path,
+      :taxonomy_path,
+      :vanity_path,
+      :shallow,
+      :strong_post_suffix,
+      :high_confidence_junk_path,
+      :high_confidence_utility_destination
+    ) do
+      # @param url [Html2rss::Url] normalized destination URL
+      # @return [DestinationFacts] route facts for downstream link scoring
+      def self.build(url) # rubocop:disable Metrics/MethodLength
+        classifier = PathClassifier.new(url.path_segments)
+
+        new(
+          url:,
+          destination: url.to_s,
+          segments: classifier.segments,
+          strong_post_suffix: classifier.strong_post_suffix?,
+          content_path: classifier.content_path?,
+          utility_path: classifier.utility_path?,
+          taxonomy_path: classifier.taxonomy_path?,
+          vanity_path: classifier.vanity_path?,
+          shallow: classifier.shallow?,
+          high_confidence_junk_path: classifier.junk_path?,
+          high_confidence_utility_destination: classifier.utility_destination?
+        )
+      end
+    end
+  end
+end

data/lib/html2rss/link_destination/noise_policy.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module LinkDestination
+    ##
+    # Noise / junk chrome eligibility for SST anchors (port of LinkHeuristics#noise_anchor?).
+    class NoisePolicy
+      # Child tag names that mark icon-only anchors as noise.
+      ICON_NAMES = %i[img svg].freeze
+
+      # @param link_resolver [LinkResolver]
+      def initialize(link_resolver:)
+        @link_resolver = link_resolver
+      end
+
+      ##
+      # @param text [String]
+      # @param destination_facts [LinkDestination::DestinationFacts, nil]
+      # @param anchor [SST::Node, nil]
+      # @param container [SST::Node, nil]
+      # @param heading_anchor [Boolean]
+      # @param utility_landmark_ancestor [Boolean] pre-computed by Segmenter tree walk
+      # @return [Boolean]
+      # rubocop:disable Metrics/CyclomaticComplexity, Metrics/ParameterLists, Metrics/PerceivedComplexity, Lint/UnusedMethodArgument
+      def noise_anchor?(text:, destination_facts:, anchor: nil, container: nil,
+                        heading_anchor: false, utility_landmark_ancestor: false)
+        return true unless destination_facts
+
+        destination_facts.taxonomy_path ||
+          short_utility_label?(text, destination_facts) ||
+          recommended_chrome?(text, destination_facts, heading_anchor:) ||
+          (@link_resolver.utility_prefix_text?(text) && destination_facts.high_confidence_utility_destination) ||
+          (@link_resolver.utility_text?(text) && destination_facts.vanity_path) ||
+          utility_text_chrome?(text, destination_facts, heading_anchor:) ||
+          icon_only_anchor?(anchor, text) ||
+          utility_landmark_ancestor
+      end
+      # rubocop:enable Metrics/CyclomaticComplexity, Metrics/ParameterLists, Metrics/PerceivedComplexity, Lint/UnusedMethodArgument
+
+      private
+
+      def short_utility_label?(text, destination_facts)
+        destination_facts.utility_path &&
+          !destination_facts.content_path &&
+          !destination_facts.strong_post_suffix &&
+          text.to_s.scan(/\p{Alnum}+/).size <= 3
+      end
+
+      def recommended_chrome?(text, destination_facts, heading_anchor:)
+        !heading_anchor && @link_resolver.recommended_text?(text) && destination_facts.shallow
+      end
+
+      def utility_text_chrome?(text, destination_facts, heading_anchor:)
+        return false if destination_facts.content_path
+        return false unless @link_resolver.utility_text?(text)
+
+        !heading_anchor && !destination_facts.strong_post_suffix
+      end
+
+      def icon_only_anchor?(anchor, text)
+        return false unless anchor
+        return false if text.to_s.match?(/\p{Alnum}/)
+
+        !!anchor.find { |n| ICON_NAMES.include?(n.name) }
+      end
+    end
+  end
+end