html2rss 0.20.1 → 0.22.0

data/lib/html2rss/auto_source/scraper/link_heuristics.rb

@@ -24,19 +24,30 @@ module Html2rss
         ) do
           # @param url [Html2rss::Url] normalized destination URL
           # @return [DestinationFacts] route facts for downstream link scoring
-          def self.build(url)
+          def self.build(url) # rubocop:disable Metrics/MethodLength
             classifier = PathClassifier.new(url.path_segments)
 
             new(
               url:,
               destination: url.to_s,
-              **classifier.destination_attributes
+              segments: classifier.segments,
+              strong_post_suffix: classifier.strong_post_suffix?,
+              content_path: classifier.content_path?,
+              utility_path: classifier.utility_path?,
+              taxonomy_path: classifier.taxonomy_path?,
+              vanity_path: classifier.vanity_path?,
+              shallow: classifier.shallow?,
+              high_confidence_junk_path: classifier.junk_path?,
+              high_confidence_utility_destination: classifier.utility_destination?
             )
           end
         end
 
         # Extracts a normalized href from a Nokogiri anchor or raw href value.
         class HrefExtractor
+          # Regexp to capture everything before the first '#'
+          HREF_BASE_PATTERN = /\A([^#]*)/
+
           # @param anchor_or_href [Nokogiri::XML::Element, String, #to_s] anchor element or href-like value
           # @return [String, nil] href without fragment, or nil when blank
           def self.call(anchor_or_href) = new(anchor_or_href).call
@@ -48,20 +59,18 @@ module Html2rss
 
           # @return [String, nil] href without fragment, or nil when blank
           def call
-            raw_href.to_s.split('#', 2).first.to_s.strip.then do |href|
-              href unless href.empty?
-            end
-          end
+            href = case @anchor_or_href
+                   when Nokogiri::XML::Node
+                     @anchor_or_href['href']
+                   else
+                     @anchor_or_href
+                   end
 
-          private
+            return unless href
 
-          def raw_href
-            case @anchor_or_href
-            when Nokogiri::XML::Node
-              @anchor_or_href['href']
-            else
-              @anchor_or_href
-            end
+            # Extract base part before # and strip whitespace
+            base = href.to_s[HREF_BASE_PATTERN, 1].strip
+            base unless base.empty?
           end
         end
 
@@ -125,8 +134,7 @@ module Html2rss
         end
 
         # Classifies normalized destination path segments for scoring.
-        # rubocop:disable Metrics/ClassLength
-        class PathClassifier
+        class PathClassifier # rubocop:disable Metrics/ClassLength
           attr_reader :segments
 
           # Segment groups used to classify article, taxonomy, utility, and vanity routes.
@@ -206,48 +214,25 @@ module Html2rss
             @segments = segments
           end
 
-          # @return [Hash] destination attributes consumed by DestinationFacts
-          def destination_attributes
-            route_attributes.merge(confidence_attributes)
-          end
-
-          # @return [Hash] baseline path classification attributes
-          def route_attributes
-            {
-              segments:,
-              content_path: content_path?,
-              utility_path: utility_path?,
-              taxonomy_path: taxonomy_path?,
-              vanity_path: vanity_path?,
-              shallow: shallow?,
-              strong_post_suffix: strong_post_suffix?
-            }
-          end
-
-          # @return [Hash] high-confidence noise classification attributes
-          def confidence_attributes
-            ConfidenceClassifier.new(self).attributes
-          end
-
           # @return [Boolean] true when the route has article-like path evidence
           def content_path?
-            @content_path ||= SEGMENT_SETS.fetch(:content).intersect?(segments.to_set) ||
+            @content_path ||= segments.any? { |s| SEGMENT_SETS[:content].include?(s) } ||
                               yearish_content_context?
           end
 
           # @return [Boolean] true when the route includes utility/navigation evidence
           def utility_path?
-            @utility_path ||= SEGMENT_SETS.fetch(:utility).intersect?(segments.to_set)
+            @utility_path ||= segments.any? { |s| SEGMENT_SETS[:utility].include?(s) }
           end
 
           # @return [Boolean] true when the route points at conversion or account chrome
           def vanity_path?
-            @vanity_path ||= SEGMENT_SETS.fetch(:vanity).intersect?(segments.to_set)
+            @vanity_path ||= segments.any? { |s| SEGMENT_SETS[:vanity].include?(s) }
           end
 
           # @return [Boolean] true when the route points at taxonomy/listing chrome
           def taxonomy_path?
-            @taxonomy_path ||= SEGMENT_SETS.fetch(:taxonomy).intersect?(segments.to_set)
+            @taxonomy_path ||= segments.any? { |s| SEGMENT_SETS[:taxonomy].include?(s) }
           end
 
           # @return [Boolean] true when the route is too shallow to strongly indicate an article
@@ -260,7 +245,9 @@ module Html2rss
 
           # @return [Boolean] true when the final path segment looks like a post slug
           def strong_post_suffix?
-            PostSuffixClassifier.new(segments).strong?
+            @strong_post_suffix ||= segments.any? &&
+                                    included_last_segment? &&
+                                    trusted_post_context?(segments.size - 1)
           end
 
           # @return [Boolean] true when every path segment is utility chrome
@@ -282,131 +269,81 @@ module Html2rss
 
           # @return [Boolean] true when the leading segments are all utility chrome
           def deep_utility_context_route?
-            LeadingSegments.new(segments).all_junk?
+            all_junk?(segments.size - 1)
           end
 
-          private
-
-          def yearish_content_context?
-            segments.any? { |segment| segment.match?(YEARISH_SEGMENT) } &&
-              (strong_post_suffix? || LeadingSegments.new(segments).trusted_post_context?)
-          end
-        end
-        # rubocop:enable Metrics/ClassLength
-
-        # Classifies high-confidence junk and utility routes from path facts.
-        class ConfidenceClassifier
-          # @param path [PathClassifier] classified destination path
-          def initialize(path)
-            @path = path
-          end
-
-          # @return [Hash] high-confidence route classification attributes
-          def attributes
-            {
-              high_confidence_junk_path: junk_path?,
-              high_confidence_utility_destination: utility_destination?
-            }
-          end
-
-          private
-
+          # @return [Boolean] true when the route is shallow and contains high-confidence noise
           def junk_path?
             return false if excluded_content_route?
 
-            @path.taxonomy_path? ||
-              @path.utility_only_route? ||
-              @path.deep_utility_context_route? ||
-              @path.shallow_high_confidence_route?
+            taxonomy_path? ||
+              utility_only_route? ||
+              deep_utility_context_route? ||
+              shallow_high_confidence_route?
           end
 
+          # @return [Boolean] true when the route points at conversion or account chrome
           def utility_destination?
             return false if excluded_content_route?
 
-            @path.vanity_path? || utility_route?
+            vanity_path? || utility_route?
+          end
+
+          private
+
+          def yearish_content_context?
+            segments.any? { |segment| segment.match?(YEARISH_SEGMENT) } &&
+              (strong_post_suffix? || trusted_post_context?(segments.size - 1))
           end
 
           def excluded_content_route?
-            @path.segments.empty? || @path.content_path? || @path.strong_post_suffix?
+            segments.empty? || content_path? || strong_post_suffix?
           end
 
           def utility_route?
-            @path.taxonomy_path? ||
-              @path.utility_only_route? ||
-              @path.deep_utility_context_route? ||
+            taxonomy_path? ||
+              utility_only_route? ||
+              deep_utility_context_route? ||
               shallow_utility_route?
           end
 
           def shallow_utility_route?
-            @path.shallow? && @path.utility_path?
-          end
-        end
-
-        # Classifies route context before the final segment.
-        class LeadingSegments
-          # @param segments [Array<String>] normalized URL path segments
-          def initialize(segments)
-            @segments = segments[0...-1]
+            shallow? && utility_path?
           end
 
-          # @return [Boolean] true when every leading segment is utility chrome
-          def all_junk?
-            junk_segments = PathClassifier::SEGMENT_SETS.fetch(:high_confidence_junk)
+          def all_junk?(limit)
+            return false if limit <= 0
 
-            @segments.any? && @segments.all? { |segment| junk_segments.include?(segment) }
+            junk_segments = SEGMENT_SETS.fetch(:high_confidence_junk)
+            (0...limit).all? { |i| junk_segments.include?(segments[i]) }
           end
 
-          # @return [Boolean] true when leading segments provide article context
-          def trusted_post_context?
-            content_segments = PathClassifier::SEGMENT_SETS.fetch(:content)
-            context_segments = PathClassifier::SEGMENT_SETS.fetch(:deep_post_context)
+          def trusted_post_context?(limit)
+            return false if limit <= 0
+
+            content_segments = SEGMENT_SETS.fetch(:content)
+            context_segments = SEGMENT_SETS.fetch(:deep_post_context)
 
-            @segments.any? do |segment|
+            (0...limit).any? do |i|
+              segment = segments[i]
               content_segments.include?(segment) ||
                 segment.match?(PathClassifier::YEARISH_SEGMENT) ||
                 context_segments.include?(segment)
             end
           end
-        end
-
-        # Classifies whether the final segment is a strong post-like suffix.
-        class PostSuffixClassifier
-          # @param segments [Array<String>] normalized URL path segments
-          def initialize(segments)
-            @segments = segments
-          end
-
-          # @return [Boolean] true when the final path segment looks like a post slug
-          def strong?
-            @segments.any? &&
-              included_last_segment? &&
-              LeadingSegments.new(@segments).trusted_post_context?
-          end
-
-          private
 
           def included_last_segment?
             !excluded_last_segment? && slug_last_segment?
           end
 
           def excluded_last_segment?
-            excluded_segments.any? { |segment| segment.include?(last_segment) }
-          end
-
-          def excluded_segments
-            [
-              PathClassifier::SEGMENT_SETS.fetch(:high_confidence_junk),
-              PathClassifier::SEGMENT_SETS.fetch(:vanity)
-            ]
+            last = segments.last
+            [SEGMENT_SETS[:high_confidence_junk], SEGMENT_SETS[:vanity]].any? { |set| set.include?(last) }
           end
 
           def slug_last_segment?
-            last_segment.match?(PathClassifier::YEARISH_SEGMENT) ||
-              last_segment.match?(PathClassifier::POST_SLUG_SEGMENT)
-          end
-
-          def last_segment
-            @segments.last
+            last = segments.last
+            last.match?(YEARISH_SEGMENT) || last.match?(POST_SLUG_SEGMENT)
           end
         end
 
@@ -421,11 +358,15 @@ module Html2rss
         # @param anchor_or_href [Nokogiri::XML::Element, String, #to_s] anchor element or href-like value
         # @return [DestinationFacts, nil] normalized destination facts, or nil for blank/invalid URLs
         def destination_facts(anchor_or_href)
+          return node_facts[anchor_or_href] if node_facts.key?(anchor_or_href)
+
           href = HrefExtractor.call(anchor_or_href)
           return unless href
 
-          url = Html2rss::Url.from_relative(href, @base_url)
-          DestinationFacts.build(url)
+          res = memoized_destination_facts(href)
+
+          node_facts[anchor_or_href] = res if anchor_or_href.is_a?(Nokogiri::XML::Node)
+          res
         rescue ArgumentError
           nil
         end
@@ -441,6 +382,19 @@ module Html2rss
         # @param text [String, #to_s] visible anchor text
         # @return [Boolean] true when text identifies recommendation chrome
         def recommended_text?(text) = @text_classifier.recommended?(text)
+
+        private
+
+        def node_facts
+          @node_facts ||= {}.compare_by_identity
+        end
+
+        def memoized_destination_facts(href)
+          (@destination_facts ||= {})[href] ||= begin
+            url = Html2rss::Url.from_relative(href, @base_url)
+            DestinationFacts.build(url)
+          end
+        end
       end
     end
   end

data/lib/html2rss/auto_source/scraper/microdata.rb

@@ -92,7 +92,7 @@ module Html2rss
         attr_reader :parsed_body, :url
 
         # @param root [Nokogiri::XML::Element] supported Microdata root node
-        # @return [Hash{Symbol => Object}, nil] normalized article hash
+        # @return [Hash{Symbol => Object, nil}] normalized article hash
         def article_from(root)
           schema_object = SchemaObjectBuilder.call(root)
           return unless schema_object
@@ -378,7 +378,7 @@ module Html2rss
           extend ValueNormalizer
 
           # @param root [Nokogiri::XML::Element] supported microdata root node
-          # @return [Hash{Symbol => Object}, nil] compact schema-like object
+          # @return [Hash{Symbol => Object, nil}] compact schema-like object
           def call(root)
             type = Microdata.supported_type_name(root)
             return unless type

data/lib/html2rss/auto_source/scraper/schema/category_extractor.rb

@@ -13,11 +13,10 @@ module Html2rss
           # @param schema_object [Hash] The schema object
           # @return [Array<String>] Array of category strings
           def self.call(schema_object)
-            # Build union of all category sources
-            field_categories = extract_field_categories(schema_object)
-            about_categories = extract_about_categories(schema_object)
-
-            (field_categories | about_categories).to_a
+            Set.new.tap do |categories|
+              extract_field_categories!(categories, schema_object)
+              extract_about_categories!(categories, schema_object)
+            end.to_a
           end
 
           ##
@@ -26,10 +25,18 @@ module Html2rss
           # @param schema_object [Hash] The schema object
           # @return [Set<String>] Set of category strings
           def self.extract_field_categories(schema_object)
-            Set.new.tap do |categories|
-              %w[keywords categories tags].each do |field|
-                categories.merge(extract_field_value(schema_object, field))
-              end
+            Set.new.tap { |categories| extract_field_categories!(categories, schema_object) }
+          end
+
+          ##
+          # Extracts categories from keywords, categories, and tags fields.
+          #
+          # @param categories [Set<String>] Accumulator set
+          # @param schema_object [Hash] The schema object
+          # @return [void]
+          def self.extract_field_categories!(categories, schema_object)
+            %i[keywords categories tags].each do |field|
+              extract_field_value!(categories, schema_object[field])
             end
           end
 
@@ -39,15 +46,23 @@ module Html2rss
           # @param schema_object [Hash] The schema object
           # @return [Set<String>] Set of category strings
           def self.extract_about_categories(schema_object)
+            Set.new.tap { |categories| extract_about_categories!(categories, schema_object) }
+          end
+
+          ##
+          # Extracts categories from the about field.
+          #
+          # @param categories [Set<String>] Accumulator set
+          # @param schema_object [Hash] The schema object
+          # @return [void]
+          def self.extract_about_categories!(categories, schema_object)
             about = schema_object[:about]
-            return Set.new unless about
+            return unless about
 
             if about.is_a?(Array)
-              extract_about_array(about)
+              extract_about_array!(categories, about)
             elsif about.is_a?(String)
-              extract_string_categories(about)
-            else
-              Set.new
+              extract_string_categories!(categories, about)
             end
           end
 
@@ -58,15 +73,25 @@ module Html2rss
           # @param field [String] The field name
           # @return [Set<String>] Set of category strings
           def self.extract_field_value(schema_object, field)
-            value = schema_object[field.to_sym]
-            return Set.new unless value
+            Set.new.tap { |categories| extract_field_value!(categories, schema_object[field.to_sym]) }
+          end
+
+          ##
+          # Extracts categories from a single field value.
+          #
+          # @param categories [Set<String>] Accumulator set
+          # @param value [Object] The field value
+          # @return [void]
+          def self.extract_field_value!(categories, value)
+            return unless value
 
             if value.is_a?(Array)
-              Set.new(value.map(&:to_s).reject(&:empty?))
+              value.each do |item|
+                s = item.to_s
+                categories.add(s) unless s.empty?
+              end
             elsif value.is_a?(String)
-              extract_string_categories(value)
-            else
-              Set.new
+              extract_string_categories!(categories, value)
             end
           end
 
@@ -76,13 +101,21 @@ module Html2rss
           # @param about [Array] The about array
           # @return [Set<String>] Set of category strings
           def self.extract_about_array(about)
-            Set.new.tap do |categories|
-              about.each do |item|
-                if item.is_a?(Hash) && item[:name]
-                  categories.add(item[:name].to_s)
-                elsif item.is_a?(String)
-                  categories.add(item)
-                end
+            Set.new.tap { |categories| extract_about_array!(categories, about) }
+          end
+
+          ##
+          # Extracts categories from an about array.
+          #
+          # @param categories [Set<String>] Accumulator set
+          # @param about [Array] The about array
+          # @return [void]
+          def self.extract_about_array!(categories, about)
+            about.each do |item|
+              if item.is_a?(Hash) && item[:name]
+                categories.add(item[:name].to_s)
+              elsif item.is_a?(String)
+                categories.add(item)
               end
             end
           end
@@ -93,7 +126,20 @@ module Html2rss
           # @param string [String] source string that may contain category delimiters
           # @return [Set<String>] Set of category strings
           def self.extract_string_categories(string)
-            Set.new(string.split(/[,;|]/).map(&:strip).reject(&:empty?))
+            Set.new.tap { |categories| extract_string_categories!(categories, string) }
+          end
+
+          ##
+          # Extracts categories from a string by splitting on separators.
+          #
+          # @param categories [Set<String>] Accumulator set
+          # @param string [String] source string that may contain category delimiters
+          # @return [void]
+          def self.extract_string_categories!(categories, string)
+            string.split(/[,;|]/).each do |part|
+              s = part.strip
+              categories.add(s) unless s.empty?
+            end
           end
         end
       end

data/lib/html2rss/auto_source/scraper/schema/list_item.rb

@@ -16,9 +16,10 @@ module Html2rss
 
           # @return [Html2rss::Url, nil]
           def url
-            url = schema_object.dig(:item, :url) || super
+            return @url if defined?(@url)
 
-            Url.from_relative(url, base_url || url) if url
+            item_url = schema_object.dig(:item, :url)
+            @url = item_url ? Url.from_relative(item_url, base_url || item_url) : super
           end
         end
       end