html2rss 0.24.0 → 0.25.0

data/lib/html2rss/auto_source/discovery/sitemap.rb

@@ -1,159 +0,0 @@
-# frozen_string_literal: true
-
-require 'date'
-require 'nokogiri'
-
-module Html2rss
-  class AutoSource
-    module Discovery
-      ##
-      # Parses XML sitemap documents and extracts prioritized, recent article entries.
-      class Sitemap
-        # Default minimum priority threshold (0.0 to 1.0)
-        DEFAULT_MIN_PRIORITY = 0.3
-        # Default maximum age in days for sitemap entries
-        DEFAULT_MAX_AGE_DAYS = 30
-        # Google News sitemap XML namespace
-        NEWS_NS = 'http://www.google.com/schemas/sitemap-news/0.9'
-
-        # Result value object returned by .call
-        # @!attribute entries [Array<SitemapEntry>] parsed entries (empty for sitemapindex docs)
-        # @!attribute sub_sitemap_urls [Array<String>] child sitemap URLs (empty for urlset docs)
-        Result = Data.define(:entries, :sub_sitemap_urls)
-
-        # Immutable facts for one sitemap entry
-        SitemapEntry = Data.define(:url, :title, :published_at, :priority, :changefreq)
-
-        class << self
-          ##
-          # Parses sitemap XML body and returns a Result.
-          #
-          # @param xml_content [String] sitemap XML body string
-          # @param min_priority [Float] minimum priority score (0.0..1.0)
-          # @param max_age_days [Integer] maximum age in days for entries
-          # @return [Result] result with entries and/or sub_sitemap_urls
-          def call(xml_content, min_priority: DEFAULT_MIN_PRIORITY, max_age_days: DEFAULT_MAX_AGE_DAYS)
-            new(xml_content, min_priority:, max_age_days:).call
-          end
-        end
-
-        ##
-        # @param xml_content [String]
-        # @param min_priority [Float]
-        # @param max_age_days [Integer]
-        def initialize(xml_content, min_priority: DEFAULT_MIN_PRIORITY, max_age_days: DEFAULT_MAX_AGE_DAYS)
-          @xml_content = xml_content
-          @min_priority = min_priority
-          @max_age_days = max_age_days
-        end
-
-        ##
-        # @return [Result]
-        def call
-          document = Nokogiri::XML(xml_content)
-          return Result.new(entries: [], sub_sitemap_urls: []) if invalid_xml?(document)
-
-          if sitemapindex?(document)
-            Result.new(entries: [], sub_sitemap_urls: extract_sub_sitemap_urls(document))
-          else
-            Result.new(entries: extract_entries(document), sub_sitemap_urls: [])
-          end
-        end
-
-        private
-
-        attr_reader :xml_content, :min_priority, :max_age_days
-
-        def invalid_xml?(document)
-          document.errors.any? && document.at_css('urlset, sitemapindex').nil?
-        end
-
-        def sitemapindex?(document)
-          !document.at_xpath('//*[local-name()="sitemapindex"]').nil?
-        end
-
-        def extract_sub_sitemap_urls(document)
-          document.xpath('//*[local-name()="sitemap"]/*[local-name()="loc"]').filter_map do |node|
-            text = node.text.strip
-            text unless text.empty?
-          end
-        end
-
-        def extract_entries(document)
-          document.xpath('//*[local-name()="url"]').filter_map { build_entry(_1) }
-        end
-
-        def build_entry(url_node)
-          loc = parse_loc(url_node)
-          return unless loc
-          return if invalid_entry_signals?(url_node)
-
-          published_at = parse_date(url_node)
-          return if stale_date?(published_at)
-
-          build_entry_facts(url_node, loc, published_at)
-        end
-
-        def build_entry_facts(url_node, loc, published_at)
-          SitemapEntry.new(
-            url: loc,
-            title: parse_title(url_node),
-            published_at:,
-            priority: parse_priority(url_node),
-            changefreq: parse_changefreq(url_node)
-          )
-        end
-
-        def parse_loc(url_node)
-          node = url_node.at_xpath('./*[local-name()="loc"]')
-          text = node&.text&.strip
-          text unless text.to_s.empty?
-        end
-
-        def parse_changefreq(url_node)
-          node = url_node.at_xpath('./*[local-name()="changefreq"]')
-          node ? node.text.strip.downcase : nil
-        end
-
-        def invalid_entry_signals?(url_node)
-          parse_priority(url_node) < min_priority || parse_changefreq(url_node) == 'never'
-        end
-
-        def parse_priority(url_node)
-          node = url_node.at_xpath('./*[local-name()="priority"]')
-          text = node&.text&.strip
-          text ? text.to_f : 0.5
-        end
-
-        def parse_date(url_node)
-          raw_date = raw_date_from(url_node)
-          return unless raw_date
-
-          Time.parse(raw_date).utc.iso8601
-        rescue ArgumentError
-          nil
-        end
-
-        def raw_date_from(url_node)
-          news_node = url_node.at_xpath('.//news:publication_date', 'news' => NEWS_NS)
-          lastmod_node = url_node.at_xpath('./*[local-name()="lastmod"]')
-          (news_node || lastmod_node)&.text&.strip
-        end
-
-        def parse_title(url_node)
-          node = url_node.at_xpath('.//news:title', 'news' => NEWS_NS)
-          node&.text&.strip
-        end
-
-        def stale_date?(published_at)
-          return false unless published_at
-
-          time = Time.parse(published_at)
-          (Time.now.utc - time) > (max_age_days * 86_400)
-        rescue ArgumentError
-          false
-        end
-      end
-    end
-  end
-end

data/lib/html2rss/auto_source/discovery.rb

@@ -1,14 +0,0 @@
-# frozen_string_literal: true
-
-module Html2rss
-  class AutoSource
-    ##
-    # Card/list discovery helpers used by AutoSource scrapers before field extraction.
-    #
-    # Html2rss::Html::ArticleExtractor fills article fields from a container; discovery finds those containers.
-    #
-    # DOM list discovery for anchorless or classless pages is owned by {Discovery::DomClustering}.
-    module Discovery
-    end
-  end
-end

data/lib/html2rss/auto_source/link_heuristics/anchor_signals.rb

@@ -1,28 +0,0 @@
-# frozen_string_literal: true
-
-module Html2rss
-  class AutoSource
-    class LinkHeuristics
-      # Score weights keyed by AnchorSignals member name.
-      ANCHOR_SCORE_RULES = {
-        heading_anchor: 100,
-        heading_text_match: 20,
-        meaningful_text: 10,
-        content_like_destination: 10
-      }.freeze
-
-      # Anchor ranking signals used by semantic primary-link selection.
-      AnchorSignals = Data.define(
-        :heading_anchor,
-        :heading_text_match,
-        :meaningful_text,
-        :content_like_destination
-      ) do
-        # @return [Integer] ranking score for one eligible anchor
-        def score
-          ANCHOR_SCORE_RULES.sum { |signal, weight| public_send(signal) ? weight : 0 }
-        end
-      end
-    end
-  end
-end

data/lib/html2rss/auto_source/link_heuristics/container_assessor.rb

@@ -1,106 +0,0 @@
-# frozen_string_literal: true
-
-module Html2rss
-  class AutoSource
-    class LinkHeuristics
-      ##
-      # Turns container DOM observations into ContainerSignals inputs.
-      #
-      # Owns publish markers, headings, visible text, and class/id tokens so
-      # LinkHeuristics can keep eligibility policy and ContainerSignals can
-      # keep scoring without re-reading the DOM.
-      class ContainerAssessor
-        # Token pattern for content-like class/id markers on containers.
-        CONTENT_TOKEN_REGEXP = begin
-          words = PathClassifier::SEGMENT_SETS.fetch(:content)
-          /(?:^|\s|[-_])(#{Regexp.union(words.to_a).source})(?:\s|[-_]|$)/i
-        end.freeze
-
-        # Token pattern for junk/utility class/id markers on containers.
-        JUNK_TOKEN_REGEXP = begin
-          words = PathClassifier::SEGMENT_SETS.fetch(:utility)
-          /(?:^|\s|[-_])(#{Regexp.union(words.to_a).source})(?:\s|[-_]|$)/i
-        end.freeze
-
-        # @param text_classifier [TextClassifier] shared utility/recommended text policy
-        def initialize(text_classifier:)
-          @text_classifier = text_classifier
-        end
-
-        ##
-        # Observes a container and builds ranking signals, including hard-junk.
-        #
-        # @param container [Nokogiri::XML::Node] semantic container node
-        # @param selected_anchor [Nokogiri::XML::Node, nil] primary anchor for the container
-        # @param destination_facts [DestinationFacts, nil] route facts for the selected anchor
-        # @return [ContainerSignals] observation + scoring signals for the container
-        # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
-        def call(container, selected_anchor, destination_facts:)
-          title = entry_title(container, selected_anchor)
-          tokens = container_tokens(container)
-
-          ContainerSignals.new(
-            title_word_count: word_count(title),
-            path_length: destination_facts&.url&.path.to_s.length,
-            content_path: destination_facts&.content_path,
-            publish_marker: publish_marker?(container),
-            descriptive_context: descriptive_context?(visible_text(container), title),
-            article_container: container.name == 'article',
-            content_tokens: content_tokens?(tokens),
-            junk_tokens: junk_tokens?(tokens),
-            utility_prefix_title: @text_classifier.utility_prefix?(title),
-            recommended_title: @text_classifier.recommended?(title),
-            utility_path: destination_facts&.utility_path,
-            strong_post_suffix: destination_facts&.strong_post_suffix,
-            shallow: destination_facts&.shallow,
-            high_confidence_junk_path: destination_facts&.high_confidence_junk_path,
-            high_confidence_utility_destination: destination_facts&.high_confidence_utility_destination,
-            selected_anchor_present: !selected_anchor.nil?
-          )
-        end
-        # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
-
-        private
-
-        def publish_marker?(container)
-          (@publish_markers ||= {}.compare_by_identity)[container] ||=
-            !!container.at_css('time, [datetime], [itemprop="datePublished"], [itemprop="dateModified"]')
-        end
-
-        def descriptive_context?(container_text, title)
-          snippet = container_text.to_s.sub(/\A#{Regexp.escape(title.to_s)}/i, '')
-          snippet.length > 30 && word_count(snippet) >= 8
-        end
-
-        def heading_for(container)
-          (@headings ||= {}.compare_by_identity)[container] ||=
-            container.at_css(Html2rss::Html::Navigator::HEADING_TAGS.join(','))
-        end
-
-        def visible_text(node)
-          return '' unless node
-
-          (@visible_texts ||= {}.compare_by_identity)[node] ||= Html2rss::Html::Navigator.extract_visible_text(node).to_s.strip
-        end
-
-        def entry_title(container, selected_anchor) = visible_text(heading_for(container) || selected_anchor)
-
-        def word_count(text)
-          (@word_counts ||= {})[text] ||= begin
-            count = 0
-            text.to_s.scan(/\p{Alnum}+/) { count += 1 }
-            count
-          end
-        end
-
-        def container_tokens(container)
-          (@container_tokens ||= {}.compare_by_identity)[container] ||= "#{container['class']} #{container['id']}"
-        end
-
-        def content_tokens?(tokens) = tokens.to_s.match?(CONTENT_TOKEN_REGEXP)
-
-        def junk_tokens?(tokens) = tokens.to_s.match?(JUNK_TOKEN_REGEXP)
-      end
-    end
-  end
-end

data/lib/html2rss/auto_source/link_heuristics/container_signals.rb

@@ -1,80 +0,0 @@
-# frozen_string_literal: true
-
-module Html2rss
-  class AutoSource
-    class LinkHeuristics
-      # Container observations used to compute quality/junk scores.
-      ContainerSignals = Data.define(
-        :title_word_count,
-        :path_length,
-        :content_path,
-        :publish_marker,
-        :descriptive_context,
-        :article_container,
-        :content_tokens,
-        :junk_tokens,
-        :utility_prefix_title,
-        :recommended_title,
-        :utility_path,
-        :strong_post_suffix,
-        :shallow,
-        :high_confidence_junk_path,
-        :high_confidence_utility_destination,
-        :selected_anchor_present
-      ) do
-        # @return [Integer] positive quality contribution for ranking
-        def quality_score # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
-          score = 0
-          score += 40 if title_word_count >= 3
-          score += 15 if title_word_count >= 7
-          score += 20 if path_length > 6
-          score += 15 if content_path
-          score += 15 if publish_marker
-          score += 10 if descriptive_context
-          score += 10 if article_container
-          score += 10 if content_tokens
-          score
-        end
-
-        # @return [Integer] junk penalty subtracted from quality
-        def junk_score # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
-          score = 0
-          score += 25 if non_content_utility_path?
-          score += 15 if utility_prefix_title && title_word_count <= 6
-          score += 10 if shallow
-          score += 10 if weak_container?
-          score += 10 if recommended_title && !content_path
-          score += 5 if high_confidence_junk_path
-          score += 15 if junk_tokens
-          score
-        end
-
-        # @return [Integer] quality minus junk for stable ranking
-        def final_score = quality_score - junk_score
-
-        # @return [Boolean] true when the entry should be dropped before ranking
-        def hard_junk? # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
-          weak = weak_article_candidate?
-
-          high_confidence_junk_path ||
-            (selected_anchor_present && recommended_title && shallow && weak) ||
-            (selected_anchor_present && utility_prefix_title &&
-              high_confidence_utility_destination && weak)
-        end
-
-        private
-
-        # @return [Boolean] true when article evidence is too weak to keep
-        def weak_article_candidate?
-          [article_container, publish_marker, descriptive_context, content_path].count(&:itself) < 2
-        end
-
-        def non_content_utility_path?
-          utility_path && !content_path && !strong_post_suffix
-        end
-
-        def weak_container? = !publish_marker && !descriptive_context
-      end
-    end
-  end
-end

data/lib/html2rss/auto_source/link_heuristics/destination_facts.rb

@@ -1,42 +0,0 @@
-# frozen_string_literal: true
-
-module Html2rss
-  class AutoSource
-    class LinkHeuristics
-      # Normalized URL plus reusable route-classification facts for one link.
-      DestinationFacts = Data.define(
-        :url,
-        :destination,
-        :segments,
-        :content_path,
-        :utility_path,
-        :taxonomy_path,
-        :vanity_path,
-        :shallow,
-        :strong_post_suffix,
-        :high_confidence_junk_path,
-        :high_confidence_utility_destination
-      ) do
-        # @param url [Html2rss::Url] normalized destination URL
-        # @return [DestinationFacts] route facts for downstream link scoring
-        def self.build(url) # rubocop:disable Metrics/MethodLength
-          classifier = PathClassifier.new(url.path_segments)
-
-          new(
-            url:,
-            destination: url.to_s,
-            segments: classifier.segments,
-            strong_post_suffix: classifier.strong_post_suffix?,
-            content_path: classifier.content_path?,
-            utility_path: classifier.utility_path?,
-            taxonomy_path: classifier.taxonomy_path?,
-            vanity_path: classifier.vanity_path?,
-            shallow: classifier.shallow?,
-            high_confidence_junk_path: classifier.junk_path?,
-            high_confidence_utility_destination: classifier.utility_destination?
-          )
-        end
-      end
-    end
-  end
-end

data/lib/html2rss/auto_source/link_heuristics/href_extractor.rb

@@ -1,38 +0,0 @@
-# frozen_string_literal: true
-
-module Html2rss
-  class AutoSource
-    class LinkHeuristics
-      # Extracts a normalized href from a Nokogiri anchor or raw href value.
-      class HrefExtractor
-        # Regexp to capture everything before the first '#'
-        HREF_BASE_PATTERN = /\A([^#]*)/
-
-        # @param anchor_or_href [Nokogiri::XML::Element, String, #to_s] anchor element or href-like value
-        # @return [String, nil] href without fragment, or nil when blank
-        def self.call(anchor_or_href) = new(anchor_or_href).call
-
-        # @param anchor_or_href [Nokogiri::XML::Element, String, #to_s] anchor element or href-like value
-        def initialize(anchor_or_href)
-          @anchor_or_href = anchor_or_href
-        end
-
-        # @return [String, nil] href without fragment, or nil when blank
-        def call
-          href = case @anchor_or_href
-                 when Nokogiri::XML::Node
-                   @anchor_or_href['href']
-                 else
-                   @anchor_or_href
-                 end
-
-          return unless href
-
-          # Extract base part before # and strip whitespace
-          base = href.to_s[HREF_BASE_PATTERN, 1].strip
-          base unless base.empty?
-        end
-      end
-    end
-  end
-end

data/lib/html2rss/auto_source/link_heuristics/path_classifier.rb

@@ -1,221 +0,0 @@
-# frozen_string_literal: true
-
-module Html2rss
-  class AutoSource
-    class LinkHeuristics
-      # Classifies normalized destination path segments for scoring.
-      class PathClassifier # rubocop:disable Metrics/ClassLength
-        attr_reader :segments
-
-        # Segment groups used to classify article, taxonomy, utility, and vanity routes.
-        SEGMENT_SETS = {
-          content: %w[
-            article articles blog blogs changelog changelogs insight insights
-            launch launches news post posts release releases story stories update updates
-            artikel beitrag beitraege nachrichten neuigkeiten aktuelles
-            articulo articulos noticia noticias entrada entradas publicacion publicaciones
-            actualite actualites nouvelle nouvelles
-            teaser teasers card cards
-          ].to_set.freeze,
-          utility: %w[
-            about account archive archives author authors category categories comment comments
-            contact feedback help login logout newsletter newsletters notification notifications
-            preference preferences profile register search settings share signup subscribe
-            tag tags topic topics
-            feed feeds comment-feed comments-feed
-            recommended
-            for-you
-            privacy terms cookie cookies
-            join member members membership plus premium plans pricing user users
-            kategorie kategorien schlagwort schlagworte thema themen autor autoren archiv
-            ueber-uns ueber ueberuns profil kontakt impressum suche hilfe anmelden registrieren
-            konto registrierung anmeldung abonnieren abo datenschutz nutzungsbedingungen agb
-            categoria categorias etiqueta etiquetas tema temas autores archivos
-            sobre-nosotros sobre quienes-somos buscar busqueda ayuda entrar ingresar
-            registrarse registro cuenta suscribirse boletin privacidad condiciones
-            categorie etiquette etiquettes sujet sujets theme themes auteur auteurs
-            a-propos apropos recherche rechercher aide connexion s-inscrire
-            sinscrire inscription compte s-abonner saboner lettre-information confidentialite mentions-legales cgu
-            menu sidebar widget social modal popup banner promo ad ads
-            related recommendation recommendations pagination pager
-          ].to_set.freeze,
-          high_confidence_junk: %w[
-            about account archive archives author authors category categories comment comments
-            contact cookie cookies feedback feed feeds help login logout notification notifications
-            preference preferences privacy profile register search settings share signup subscribe
-            tag tags terms topic topics comment-feed comments-feed user users
-            kategorie kategorien schlagwort schlagworte thema themen autor autoren archiv
-            ueber-uns ueber ueberuns profil kontakt impressum suche hilfe anmelden registrieren
-            konto registrierung anmeldung abonnieren abo datenschutz nutzungsbedingungen agb
-            categoria categorias etiqueta etiquetas tema temas autores archivos
-            sobre-nosotros sobre quienes-somos buscar busqueda ayuda entrar ingresar
-            registrarse registro cuenta suscribirse boletin privacidad condiciones
-            categorie etiquette etiquettes sujet sujets theme themes auteur auteurs
-            a-propos apropos recherche rechercher aide connexion s-inscrire
-            sinscrire inscription compte s-abonner saboner lettre-information confidentialite mentions-legales cgu
-            menu sidebar widget social modal popup banner promo ad ads
-            related recommendation recommendations pagination pager
-          ].to_set.freeze,
-          taxonomy: %w[
-            category categories tag tags topic topics
-            kategorie kategorien schlagwort schlagworte thema themen
-            categoria categorias etiqueta etiquetas tema temas
-            categorie etiquette etiquettes sujet sujets theme themes
-          ].to_set.freeze,
-          vanity: %w[
-            join membership plus premium pricing plans subscribe signup
-            abonnieren abo
-            suscribirse boletin
-            s-abonner saboner
-          ].to_set.freeze,
-          deep_post_context: %w[
-            press newsroom
-            presse pressemitteilungen
-            prensa
-          ].to_set.freeze
-        }.freeze
-        # Path segment that begins with a year-like publishing marker.
-        YEARISH_SEGMENT = /\A\d{4,}[\w-]*\z/
-        # Hyphenated slug shape common to article permalinks.
-        POST_SLUG_SEGMENT = /\A[a-z0-9]+(?:-[a-z0-9]+){2,}\z/i
-
-        # @param segments [Array<String>] normalized URL path segments
-        def initialize(segments)
-          @segments = segments
-        end
-
-        # @return [Boolean] true when the route has article-like path evidence
-        def content_path?
-          @content_path ||= SEGMENT_SETS[:content].intersect?(segments) ||
-                            yearish_content_context?
-        end
-
-        # @return [Boolean] true when the route includes utility/navigation evidence
-        def utility_path?
-          @utility_path ||= SEGMENT_SETS[:utility].intersect?(segments)
-        end
-
-        # @return [Boolean] true when the route points at conversion or account chrome
-        def vanity_path?
-          @vanity_path ||= SEGMENT_SETS[:vanity].intersect?(segments)
-        end
-
-        # @return [Boolean] true when the route points at taxonomy/listing chrome
-        def taxonomy_path?
-          @taxonomy_path ||= SEGMENT_SETS[:taxonomy].intersect?(segments)
-        end
-
-        # @return [Boolean] true when the route is too shallow to strongly indicate an article
-        def shallow?
-          segment_count = segments.size
-          junk_segments = SEGMENT_SETS.fetch(:high_confidence_junk)
-
-          segment_count <= 1 || (segment_count == 2 && junk_segments.include?(segments.last))
-        end
-
-        # @return [Boolean] true when the final path segment looks like a post slug
-        def strong_post_suffix?
-          @strong_post_suffix ||= segments.any? &&
-                                  included_last_segment? &&
-                                  trusted_post_context?(segments.size - 1)
-        end
-
-        # @return [Boolean] true when every path segment is utility chrome
-        def utility_only_route?
-          junk_segments = SEGMENT_SETS.fetch(:high_confidence_junk)
-
-          segments.all? { |segment| junk_segments.include?(segment) }
-        end
-
-        # @return [Boolean] true when the route is shallow and contains high-confidence noise
-        def shallow_high_confidence_route?
-          junk_segments = SEGMENT_SETS.fetch(:high_confidence_junk)
-          vanity_segments = SEGMENT_SETS.fetch(:vanity)
-
-          shallow? && segments.any? do |segment|
-            junk_segments.include?(segment) || vanity_segments.include?(segment)
-          end
-        end
-
-        # @return [Boolean] true when the leading segments are all utility chrome
-        def deep_utility_context_route?
-          all_junk?(segments.size - 1)
-        end
-
-        # @return [Boolean] true when the route is shallow and contains high-confidence noise
-        def junk_path?
-          return false if excluded_content_route?
-
-          taxonomy_path? ||
-            utility_only_route? ||
-            deep_utility_context_route? ||
-            shallow_high_confidence_route?
-        end
-
-        # @return [Boolean] true when the route points at conversion or account chrome
-        def utility_destination?
-          return false if excluded_content_route?
-
-          vanity_path? || utility_route?
-        end
-
-        private
-
-        def yearish_content_context?
-          segments.any? { |segment| segment.match?(YEARISH_SEGMENT) } &&
-            (strong_post_suffix? || trusted_post_context?(segments.size - 1))
-        end
-
-        def excluded_content_route?
-          segments.empty? || content_path? || strong_post_suffix?
-        end
-
-        def utility_route?
-          taxonomy_path? ||
-            utility_only_route? ||
-            deep_utility_context_route? ||
-            shallow_utility_route?
-        end
-
-        def shallow_utility_route?
-          shallow? && utility_path?
-        end
-
-        def all_junk?(limit)
-          return false if limit <= 0
-
-          junk_segments = SEGMENT_SETS.fetch(:high_confidence_junk)
-          (0...limit).all? { |i| junk_segments.include?(segments[i]) }
-        end
-
-        def trusted_post_context?(limit)
-          return false if limit <= 0
-
-          content_segments = SEGMENT_SETS.fetch(:content)
-          context_segments = SEGMENT_SETS.fetch(:deep_post_context)
-
-          (0...limit).any? do |i|
-            segment = segments[i]
-            content_segments.include?(segment) ||
-              segment.match?(PathClassifier::YEARISH_SEGMENT) ||
-              context_segments.include?(segment)
-          end
-        end
-
-        def included_last_segment?
-          !excluded_last_segment? && slug_last_segment?
-        end
-
-        def excluded_last_segment?
-          last = segments.last
-          [SEGMENT_SETS[:high_confidence_junk], SEGMENT_SETS[:vanity]].any? { |set| set.include?(last) }
-        end
-
-        def slug_last_segment?
-          last = segments.last
-          last.match?(YEARISH_SEGMENT) || last.match?(POST_SLUG_SEGMENT)
-        end
-      end
-    end
-  end
-end