html2rss 0.27.1 → 0.28.0

data/lib/html2rss/status.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-module Html2rss
+module Html2rss # rubocop:disable Metrics/ModuleLength -- Status Data.define + marshal stay co-located
   ##
   # Shared RSS +generator+ / JSON Feed +user_comment+ formatter string.
   #
@@ -9,7 +9,7 @@ module Html2rss
   # Tallies and counters are validated and frozen at construction (including Marshal load).
   Status = Data.define(
     :version, :scraper_tallies, :dedup_dropped, :selected_strategy, :attempt_count,
-    :strategy_attempts, :admission_drops
+    :strategy_attempts, :admission_drops, :entry_url, :scrape_url, :entry_resolution
   ) do
     class << self
       ##
@@ -21,10 +21,15 @@ module Html2rss
       # @param attempt_count [Integer] auto-fallback attempt count (0 when not under +:auto+)
       # @param strategy_attempts [Array<Hash>] auto-fallback attempt hashes (empty outside +:auto+)
       # @param admission_drops [Hash{String => Integer}] Cleanup reason → count (empty when unused)
+      # @param scrape_target [Html2rss::ScrapeTarget, nil] domain URL pair (mapped to wire strings)
+      # @param entry_url [String, nil] original channel URL when entry resolution ran
+      # @param scrape_url [String, nil] effective scrape URL after resolution
+      # @param entry_resolution [Html2rss::FeedResolution::Diag, Hash, nil] resolution diagnostics
       # @return [Html2rss::Status]
-      # rubocop:disable Metrics/ParameterLists -- Status kwargs stay co-located
+      # rubocop:disable Metrics/ParameterLists, Metrics/MethodLength -- Status kwargs stay co-located
       def build(articles:, dedup_dropped: 0, selected_strategy: nil, attempt_count: 0,
-                strategy_attempts: [], admission_drops: {})
+                strategy_attempts: [], admission_drops: {}, scrape_target: nil,
+                entry_url: nil, scrape_url: nil, entry_resolution: nil)
         tallies = articles.filter_map(&:scraper).tally.transform_keys { |klass| scraper_name(klass) }
         new(
           version: Html2rss::VERSION,
@@ -33,10 +38,13 @@ module Html2rss
           selected_strategy:,
           attempt_count:,
           strategy_attempts:,
-          admission_drops:
+          admission_drops:,
+          entry_url: entry_url || scrape_target&.entry_url,
+          scrape_url: scrape_url || scrape_target&.effective_url,
+          entry_resolution:
         )
       end
-      # rubocop:enable Metrics/ParameterLists
+      # rubocop:enable Metrics/ParameterLists, Metrics/MethodLength
 
       ##
       # @param klass [Class, #to_s] scraper class
@@ -54,10 +62,13 @@ module Html2rss
     # @param attempt_count [Integer]
     # @param strategy_attempts [Array<Hash>]
     # @param admission_drops [Hash{String => Integer}]
+    # @param entry_url [String, nil]
+    # @param scrape_url [String, nil]
+    # @param entry_resolution [Hash, nil]
     # rubocop:disable Metrics/ParameterLists, Metrics/MethodLength -- Status Data.define members
     def initialize(
       version:, scraper_tallies:, dedup_dropped:, selected_strategy: nil, attempt_count: 0,
-      strategy_attempts: [], admission_drops: {}
+      strategy_attempts: [], admission_drops: {}, entry_url: nil, scrape_url: nil, entry_resolution: nil
     )
       dedup = Integer(dedup_dropped)
       attempts = Integer(attempt_count)
@@ -70,21 +81,19 @@ module Html2rss
         selected_strategy:,
         attempt_count: attempts,
         strategy_attempts: freeze_attempts(strategy_attempts),
-        admission_drops: freeze_tallies(admission_drops)
+        admission_drops: freeze_tallies(admission_drops),
+        entry_url: freeze_optional_string(entry_url),
+        scrape_url: freeze_optional_string(scrape_url),
+        entry_resolution: freeze_entry_resolution(entry_resolution)
       )
     end
     # rubocop:enable Metrics/ParameterLists, Metrics/MethodLength
 
     ##
-    # Observability hash for web (+scraper_status+). Omits empty/absent optional keys:
-    # +:scraper_tallies+ when empty, +:selected_strategy+ when +nil+, +:attempt_count+ when zero,
-    # +:strategy_attempts+ / +:admission_drops+ when empty.
-    # Data members remain available via readers even when omitted here.
+    # Observability hash for web (+scraper_status+). Omits empty/absent optional keys.
     #
-    # @return [Hash{Symbol => Object}] always +:version+ (String), +:dedup_dropped+ (Integer);
-    #   optionally +:scraper_tallies+, +:selected_strategy+, +:attempt_count+,
-    #   +:strategy_attempts+, +:admission_drops+
-    # rubocop:disable Metrics/AbcSize -- omit-empty optional keys stay explicit
+    # @return [Hash{Symbol => Object}]
+    # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity -- omit-empty optional keys stay explicit
     def to_h
       {
         version:,
@@ -93,10 +102,13 @@ module Html2rss
         **(selected_strategy.nil? ? {} : { selected_strategy: }),
         **(attempt_count.positive? ? { attempt_count: } : {}),
         **(strategy_attempts.any? ? { strategy_attempts: } : {}),
-        **(admission_drops.any? ? { admission_drops: } : {})
+        **(admission_drops.any? ? { admission_drops: } : {}),
+        **(entry_url ? { entry_url: } : {}),
+        **(scrape_url ? { scrape_url: } : {}),
+        **(entry_resolution ? { entry_resolution: } : {})
       }
     end
-    # rubocop:enable Metrics/AbcSize
+    # rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
 
     ##
     # Formats the RSS +generator+ string and JSON Feed +user_comment+.
@@ -115,13 +127,15 @@ module Html2rss
 
     def marshal_dump
       [version, scraper_tallies, dedup_dropped, selected_strategy, attempt_count, strategy_attempts,
-       admission_drops]
+       admission_drops, entry_url, scrape_url, entry_resolution]
     end
 
-    def marshal_load((version, scraper_tallies, dedup_dropped, selected_strategy, attempt_count,
-                      strategy_attempts, admission_drops))
+    def marshal_load(payload)
+      version, scraper_tallies, dedup_dropped, selected_strategy, attempt_count,
+        strategy_attempts, admission_drops, entry_url, scrape_url, entry_resolution = payload
       initialize(version:, scraper_tallies:, dedup_dropped:, selected_strategy:, attempt_count:,
-                 strategy_attempts:, admission_drops: admission_drops || {})
+                 strategy_attempts:, admission_drops: admission_drops || {}, entry_url:, scrape_url:,
+                 entry_resolution:)
     end
 
     def freeze_tallies(tallies)
@@ -132,6 +146,19 @@ module Html2rss
       Array(attempts).map { |attempt| attempt.to_h.freeze }.freeze
     end
 
+    def freeze_optional_string(value)
+      return if value.nil?
+
+      value.to_s.dup.freeze
+    end
+
+    def freeze_entry_resolution(value)
+      return if value.nil?
+
+      hash = value.respond_to?(:to_h) ? value.to_h : value
+      hash.to_h.transform_keys(&:to_sym).freeze
+    end
+
     def validate_counters!(dedup:, attempts:, selected_strategy:)
       raise ArgumentError, 'dedup_dropped must be >= 0' if dedup.negative?
       raise ArgumentError, 'attempt_count must be >= 0' if attempts.negative?

data/lib/html2rss/surface_category.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module Html2rss
+  ##
+  # Closed surface class for no-scraper / page-assessment gates.
+  #
+  # Construction: {AutoSource::Scraper.classify_no_scraper_surface} and {PageRecon.assess}.
+  # Predicates own weak/blocked/listing-bonus decisions — do not re-list WEAK sets in Policy/Scorer.
+  class SurfaceCategory
+    # Surfaces that warrant listing/feed resolution (hubs / shells — not blocked).
+    WEAK = Set[:high_entropy_surface, :app_shell, :unsupported_surface].freeze
+
+    class << self
+      ##
+      # @param value [SurfaceCategory, Symbol, String, nil]
+      # @return [SurfaceCategory]
+      def coerce(value)
+        return value if value.is_a?(self)
+        return new(name: nil) if value.nil?
+
+        new(name: value.to_sym)
+      end
+    end
+
+    ##
+    # @return [Symbol, nil]
+    attr_reader :name
+
+    ##
+    # @param name [Symbol, nil]
+    def initialize(name:)
+      @name = name
+    end
+
+    ##
+    # @return [Boolean]
+    def weak? = WEAK.include?(name)
+
+    ##
+    # @return [Boolean]
+    def blocked? = name == :blocked_surface
+
+    ##
+    # @return [Boolean] non-weak, non-blocked surface eligible for listing bonus
+    def listing_bonus? = !name.nil? && !weak? && !blocked?
+
+    ##
+    # @return [Symbol, nil]
+    def to_sym = name
+
+    ##
+    # @param other [Object]
+    # @return [Boolean]
+    def ==(other)
+      other.is_a?(self.class) && name == other.name
+    end
+    alias eql? ==
+
+    ##
+    # @return [Integer]
+    def hash = [self.class, name].hash
+  end
+end

data/lib/html2rss/syndication/README.md

@@ -0,0 +1,21 @@
+# Syndication
+
+Native RSS/Atom discovery and parse for auto-source promotion and direct feed URLs.
+
+## Ownership
+
+| Concern | Owner |
+| --- | --- |
+| Head `rel=alternate` + common path probes + status-gated feedish | `Syndication::Discovery` |
+| RSS 2.0 / Atom → article hashes | `Syndication::Parser` |
+| Strict head-only link parse (no path guessing) | `Html::FeedLink` |
+| Response Content-Type / body feed sniff | `RequestService::Response#feed_response?` (sole sniff owner; Discovery only adds HTTP status gate) |
+
+## Non-goals
+
+- Entry-URL tournament / listing resolution (`FeedResolution`)
+- Page recon / surface classification (`PageRecon`)
+- Feed channel metadata for output (`Channel` / `FeedBuilder`)
+- Ranking or quality scoring of competing feeds beyond first feedish hit
+
+Discovery stops at the first same-origin feedish URL (lazy probe). Path guessing mirrors the legacy configs `probe_rss` script; that script should eventually thin-wrap this module.

data/lib/html2rss/syndication/candidate_catalog.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Syndication
+    ##
+    # Shared path lexicon for syndication discovery and entry-resolution candidates.
+    module CandidateCatalog
+      # Common feed path suffixes probed after head +rel=alternate+ hints.
+      FEED_PATHS = %w[
+        /feed
+        /feed.xml
+        /rss
+        /rss.xml
+        /atom.xml
+        /index.xml
+        /news/rss
+        /news/feed
+        /blog/feed
+        /blog/rss.xml
+      ].freeze
+
+      # HTML listing paths probed only by {FeedResolution::CandidateGenerator}.
+      LISTING_PATHS = %w[/news /blog /releases /changelog /updates].freeze
+    end
+  end
+end

data/lib/html2rss/syndication/discovery.rb

@@ -0,0 +1,217 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Syndication
+    ##
+    # Finds a same-origin RSS/Atom URL for a page via head alternates and path guesses.
+    #
+    # Ports configs +probe_rss+ path/alternate logic onto {RequestSession#follow_up}.
+    module Discovery # rubocop:disable Metrics/ModuleLength -- discovery + path probes stay co-located
+      # Common feed path suffixes probed after head +rel=alternate+ hints.
+      DEFAULT_PATHS = CandidateCatalog::FEED_PATHS
+
+      LINK_TAG_RE = /<link\b[^>]*>/i
+      HREF_RE = /\bhref\s*=\s*["']([^"']+)["']/i
+      TYPE_RE = /\btype\s*=\s*["']([^"']+)["']/i
+      REL_RE = /\brel\s*=\s*["']([^"']+)["']/i
+      private_constant :LINK_TAG_RE, :HREF_RE, :TYPE_RE, :REL_RE
+
+      module_function
+
+      ##
+      # Probes candidates lazily and returns the first feedish URL.
+      #
+      # @param page_url [String, Html2rss::Url] page or origin URL
+      # @param request_session [Html2rss::RequestSession] shared session for follow-ups
+      # @param parsed_body [Nokogiri::HTML::Document, nil] parsed HTML when available
+      # @param html [String, nil] raw HTML when a document is unavailable
+      # @param extra_paths [Array<String>] additional path guesses
+      # @param max_probes [Integer, nil] max candidate GETs (nil = uncapped)
+      # @return [Html2rss::Url, nil]
+      # rubocop:disable Metrics/ParameterLists -- discovery kwargs stay co-located
+      def best_feed_url(page_url:, request_session:, parsed_body: nil, html: nil, extra_paths: [],
+                        max_probes: nil)
+        best_feed_response(
+          page_url:, request_session:, parsed_body:, html:, extra_paths:, max_probes:
+        )&.url
+      end
+      # rubocop:enable Metrics/ParameterLists
+
+      ##
+      # Like {#best_feed_url} but returns the validated syndication response for parsing.
+      #
+      # @param page_url [String, Html2rss::Url]
+      # @param request_session [Html2rss::RequestSession]
+      # @param parsed_body [Nokogiri::HTML::Document, nil]
+      # @param html [String, nil]
+      # @param extra_paths [Array<String>]
+      # @param max_probes [Integer, nil] max candidate GETs (nil = uncapped)
+      # @return [Html2rss::RequestService::Response, nil]
+      # rubocop:disable Metrics/ParameterLists, Metrics/MethodLength -- discovery kwargs stay co-located
+      def best_feed_response(page_url:, request_session:, parsed_body: nil, html: nil, extra_paths: [],
+                             max_probes: nil)
+        page = Html2rss::Url.from_absolute(page_url)
+        candidates = candidate_urls(page_url: page, parsed_body:, html:, extra_paths:)
+        Log.debug(
+          "Syndication::Discovery: host=#{page.host} candidate_count=#{candidates.size}"
+        )
+
+        first_feedish_response(
+          candidates, request_session:, origin_url: page, max_probes:
+        ).tap do |response|
+          next unless response
+
+          Log.info("Syndication::Discovery: host=#{page.host} selected_feed_url=#{response.url}")
+        end
+      end
+      # rubocop:enable Metrics/ParameterLists, Metrics/MethodLength
+
+      ##
+      # Ordered candidate feed URLs (head alternates first, then path guesses).
+      #
+      # @param page_url [String, Html2rss::Url]
+      # @param parsed_body [Nokogiri::HTML::Document, nil]
+      # @param html [String, nil]
+      # @param extra_paths [Array<String>]
+      # @return [Array<Html2rss::Url>]
+      def candidate_urls(page_url:, parsed_body: nil, html: nil, extra_paths: [])
+        page = Html2rss::Url.from_absolute(page_url)
+        (alternate_feed_urls(page, parsed_body:, html:) +
+          path_guess_urls(page, extra_paths:)).uniq
+      end
+
+      ##
+      # @param response [Html2rss::RequestService::Response, nil]
+      # @return [Boolean]
+      def feedish?(response)
+        return false unless response
+        return false unless successful_status?(response)
+
+        response.feed_response?
+      end
+
+      def successful_status?(response)
+        status = response.status
+        status.nil? || status.between?(200, 299)
+      end
+      module_function :successful_status?
+      private_class_method :successful_status?
+
+      ##
+      # Directory-relative feed path guesses for a page URL.
+      #
+      # @param page_url [String, Html2rss::Url]
+      # @return [Array<String>] relative paths
+      def page_dir_paths(page_url)
+        path = Html2rss::Url.from_absolute(page_url).path
+        return [] if path.nil? || path.empty? || path == '/'
+
+        dir = path.sub(%r{/[^/]*$}, '/')
+        ["#{dir}feed", "#{dir}rss.xml", "#{dir}atom.xml"]
+      end
+
+      def first_feedish_response(urls, request_session:, origin_url:, max_probes: nil)
+        scoped = max_probes.nil? ? urls.lazy : urls.lazy.take(max_probes)
+        scoped.filter_map do |url|
+          response = probe(url, request_session:, origin_url:)
+          response if feedish?(response)
+        end.first
+      end
+      module_function :first_feedish_response
+      private_class_method :first_feedish_response
+
+      def first_feedish_url(urls, request_session:, origin_url:, max_probes: nil)
+        first_feedish_response(urls, request_session:, origin_url:, max_probes:)&.url
+      end
+      module_function :first_feedish_url
+      private_class_method :first_feedish_url
+
+      def probe(url, request_session:, origin_url:)
+        request_session.follow_up(url:, relation: :auto_source, origin_url:)
+      rescue Html2rss::Error => error
+        Log.debug(
+          "Syndication::Discovery: probe skipped host=#{origin_url.host} " \
+          "(#{error.class})"
+        )
+        nil
+      end
+      module_function :probe
+      private_class_method :probe
+
+      def alternate_feed_urls(page, parsed_body:, html:)
+        from_document = feed_links_from_document(parsed_body, page)
+        return from_document if from_document.any?
+
+        alternate_feed_hrefs(html.to_s, page)
+      end
+      module_function :alternate_feed_urls
+      private_class_method :alternate_feed_urls
+
+      def feed_links_from_document(parsed_body, page)
+        return [] unless parsed_body.is_a?(Nokogiri::HTML::Document)
+
+        Html::FeedLink.from_document(parsed_body).filter_map do |link|
+          absolute_url(page, link.href)
+        end
+      end
+      module_function :feed_links_from_document
+      private_class_method :feed_links_from_document
+
+      def path_guess_urls(page, extra_paths:)
+        paths = (page_dir_paths(page) + DEFAULT_PATHS + Array(extra_paths)).uniq
+        paths.filter_map { |path| absolute_url(page, path) }
+      end
+      module_function :path_guess_urls
+      private_class_method :path_guess_urls
+
+      def alternate_feed_hrefs(html, page)
+        return [] if html.nil? || html.empty?
+
+        html.scan(LINK_TAG_RE).filter_map { |tag| href_from_alternate_link(tag, page) }.uniq
+      end
+      module_function :alternate_feed_hrefs
+      private_class_method :alternate_feed_hrefs
+
+      def href_from_alternate_link(tag, page)
+        return unless alternate_rel?(tag)
+
+        href = tag[HREF_RE, 1]
+        return if href.nil? || href.empty?
+        return unless syndication_type?(tag[TYPE_RE, 1]) || feed_like_href?(href)
+
+        absolute_url(page, href.strip)
+      end
+      module_function :href_from_alternate_link
+      private_class_method :href_from_alternate_link
+
+      def alternate_rel?(tag)
+        tag[REL_RE, 1].to_s.downcase.split(/\s+/).include?('alternate')
+      end
+      module_function :alternate_rel?
+      private_class_method :alternate_rel?
+
+      def syndication_type?(type)
+        return false if type.nil? || type.empty?
+
+        t = type.downcase
+        t.include?('rss+xml') || t.include?('atom+xml') || t.include?('rss') || t.include?('atom')
+      end
+      module_function :syndication_type?
+      private_class_method :syndication_type?
+
+      def feed_like_href?(href)
+        href.match?(/rss|atom|feed|\.xml/i)
+      end
+      module_function :feed_like_href?
+      private_class_method :feed_like_href?
+
+      def absolute_url(base, path)
+        Html2rss::Url.from_relative(path, base)
+      rescue ArgumentError
+        nil
+      end
+      module_function :absolute_url
+      private_class_method :absolute_url
+    end
+  end
+end

data/lib/html2rss/syndication/parser.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+require 'rss'
+
+module Html2rss
+  module Syndication
+    ##
+    # Parses RSS 2.0 and Atom documents into AutoSource article hashes.
+    module Parser
+      module_function
+
+      ##
+      # @param response [Html2rss::RequestService::Response]
+      # @return [Array<Hash{Symbol => Object}>] article hashes (may be empty)
+      # @raise [ArgumentError] when the response is not a syndication feed
+      def parse_response(response)
+        raise ArgumentError, 'response is not a syndication feed' unless response.feed_response?
+
+        parse(response.body, base_url: response.url)
+      end
+
+      ##
+      # @param body [String] raw RSS/Atom document
+      # @param base_url [String, Html2rss::Url, nil] fallback base for relative item links
+      # @return [Array<Hash{Symbol => Object}>]
+      def parse(body, base_url: nil)
+        feed = RSS::Parser.parse(body.to_s, false)
+        return [] unless feed
+
+        items_from(feed, base_url:).filter_map { |item| article_hash(item, base_url:) }
+      rescue RSS::Error => error
+        Log.warn("Syndication::Parser: failed to parse feed (#{error.class}: #{error.message})")
+        []
+      end
+
+      def items_from(feed, base_url:) # rubocop:disable Lint/UnusedMethodArgument -- base reserved for callers
+        case feed
+        when RSS::Rss then Array(feed.items)
+        when RSS::Atom::Feed then Array(feed.entries)
+        else []
+        end
+      end
+      module_function :items_from
+      private_class_method :items_from
+
+      def article_hash(item, base_url:)
+        case item
+        when RSS::Rss::Channel::Item then rss_item_hash(item, base_url:)
+        when RSS::Atom::Entry, RSS::Atom::Feed::Entry then atom_entry_hash(item, base_url:)
+        end
+      end
+      module_function :article_hash
+      private_class_method :article_hash
+
+      def rss_item_hash(item, base_url:) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength -- RSS item mapping
+        url = absolute_url(item.link, base_url:)
+        return unless url
+
+        {
+          id: present(item.guid&.content) || url.to_s,
+          title: present(item.title),
+          description: present(item.description),
+          url:,
+          published_at: item.date || item.pubDate,
+          author: present(item.author),
+          categories: Array(item.categories).filter_map { |cat| present(cat.content) }
+        }.compact
+      end
+      module_function :rss_item_hash
+      private_class_method :rss_item_hash
+
+      def atom_entry_hash(entry, base_url:) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity -- Atom entry mapping
+        url = atom_link(entry, base_url:)
+        return unless url
+
+        {
+          id: present(entry.id&.content) || url.to_s,
+          title: present(atom_text(entry.title)),
+          description: present(atom_text(entry.content) || atom_text(entry.summary)),
+          url:,
+          published_at: entry.published&.content || entry.updated&.content,
+          author: present(atom_author(entry)),
+          categories: Array(entry.categories).filter_map { |cat| present(cat.term) }
+        }.compact
+      end
+      module_function :atom_entry_hash
+      private_class_method :atom_entry_hash
+
+      def atom_link(entry, base_url:)
+        links = Array(entry.links)
+        link = links.find { |node| node.rel.nil? || node.rel == 'alternate' } || links.first
+        absolute_url(link&.href, base_url:)
+      end
+      module_function :atom_link
+      private_class_method :atom_link
+
+      def atom_text(node)
+        return if node.nil?
+
+        node.respond_to?(:content) ? node.content : node.to_s
+      end
+      module_function :atom_text
+      private_class_method :atom_text
+
+      def atom_author(entry)
+        author = Array(entry.author).first
+        return unless author
+
+        present(author.name&.content) || present(author.email&.content)
+      end
+      module_function :atom_author
+      private_class_method :atom_author
+
+      def absolute_url(raw, base_url:)
+        value = present(raw)
+        return unless value
+
+        if base_url
+          Html2rss::Url.from_relative(value, base_url)
+        else
+          Html2rss::Url.from_absolute(value)
+        end
+      rescue ArgumentError
+        nil
+      end
+      module_function :absolute_url
+      private_class_method :absolute_url
+
+      def present(value)
+        text = value.to_s.strip
+        text unless text.empty?
+      end
+      module_function :present
+      private_class_method :present
+    end
+  end
+end

data/lib/html2rss/syndication.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module Html2rss
+  ##
+  # Native RSS/Atom discovery and parse (not HTML AutoSource ownership).
+  #
+  # {include:file:lib/html2rss/syndication/README.md}
+  module Syndication
+  end
+end

data/lib/html2rss/url.rb

@@ -281,6 +281,18 @@ module Html2rss
     def ==(other) = other.is_a?(Url) && to_s == other.to_s
     alias eql? ==
 
+    ##
+    # Whether +other+ names the same document (trailing slash and fragment ignored).
+    # Differs from {#==}: `/blog` and `/blog/` match; query strings must still match.
+    #
+    # @param other [Object]
+    # @return [Boolean]
+    def same_document?(other)
+      return false unless other.is_a?(Url)
+
+      document_identity == other.document_identity
+    end
+
     ##
     # Returns the hash code for this URL.
     #
@@ -293,6 +305,16 @@ module Html2rss
     # @return [String] the debug representation
     def inspect = "#<#{self.class}:#{object_id} @uri=#{@uri.inspect}>"
 
+    protected
+
+    # Scheme, authority, slash-stripped path, and query (fragment omitted).
+    #
+    # @return [String]
+    def document_identity
+      uri = @uri.omit(:fragment)
+      "#{uri.scheme}\0#{uri.authority}\0#{uri.path.to_s.chomp('/')}\0#{uri.query}"
+    end
+
     private
 
     def ip_address_host?

data/lib/html2rss/version.rb

@@ -2,6 +2,6 @@
 
 module Html2rss
   # Current application version.
-  VERSION = '0.27.1'
+  VERSION = '0.28.0'
   public_constant :VERSION
 end