html2rss 0.27.2 → 0.28.0

data/lib/html2rss/syndication/discovery.rb

@@ -0,0 +1,217 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Syndication
+    ##
+    # Finds a same-origin RSS/Atom URL for a page via head alternates and path guesses.
+    #
+    # Ports configs +probe_rss+ path/alternate logic onto {RequestSession#follow_up}.
+    module Discovery # rubocop:disable Metrics/ModuleLength -- discovery + path probes stay co-located
+      # Common feed path suffixes probed after head +rel=alternate+ hints.
+      DEFAULT_PATHS = CandidateCatalog::FEED_PATHS
+
+      LINK_TAG_RE = /<link\b[^>]*>/i
+      HREF_RE = /\bhref\s*=\s*["']([^"']+)["']/i
+      TYPE_RE = /\btype\s*=\s*["']([^"']+)["']/i
+      REL_RE = /\brel\s*=\s*["']([^"']+)["']/i
+      private_constant :LINK_TAG_RE, :HREF_RE, :TYPE_RE, :REL_RE
+
+      module_function
+
+      ##
+      # Probes candidates lazily and returns the first feedish URL.
+      #
+      # @param page_url [String, Html2rss::Url] page or origin URL
+      # @param request_session [Html2rss::RequestSession] shared session for follow-ups
+      # @param parsed_body [Nokogiri::HTML::Document, nil] parsed HTML when available
+      # @param html [String, nil] raw HTML when a document is unavailable
+      # @param extra_paths [Array<String>] additional path guesses
+      # @param max_probes [Integer, nil] max candidate GETs (nil = uncapped)
+      # @return [Html2rss::Url, nil]
+      # rubocop:disable Metrics/ParameterLists -- discovery kwargs stay co-located
+      def best_feed_url(page_url:, request_session:, parsed_body: nil, html: nil, extra_paths: [],
+                        max_probes: nil)
+        best_feed_response(
+          page_url:, request_session:, parsed_body:, html:, extra_paths:, max_probes:
+        )&.url
+      end
+      # rubocop:enable Metrics/ParameterLists
+
+      ##
+      # Like {#best_feed_url} but returns the validated syndication response for parsing.
+      #
+      # @param page_url [String, Html2rss::Url]
+      # @param request_session [Html2rss::RequestSession]
+      # @param parsed_body [Nokogiri::HTML::Document, nil]
+      # @param html [String, nil]
+      # @param extra_paths [Array<String>]
+      # @param max_probes [Integer, nil] max candidate GETs (nil = uncapped)
+      # @return [Html2rss::RequestService::Response, nil]
+      # rubocop:disable Metrics/ParameterLists, Metrics/MethodLength -- discovery kwargs stay co-located
+      def best_feed_response(page_url:, request_session:, parsed_body: nil, html: nil, extra_paths: [],
+                             max_probes: nil)
+        page = Html2rss::Url.from_absolute(page_url)
+        candidates = candidate_urls(page_url: page, parsed_body:, html:, extra_paths:)
+        Log.debug(
+          "Syndication::Discovery: host=#{page.host} candidate_count=#{candidates.size}"
+        )
+
+        first_feedish_response(
+          candidates, request_session:, origin_url: page, max_probes:
+        ).tap do |response|
+          next unless response
+
+          Log.info("Syndication::Discovery: host=#{page.host} selected_feed_url=#{response.url}")
+        end
+      end
+      # rubocop:enable Metrics/ParameterLists, Metrics/MethodLength
+
+      ##
+      # Ordered candidate feed URLs (head alternates first, then path guesses).
+      #
+      # @param page_url [String, Html2rss::Url]
+      # @param parsed_body [Nokogiri::HTML::Document, nil]
+      # @param html [String, nil]
+      # @param extra_paths [Array<String>]
+      # @return [Array<Html2rss::Url>]
+      def candidate_urls(page_url:, parsed_body: nil, html: nil, extra_paths: [])
+        page = Html2rss::Url.from_absolute(page_url)
+        (alternate_feed_urls(page, parsed_body:, html:) +
+          path_guess_urls(page, extra_paths:)).uniq
+      end
+
+      ##
+      # @param response [Html2rss::RequestService::Response, nil]
+      # @return [Boolean]
+      def feedish?(response)
+        return false unless response
+        return false unless successful_status?(response)
+
+        response.feed_response?
+      end
+
+      def successful_status?(response)
+        status = response.status
+        status.nil? || status.between?(200, 299)
+      end
+      module_function :successful_status?
+      private_class_method :successful_status?
+
+      ##
+      # Directory-relative feed path guesses for a page URL.
+      #
+      # @param page_url [String, Html2rss::Url]
+      # @return [Array<String>] relative paths
+      def page_dir_paths(page_url)
+        path = Html2rss::Url.from_absolute(page_url).path
+        return [] if path.nil? || path.empty? || path == '/'
+
+        dir = path.sub(%r{/[^/]*$}, '/')
+        ["#{dir}feed", "#{dir}rss.xml", "#{dir}atom.xml"]
+      end
+
+      def first_feedish_response(urls, request_session:, origin_url:, max_probes: nil)
+        scoped = max_probes.nil? ? urls.lazy : urls.lazy.take(max_probes)
+        scoped.filter_map do |url|
+          response = probe(url, request_session:, origin_url:)
+          response if feedish?(response)
+        end.first
+      end
+      module_function :first_feedish_response
+      private_class_method :first_feedish_response
+
+      def first_feedish_url(urls, request_session:, origin_url:, max_probes: nil)
+        first_feedish_response(urls, request_session:, origin_url:, max_probes:)&.url
+      end
+      module_function :first_feedish_url
+      private_class_method :first_feedish_url
+
+      def probe(url, request_session:, origin_url:)
+        request_session.follow_up(url:, relation: :auto_source, origin_url:)
+      rescue Html2rss::Error => error
+        Log.debug(
+          "Syndication::Discovery: probe skipped host=#{origin_url.host} " \
+          "(#{error.class})"
+        )
+        nil
+      end
+      module_function :probe
+      private_class_method :probe
+
+      def alternate_feed_urls(page, parsed_body:, html:)
+        from_document = feed_links_from_document(parsed_body, page)
+        return from_document if from_document.any?
+
+        alternate_feed_hrefs(html.to_s, page)
+      end
+      module_function :alternate_feed_urls
+      private_class_method :alternate_feed_urls
+
+      def feed_links_from_document(parsed_body, page)
+        return [] unless parsed_body.is_a?(Nokogiri::HTML::Document)
+
+        Html::FeedLink.from_document(parsed_body).filter_map do |link|
+          absolute_url(page, link.href)
+        end
+      end
+      module_function :feed_links_from_document
+      private_class_method :feed_links_from_document
+
+      def path_guess_urls(page, extra_paths:)
+        paths = (page_dir_paths(page) + DEFAULT_PATHS + Array(extra_paths)).uniq
+        paths.filter_map { |path| absolute_url(page, path) }
+      end
+      module_function :path_guess_urls
+      private_class_method :path_guess_urls
+
+      def alternate_feed_hrefs(html, page)
+        return [] if html.nil? || html.empty?
+
+        html.scan(LINK_TAG_RE).filter_map { |tag| href_from_alternate_link(tag, page) }.uniq
+      end
+      module_function :alternate_feed_hrefs
+      private_class_method :alternate_feed_hrefs
+
+      def href_from_alternate_link(tag, page)
+        return unless alternate_rel?(tag)
+
+        href = tag[HREF_RE, 1]
+        return if href.nil? || href.empty?
+        return unless syndication_type?(tag[TYPE_RE, 1]) || feed_like_href?(href)
+
+        absolute_url(page, href.strip)
+      end
+      module_function :href_from_alternate_link
+      private_class_method :href_from_alternate_link
+
+      def alternate_rel?(tag)
+        tag[REL_RE, 1].to_s.downcase.split(/\s+/).include?('alternate')
+      end
+      module_function :alternate_rel?
+      private_class_method :alternate_rel?
+
+      def syndication_type?(type)
+        return false if type.nil? || type.empty?
+
+        t = type.downcase
+        t.include?('rss+xml') || t.include?('atom+xml') || t.include?('rss') || t.include?('atom')
+      end
+      module_function :syndication_type?
+      private_class_method :syndication_type?
+
+      def feed_like_href?(href)
+        href.match?(/rss|atom|feed|\.xml/i)
+      end
+      module_function :feed_like_href?
+      private_class_method :feed_like_href?
+
+      def absolute_url(base, path)
+        Html2rss::Url.from_relative(path, base)
+      rescue ArgumentError
+        nil
+      end
+      module_function :absolute_url
+      private_class_method :absolute_url
+    end
+  end
+end

data/lib/html2rss/syndication/parser.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+require 'rss'
+
+module Html2rss
+  module Syndication
+    ##
+    # Parses RSS 2.0 and Atom documents into AutoSource article hashes.
+    module Parser
+      module_function
+
+      ##
+      # @param response [Html2rss::RequestService::Response]
+      # @return [Array<Hash{Symbol => Object}>] article hashes (may be empty)
+      # @raise [ArgumentError] when the response is not a syndication feed
+      def parse_response(response)
+        raise ArgumentError, 'response is not a syndication feed' unless response.feed_response?
+
+        parse(response.body, base_url: response.url)
+      end
+
+      ##
+      # @param body [String] raw RSS/Atom document
+      # @param base_url [String, Html2rss::Url, nil] fallback base for relative item links
+      # @return [Array<Hash{Symbol => Object}>]
+      def parse(body, base_url: nil)
+        feed = RSS::Parser.parse(body.to_s, false)
+        return [] unless feed
+
+        items_from(feed, base_url:).filter_map { |item| article_hash(item, base_url:) }
+      rescue RSS::Error => error
+        Log.warn("Syndication::Parser: failed to parse feed (#{error.class}: #{error.message})")
+        []
+      end
+
+      def items_from(feed, base_url:) # rubocop:disable Lint/UnusedMethodArgument -- base reserved for callers
+        case feed
+        when RSS::Rss then Array(feed.items)
+        when RSS::Atom::Feed then Array(feed.entries)
+        else []
+        end
+      end
+      module_function :items_from
+      private_class_method :items_from
+
+      def article_hash(item, base_url:)
+        case item
+        when RSS::Rss::Channel::Item then rss_item_hash(item, base_url:)
+        when RSS::Atom::Entry, RSS::Atom::Feed::Entry then atom_entry_hash(item, base_url:)
+        end
+      end
+      module_function :article_hash
+      private_class_method :article_hash
+
+      def rss_item_hash(item, base_url:) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength -- RSS item mapping
+        url = absolute_url(item.link, base_url:)
+        return unless url
+
+        {
+          id: present(item.guid&.content) || url.to_s,
+          title: present(item.title),
+          description: present(item.description),
+          url:,
+          published_at: item.date || item.pubDate,
+          author: present(item.author),
+          categories: Array(item.categories).filter_map { |cat| present(cat.content) }
+        }.compact
+      end
+      module_function :rss_item_hash
+      private_class_method :rss_item_hash
+
+      def atom_entry_hash(entry, base_url:) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity -- Atom entry mapping
+        url = atom_link(entry, base_url:)
+        return unless url
+
+        {
+          id: present(entry.id&.content) || url.to_s,
+          title: present(atom_text(entry.title)),
+          description: present(atom_text(entry.content) || atom_text(entry.summary)),
+          url:,
+          published_at: entry.published&.content || entry.updated&.content,
+          author: present(atom_author(entry)),
+          categories: Array(entry.categories).filter_map { |cat| present(cat.term) }
+        }.compact
+      end
+      module_function :atom_entry_hash
+      private_class_method :atom_entry_hash
+
+      def atom_link(entry, base_url:)
+        links = Array(entry.links)
+        link = links.find { |node| node.rel.nil? || node.rel == 'alternate' } || links.first
+        absolute_url(link&.href, base_url:)
+      end
+      module_function :atom_link
+      private_class_method :atom_link
+
+      def atom_text(node)
+        return if node.nil?
+
+        node.respond_to?(:content) ? node.content : node.to_s
+      end
+      module_function :atom_text
+      private_class_method :atom_text
+
+      def atom_author(entry)
+        author = Array(entry.author).first
+        return unless author
+
+        present(author.name&.content) || present(author.email&.content)
+      end
+      module_function :atom_author
+      private_class_method :atom_author
+
+      def absolute_url(raw, base_url:)
+        value = present(raw)
+        return unless value
+
+        if base_url
+          Html2rss::Url.from_relative(value, base_url)
+        else
+          Html2rss::Url.from_absolute(value)
+        end
+      rescue ArgumentError
+        nil
+      end
+      module_function :absolute_url
+      private_class_method :absolute_url
+
+      def present(value)
+        text = value.to_s.strip
+        text unless text.empty?
+      end
+      module_function :present
+      private_class_method :present
+    end
+  end
+end

data/lib/html2rss/syndication.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module Html2rss
+  ##
+  # Native RSS/Atom discovery and parse (not HTML AutoSource ownership).
+  #
+  # {include:file:lib/html2rss/syndication/README.md}
+  module Syndication
+  end
+end

data/lib/html2rss/url.rb

@@ -281,6 +281,18 @@ module Html2rss
     def ==(other) = other.is_a?(Url) && to_s == other.to_s
     alias eql? ==
 
+    ##
+    # Whether +other+ names the same document (trailing slash and fragment ignored).
+    # Differs from {#==}: `/blog` and `/blog/` match; query strings must still match.
+    #
+    # @param other [Object]
+    # @return [Boolean]
+    def same_document?(other)
+      return false unless other.is_a?(Url)
+
+      document_identity == other.document_identity
+    end
+
     ##
     # Returns the hash code for this URL.
     #
@@ -293,6 +305,16 @@ module Html2rss
     # @return [String] the debug representation
     def inspect = "#<#{self.class}:#{object_id} @uri=#{@uri.inspect}>"
 
+    protected
+
+    # Scheme, authority, slash-stripped path, and query (fragment omitted).
+    #
+    # @return [String]
+    def document_identity
+      uri = @uri.omit(:fragment)
+      "#{uri.scheme}\0#{uri.authority}\0#{uri.path.to_s.chomp('/')}\0#{uri.query}"
+    end
+
     private
 
     def ip_address_host?

data/lib/html2rss/version.rb

@@ -2,6 +2,6 @@
 
 module Html2rss
   # Current application version.
-  VERSION = '0.27.2'
+  VERSION = '0.28.0'
   public_constant :VERSION
 end

data/schema/html2rss-config.schema.json

@@ -157,9 +157,40 @@
           },
           "exclusiveMinimum": 0
         },
+        "entry_resolution": {
+          "type": "object",
+          "properties": {
+            "enabled": {
+              "type": "boolean",
+              "not": {
+                "type": "null"
+              }
+            },
+            "max_probes": {
+              "type": "integer",
+              "not": {
+                "type": "null"
+              },
+              "exclusiveMinimum": 0
+            }
+          },
+          "required": []
+        },
         "scraper": {
           "type": "object",
           "properties": {
+            "native_feed": {
+              "type": "object",
+              "properties": {
+                "enabled": {
+                  "type": "boolean",
+                  "not": {
+                    "type": "null"
+                  }
+                }
+              },
+              "required": []
+            },
             "wordpress_api": {
               "type": "object",
               "properties": {
@@ -338,7 +369,14 @@
       "required": [],
       "default": {
         "limit": 25,
+        "entry_resolution": {
+          "enabled": true,
+          "max_probes": 5
+        },
         "scraper": {
+          "native_feed": {
+            "enabled": true
+          },
           "wordpress_api": {
             "enabled": true
           },

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: html2rss
 version: !ruby/object:Gem::Version
-  version: 0.27.2
+  version: 0.28.0
 platform: ruby
 authors:
 - Gil Desmarais
@@ -331,6 +331,7 @@ files:
 - lib/html2rss/auto_source/scraper/meta_oembed.rb
 - lib/html2rss/auto_source/scraper/microdata.rb
 - lib/html2rss/auto_source/scraper/microformats2.rb
+- lib/html2rss/auto_source/scraper/native_feed.rb
 - lib/html2rss/auto_source/scraper/schema.rb
 - lib/html2rss/auto_source/scraper/schema/category_extractor.rb
 - lib/html2rss/auto_source/scraper/schema/item_list.rb
@@ -376,6 +377,14 @@ files:
 - lib/html2rss/feed_pipeline/auto_fallback.rb
 - lib/html2rss/feed_pipeline/runtime_policy.rb
 - lib/html2rss/feed_pipeline/strategy_plan.rb
+- lib/html2rss/feed_resolution.rb
+- lib/html2rss/feed_resolution/README.md
+- lib/html2rss/feed_resolution/candidate_generator.rb
+- lib/html2rss/feed_resolution/diag.rb
+- lib/html2rss/feed_resolution/options.rb
+- lib/html2rss/feed_resolution/policy.rb
+- lib/html2rss/feed_resolution/probe.rb
+- lib/html2rss/feed_resolution/scorer.rb
 - lib/html2rss/feed_result.rb
 - lib/html2rss/hash_util.rb
 - lib/html2rss/html.rb
@@ -416,6 +425,7 @@ files:
 - lib/html2rss/mcp/inspect.rb
 - lib/html2rss/mcp/outcome.rb
 - lib/html2rss/mcp/server.rb
+- lib/html2rss/page_recon.rb
 - lib/html2rss/request_service.rb
 - lib/html2rss/request_service/blocked_surface.rb
 - lib/html2rss/request_service/botasaurus_contract.rb
@@ -446,6 +456,7 @@ files:
 - lib/html2rss/scoring/observation.rb
 - lib/html2rss/scoring/ranked_segment.rb
 - lib/html2rss/scoring/score.rb
+- lib/html2rss/scrape_target.rb
 - lib/html2rss/selectors.rb
 - lib/html2rss/selectors/extractors.rb
 - lib/html2rss/selectors/extractors/attribute.rb
@@ -477,6 +488,12 @@ files:
 - lib/html2rss/sst/tags.rb
 - lib/html2rss/sst/text.rb
 - lib/html2rss/status.rb
+- lib/html2rss/surface_category.rb
+- lib/html2rss/syndication.rb
+- lib/html2rss/syndication/README.md
+- lib/html2rss/syndication/candidate_catalog.rb
+- lib/html2rss/syndication/discovery.rb
+- lib/html2rss/syndication/parser.rb
 - lib/html2rss/url.rb
 - lib/html2rss/version.rb
 - lib/tasks/config_schema.rake
@@ -486,7 +503,7 @@ licenses:
 - MIT
 metadata:
   allowed_push_host: https://rubygems.org
-  changelog_uri: https://github.com/html2rss/html2rss/releases/tag/v0.27.2
+  changelog_uri: https://github.com/html2rss/html2rss/releases/tag/v0.28.0
   rubygems_mfa_required: 'true'
 rdoc_options: []
 require_paths: