html2rss 0.33.0 → 0.34.0

data/lib/html2rss/sst/tags.rb

@@ -11,16 +11,127 @@ module Html2rss
       BLOCK_NAMES = Set[:p, :div, :li, :ul, :ol, :h1, :h2, :h3, :h4, :h5, :h6, :tr, :br].freeze
       # Names skipped when collecting visible text.
       INVISIBLE_NAMES = Set[:svg, :script, :noscript, :style, :template].freeze
-      # Landmark names treated as utility chrome.
+      # Utility landmark names: peripheral unless chrome (see {.chrome_scope}).
       UTILITY_LANDMARK_NAMES = Set[:nav, :aside, :footer, :menu].freeze
       # Container names ignored as article roots.
-      IGNORED_CONTAINER_NAMES = Set[:nav, :footer, :header, :svg, :script, :style].freeze
+      IGNORED_CONTAINER_NAMES = Set[:nav, :footer, :header, :dialog, :svg, :script, :style].freeze
+      # Ignored containers that are chrome only at page level (ARIA banner/contentinfo scoping).
+      PAGE_CHROME_NAMES = Set[:header, :footer].freeze
+      # Ancestors that scope a header/footer to their own section instead of the page.
+      SECTIONING_NAMES = Set[:article, :section, :aside, :main].freeze
       # Names excluded from class/structure clustering.
       CLUSTER_EXCLUDED_NAMES = Set[:html, :body, :nav, :footer, :header, :svg, :script, :style].freeze
       # Href prefixes that never count as content destinations.
       SKIP_HREF_PREFIXES = ['#', 'javascript:', 'mailto:', 'tel:', 'file://', 'sms:', 'data:'].freeze
       # Layout-ish names used when resolving nested wrapper groups.
       LAYOUT_NAMES = Set[:div, :section, :article, :li, :ul, :ol].freeze
+
+      # header/footer/nav: chrome at page level, their parent's scope inside sectioning content.
+      PAGE_CHROME = PAGE_CHROME_NAMES.to_set(&:to_s).freeze
+      # +noscript+ too: the SST keeps one only for its file links (enclosures), never as content anchors.
+      ALWAYS_CHROME = ((IGNORED_CONTAINER_NAMES.to_set(&:to_s) - PAGE_CHROME - Set['nav']) | Set['noscript']).freeze
+      UTILITY_LANDMARK = UTILITY_LANDMARK_NAMES.to_set(&:to_s).freeze
+      SECTIONING = SECTIONING_NAMES.to_set(&:to_s).freeze
+      # ARIA roles whose subtree is chrome anywhere (+search+ is not: it may wrap the result list).
+      CHROME_ROLES = Set['navigation', 'menu', 'menubar', 'dialog', 'alertdialog'].freeze
+      # ARIA landmark roles that, like header/footer, are chrome only at page level.
+      PAGE_CHROME_ROLES = Set['banner', 'contentinfo'].freeze
+      # Utility landmark roles: the chrome roles plus +complementary+ (the aside role).
+      UTILITY_ROLES = (CHROME_ROLES | Set['complementary']).freeze
+      # A nav or list labelled (aria-label) or classed as pagination is chrome in any scope.
+      PAGER_NAMES = Set['nav', 'ul', 'ol'].freeze
+      # Pagination nouns ("Results pagination", "paginación"), never +paginated+.
+      PAGER_LABEL = /\bpagina(?:tion|tie|ción|cion|zione|cja|ção|cao)(?!\p{L})|\bpaginering\b|\bpaginierung\b|
+                     \bpager\b|\bpaging\b|\bpage\snavigation\b/ix
+      # nav: a class-token part (+pagination+, +gsc-pagination+, +pager__list+), not +paginated-list+.
+      PAGER_CLASS = /(?:\A|\s|[-_])(?:pagination|pager|paging)(?:\z|\s|[-_])/i
+      # ul/ol: only a whole class token (+infinite-pager+ or +js-pagination-container+ is a feed list).
+      PAGER_TOKENS = Set['pagination', 'pager', 'paging'].freeze
+      PAGER_RELS = Set['next', 'prev', 'previous'].freeze
+      # A nav linking rel=next/prev is a pager.
+      PAGER_LINK_XPATH = PAGER_RELS.map { "contains(concat(' ', normalize-space(@rel), ' '), ' #{_1} ')" }
+                                   .join(' or ').then { ".//a[#{_1}]" }.freeze
+      # Page-navigation link shapes: a page number, an arrow, prev/next wording, or a page query/path.
+      PAGE_LINK_TEXT = /\A(?:\d{1,4}|[«‹›»←→<>]+|\W*(?:prev(?:ious)?|next|older|newer|first|last)\b.*)\z/im
+      PAGE_LINK_HREF = %r{[?&](?:page|paged|pg|p)=\d+|/page/\d+}i
+      private_constant :PAGE_CHROME, :ALWAYS_CHROME, :UTILITY_LANDMARK, :SECTIONING, :CHROME_ROLES,
+                       :PAGE_CHROME_ROLES, :UTILITY_ROLES, :PAGER_NAMES, :PAGER_LABEL, :PAGER_CLASS,
+                       :PAGER_TOKENS, :PAGER_RELS, :PAGER_LINK_XPATH, :PAGE_LINK_TEXT, :PAGE_LINK_HREF
+
+      ##
+      # The single chrome-scope rule, folded down the tree by both the SST normalizer
+      # and the Nokogiri ancestor walk. +:chrome+ sticks: dialog, noscript and the ARIA navigation,
+      # menu and dialog roles are chrome; header/footer/nav and the banner/contentinfo roles
+      # are chrome only at +:page+ level; a pager (a nav labelled or classed as pagination or
+      # linking rel=next/prev, or a list labelled or whole-token classed as pagination whose
+      # links mostly look like page navigation) is chrome in any scope. Otherwise a
+      # utility landmark ({.utility_landmark?}) opens a sticky +:peripheral+ scope, and
+      # article/section/main (or role=main) open +:sectioned+, so card headers and in-content
+      # nav lists stay out of chrome.
+      #
+      # @param element [Nokogiri::XML::Element] element being entered
+      # @param scope [Symbol] parent scope: +:page+, +:sectioned+, +:peripheral+, or +:chrome+
+      # @return [Symbol] scope the element's subtree inherits
+      def self.chrome_scope(element, scope)
+        tag = Html::Probe.tag(element)
+        roles = role_tokens(element['role'])
+        return :chrome if scope == :chrome || chrome?(element, tag, roles, scope)
+        return :peripheral if scope == :peripheral || utility?(tag, roles)
+
+        SECTIONING.include?(tag) || roles.include?('main') ? :sectioned : scope
+      end
+
+      ##
+      # Utility landmark: nav/aside/footer/menu, or a navigation, menu, dialog or
+      # complementary role. Peripheral content when not chrome; never an article card.
+      #
+      # @param tag [String] folded tag name
+      # @param role [String, nil] raw +role+ attribute
+      # @return [Boolean]
+      def self.utility_landmark?(tag, role) = utility?(tag, role_tokens(role))
+
+      ##
+      # Chrome only at page level: header/footer, or the banner/contentinfo role.
+      #
+      # @param tag [String] folded tag name
+      # @param role [String, nil] raw +role+ attribute
+      # @return [Boolean]
+      def self.page_level_chrome?(tag, role) = page_level?(tag, role_tokens(role))
+
+      def self.role_tokens(role) = role ? role.downcase.split : []
+
+      def self.utility?(tag, roles) = UTILITY_LANDMARK.include?(tag) || roles.any? { UTILITY_ROLES.include?(_1) }
+
+      def self.page_level?(tag, roles) = PAGE_CHROME.include?(tag) || roles.any? { PAGE_CHROME_ROLES.include?(_1) }
+
+      def self.chrome?(element, tag, roles, scope)
+        return true if ALWAYS_CHROME.include?(tag) || roles.any? { CHROME_ROLES.include?(_1) }
+
+        (scope == :page && (tag == 'nav' || page_level?(tag, roles))) || pager?(element, tag)
+      end
+
+      def self.pager?(element, tag)
+        return false unless PAGER_NAMES.include?(tag)
+        return list_pager?(element) unless tag == 'nav'
+        return true if element['aria-label'].to_s.match?(PAGER_LABEL) || element['class'].to_s.match?(PAGER_CLASS)
+
+        !element.at_xpath(PAGER_LINK_XPATH).nil?
+      end
+
+      # A list is a pager only when labelled or whole-token classed as one and at least
+      # half of its links look like page navigation.
+      def self.list_pager?(element)
+        named = element['aria-label'].to_s.match?(PAGER_LABEL) ||
+                element['class'].to_s.downcase.split.any? { PAGER_TOKENS.include?(_1) }
+        links = named ? element.css('a') : []
+        !links.empty? && links.count { page_link?(_1) } * 2 >= links.size
+      end
+
+      def self.page_link?(link)
+        link['rel'].to_s.downcase.split.any? { PAGER_RELS.include?(_1) } ||
+          link['href'].to_s.match?(PAGE_LINK_HREF) || link.text.strip.match?(PAGE_LINK_TEXT)
+      end
+      private_class_method :role_tokens, :utility?, :page_level?, :chrome?, :pager?, :list_pager?, :page_link?
     end
   end
 end

data/lib/html2rss/sst/text.rb

@@ -4,7 +4,18 @@ module Html2rss
   module SST
     ##
     # Visible-text extraction for SST nodes (port of Navigator::TextExtractor).
+    #
+    # Block children separate by a newline (or the given separator); other neighbours by one space,
+    # except where an inline element embedded in direct text touches it with no whitespace between them
+    # in the source: +Straßenbah<span>n</span> für+ stays one word. Two adjacent elements, and an element
+    # opening or closing its parent's text (+<span>Live</span>Headline+), always separate, since sites
+    # lay out such spans as blocks with CSS.
     module Text
+      # One piece of a node's text: direct text or one child's text, whether it is a block, and whether it
+      # glues to the piece before it (no whitespace between them in the source).
+      Fragment = Data.define(:text, :block, :glue)
+      private_constant :Fragment
+
       class << self
         ##
         # @param node [Node]
@@ -12,7 +23,7 @@ module Html2rss
         # @param exclude [Array<Node>, nil]
         # @return [String, nil]
         def extract(node, separator: ' ', exclude: nil)
-          raise ArgumentError, 'node is required' unless node
+          raise InvalidError, 'node is required' unless node
 
           excluded = exclude&.each_with_object({}.compare_by_identity) { |n, h| h[n] = true }
           join_fragments(fragments(node, separator, excluded), separator)
@@ -20,51 +31,80 @@ module Html2rss
 
         private
 
-        # Ordered [text, block?] pairs: lead text, then each child followed by the text after it.
+        # Ordered fragments: lead text, then each shown child followed by the text after it.
         def fragments(node, separator, excluded)
-          pairs = [[squish(node.lead_text), false]]
-          node.children.zip(node.tails).each do |child, tail|
-            pairs << child_text_and_block(child, separator, excluded) if shown?(child, excluded)
-            pairs << [squish(tail), false]
+          raws = [node.lead_text, *node.tails]
+          pieces = [direct_fragment(raws.first, false)]
+          node.children.each_with_index do |child, index|
+            next pieces << direct_fragment(raws[index + 1], false) unless shown?(child, excluded)
+
+            pieces.push(*shown_fragments(child, raws, index, separator, excluded))
           end
-          pairs
+          pieces
         end
 
-        def join_fragments(pairs, separator)
-          parts = []
-          last_block = false
-          pairs.each do |text, block|
-            next if text.empty?
+        # A shown child and the direct text after it. Only a child embedded in direct text (text on both
+        # sides) glues to it, where they meet with no whitespace between them: a leading or trailing span
+        # beside text (+<span>Live</span>Headline+) is a styled label more often than part of a word.
+        def shown_fragments(child, raws, index, separator, excluded)
+          before = raws[index].to_s
+          after = raws[index + 1].to_s
+          embedded = !before.empty? && !after.empty?
+          [child_fragment(child, separator, excluded, embedded && before.match?(/\S\z/) && opens_word?(child)),
+           direct_fragment(after, embedded && closes_word?(child) && after.match?(/\A\S/))]
+        end
+
+        # Whether the raw text of +node+ starts with a non-space, descending into its first child while it
+        # has no lead text.
+        def opens_word?(node)
+          first = node.children.first
+          node.lead_text.empty? && first ? opens_word?(first) : node.lead_text.match?(/\A\S/)
+        end
 
-            append_separator!(parts, separator, block, last_block)
-            parts << text
-            last_block = block
+        # Whether the raw text of +node+ ends with a non-space, descending into its last child while no
+        # text follows it.
+        def closes_word?(node)
+          raw = node.tails.last || node.own_text
+          last = node.children.last
+          raw.empty? && last ? closes_word?(last) : raw.match?(/\S\z/)
+        end
+
+        def join_fragments(pieces, separator)
+          last_block = nil
+          parts = pieces.each_with_object([]) do |piece, out|
+            next if piece.text.empty?
+
+            out << gap(separator, piece, last_block) unless last_block.nil?
+            out << piece.text
+            last_block = piece.block
           end
           parts.join unless parts.empty?
         end
 
+        def gap(separator, piece, last_block)
+          return (separator == ' ' ? "\n" : separator) if piece.block || last_block
+
+          piece.glue ? '' : ' '
+        end
+
         def shown?(child, excluded) = !excluded&.[](child) && visible_child?(child)
 
         def squish(text) = text.to_s.gsub(/\s+/, ' ').strip
 
-        def child_text_and_block(child, separator, excluded)
+        def direct_fragment(raw, glue) = Fragment.new(squish(raw), false, glue)
+
+        def child_fragment(child, separator, excluded, glue)
+          Fragment.new(child_text(child, separator, excluded), Tags::BLOCK_NAMES.include?(child.name), glue)
+        end
+
+        # A leaf's text is read directly (no fragments to join), the hot path of every text read.
+        def child_text(child, separator, excluded)
           text = if child.children.empty?
                    squish(child.own_text)
                  else
-                   extract(child, separator:, exclude: excluded&.keys).to_s.strip
+                   extract(child, separator:, exclude: excluded&.keys).to_s
                  end
-          text = "- #{text}" if child.name == :li && !text.empty?
-          [text, Tags::BLOCK_NAMES.include?(child.name)]
-        end
-
-        def append_separator!(parts, separator, is_block, last_was_block)
-          return if parts.empty?
-
-          parts << if is_block || last_was_block
-                     (separator == ' ' ? "\n" : separator)
-                   else
-                     ' '
-                   end
+          child.name == :li && !text.empty? ? "- #{text}" : text
         end
 
         def visible_child?(node)

data/lib/html2rss/sst.rb

@@ -4,5 +4,7 @@ module Html2rss
   ##
   # Simplified Semantic Tree namespace for the heuristic auto-source pipeline.
   module SST
+    # Raised when a tree node, attrs, text or document is built from invalid parts.
+    class InvalidError < Html2rss::ContractError; end
   end
 end

data/lib/html2rss/status.rb

@@ -13,6 +13,9 @@ module Html2rss
   # Stable telemetry payload for cross-repo consumers (e.g. html2rss-web observability).
   # Tallies and counters are validated and frozen at construction (including Marshal load).
   class Status
+    # Raised when telemetry fields fail validation at construction.
+    class InvalidError < Html2rss::ContractError; end
+
     # Keys kept in {#to_h} even when zero or empty.
     ALWAYS_REPORTED = %i[version dedup_dropped].freeze
     private_constant :ALWAYS_REPORTED
@@ -157,14 +160,14 @@ module Html2rss
       case value
       when Hash, FeedResolution::Diag then value.to_h
       else
-        raise ArgumentError, "entry_resolution must be a Hash or FeedResolution::Diag, got #{value.class}"
+        raise InvalidError, "entry_resolution must be a Hash or FeedResolution::Diag, got #{value.class}"
       end.transform_keys(&:to_sym).freeze
     end
 
     def validate_selected_strategy!(selected_strategy, attempts)
       return if selected_strategy.nil?
-      raise ArgumentError, 'selected_strategy must be a Symbol or nil' unless selected_strategy.is_a?(Symbol)
-      raise ArgumentError, 'attempt_count must be >= 1 when selected_strategy is set' if attempts < 1
+      raise InvalidError, 'selected_strategy must be a Symbol or nil' unless selected_strategy.is_a?(Symbol)
+      raise InvalidError, 'attempt_count must be >= 1 when selected_strategy is set' if attempts < 1
     end
   end
 end

data/lib/html2rss/surface_category.rb

@@ -20,7 +20,7 @@ module Html2rss
     ##
     # @param name [Symbol, String]
     # @return [SurfaceCategory]
-    def self.[](name) = ALL.fetch(name.to_sym) { raise ArgumentError, "unknown surface category: #{name.inspect}" }
+    def self.[](name) = ALL.fetch(name.to_sym) { raise InvalidOption, "unknown surface category: #{name.inspect}" }
 
     ##
     # @return [Boolean]

data/lib/html2rss/syndication/README.md

@@ -1,13 +1,17 @@
 # Syndication
 
-Native RSS/Atom discovery and parse for auto-source promotion and direct feed URLs.
+Native RSS/Atom/RDF discovery and parse for auto-source promotion and direct feed URLs.
 
 ## Ownership
 
 | Concern                                                          | Owner                                                                                              |
 | ---------------------------------------------------------------- | -------------------------------------------------------------------------------------------------- |
-| Head `rel=alternate` + common path probes + status-gated feedish | `Syndication::Discovery`                                                                           |
-| RSS 2.0 / Atom → article hashes                                  | `Syndication::Parser`                                                                              |
+| Head `rel=alternate` (audio-hinted first on a podcast page, after the path guesses elsewhere) + common path probes + status-gated feedish | `Syndication::Discovery`                                      |
+| Native feed of a fetched page for Recon, Capture and Test        | `Syndication::Discovery.native_feed_url` (one entry point: the page response they already hold, at the URL a redirect landed on) |
+| Cross-origin permission for a page's advertised feed             | `RequestService::Policy` (proof: `RequestService::FeedAdvertisement`)                              |
+| RSS 2.0 / RSS 1.0 (RDF) / Atom → `Syndication::Feed` (article hashes + channel facts); item URL: an http(s) link, else an http(s) guid, else the first audio/video file | `Syndication::Parser` |
+| Item media files and image (enclosure, Media RSS, iTunes) → item | `Syndication::ItemMedia` (called by `Parser`; http(s) only via `Url.resolve_web`; sizes, durations and player pages via `Article::Enclosure`) |
+| Channel facts (author, `itunes:category`, `itunes:explicit`) → `Feed#channel` | `Syndication::ChannelFacts` (called by `Parser` on the same parse; precedence over page meta is `Channel`'s) |
 | Strict head-only link parse (no path guessing)                   | `Html::FeedLink`                                                                                   |
 | Response Content-Type / body feed sniff                          | `RequestService::Response#feed_response?` (sole sniff owner; Discovery only adds HTTP status gate) |
 
@@ -15,7 +19,9 @@ Native RSS/Atom discovery and parse for auto-source promotion and direct feed UR
 
 - Entry-URL tournament / listing resolution (`FeedResolution`)
 - Page recon / surface classification (`PageRecon`)
-- Feed channel metadata for output (`Channel` / `FeedBuilder`)
-- Ranking or quality scoring of competing feeds beyond first feedish hit
+- Feed channel metadata for output and its precedence (`Channel` / `FeedBuilder`); Syndication only reads what a feed states
+- Ranking or quality scoring of competing feeds beyond the audio-hint order and first feedish hit
 
-Discovery stops at the first same-origin feedish URL (lazy probe). Path guessing mirrors the legacy configs `probe_rss` script; that script should eventually thin-wrap this module.
+Discovery stops at the first feedish URL (lazy probe). Head alternates are fetched as `:advertised_feed` with the page's `FeedAdvertisement`, so a feed on another host (a podcast platform) is followed. An advertised `http:` feed off the page origin is fetched as `https:` or skipped. Path guesses stay same-origin. Alternates keep document order, except on a podcast page: `podcast`, `episode` or `audio` (word-ish, plural allowed) in its URL host, path or `<title>`, an `og:type` of `music.*` or `video.*`, or a JSON-LD `@type` `PodcastSeries`, `PodcastEpisode` or `AudioObject` anywhere (regex on the script text, no JSON parse). There, an alternate whose title or URL hints audio (the same words plus `mp3`, `m4a`, `aac`, `opus`) goes first. On any other page those rank after the same-origin path guesses, and other alternates still precede them. A news front page that advertises only its podcast ships its page items or a same-origin feed, not the episodes; the podcast feed is its last candidate.
+
+An item without a link takes its guid (Atom `id`, RDF `rdf:about`) when that is an absolute http(s) URL, whatever `isPermaLink` says, else its first audio or video file. Relative, `urn:` and `tag:` ids name no page. An item with none of these is skipped. Links, files and images resolve against the feed URL through `Url.resolve_web`, so a `javascript:`, `data:` or `ftp:` one counts as absent. A followed feed's items may link to the feed's own domain as well as the page's (AutoSource `Cleanup`, which owns that rule). Path guessing mirrors the legacy configs `probe_rss` script; that script should eventually thin-wrap this module.

data/lib/html2rss/syndication/alternate_links.rb

@@ -31,7 +31,7 @@ module Html2rss
         end
 
         def syndication_type?(type)
-          type.to_s.downcase.match?(/rss|atom/)
+          type.to_s.downcase.match?(/rss|atom|rdf/)
         end
       end
     end

data/lib/html2rss/syndication/channel_facts.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+require 'rss'
+
+module Html2rss
+  module Syndication
+    ChannelFacts = Data.define(:author, :categories, :explicit)
+
+    ##
+    # What a native feed states about its show or publication, for {Html2rss::Channel}: the author
+    # name, the iTunes categories and the explicit flag. A fact the feed does not state stays unknown
+    # (+nil+, no categories); nothing is guessed.
+    #
+    # RSS 2.0: the author is +itunes:author+, else the name part of +managingEditor+
+    # (+editor@example.com (Name)+ gives +Name+, a plain name is kept, a bare email address is no name);
+    # each +itunes:category+ is its +text+ followed by its nested subcategory texts; +itunes:explicit+
+    # +true+, +yes+ or +explicit+ is true, +false+, +no+ or +clean+ false, any other value unstated.
+    # Atom: the author is the feed +author+ name. RSS 1.0 (RDF) states none of these.
+    #
+    # @!attribute [r] author
+    #   @return [String, nil] the channel author name
+    # @!attribute [r] categories
+    #   @return [Array<Array<String>>] each category followed by its subcategories
+    # @!attribute [r] explicit
+    #   @return [Boolean, nil] whether the feed declares explicit content; nil when unstated
+    class ChannelFacts
+      # +itunes:explicit+ values, lowercased, to the flag they state.
+      EXPLICIT = {
+        'true' => true, 'yes' => true, 'explicit' => true,
+        'false' => false, 'no' => false, 'clean' => false
+      }.freeze
+      # A +managingEditor+ written as an email address followed by the name in parentheses.
+      NAMED_EMAIL = /\A\s*[^\s@]+@\S+\s+\((.+)\)\s*\z/
+
+      class << self
+        ##
+        # @param feed [RSS::Rss, RSS::RDF, RSS::Atom::Feed, nil] a document the +rss+ library parsed
+        # @return [ChannelFacts, nil] the facts an RSS 2.0 or Atom feed states; nil for any other document
+        def from(feed)
+          case feed
+          when RSS::Rss then from_rss(feed.channel)
+          when RSS::Atom::Feed then new(author: feed.author&.name&.content)
+          end
+        end
+
+        private
+
+        def from_rss(channel)
+          new(author: rss_author(channel),
+              categories: channel.itunes_categories.map { [_1.text, *_1.itunes_categories.map(&:text)] },
+              explicit: EXPLICIT[channel.itunes_explicit.to_s.strip.downcase])
+        end
+
+        # Blank and padded values are normalized by {#initialize}.
+        def rss_author(channel)
+          author = channel.itunes_author.to_s
+          return author if author.match?(/\S/)
+
+          editor = channel.managingEditor.to_s
+          editor[NAMED_EMAIL, 1] || (editor unless editor.include?('@'))
+        end
+      end
+
+      ##
+      # @param author [String, nil] blank becomes nil
+      # @param categories [Array<Array<String, nil>>] a category without a name is dropped, blank
+      #   subcategories are skipped
+      # @param explicit [Boolean, nil]
+      def initialize(author: nil, categories: [], explicit: nil)
+        super(author: present(author), categories: categories.filter_map { path(_1) }.freeze, explicit:)
+      end
+
+      private
+
+      def path((name, *subcategories))
+        return unless (name = present(name))
+
+        [name, *subcategories.filter_map { present(_1) }].freeze
+      end
+
+      def present(value)
+        text = value.to_s.strip
+        text.freeze unless text.empty?
+      end
+    end
+  end
+end

data/lib/html2rss/syndication/discovery.rb

@@ -3,12 +3,34 @@
 module Html2rss
   module Syndication
     ##
-    # Finds a same-origin RSS/Atom URL for a page via head alternates and path guesses.
+    # Finds an RSS/Atom URL for a page via head alternates and same-origin path guesses.
     #
     # Ports configs +probe_rss+ path/alternate logic onto {RequestSession#follow_up}.
-    module Discovery
+    # Head alternates keep document order within two groups: those whose title or URL
+    # hints audio ({AUDIO_HINT}) and the rest. On a page that shows podcast signals
+    # ({PODCAST_PAGE_HINT} in its URL host, path or +<title>+, an +og:type+ matching {MEDIA_OG_TYPE},
+    # or a {PODCAST_SCHEMA_TYPE} in its JSON-LD) the order is audio-hinted alternates, other
+    # alternates, path guesses. On any other page it is other alternates, path guesses, then the
+    # audio-hinted alternates, so a news page that advertises only a podcast feed does not become
+    # a media feed. This is the one ordering owner: {.best_feed_url}, {.candidate_urls} and the
+    # feed-resolution feed slot all read it. Alternates are fetched as +:advertised_feed+ with the
+    # page's {RequestService::FeedAdvertisement}, so a feed the page advertises on another host
+    # is followed (see {RequestService::Policy}). Path guesses stay same-origin.
+    module Discovery # rubocop:disable Metrics/ModuleLength -- discovery ordering + probe stay on one owner
       # Common feed path suffixes probed after head +rel=alternate+ hints.
       DEFAULT_PATHS = CandidateCatalog::FEED_PATHS
+      # Words marking a podcast page in its URL host, path or title; {AUDIO_HINT} adds audio file formats.
+      PODCAST_WORDS = %w[podcast episode audio].freeze
+      # Word-ish podcast hint in a page's URL host, path or +<title>+ (plural allowed, case-insensitive).
+      PODCAST_PAGE_HINT = /(?<![[:alpha:]])(?:#{PODCAST_WORDS.join('|')})s?(?![[:alpha:]])/i
+      # Word-ish audio hint in an alternate's title or URL (Podlove-style per-format podcast feeds).
+      AUDIO_HINT = /(?<![[:alpha:]])(?:#{(PODCAST_WORDS + %w[mp3 m4a aac opus]).join('|')})s?(?![[:alpha:]])/i
+      # +og:type+ values (+music.song+, +video.episode+, ...) marking a podcast or media page.
+      MEDIA_OG_TYPE = /\A\s*(?:music|video)\./i
+      # A JSON-LD +@type+ naming podcast content (short name or schema.org URL, alone or in an
+      # array), matched on the script text anywhere in the page without parsing the JSON.
+      PODCAST_SCHEMA_TYPE = %r{"@type"\s*:\s*(?:\[\s*(?:"[^"]*"\s*,\s*)*)?
+                               "(?:https?://schema\.org/)?(?:PodcastSeries|PodcastEpisode|AudioObject)"}ix
 
       class << self
         ##
@@ -24,9 +46,23 @@ module Html2rss
         # rubocop:disable-next Metrics/ParameterLists -- discovery kwargs stay co-located
         def best_feed_url(page_url:, request_session:, parsed_body: nil, html: nil, extra_paths: [],
                           max_probes: nil)
-          best_feed_response(
-            page_url:, request_session:, parsed_body:, html:, extra_paths:, max_probes:
-          )&.url
+          best_feed_response(page_url:, request_session:, parsed_body:, html:, extra_paths:, max_probes:)&.url
+        end
+
+        ##
+        # The native feed of a page already fetched: the one entry point for the curation gates
+        # (Recon, Capture, Test), so each reads the page's head alternates (cross-origin only
+        # through its {RequestService::FeedAdvertisement}) before same-origin path guesses. The page
+        # is the response's own URL, where a redirect landed, so relative alternates and the origin of
+        # the follow-ups are the fetched page's, never the entry URL's.
+        #
+        # @param response [Html2rss::RequestService::Response] the fetched page; a non-HTML
+        #   response contributes its raw body only
+        # @param request_session [Html2rss::RequestSession] session for the probes
+        # @return [Html2rss::Url, nil]
+        def native_feed_url(response:, request_session:)
+          parsed_body = response.parsed_body if response.html_response?
+          best_feed_url(page_url: response.url, request_session:, parsed_body:, html: response.body)
         end
 
         ##
@@ -43,13 +79,13 @@ module Html2rss
         def best_feed_response(page_url:, request_session:, parsed_body: nil, html: nil, extra_paths: [],
                                max_probes: nil)
           page = Html2rss::Url.from_absolute(page_url)
-          candidates = candidate_urls(page_url: page, parsed_body:, html:, extra_paths:)
-          Log.debug(
-            "Syndication::Discovery: host=#{page.host} candidate_count=#{candidates.size}"
-          )
+          doc = document(parsed_body)
+          advertisement = RequestService::FeedAdvertisement.from_document(doc, page_url: page)
+          candidates = candidates(page, advertisement, doc:, html:, extra_paths:)
+          Log.debug("Syndication::Discovery: host=#{page.host} candidate_count=#{candidates.size}")
 
           first_feedish_response(
-            candidates, request_session:, origin_url: page, max_probes:
+            candidates, request_session:, origin_url: page, advertisement:, max_probes:
           ).tap do |response|
             next unless response
 
@@ -62,7 +98,8 @@ module Html2rss
         end
 
         ##
-        # Ordered candidate feed URLs (head alternates first, then path guesses).
+        # Ordered candidate feed URLs (head alternates, then path guesses; on a page without podcast
+        # signals the audio-hinted alternates come after the guesses).
         #
         # @param page_url [String, Html2rss::Url]
         # @param parsed_body [Nokogiri::HTML::Document, nil]
@@ -71,8 +108,9 @@ module Html2rss
         # @return [Array<Html2rss::Url>]
         def candidate_urls(page_url:, parsed_body: nil, html: nil, extra_paths: [])
           page = Html2rss::Url.from_absolute(page_url)
-          (alternate_feed_urls(page, parsed_body:, html:) +
-            path_guess_urls(page, extra_paths:)).uniq
+          doc = document(parsed_body)
+          advertisement = RequestService::FeedAdvertisement.from_document(doc, page_url: page)
+          candidates(page, advertisement, doc:, html:, extra_paths:)
         end
 
         ##
@@ -100,37 +138,53 @@ module Html2rss
 
         private
 
-        def successful_status?(response)
-          status = response.status
-          status.nil? || status.between?(200, 299)
-        end
+        def successful_status?(response) = response.status.nil? || response.status.between?(200, 299)
 
-        def first_feedish_response(urls, request_session:, origin_url:, max_probes: nil)
+        def first_feedish_response(urls, request_session:, origin_url:, advertisement:, max_probes: nil)
           scoped = max_probes.nil? ? urls.lazy : urls.lazy.take(max_probes)
           scoped.filter_map do |url|
-            response = probe(url, request_session:, origin_url:)
+            response = probe(url, request_session:, origin_url:, advertisement:)
             response if feedish?(response)
           end.first
         end
 
-        def probe(url, request_session:, origin_url:)
-          request_session.follow_up(url:, relation: :auto_source, origin_url:)
+        def probe(url, request_session:, origin_url:, advertisement:)
+          if advertisement.advertises?(url)
+            request_session.follow_up(url:, relation: :advertised_feed, origin_url:, advertisement:)
+          else
+            request_session.follow_up(url:, relation: :auto_source, origin_url:)
+          end
+        rescue Html2rss::ContractError
+          raise
         rescue Html2rss::Error => error
           Log.debug("Syndication::Discovery: probe skipped host=#{origin_url.host} (#{error.class})")
           nil
         end
 
-        def alternate_feed_urls(page, parsed_body:, html:)
-          from_document = feed_links_from_document(parsed_body, page)
-          return from_document if from_document.any?
+        def candidates(page, advertisement, doc:, html:, extra_paths:)
+          leading, trailing = alternate_feed_urls(page, advertisement, doc:, html:)
+          (leading + path_guess_urls(page, extra_paths:) + trailing).uniq
+        end
 
-          AlternateLinks.hrefs(html.to_s).filter_map { |href| absolute_url(page, href) }.uniq
+        # An HTML page without a parsed document still gets an (empty) advertisement.
+        def document(parsed_body)
+          parsed_body.is_a?(Nokogiri::HTML::Document) ? parsed_body : Nokogiri::HTML::Document.new
         end
 
-        def feed_links_from_document(parsed_body, page)
-          return [] unless parsed_body.is_a?(Nokogiri::HTML::Document)
+        # @return [Array(Array, Array)] alternates ranked before the path guesses, and those after them
+        def alternate_feed_urls(page, advertisement, doc:, html:)
+          feeds = advertisement.feeds
+          return [AlternateLinks.hrefs(html.to_s).filter_map { |href| absolute_url(page, href) }, []] if feeds.empty?
+
+          audio, other = feeds.keys.partition { |url| AUDIO_HINT.match?("#{feeds.fetch(url)} #{url}") }
+          podcast_page?(doc, page) ? [audio + other, []] : [other, audio]
+        end
 
-          Html::FeedLink.from_document(parsed_body).filter_map { |link| absolute_url(page, link.href) }
+        # The page itself looks like a podcast page: cheap signals only (no request, no JSON parse).
+        def podcast_page?(doc, page)
+          PODCAST_PAGE_HINT.match?("#{page.host}#{page.path}") || PODCAST_PAGE_HINT.match?(doc.title) ||
+            MEDIA_OG_TYPE.match?(doc.at_css('meta[property="og:type"]')&.[]('content')) ||
+            Html::Probe.scripts(doc, Html::Probe::APPLICATION_LD_JSON).any? { PODCAST_SCHEMA_TYPE.match?(_1.text) }
         end
 
         def path_guess_urls(page, extra_paths:)

data/lib/html2rss/syndication/feed.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Syndication
+    Feed = Data.define(:articles, :channel)
+
+    ##
+    # One parsed native feed ({Parser.parse}): its items as AutoSource article hashes and what it
+    # states about its channel ({ChannelFacts}).
+    #
+    # @!attribute [r] articles
+    #   @return [Array<Hash{Symbol => Object}>] article hashes (may be empty)
+    # @!attribute [r] channel
+    #   @return [ChannelFacts, nil] nil when the document is no RSS 2.0 or Atom feed or failed to parse
+    class Feed
+      # No feed read: no articles, no channel facts.
+      NONE = new(articles: [].freeze, channel: nil)
+    end
+  end
+end