html2rss 0.32.4 → 0.34.0

data/lib/html2rss/sst/text.rb

@@ -4,78 +4,114 @@ module Html2rss
   module SST
     ##
     # Visible-text extraction for SST nodes (port of Navigator::TextExtractor).
+    #
+    # Block children separate by a newline (or the given separator); other neighbours by one space,
+    # except where an inline element embedded in direct text touches it with no whitespace between them
+    # in the source: +Straßenbah<span>n</span> für+ stays one word. Two adjacent elements, and an element
+    # opening or closing its parent's text (+<span>Live</span>Headline+), always separate, since sites
+    # lay out such spans as blocks with CSS.
     module Text
-      module_function
-
-      ##
-      # @param node [Node]
-      # @param separator [String]
-      # @param exclude [Array<Node>, nil]
-      # @return [String, nil]
-      def extract(node, separator: ' ', exclude: nil) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength
-        raise ArgumentError, 'node is required' unless node
-
-        excluded = exclude&.each_with_object({}.compare_by_identity) { |n, h| h[n] = true }
-        if node.children.empty?
-          text = node.own_text.to_s.gsub(/\s+/, ' ').strip
-          return text.empty? ? nil : text
+      # One piece of a node's text: direct text or one child's text, whether it is a block, and whether it
+      # glues to the piece before it (no whitespace between them in the source).
+      Fragment = Data.define(:text, :block, :glue)
+      private_constant :Fragment
+
+      class << self
+        ##
+        # @param node [Node]
+        # @param separator [String]
+        # @param exclude [Array<Node>, nil]
+        # @return [String, nil]
+        def extract(node, separator: ' ', exclude: nil)
+          raise InvalidError, 'node is required' unless node
+
+          excluded = exclude&.each_with_object({}.compare_by_identity) { |n, h| h[n] = true }
+          join_fragments(fragments(node, separator, excluded), separator)
         end
 
-        parts = collect_parts(node, separator, excluded)
-        own = node.own_text.to_s.gsub(/\s+/, ' ').strip
-        parts = [own, *parts] unless own.empty?
-        return if parts.empty?
+        private
 
-        parts.join.squeeze(' ').strip
-      end
+        # Ordered fragments: lead text, then each shown child followed by the text after it.
+        def fragments(node, separator, excluded)
+          raws = [node.lead_text, *node.tails]
+          pieces = [direct_fragment(raws.first, false)]
+          node.children.each_with_index do |child, index|
+            next pieces << direct_fragment(raws[index + 1], false) unless shown?(child, excluded)
 
-      def collect_parts(node, separator, excluded)
-        last_block = false
-        node.children.each_with_object([]) do |child, parts|
-          next if excluded&.[](child)
-          next unless visible_child?(child)
+            pieces.push(*shown_fragments(child, raws, index, separator, excluded))
+          end
+          pieces
+        end
 
-          text, block = child_text_and_block(child, separator, excluded)
-          next if text.empty?
+        # A shown child and the direct text after it. Only a child embedded in direct text (text on both
+        # sides) glues to it, where they meet with no whitespace between them: a leading or trailing span
+        # beside text (+<span>Live</span>Headline+) is a styled label more often than part of a word.
+        def shown_fragments(child, raws, index, separator, excluded)
+          before = raws[index].to_s
+          after = raws[index + 1].to_s
+          embedded = !before.empty? && !after.empty?
+          [child_fragment(child, separator, excluded, embedded && before.match?(/\S\z/) && opens_word?(child)),
+           direct_fragment(after, embedded && closes_word?(child) && after.match?(/\A\S/))]
+        end
 
-          append_separator!(parts, separator, block, last_block)
-          parts << text
-          last_block = block
+        # Whether the raw text of +node+ starts with a non-space, descending into its first child while it
+        # has no lead text.
+        def opens_word?(node)
+          first = node.children.first
+          node.lead_text.empty? && first ? opens_word?(first) : node.lead_text.match?(/\A\S/)
         end
-      end
-      module_function :collect_parts
-      private_class_method :collect_parts
-
-      def child_text_and_block(child, separator, excluded)
-        text = if child.children.empty?
-                 child.own_text.gsub(/\s+/, ' ').strip
-               else
-                 extract(child, separator:, exclude: excluded&.keys).to_s.strip
-               end
-        text = "- #{text}" if child.name == :li && !text.empty?
-        [text, Tags::BLOCK_NAMES.include?(child.name)]
-      end
-      module_function :child_text_and_block
-      private_class_method :child_text_and_block
 
-      def append_separator!(parts, separator, is_block, last_was_block)
-        return if parts.empty?
+        # Whether the raw text of +node+ ends with a non-space, descending into its last child while no
+        # text follows it.
+        def closes_word?(node)
+          raw = node.tails.last || node.own_text
+          last = node.children.last
+          raw.empty? && last ? closes_word?(last) : raw.match?(/\S\z/)
+        end
+
+        def join_fragments(pieces, separator)
+          last_block = nil
+          parts = pieces.each_with_object([]) do |piece, out|
+            next if piece.text.empty?
+
+            out << gap(separator, piece, last_block) unless last_block.nil?
+            out << piece.text
+            last_block = piece.block
+          end
+          parts.join unless parts.empty?
+        end
+
+        def gap(separator, piece, last_block)
+          return (separator == ' ' ? "\n" : separator) if piece.block || last_block
+
+          piece.glue ? '' : ' '
+        end
+
+        def shown?(child, excluded) = !excluded&.[](child) && visible_child?(child)
+
+        def squish(text) = text.to_s.gsub(/\s+/, ' ').strip
+
+        def direct_fragment(raw, glue) = Fragment.new(squish(raw), false, glue)
 
-        parts << if is_block || last_was_block
-                   (separator == ' ' ? "\n" : separator)
+        def child_fragment(child, separator, excluded, glue)
+          Fragment.new(child_text(child, separator, excluded), Tags::BLOCK_NAMES.include?(child.name), glue)
+        end
+
+        # A leaf's text is read directly (no fragments to join), the hot path of every text read.
+        def child_text(child, separator, excluded)
+          text = if child.children.empty?
+                   squish(child.own_text)
                  else
-                   ' '
+                   extract(child, separator:, exclude: excluded&.keys).to_s
                  end
-      end
-      module_function :append_separator!
-      private_class_method :append_separator!
+          child.name == :li && !text.empty? ? "- #{text}" : text
+        end
 
-      def visible_child?(node)
-        !Tags::INVISIBLE_NAMES.include?(node.name) &&
-          !(node.name == :a && node.attrs.href&.start_with?('#'))
+        def visible_child?(node)
+          !Tags::INVISIBLE_NAMES.include?(node.name) &&
+            !(node.name == :a && node.attrs.href&.start_with?('#'))
+        end
       end
-      module_function :visible_child?
-      private_class_method :visible_child?
     end
   end
 end

data/lib/html2rss/sst.rb

@@ -4,5 +4,7 @@ module Html2rss
   ##
   # Simplified Semantic Tree namespace for the heuristic auto-source pipeline.
   module SST
+    # Raised when a tree node, attrs, text or document is built from invalid parts.
+    class InvalidError < Html2rss::ContractError; end
   end
 end

data/lib/html2rss/status.rb

@@ -13,6 +13,13 @@ module Html2rss
   # Stable telemetry payload for cross-repo consumers (e.g. html2rss-web observability).
   # Tallies and counters are validated and frozen at construction (including Marshal load).
   class Status
+    # Raised when telemetry fields fail validation at construction.
+    class InvalidError < Html2rss::ContractError; end
+
+    # Keys kept in {#to_h} even when zero or empty.
+    ALWAYS_REPORTED = %i[version dedup_dropped].freeze
+    private_constant :ALWAYS_REPORTED
+
     class << self
       ##
       # Builds status from extracted articles and scrape telemetry.
@@ -71,9 +78,9 @@ module Html2rss
       version:, scraper_tallies:, dedup_dropped:, selected_strategy: nil, attempt_count: 0,
       strategy_attempts: [], admission_drops: {}, entry_url: nil, scrape_url: nil, entry_resolution: nil
     )
-      dedup = Integer(dedup_dropped)
-      attempts = Integer(attempt_count)
-      validate_counters!(dedup:, attempts:, selected_strategy:)
+      dedup = Option.integer(:dedup_dropped, dedup_dropped, min: 0)
+      attempts = Option.integer(:attempt_count, attempt_count, min: 0)
+      validate_selected_strategy!(selected_strategy, attempts)
 
       super(
         version: version.to_s.dup.freeze,
@@ -93,20 +100,8 @@ module Html2rss
     # Observability hash for web (+scraper_status+). Omits empty/absent optional keys.
     #
     # @return [Hash{Symbol => Object}]
-    # rubocop:disable-next Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity -- omit-empty optional keys stay explicit
     def to_h
-      {
-        version:,
-        dedup_dropped:,
-        **(scraper_tallies.any? ? { scraper_tallies: } : {}),
-        **(selected_strategy.nil? ? {} : { selected_strategy: }),
-        **(attempt_count.positive? ? { attempt_count: } : {}),
-        **(strategy_attempts.any? ? { strategy_attempts: } : {}),
-        **(admission_drops.any? ? { admission_drops: } : {}),
-        **(entry_url ? { entry_url: } : {}),
-        **(scrape_url ? { scrape_url: } : {}),
-        **(entry_resolution ? { entry_resolution: } : {})
-      }
+      super.reject { |key, value| !ALWAYS_REPORTED.include?(key) && omit?(value) }
     end
 
     ##
@@ -124,6 +119,14 @@ module Html2rss
 
     private
 
+    def omit?(value)
+      case value
+      when nil then true
+      when Integer then value.zero?
+      else value.empty?
+      end
+    end
+
     def marshal_dump
       [version, scraper_tallies, dedup_dropped, selected_strategy, attempt_count, strategy_attempts,
        admission_drops, entry_url, scrape_url, entry_resolution]
@@ -133,16 +136,16 @@ module Html2rss
       version, scraper_tallies, dedup_dropped, selected_strategy, attempt_count,
         strategy_attempts, admission_drops, entry_url, scrape_url, entry_resolution = payload
       initialize(version:, scraper_tallies:, dedup_dropped:, selected_strategy:, attempt_count:,
-                 strategy_attempts:, admission_drops: admission_drops || {}, entry_url:, scrape_url:,
+                 strategy_attempts:, admission_drops:, entry_url:, scrape_url:,
                  entry_resolution:)
     end
 
     def freeze_tallies(tallies)
-      tallies.to_h.transform_keys(&:to_s).transform_values { |count| Integer(count) }.freeze
+      tallies.to_h { |name, count| [name.to_s, count] }.freeze
     end
 
     def freeze_attempts(attempts)
-      Array(attempts).map { |attempt| attempt.to_h.freeze }.freeze
+      attempts.map { |attempt| attempt.dup.freeze }.freeze
     end
 
     def freeze_optional_string(value)
@@ -157,21 +160,14 @@ module Html2rss
       case value
       when Hash, FeedResolution::Diag then value.to_h
       else
-        raise ArgumentError, "entry_resolution must be a Hash or FeedResolution::Diag, got #{value.class}"
+        raise InvalidError, "entry_resolution must be a Hash or FeedResolution::Diag, got #{value.class}"
       end.transform_keys(&:to_sym).freeze
     end
 
-    def validate_counters!(dedup:, attempts:, selected_strategy:)
-      raise ArgumentError, 'dedup_dropped must be >= 0' if dedup.negative?
-      raise ArgumentError, 'attempt_count must be >= 0' if attempts.negative?
-
-      unless selected_strategy.nil? || selected_strategy.is_a?(Symbol)
-        raise ArgumentError, 'selected_strategy must be a Symbol or nil'
-      end
-
-      return unless selected_strategy && attempts < 1
-
-      raise ArgumentError, 'attempt_count must be >= 1 when selected_strategy is set'
+    def validate_selected_strategy!(selected_strategy, attempts)
+      return if selected_strategy.nil?
+      raise InvalidError, 'selected_strategy must be a Symbol or nil' unless selected_strategy.is_a?(Symbol)
+      raise InvalidError, 'attempt_count must be >= 1 when selected_strategy is set' if attempts < 1
     end
   end
 end

data/lib/html2rss/surface_category.rb

@@ -20,7 +20,7 @@ module Html2rss
     ##
     # @param name [Symbol, String]
     # @return [SurfaceCategory]
-    def self.[](name) = ALL.fetch(name.to_sym) { raise ArgumentError, "unknown surface category: #{name.inspect}" }
+    def self.[](name) = ALL.fetch(name.to_sym) { raise InvalidOption, "unknown surface category: #{name.inspect}" }
 
     ##
     # @return [Boolean]

data/lib/html2rss/syndication/README.md

@@ -1,13 +1,17 @@
 # Syndication
 
-Native RSS/Atom discovery and parse for auto-source promotion and direct feed URLs.
+Native RSS/Atom/RDF discovery and parse for auto-source promotion and direct feed URLs.
 
 ## Ownership
 
 | Concern                                                          | Owner                                                                                              |
 | ---------------------------------------------------------------- | -------------------------------------------------------------------------------------------------- |
-| Head `rel=alternate` + common path probes + status-gated feedish | `Syndication::Discovery`                                                                           |
-| RSS 2.0 / Atom → article hashes                                  | `Syndication::Parser`                                                                              |
+| Head `rel=alternate` (audio-hinted first on a podcast page, after the path guesses elsewhere) + common path probes + status-gated feedish | `Syndication::Discovery`                                      |
+| Native feed of a fetched page for Recon, Capture and Test        | `Syndication::Discovery.native_feed_url` (one entry point: the page response they already hold, at the URL a redirect landed on) |
+| Cross-origin permission for a page's advertised feed             | `RequestService::Policy` (proof: `RequestService::FeedAdvertisement`)                              |
+| RSS 2.0 / RSS 1.0 (RDF) / Atom → `Syndication::Feed` (article hashes + channel facts); item URL: an http(s) link, else an http(s) guid, else the first audio/video file | `Syndication::Parser` |
+| Item media files and image (enclosure, Media RSS, iTunes) → item | `Syndication::ItemMedia` (called by `Parser`; http(s) only via `Url.resolve_web`; sizes, durations and player pages via `Article::Enclosure`) |
+| Channel facts (author, `itunes:category`, `itunes:explicit`) → `Feed#channel` | `Syndication::ChannelFacts` (called by `Parser` on the same parse; precedence over page meta is `Channel`'s) |
 | Strict head-only link parse (no path guessing)                   | `Html::FeedLink`                                                                                   |
 | Response Content-Type / body feed sniff                          | `RequestService::Response#feed_response?` (sole sniff owner; Discovery only adds HTTP status gate) |
 
@@ -15,7 +19,9 @@ Native RSS/Atom discovery and parse for auto-source promotion and direct feed UR
 
 - Entry-URL tournament / listing resolution (`FeedResolution`)
 - Page recon / surface classification (`PageRecon`)
-- Feed channel metadata for output (`Channel` / `FeedBuilder`)
-- Ranking or quality scoring of competing feeds beyond first feedish hit
+- Feed channel metadata for output and its precedence (`Channel` / `FeedBuilder`); Syndication only reads what a feed states
+- Ranking or quality scoring of competing feeds beyond the audio-hint order and first feedish hit
 
-Discovery stops at the first same-origin feedish URL (lazy probe). Path guessing mirrors the legacy configs `probe_rss` script; that script should eventually thin-wrap this module.
+Discovery stops at the first feedish URL (lazy probe). Head alternates are fetched as `:advertised_feed` with the page's `FeedAdvertisement`, so a feed on another host (a podcast platform) is followed. An advertised `http:` feed off the page origin is fetched as `https:` or skipped. Path guesses stay same-origin. Alternates keep document order, except on a podcast page: `podcast`, `episode` or `audio` (word-ish, plural allowed) in its URL host, path or `<title>`, an `og:type` of `music.*` or `video.*`, or a JSON-LD `@type` `PodcastSeries`, `PodcastEpisode` or `AudioObject` anywhere (regex on the script text, no JSON parse). There, an alternate whose title or URL hints audio (the same words plus `mp3`, `m4a`, `aac`, `opus`) goes first. On any other page those rank after the same-origin path guesses, and other alternates still precede them. A news front page that advertises only its podcast ships its page items or a same-origin feed, not the episodes; the podcast feed is its last candidate.
+
+An item without a link takes its guid (Atom `id`, RDF `rdf:about`) when that is an absolute http(s) URL, whatever `isPermaLink` says, else its first audio or video file. Relative, `urn:` and `tag:` ids name no page. An item with none of these is skipped. Links, files and images resolve against the feed URL through `Url.resolve_web`, so a `javascript:`, `data:` or `ftp:` one counts as absent. A followed feed's items may link to the feed's own domain as well as the page's (AutoSource `Cleanup`, which owns that rule). Path guessing mirrors the legacy configs `probe_rss` script; that script should eventually thin-wrap this module.

data/lib/html2rss/syndication/alternate_links.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Syndication
+    ##
+    # Scans raw HTML for +<link rel="alternate">+ hints that point at RSS/Atom feeds.
+    module AlternateLinks
+      LINK_TAG_RE = /<link\b[^>]*>/i
+      HREF_RE = /\bhref\s*=\s*["']([^"']+)["']/i
+      TYPE_RE = /\btype\s*=\s*["']([^"']+)["']/i
+      REL_RE = /\brel\s*=\s*["']([^"']+)["']/i
+      private_constant :LINK_TAG_RE, :HREF_RE, :TYPE_RE, :REL_RE
+
+      class << self
+        ##
+        # @param html [String] raw HTML
+        # @return [Array<String>] hrefs of alternate links that look like feeds
+        def hrefs(html)
+          html.scan(LINK_TAG_RE).filter_map { |tag| feed_href(tag) }
+        end
+
+        private
+
+        def feed_href(tag)
+          return unless tag[REL_RE, 1].to_s.downcase.split(/\s+/).include?('alternate')
+
+          href = tag[HREF_RE, 1]
+          return if href.nil? || href.empty?
+
+          href.strip if syndication_type?(tag[TYPE_RE, 1]) || href.match?(/rss|atom|feed|\.xml/i)
+        end
+
+        def syndication_type?(type)
+          type.to_s.downcase.match?(/rss|atom|rdf/)
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/syndication/channel_facts.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+require 'rss'
+
+module Html2rss
+  module Syndication
+    ChannelFacts = Data.define(:author, :categories, :explicit)
+
+    ##
+    # What a native feed states about its show or publication, for {Html2rss::Channel}: the author
+    # name, the iTunes categories and the explicit flag. A fact the feed does not state stays unknown
+    # (+nil+, no categories); nothing is guessed.
+    #
+    # RSS 2.0: the author is +itunes:author+, else the name part of +managingEditor+
+    # (+editor@example.com (Name)+ gives +Name+, a plain name is kept, a bare email address is no name);
+    # each +itunes:category+ is its +text+ followed by its nested subcategory texts; +itunes:explicit+
+    # +true+, +yes+ or +explicit+ is true, +false+, +no+ or +clean+ false, any other value unstated.
+    # Atom: the author is the feed +author+ name. RSS 1.0 (RDF) states none of these.
+    #
+    # @!attribute [r] author
+    #   @return [String, nil] the channel author name
+    # @!attribute [r] categories
+    #   @return [Array<Array<String>>] each category followed by its subcategories
+    # @!attribute [r] explicit
+    #   @return [Boolean, nil] whether the feed declares explicit content; nil when unstated
+    class ChannelFacts
+      # +itunes:explicit+ values, lowercased, to the flag they state.
+      EXPLICIT = {
+        'true' => true, 'yes' => true, 'explicit' => true,
+        'false' => false, 'no' => false, 'clean' => false
+      }.freeze
+      # A +managingEditor+ written as an email address followed by the name in parentheses.
+      NAMED_EMAIL = /\A\s*[^\s@]+@\S+\s+\((.+)\)\s*\z/
+
+      class << self
+        ##
+        # @param feed [RSS::Rss, RSS::RDF, RSS::Atom::Feed, nil] a document the +rss+ library parsed
+        # @return [ChannelFacts, nil] the facts an RSS 2.0 or Atom feed states; nil for any other document
+        def from(feed)
+          case feed
+          when RSS::Rss then from_rss(feed.channel)
+          when RSS::Atom::Feed then new(author: feed.author&.name&.content)
+          end
+        end
+
+        private
+
+        def from_rss(channel)
+          new(author: rss_author(channel),
+              categories: channel.itunes_categories.map { [_1.text, *_1.itunes_categories.map(&:text)] },
+              explicit: EXPLICIT[channel.itunes_explicit.to_s.strip.downcase])
+        end
+
+        # Blank and padded values are normalized by {#initialize}.
+        def rss_author(channel)
+          author = channel.itunes_author.to_s
+          return author if author.match?(/\S/)
+
+          editor = channel.managingEditor.to_s
+          editor[NAMED_EMAIL, 1] || (editor unless editor.include?('@'))
+        end
+      end
+
+      ##
+      # @param author [String, nil] blank becomes nil
+      # @param categories [Array<Array<String, nil>>] a category without a name is dropped, blank
+      #   subcategories are skipped
+      # @param explicit [Boolean, nil]
+      def initialize(author: nil, categories: [], explicit: nil)
+        super(author: present(author), categories: categories.filter_map { path(_1) }.freeze, explicit:)
+      end
+
+      private
+
+      def path((name, *subcategories))
+        return unless (name = present(name))
+
+        [name, *subcategories.filter_map { present(_1) }].freeze
+      end
+
+      def present(value)
+        text = value.to_s.strip
+        text.freeze unless text.empty?
+      end
+    end
+  end
+end