html2rss 0.33.0 → 0.34.0

data/lib/html2rss/selectors/post_processors/parse_uri.rb

@@ -44,7 +44,7 @@ module Html2rss
         # @param _context [Selectors::StepEnv] post-processor context
         # @return [void]
         def self.validate_args!(value, _context)
-          raise ArgumentError, 'The `value` option is missing or empty.' if value.to_s.empty?
+          raise MissingOption, 'The `value` option is missing or empty.' if value.to_s.empty?
         end
 
         ##

data/lib/html2rss/selectors/post_processors/sanitize_html.rb

@@ -17,12 +17,14 @@ module Html2rss
       # - `referrer-policy='no-referrer'` to <img> tags
       # - wraps all <img> tags, whose direct parent is not an <a>, into an <a>
       #   linking to the <img>'s `src`.
+      # - keeps <audio>, <video> and <source> with an absolute http(s) `src`; a player
+      #   without a playable `src` (its own or a <source>'s) is removed.
       #
       # Imagine this HTML structure:
       #
       #     <section>
       #       Lorem <b>ipsum</b> dolor...
-      #       <iframe src="https://evil.corp/miner"></iframe>
+      #       <iframe src="https://evil.example/miner"></iframe>
       #       <script>alert();</script>
       #     </section>
       #
@@ -93,10 +95,20 @@ module Html2rss
           'audio' => {
             'referrerpolicy' => 'no-referrer',
             'crossorigin' => 'anonymous',
-            'preload' => 'none'
+            'preload' => 'none',
+            'controls' => 'true'
           }
         }.freeze
 
+        # Media elements kept in descriptions and the attributes they keep besides the html2rss defaults
+        # ({TAG_ATTRIBUTES}); {HtmlTransformers::KeepPlayableMedia} keeps +src+ only as an absolute
+        # +http(s)+ URL and drops a player left without one.
+        MEDIA_ATTRIBUTES = {
+          'audio' => %w[src],
+          'video' => %w[src],
+          'source' => %w[src type]
+        }.freeze
+
         # @return [Hash{Symbol => Object}] JSON Schema fragment for this post-processor
         def self.schema_export = SchemaExport.for_post_processor(name: :sanitize_html, klass: self)
 
@@ -145,24 +157,25 @@ module Html2rss
         ##
         # @param base_url [String, Html2rss::Url]
         # @return [Hash] the memoized sanitize configuration
-        # rubocop:disable-next Metrics/MethodLength, ThreadSafety/ClassInstanceVariable
+        # rubocop:disable-next Metrics/MethodLength, Metrics/AbcSize, ThreadSafety/ClassInstanceVariable
         def self.sanitize_config(base_url)
           @sanitize_configs ||= {}
           @sanitize_configs[base_url] ||= begin
             config = Sanitize::Config.merge(
               Sanitize::Config::RELAXED,
-              attributes: { all: %w[dir lang alt title translate] },
+              attributes: { all: %w[dir lang alt title translate], **MEDIA_ATTRIBUTES },
               add_attributes: TAG_ATTRIBUTES,
               transformers: [
                 lambda { |env|
                   HtmlTransformers::TransformUrlsToAbsoluteOnes.new(base_url).call(**env)
                 },
+                ->(env) { HtmlTransformers::KeepPlayableMedia.new(base_url).call(**env) },
                 ->(env) { HtmlTransformers::WrapImgInA.new.call(**env) }
               ]
             )
             config[:elements].delete('style')
             config.delete(:css)
-            config[:elements].push('audio', 'video', 'source')
+            config[:elements].push(*MEDIA_ATTRIBUTES.keys)
             config.freeze
           end
         end

data/lib/html2rss/selectors/post_processors/substring.rb

@@ -70,7 +70,7 @@ module Html2rss
           return (start..) unless step_config.key?(:end)
 
           finish = step_config[:end]
-          raise ArgumentError, 'The `start` value must be unequal to the `end` value.' if start == finish
+          raise InvalidOption, 'The `start` value must be unequal to the `end` value.' if start == finish
 
           (start..finish)
         end

data/lib/html2rss/selectors.rb

@@ -35,6 +35,10 @@ module Html2rss
     # Config selector keys that map onto a different {Article} attribute.
     # +:enclosure+ stays singular in YAML; Article stores +:enclosures+.
     SELECTOR_TO_ARTICLE_KEY = { enclosure: :enclosures }.freeze
+    # Admission-drop reason tallied for selector items that carry no URL (same shape as
+    # {AutoSource#admission_drops}).
+    MISSING_URL_REASON = 'missing_url'
+
     # Selector keys that may be copied onto an Article (PROVIDED_KEYS + mapped aliases).
     SELECTABLE_SELECTOR_KEYS = Set[*(Html2rss::Article::PROVIDED_KEYS + SELECTOR_TO_ARTICLE_KEY.keys)].freeze
     # ArticleExtractor keys that enhance can fill. Skip the extractor when all are already present.
@@ -56,6 +60,7 @@ module Html2rss
       @time_zone = time_zone
       @rss_item_attributes = @selectors.keys.select { SELECTABLE_SELECTOR_KEYS.include?(_1) }
       @parsed_body = nil
+      @unlinked_count = 0
       @compiled_fields = compile_selector_fields
       @compiled_item_fields = @rss_item_attributes.filter_map { @compiled_fields[_1] }
     end
@@ -70,39 +75,52 @@ module Html2rss
     end
 
     ##
-    # Iterates over each scraped article.
+    # Drop tallies for the last iteration: items skipped for having no URL, as {AutoSource#admission_drops}.
+    #
+    # @return [Hash{String => Integer}] empty when every item was linked
+    def admission_drops
+      @unlinked_count.positive? ? { MISSING_URL_REASON => @unlinked_count }.freeze : {}.freeze
+    end
+
+    ##
+    # Iterates over each scraped article. An item without a URL is skipped and tallied in {#admission_drops},
+    # as in auto-source ({Html2rss::Article#linked?}).
     #
     # @yield [article] Gives each article as an Html2rss::Article.
     # @return [Enumerator] An enumerator if no block is given.
     def each(&)
-      return enum_for(:each) unless block_given?
+      return enum_for unless block_given?
 
       enhance = enhance?
+      @unlinked_count = 0
 
       parsed_body.css(items_selector).each do |item|
         article_hash = extract_article(item)
 
         enhance_article_hash(article_hash, item) if enhance
 
-        yield Html2rss::Article.new(**article_hash, scraper: self.class)
+        article = build_article(article_hash)
+        article.linked? ? yield(article) : @unlinked_count += 1
       end
     end
 
     ##
     # Yields +baseline+, +enhanced+, and the item node per list entry for enhance diagnostics.
+    #
+    # @param linked_only [Boolean] skip items {#each} would drop for having no URL
     # Consumed by {Test::EnhanceAudit} so the extraction loop has one owner.
     #
     # @yieldparam baseline [Hash]
     # @yieldparam enhanced [Hash]
     # @yieldparam item [Nokogiri::XML::Element]
     # @return [Enumerator] if no block is given
-    def each_enhance_pair
-      return enum_for(:each_enhance_pair) unless block_given?
+    def each_enhance_pair(linked_only: false)
+      return enum_for(:each_enhance_pair, linked_only:) unless block_given?
 
       parsed_body.css(items_selector).each do |item|
         baseline = extract_article(item)
         enhanced = enhance_article_hash(baseline.dup, item)
-        yield baseline, enhanced, item
+        yield baseline, enhanced, item unless linked_only && !build_article(enhanced).linked?
       end
     end
 
@@ -168,6 +186,12 @@ module Html2rss
       text.empty? || text == DEFAULT_ITEMS_SELECTOR
     end
 
+    ##
+    # Whether +selectors.items.enhance+ is on: each matched item node fills fields that explicit
+    # selectors left empty (see {ENHANCEABLE_KEYS}) via {Html::ArticleExtractor}, using only the
+    # item's own markup and making no extra requests.
+    #
+    # @return [Boolean]
     def enhance? = !!@selectors.dig(ITEMS_SELECTOR_KEY, :enhance)
 
     def parsed_body
@@ -183,6 +207,8 @@ module Html2rss
       end
     end
 
+    def build_article(article_hash) = Article.new(**article_hash, scraper: self.class)
+
     def extract_article(item)
       scope = item_env_for(item, @url)
       hash = @compiled_item_fields.each_with_object({}) do |field, h|
@@ -190,13 +216,30 @@ module Html2rss
         next if value.nil?
 
         article_key = field.article_key
-        h[article_key] = article_key == :enclosures ? wrap_enclosure_value(value) : value
+        h[article_key] = article_value(article_key, value, scope)
       end
 
       hash[:url] ||= default_item_url(item) if anchor_element?(item)
       hash
     end
 
+    def article_value(article_key, value, scope)
+      case article_key
+      when :enclosures then wrap_enclosure_value(value)
+      when :url then resolve_url_value(value, scope.base_url)
+      else value
+      end
+    end
+
+    # A +url+ selector value that is a single whitespace-free string token (a +static+ value or a post-processor
+    # result) resolves against the page URL like an +href+; a value that is no http(s) reference stays as given.
+    # Text with surrounding words is left to +Article#url+'s sanitizer, which extracts the embedded URL.
+    def resolve_url_value(value, base_url)
+      return value unless value.is_a?(String) && value.match?(/\A\S+\z/)
+
+      Url.resolve_web(value, base_url) || value
+    end
+
     def enhance_article_hash(article_hash, article_tag) # rubocop:disable Metrics/MethodLength -- merge extracted keys
       return article_hash if enhanceable_fields_present?(article_hash)
 
@@ -205,7 +248,6 @@ module Html2rss
         article_tag,
         base_url: @url,
         selected_anchor:,
-        fallback_anchorless: true,
         time_zone: @time_zone
       )
       return article_hash unless extracted
@@ -392,12 +434,11 @@ module Html2rss
       Extractors.element(item, config[:selector])
     end
 
-    # @return [Hash, nil] enclosure details, or nil when the selector yields nothing.
+    # @return [Hash, nil] enclosure details, or nil when the selector yields nothing or no http(s) URL
+    #   ({Url.resolve_web})
     def enclosure(scope:, config:)
       selected = select_regular(:enclosure, scope:, config:)
-      return if selected.nil? || selected.to_s.strip.empty?
-
-      url = Url.from_relative(selected, scope.base_url)
+      return unless (url = Url.resolve_web(selected, scope.base_url))
 
       { url:, type: config[:content_type] }
     end

data/lib/html2rss/sst/attrs.rb

@@ -65,7 +65,7 @@ module Html2rss
           case class_names
           when nil then AttrDefaults::EMPTY_CLASS_NAMES
           when Array then class_names.map { |token| token.to_s.freeze }.freeze
-          else raise ArgumentError, "class_names must be an Array, got #{class_names.class}"
+          else raise InvalidError, "class_names must be an Array, got #{class_names.class}"
           end
         end
 
@@ -73,7 +73,7 @@ module Html2rss
           case raw
           when nil then AttrDefaults::EMPTY_RAW
           when Hash then raw.transform_values { |v| v.to_s.freeze }.freeze
-          else raise ArgumentError, "raw must be a Hash, got #{raw.class}"
+          else raise InvalidError, "raw must be a Hash, got #{raw.class}"
           end
         end
       end

data/lib/html2rss/sst/document.rb

@@ -13,10 +13,10 @@ module Html2rss
       # @param degraded [Boolean] true when MAX_NODES forced semantic-tag-only mode
       # @param node_count [Integer]
       # @return [Document]
-      # @raise [ArgumentError] when root/index missing
+      # @raise [InvalidError] when root/index missing
       def self.build(root:, index:, degraded: false, node_count: 0)
-        raise ArgumentError, 'root is required' unless root.is_a?(Node)
-        raise ArgumentError, 'index is required' unless index.is_a?(Index)
+        raise InvalidError, 'root is required' unless root.is_a?(Node)
+        raise InvalidError, 'index is required' unless index.is_a?(Index)
 
         new(root:, index:, degraded: !!degraded, node_count: Integer(node_count))
       end

data/lib/html2rss/sst/index.rb

@@ -14,18 +14,32 @@ module Html2rss
       # @return [Index, nil]
       def self.for_node(node) = NODE_BINDINGS[node]
 
+      ##
+      # Whether +node+ is +ancestor+ or inside it: parent pointers when +node+ is indexed, else a
+      # scan of +ancestor+'s subtree.
+      #
+      # @param node [Node]
+      # @param ancestor [Node]
+      # @return [Boolean]
+      def self.descendant_or_self?(node, ancestor)
+        return true if node.equal?(ancestor)
+
+        index = for_node(node)
+        index ? index.descendant_of?(node, ancestor) : !ancestor.find { _1.equal?(node) }.nil?
+      end
+
       ##
       # @param root [Node]
       # @param parents [Hash{Node => Node, nil}]
       # @param depths [Hash{Node => Integer}]
-      # @param ignored_chrome [Hash{Node => Boolean}]
-      def initialize(root:, parents:, depths:, ignored_chrome:)
+      # @param scopes [Hash{Node => Symbol}] {Tags.chrome_scope} of each node
+      def initialize(root:, parents:, depths:, scopes:)
         @root = root
         @parents = parents
         @depths = depths
-        @ignored_chrome = ignored_chrome
+        @scopes = scopes
         @visible_text = {}.compare_by_identity
-        @word_count = {}.compare_by_identity
+        @run_count = {}.compare_by_identity
         bind_tree(root)
       end
 
@@ -44,7 +58,14 @@ module Html2rss
       ##
       # @param node [Node]
       # @return [Boolean]
-      def ignored_chrome?(node) = @ignored_chrome.fetch(node, false)
+      def ignored_chrome?(node) = @scopes[node] == :chrome
+
+      ##
+      # Inside a utility landmark that is not chrome (aside, in-content nav, sectioned footer).
+      #
+      # @param node [Node]
+      # @return [Boolean]
+      def peripheral?(node) = @scopes[node] == :peripheral
 
       ##
       # Memoized default visible text (separator space, no excludes).
@@ -59,13 +80,11 @@ module Html2rss
 
       ##
       # @param node [Node]
-      # @return [Integer]
-      def memo_word_count(node)
-        return @word_count[node] if @word_count.key?(node)
+      # @return [Integer] memoized letter/digit runs of the default visible text ({WordCount.runs})
+      def memo_run_count(node)
+        return @run_count[node] if @run_count.key?(node)
 
-        count = 0
-        memo_visible_text(node).to_s.scan(/\p{Alnum}+/) { count += 1 }
-        @word_count[node] = count
+        @run_count[node] = WordCount.runs(memo_visible_text(node))
       end
 
       ##

data/lib/html2rss/sst/node.rb

@@ -16,7 +16,7 @@ module Html2rss
       # @param children [Array<Node>, nil]
       # @param tag_path [String, nil] index-free ancestry path for list clustering
       # @return [Node]
-      # @raise [ArgumentError] on invalid name/attrs/children
+      # @raise [InvalidError] on invalid name/attrs/children
       def self.build(name:, attrs: nil, own_text: nil, children: nil, tag_path: nil, tails: nil) # rubocop:disable Metrics/ParameterLists
         attrs_value = coerce_attrs(attrs)
         kids = coerce_children(children)
@@ -32,7 +32,7 @@ module Html2rss
       end
 
       def self.coerce_name(name)
-        raise ArgumentError, 'name is required' if name.nil? || name.to_s.strip.empty?
+        raise InvalidError, 'name is required' if name.nil? || name.to_s.strip.empty?
 
         name.to_sym
       end
@@ -40,7 +40,7 @@ module Html2rss
 
       def self.coerce_attrs(attrs)
         attrs_value = attrs.nil? ? Attrs.empty : attrs
-        raise ArgumentError, 'attrs must be SST::Attrs' unless attrs_value.is_a?(Attrs)
+        raise InvalidError, 'attrs must be SST::Attrs' unless attrs_value.is_a?(Attrs)
 
         attrs_value
       end
@@ -48,7 +48,7 @@ module Html2rss
 
       def self.coerce_children(children)
         kids = Array(children).freeze
-        raise ArgumentError, 'children must be SST::Node instances' unless kids.all?(Node)
+        raise InvalidError, 'children must be SST::Node instances' unless kids.all?(Node)
 
         kids
       end
@@ -74,11 +74,7 @@ module Html2rss
 
       ##
       # @return [Boolean]
-      def utility_landmark? = Tags::UTILITY_LANDMARK_NAMES.include?(name)
-
-      ##
-      # @return [Boolean]
-      def ignored_container_name? = Tags::IGNORED_CONTAINER_NAMES.include?(name)
+      def utility_landmark? = Tags.utility_landmark?(name.name, attrs.raw['role'])
 
       ##
       # @return [Boolean]
@@ -123,15 +119,23 @@ module Html2rss
       end
 
       ##
-      # @return [Array<Node>]
+      # @yieldparam node [Node]
+      # @return [Array<Node>] self and descendants (depth-first) for which the block is truthy
       def find_all(&)
-        each_node.select(&)
+        collect_matching([], &)
       end
 
       ##
-      # @return [Node, nil]
-      def find(&)
-        each_node.find(&)
+      # @yieldparam node [Node]
+      # @return [Node, nil] first self-or-descendant (depth-first) for which the block is truthy
+      def find(&block)
+        return self if yield(self)
+
+        children.each do |child|
+          hit = child.find(&block)
+          return hit if hit
+        end
+        nil
       end
 
       ##
@@ -149,23 +153,26 @@ module Html2rss
       end
 
       ##
-      # @return [Integer] alphanumeric word count of visible text
-      def word_count
-        if (index = Index.for_node(self))
-          return index.memo_word_count(self)
-        end
-
-        count = 0
-        visible_text.to_s.scan(/\p{Alnum}+/) { count += 1 }
-        count
+      # @return [Integer] letter/digit runs of visible text ({WordCount.runs})
+      def run_count
+        index = Index.for_node(self)
+        index ? index.memo_run_count(self) : WordCount.runs(visible_text)
       end
 
       ##
-      # @return [Float] words per descendant link
+      # @return [Float] letter/digit runs per descendant link
       def text_density
-        words = word_count
+        runs = run_count
         links = count_descendants(&:link?)
-        links.zero? ? words.to_f : words.to_f / links
+        links.zero? ? runs.to_f : runs.to_f / links
+      end
+
+      protected
+
+      def collect_matching(found, &block)
+        found << self if yield(self)
+        children.each { |child| child.collect_matching(found, &block) }
+        found
       end
     end
   end

data/lib/html2rss/sst/normalizer.rb

@@ -11,7 +11,7 @@ module Html2rss
       # Hard ceiling for SST node allocations; beyond this we degrade.
       MAX_NODES = 5_000
 
-      # Tags stripped entirely from the SST.
+      # Tags stripped entirely from the SST; a +noscript+ keeps only its file links (see {#child_nodes}).
       STRIPPED_TAGS = Set['script', 'style', 'noscript', 'iframe', 'svg', 'template'].freeze
 
       # Kept when MAX_NODES forces semantic-tag-only degrade.
@@ -29,9 +29,12 @@ module Html2rss
           href |
           src |
           srcset |
+          poster\z |
           style |
           datetime |
           itemprop |
+          rel\z |
+          hidden\z |
           role |
           type |
           category |
@@ -54,17 +57,14 @@ module Html2rss
       # Typed Attrs fields excluded from the leftover raw hash.
       TYPED_ATTR_NAMES = Set['href', 'src', 'id', 'class', 'datetime', 'itemprop', 'style', 'srcset', 'type'].freeze
 
-      # String form of Tags::IGNORED_CONTAINER_NAMES for chrome checks without to_sym.
-      IGNORED_CONTAINER_TAGS = Tags::IGNORED_CONTAINER_NAMES.to_set(&:to_s).freeze
-
       class << self
         ##
         # @param input [String, Nokogiri::HTML::Document, Nokogiri::XML::Node]
         #   HTML string or already-parsed node (String is parsed once here)
         # @return [Document, nil] +nil+ when no SST nodes survive normalization
-        # @raise [InvalidOption] when input is nil or unsupported
+        # @raise [InvalidError] when input is nil or unsupported
         def call(input)
-          raise InvalidOption, 'input is required' if input.nil?
+          raise InvalidError, 'input is required' if input.nil?
 
           new(coerce(input)).call
         end
@@ -78,7 +78,7 @@ module Html2rss
           when String then Nokogiri::HTML(input)
           when Nokogiri::XML::Node then input
           else
-            raise InvalidOption, "expected String or Nokogiri::XML::Node, got #{input.class}"
+            raise InvalidError, "expected String or Nokogiri::XML::Node, got #{input.class}"
           end
         end
       end
@@ -90,17 +90,17 @@ module Html2rss
         @degraded = false
         @parents = {}.compare_by_identity
         @depths = {}.compare_by_identity
-        @ignored_chrome = {}.compare_by_identity
+        @scopes = {}.compare_by_identity
       end
 
       ##
       # @return [Document, nil] +nil+ when no SST nodes survive normalization
       def call
         root_nk = resolve_root_nk
-        root = normalize_element(root_nk, depth: 0, path: '', chrome: false)
+        root = normalize_element(root_nk, depth: 0, path: '', scope: :page)
         return unless root
 
-        index = Index.new(root:, parents: @parents, depths: @depths, ignored_chrome: @ignored_chrome)
+        index = Index.new(root:, parents: @parents, depths: @depths, scopes: @scopes)
         Document.build(root:, index:, degraded: @degraded, node_count: @node_count)
       end
 
@@ -146,12 +146,12 @@ module Html2rss
         node.element? && !STRIPPED_TAGS.include?(Html2rss::Html::Probe.tag(node))
       end
 
-      # rubocop:disable-next Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
-      def normalize_element(nk_node, depth:, path:, chrome:)
+      # rubocop:disable-next Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength
+      def normalize_element(nk_node, depth:, path:, scope:)
         return unless nk_node.element?
 
         tag = Html::Probe.tag(nk_node)
-        return if STRIPPED_TAGS.include?(tag)
+        return unless (nk_children = child_nodes(nk_node, tag))
 
         if @node_count >= MAX_NODES && !@degraded
           @degraded = true
@@ -167,18 +167,30 @@ module Html2rss
         @node_count += 1
         tag_path = "#{path}/#{tag}"
         attrs = extract_attrs(nk_node)
-        chrome_here = chrome || IGNORED_CONTAINER_TAGS.include?(tag)
+        scope_here = Tags.chrome_scope(nk_node, scope)
 
-        children, lead, tails = normalize_children(nk_node, depth:, path: tag_path, chrome: chrome_here)
+        children, lead, tails = normalize_children(nk_children, depth:, path: tag_path, scope: scope_here)
 
         node = Node.build(name: tag, attrs:, own_text: [lead, *tails].join, children:, tag_path:, tails:)
         @depths[node] = depth
-        @ignored_chrome[node] = chrome_here
+        @scopes[node] = scope_here
         children.each { |child| @parents[child] = node }
 
         node
       end
 
+      # The Nokogiri children to normalize, or nil for a stripped tag. A +noscript+ is kept only for the
+      # file links inside it ({Html::ArticleRules::Enclosure.file_link?}), as its only children, so a
+      # card's no-script download link still names its enclosure; its text stays invisible
+      # ({Tags::INVISIBLE_NAMES}) and a +noscript+ without one is stripped.
+      def child_nodes(nk_node, tag)
+        return nk_node.children unless STRIPPED_TAGS.include?(tag)
+        return unless tag == 'noscript'
+
+        links = nk_node.xpath('.//a[@href]').select { Html::ArticleRules::Enclosure.file_link?(_1['href']) }
+        links unless links.empty?
+      end
+
       def extract_attrs(nk_node) # rubocop:disable Metrics/MethodLength
         Attrs.build(
           href: nk_node['href'],
@@ -206,13 +218,13 @@ module Html2rss
 
       # Walks child nodes once, keeping direct text in the slot of the preceding kept child
       # (slot 0 is the lead text) so text order survives the element-only tree.
-      def normalize_children(nk_node, depth:, path:, chrome:)
+      def normalize_children(nk_children, depth:, path:, scope:)
         children = []
         slots = [+'']
-        nk_node.children.each do |child|
+        nk_children.each do |child|
           next slots.last << child.text if child.text?
 
-          node = normalize_element(child, depth: depth + 1, path:, chrome:)
+          node = normalize_element(child, depth: depth + 1, path:, scope:)
           (children << node) && (slots << +'') if node
         end
         [children, slots.first, slots.drop(1)]