html2rss 0.32.4 → 0.34.0

data/lib/html2rss/auto_source/scraper/microformats2.rb

@@ -35,10 +35,8 @@ module Html2rss
 
         # @param parsed_body [Nokogiri::HTML::Document] parsed HTML document
         # @param url [String, Html2rss::Url] base page URL
-        # @param _opts [Hash] unused scraper-specific options
-        # @option _opts [Object] :_reserved reserved for future options
         # @return [void]
-        def initialize(parsed_body, url:, **_opts)
+        def initialize(parsed_body, url:)
           @parsed_body = parsed_body
           @url = Html2rss::Url.from_absolute(url)
         end
@@ -111,11 +109,19 @@ module Html2rss
         # @param entry [Nokogiri::XML::Element] entry DOM node
         # @return [String, nil] image URL
         def extract_image(entry)
-          node = entry.at_css('.u-featured, .u-photo')
+          node = entry.css('.u-featured, .u-photo').find { !author_node?(_1, entry) }
           raw_src = node&.attr('src')&.strip || node&.attr('href')&.strip
           resolve_url(raw_src)&.to_s
         end
 
+        # @param node [Nokogiri::XML::Element] candidate node
+        # @param entry [Nokogiri::XML::Element] entry DOM node
+        # @return [Boolean] whether the node is, or sits inside, an author card within the entry
+        def author_node?(node, entry)
+          chain = [node, *node.ancestors.take_while { _1 != entry }]
+          chain.any? { _1['class'].to_s.split.intersect?(%w[p-author h-card]) }
+        end
+
         # @param entry [Nokogiri::XML::Element] entry DOM node
         # @return [String, nil] author text
         def extract_author(entry)

data/lib/html2rss/auto_source/scraper/native_feed.rb

@@ -5,9 +5,15 @@ module Html2rss
     module Scraper
       ##
       # Promotes a page's native RSS/Atom feed via {Syndication::Discovery} + {Syndication::Parser}.
+      # After {#each}, {#channel} holds what that feed states about its channel and {#feed_url} where
+      # it was read, the provenance {Cleanup} admits the feed's own domain by.
       class NativeFeed
         include Enumerable
 
+        # Href shapes that Discovery's path probing can still resolve when the link type is not a feed MIME type.
+        FEEDISH_HREF = /rss|atom|feed|\.xml/i
+        private_constant :FEEDISH_HREF
+
         # @return [Symbol] scraper config key
         def self.options_key = :native_feed
 
@@ -27,25 +33,16 @@ module Html2rss
         def self.articles?(parsed_body)
           return false unless parsed_body.is_a?(Nokogiri::HTML::Document)
 
-          return true if ::Html2rss::Html::FeedLink.from_document(parsed_body).any?
+          return true if Html2rss::Html::FeedLink.from_document(parsed_body).any?
 
-          parsed_body.css('head link[rel~="alternate"][href]').any? do |node|
-            href = node['href'].to_s
-            ::Html2rss::Html::Probe.mime_match?(
-              node['type'],
-              ::Html2rss::Html::Probe::APPLICATION_RSS_XML,
-              ::Html2rss::Html::Probe::APPLICATION_ATOM_XML
-            ) || href.match?(/rss|atom|feed|\.xml/i)
-          end
+          parsed_body.css('head link[rel~="alternate"][href]').any? { |node| node['href'].match?(FEEDISH_HREF) }
         end
 
         ##
         # @param parsed_body [Nokogiri::HTML::Document]
         # @param url [String, Html2rss::Url]
         # @param request_session [Html2rss::RequestSession, nil]
-        # @param _opts [Hash]
-        # @option _opts [Object] :_reserved reserved for future scraper-specific options
-        def initialize(parsed_body, url:, request_session: nil, **_opts)
+        def initialize(parsed_body, url:, request_session: nil)
           @parsed_body = parsed_body
           @url = Html2rss::Url.from_absolute(url)
           @request_session = request_session
@@ -54,6 +51,16 @@ module Html2rss
         # @return [Boolean]
         def extractable? = self.class.articles?(parsed_body)
 
+        ##
+        # @return [Syndication::ChannelFacts, nil] what the followed feed states about its channel;
+        #   nil before {#each} or when no feed was read
+        def channel = @feed&.channel
+
+        ##
+        # @return [Html2rss::Url, nil] URL of the feed read (after redirects); nil before {#each} or
+        #   when no feed was read
+        attr_reader :feed_url
+
         ##
         # @yieldparam article [Hash{Symbol => Object}]
         # @return [Enumerator, void]
@@ -61,7 +68,7 @@ module Html2rss
         def each(&)
           return enum_for(:each) unless block_given?
 
-          articles = fetch_articles
+          articles = (@feed = fetch_feed).articles
           if articles.empty?
             Log.info({
               event_name: 'auto_source.native_feed.empty',
@@ -84,8 +91,8 @@ module Html2rss
         attr_reader :parsed_body, :url, :request_session
 
         # rubocop:disable-next Metrics/AbcSize, Metrics/MethodLength -- discovery + parse + structured warn
-        def fetch_articles
-          return [] unless request_session
+        def fetch_feed
+          return Syndication::Feed::NONE unless request_session
 
           response = Syndication::Discovery.best_feed_response(
             page_url: url,
@@ -93,17 +100,20 @@ module Html2rss
             parsed_body:,
             max_probes: self.class.request_slots
           )
-          return [] unless response
+          return Syndication::Feed::NONE unless response
 
+          @feed_url = response.url
           Syndication::Parser.parse_response(response)
-        rescue Html2rss::Error, ArgumentError => error
+        rescue Html2rss::ContractError
+          raise
+        rescue Html2rss::Error => error
           Log.warn({
             event_name: 'auto_source.native_feed.failed',
             host: url.host,
             error_class: error.class.name,
             message: "#{self.class}: host=#{url.host} failed (#{error.class}: #{error.message})"
           }.to_json)
-          []
+          Syndication::Feed::NONE
         end
       end
     end

data/lib/html2rss/auto_source/scraper/schema/category_extractor.rb

@@ -8,7 +8,7 @@ module Html2rss
         # Extracts categories from Schema.org structured data.
         module CategoryExtractor
           # Schema.org fields checked for category-like string values.
-          CATEGORY_FIELDS = %i[articleSection keywords categories tags].freeze
+          CATEGORY_FIELDS = %i[articleSection keywords categories tags occupationalCategory industry].freeze
 
           ##
           # @param schema_object [Hash] The schema object
@@ -26,15 +26,20 @@ module Html2rss
             def add_field(categories, value)
               case value
               when Array
-                value.each { |item| add_string(categories, item.to_s) }
+                value.each { |item| add_item(categories, item) }
               when String
                 split_string(categories, value)
               end
             end
 
+            def add_item(categories, item)
+              item.is_a?(String) ? split_string(categories, item) : add_string(categories, item.to_s)
+            end
+
             def add_about(categories, about)
               case about
               when Array then add_about_items(categories, about)
+              when Hash then add_about_items(categories, [about])
               when String then split_string(categories, about)
               end
             end

data/lib/html2rss/auto_source/scraper/schema/item_list.rb

@@ -49,11 +49,13 @@ module Html2rss
 
           # @param object [Hash] candidate schema object
           # @param from_list_item [Boolean] whether object came from a ListItem unwrap
-          # @return [Boolean] whether to emit this object as an article
+          # @return [Boolean] whether to emit this object: a ListItem unwrap admits everything except pure
+          #   entities ({Thing::ENTITY_TYPES}); a bare element must be an article, media, Event or JobPosting type
           def emit?(object, from_list_item:)
-            return true if from_list_item
+            types = Schema.normalize_types(object[:@type])
+            return types.empty? || !types.subset?(Thing::ENTITY_TYPES) if from_list_item
 
-            Schema.normalize_types(object[:@type]).intersect?(Thing::SUPPORTED_TYPES)
+            types.intersect?(Thing::ITEM_TYPES | Thing::ITEM_LIST_TYPES)
           end
         end
       end

data/lib/html2rss/auto_source/scraper/schema/thing.rb

@@ -10,7 +10,7 @@ module Html2rss
         # A Thing is kind of the 'base class' for Schema.org schema_objects.
         #
         # @see https://schema.org/Thing
-        class Thing
+        class Thing # rubocop:disable Metrics/ClassLength -- one accessor per schema.org field
           # Supported Schema.org `@type` values mapped to article extraction.
           SUPPORTED_TYPES = %w[
             AdvertiserContentArticle AnalysisNewsArticle APIReference Article
@@ -19,6 +19,41 @@ module Html2rss
             ReviewNewsArticle SatiricalArticle ScholarlyArticle SocialMediaPosting TechArticle
           ].to_set.freeze
 
+          # Schema.org media types and the file kind each proves (`contentUrl` is the file; `embedUrl`, the player,
+          # is never an enclosure nor the item URL), so an untyped `contentUrl` without a media extension is still
+          # audio or video.
+          MEDIA_KINDS = {
+            'AudioObject' => :audio, 'PodcastEpisode' => :audio, 'VideoObject' => :video, 'Clip' => :video
+          }.freeze
+
+          # Schema.org media types: items wherever they appear (standalone, in `@graph` or an array, as ItemList
+          # elements), marked `media: true`; their own `contentUrl` (when present) is a media enclosure.
+          MEDIA_TYPES = MEDIA_KINDS.keys.to_set.freeze
+
+          # Types emitted as items wherever they appear: articles and media.
+          ITEM_TYPES = (SUPPORTED_TYPES | MEDIA_TYPES).freeze
+
+          # Keys holding the media objects (object or array) whose `contentUrl` is an enclosure, in choice order.
+          MEDIA_KEYS = %i[video audio associatedMedia].freeze
+
+          # A MIME type (`type/subtype`) at the start of `encodingFormat`; a bare format name such as `mp3` maps
+          # through {Article::Enclosure::MEDIA_TYPES}.
+          MIME_TYPE = %r{\A[a-z]+/[a-z0-9][\w.+-]*}i
+
+          # Schema.org entity types that describe who or what something is, never a piece of content; a
+          # ListItem-wrapped object typed only with these is not a feed item. Organization and Place subtypes
+          # are listed because the wire form carries the subtype, not the parent.
+          ENTITY_TYPES = Set[
+            'Person',
+            'Organization', 'Corporation', 'NGO', 'LocalBusiness', 'GovernmentOrganization',
+            'EducationalOrganization', 'NewsMediaOrganization', 'Brand',
+            'Product', 'ProductGroup', 'Offer',
+            'Place', 'City', 'Country', 'PostalAddress'
+          ].freeze
+
+          # Schema.org types emitted only as elements of an ItemList (never standalone).
+          ITEM_LIST_TYPES = Set['Event', 'JobPosting'].freeze
+
           # @param schema_object [Hash{Symbol => Object}] parsed schema.org object
           # @param url [String, Html2rss::Url, nil] base URL used for relative normalization
           def initialize(schema_object, url:)
@@ -26,9 +61,10 @@ module Html2rss
             @base_url = normalized_base_url(url)
           end
 
-          # @return [Hash{Symbol => Object}] scraped article attributes
+          # @return [Hash{Symbol => Object}] scraped article attributes; a media object's carry `media: true`
           def call
-            { id:, title:, description:, author:, url:, image:, published_at:, categories: }
+            item = { id:, title:, description:, author:, url:, image:, published_at:, categories:, enclosures: }
+            own_media? ? item.merge(media: true) : item
           end
 
           # @return [String, nil] stable schema object identifier
@@ -43,31 +79,29 @@ module Html2rss
           def title
             schema_object.values_at(:headline, :name, :alternativeHeadline, :title)
                          .lazy
-                         .map { _1.to_s.strip }
+                         .map { first_scalar(_1).to_s.strip }
                          .find { !_1.empty? }
           end
 
           # @return [String, nil] longest available description field
           def description
-            schema_object.values_at(:description, :articleBody, :abstract, :schema_object_body).max_by { _1.to_s.size }
+            schema_object.values_at(:description, :articleBody, :abstract, :schema_object_body)
+                         .map { first_scalar(_1) }
+                         .max_by { _1.to_s.size }
           end
 
           # @return [String, nil] extracted author or publisher name
           def author
-            extract_author_name(schema_object[:author]) || extract_author_name(schema_object[:publisher])
+            extract_author_name(schema_object[:author]) || extract_author_name(schema_object[:publisher]) ||
+              extract_author_name(schema_object[:hiringOrganization]) ||
+              extract_author_name(schema_object[:organizer])
           end
 
           # @return [Html2rss::Url, nil] the URL of the schema object
           def url
             return @url if defined?(@url)
 
-            url = schema_object[:url]
-            if url.to_s.empty?
-              Log.debug("Schema#Thing.url: no url in schema_object: #{schema_object.inspect}")
-              return @url = nil
-            end
-
-            @url = Url.from_relative(url, base_url || url)
+            @url = build_url(first_scalar(schema_object[:url])) || fallback_url
           end
 
           # @return [Html2rss::Url, nil] normalized article image URL
@@ -78,10 +112,11 @@ module Html2rss
             @image = img_url ? Url.from_relative(img_url, base_url || img_url) : nil
           end
 
-          # @return [String, nil] published-at timestamp (datePublished, else dateModified)
+          # @return [String, nil] first non-empty of datePublished, datePosted, dateCreated, dateModified, uploadDate
           def published_at
-            value = schema_object[:datePublished] || schema_object[:dateModified]
-            value.to_s.empty? ? nil : value
+            schema_object.values_at(:datePublished, :datePosted, :dateCreated, :dateModified, :uploadDate)
+                         .map { first_scalar(_1) }
+                         .find { !_1.to_s.empty? }
           end
 
           # @return [Array<String>, nil] extracted category labels
@@ -89,15 +124,95 @@ module Html2rss
             @categories ||= CategoryExtractor.call(schema_object)
           end
 
+          # @return [Array<Hash{Symbol => Object}>, nil] enclosures from the `contentUrl` of nested media objects
+          #   ({MEDIA_KEYS}) and of a media object itself, with declared size and duration; `embedUrl` is never one
+          def enclosures
+            nodes = schema_object.values_at(*MEDIA_KEYS).flatten
+            nodes << schema_object if own_media?
+            found = nodes.filter_map { media_enclosure(_1) }
+            found unless found.empty?
+          end
+
           attr_reader :schema_object, :base_url
 
           # @return [Array<String>] normalized image URL candidates
           def image_urls
-            @image_urls ||= schema_object.values_at(:image, :thumbnailUrl).filter_map { image_url_from(_1) }
+            @image_urls ||= schema_object.values_at(:image, :thumbnailUrl, :thumbnail).filter_map { image_url_from(_1) }
           end
 
           private
 
+          # @return [Boolean] whether the object itself is a media object (an item marked media)
+          def own_media? = Schema.normalize_types(schema_object[:@type]).intersect?(MEDIA_TYPES)
+
+          # @param node [Object] media object (VideoObject, AudioObject, MediaObject) or the media item itself
+          # @return [Hash{Symbol => Object}, nil] enclosure attributes, or nil without an http(s) contentUrl
+          #   (an ImageObject is never one: images are {#image})
+          def media_enclosure(node)
+            return unless node.is_a?(Hash) && (src = first_scalar(node[:contentUrl])).is_a?(String)
+            return if (types = Schema.normalize_types(node[:@type])).include?('ImageObject')
+
+            kind = types.filter_map { MEDIA_KINDS[_1] }.first
+            type = mime_type(node[:encodingFormat])
+            Html2rss::Html::ArticleRules::Enclosure.from_media(src, type, base_url, kind:)&.merge(file_facts(node))
+          end
+
+          # @param node [Hash] media object
+          # @return [Hash{Symbol => Integer, nil}] its byte length and playback seconds, the duration else that of
+          #   the item itself (a PodcastEpisode's duration)
+          def file_facts(node)
+            duration = Article::Enclosure.parse_duration(node[:duration]) ||
+                       Article::Enclosure.parse_duration(schema_object[:duration])
+            { bytes_length: Article::Enclosure.parse_size(node[:contentSize]), duration: }
+          end
+
+          # @param format [Object] encodingFormat
+          # @return [String, nil] its lowercase MIME type, or the playable type of a media format name (+mp3+,
+          #   +.m4a+); nil for any other name, so the file name decides
+          def mime_type(format)
+            return unless format.is_a?(String)
+
+            format[MIME_TYPE]&.downcase || Article::Enclosure::MEDIA_TYPES[format.strip.downcase.delete_prefix('.')]
+          end
+
+          # @param value [Object] url candidate
+          # @return [Html2rss::Url, nil] absolute url, or nil when missing or unusable
+          def build_url(value)
+            return Url.from_relative(value, base_url || value) if value.is_a?(String) && !value.empty?
+
+            Log.debug("Schema#Thing.url: no usable url in schema_object: #{schema_object.inspect}")
+            nil
+          rescue Url::InvalidUrlError
+            Log.debug("Schema#Thing.url: unusable url #{value.inspect}")
+            nil
+          end
+
+          # @return [Html2rss::Url, nil] first absolute http(s) mainEntityOfPage or @id, without fragment; never
+          #   `contentUrl` (a file) or `embedUrl` (a player page)
+          def fallback_url
+            main = schema_object[:mainEntityOfPage]
+            candidates = main.is_a?(Hash) ? [main[:@id], main[:url]] : [main]
+            (candidates << schema_object[:@id]).filter_map { web_url(_1) }.first
+          end
+
+          # @param value [Object] url candidate
+          # @return [Html2rss::Url, nil] fragment-free url when value is an absolute http(s) string
+          def web_url(value)
+            return unless value.is_a?(String) && value.match?(%r{\Ahttps?://}i)
+
+            Url.from_absolute(value.strip).without_fragment
+          rescue Url::InvalidUrlError
+            nil
+          end
+
+          # @param value [Object] schema value that may be wrapped in a list (multilingual or multi-valued fields)
+          # @return [Object, nil] the value, or the first non-empty String of a list
+          def first_scalar(value)
+            return value unless value.is_a?(Array)
+
+            value.find { _1.is_a?(String) && !_1.strip.empty? }
+          end
+
           # @param obj [String, Hash, Array, nil] schema author or publisher field
           # @return [String, nil] extracted author name
           def extract_author_name(obj)
@@ -110,12 +225,15 @@ module Html2rss
             end
           end
 
-          # @param obj [String, Hash, nil] image node
-          # @return [String, nil] image URL string
+          # @param obj [String, Hash, Array, nil] image node or list of image nodes
+          # @return [String, nil] image URL string (first usable entry for lists)
           def image_url_from(obj)
             return obj if obj.is_a?(String)
+            return obj.filter_map { image_url_from(_1) }.first if obj.is_a?(Array)
             return unless obj.is_a?(Hash)
-            return unless Schema.normalize_types(obj[:@type]).include?('ImageObject')
+
+            types = Schema.normalize_types(obj[:@type])
+            return unless types.empty? || types.include?('ImageObject')
 
             obj[:url] || obj[:contentUrl]
           end
@@ -129,7 +247,7 @@ module Html2rss
 
             norm_url = normalized_id_url(text, reference_url:)
             reference_url && norm_url.host == reference_url.host ? normalized_id_value(norm_url) : text
-          rescue ArgumentError
+          rescue Html2rss::Url::InvalidUrlError
             text
           end
 
@@ -153,7 +271,7 @@ module Html2rss
           # @return [Html2rss::Url, nil] normalized absolute URL for schema resolution
           def normalized_base_url(url)
             Url.from_absolute(url) unless url.to_s.strip.empty?
-          rescue ArgumentError
+          rescue Html2rss::Url::InvalidUrlError
             nil
           end
         end

data/lib/html2rss/auto_source/scraper/schema.rb

@@ -25,14 +25,15 @@ module Html2rss
 
         # Canonical Schema.org type names keyed by folded wire forms (case-insensitive lookup).
         CANONICAL_BY_DOWNCASE = begin
-          canonical_types = Thing::SUPPORTED_TYPES | ItemList::SUPPORTED_TYPES | DENIED_CONTAINER_TYPES | Set['Product']
+          canonical_types = Thing::ITEM_TYPES | ItemList::SUPPORTED_TYPES | Thing::ITEM_LIST_TYPES |
+                            DENIED_CONTAINER_TYPES | Set['Product']
           canonical_types.to_h { |type| [::Html2rss::Html::Probe.fold(type), type] }.freeze
         end.freeze
 
         # Pre-compiled regex for supported schema types (short name or schema.org URL; string or array @type).
         # Allows preceding entries in a JSON @type array (e.g. ["WebPage","NewsArticle"]).
         SUPPORTED_TYPES_RE = begin
-          types = Thing::SUPPORTED_TYPES | ItemList::SUPPORTED_TYPES
+          types = Thing::ITEM_TYPES | ItemList::SUPPORTED_TYPES
           type_re = Regexp.union(types.to_a)
           %r{(?i)"@type"\s*:\s*(?:\[\s*(?:"[^"]*"\s*,\s*)*)?"(?:https?://schema\.org/)?(?:#{type_re.source})"}
         end.freeze
@@ -43,6 +44,9 @@ module Html2rss
         # Shared empty type set for nil/unsupported wire forms (avoids per-call Set.new).
         EMPTY_TYPES = Set.new.freeze
 
+        # Keys whose `{"@id": ...}` stubs are resolved against sibling nodes of the same JSON-LD document.
+        REFERENCE_KEYS = (%i[author publisher image] + Thing::MEDIA_KEYS).freeze
+
         # @return [Symbol] scraper config key
         def self.options_key = :schema
 
@@ -84,6 +88,18 @@ module Html2rss
             end
           end
 
+          ##
+          # Like {from} for a script tag, but resolves `@id` stubs for {REFERENCE_KEYS} against the same document.
+          #
+          # @param script [Nokogiri::XML::Element] schema JSON-LD script tag
+          # @return [Array<Hash>] the schema_objects with resolved references
+          # @api private
+          def from_script(script)
+            document = parse_script_tag(script)
+            index = node_index(document)
+            from(document).map { |object| resolve_references(object, index) }
+          end
+
           # @param object [Hash{Symbol => Object}] schema candidate object
           # @return [Boolean] whether an extractor exists for the candidate object
           def supported_schema_object?(object)
@@ -99,7 +115,7 @@ module Html2rss
             # Prefer article/list extractors over denied containers for multi-type @type arrays
             # (e.g. ["NewsArticle","WebPage"]).
             return ItemList if types.intersect?(ItemList::SUPPORTED_TYPES)
-            return Thing if types.intersect?(Thing::SUPPORTED_TYPES)
+            return Thing if types.intersect?(Thing::ITEM_TYPES)
             return nil if types.intersect?(DENIED_CONTAINER_TYPES)
 
             Log.debug("#{name}: unsupported schema object @type=#{schema_object[:@type].inspect}")
@@ -139,6 +155,37 @@ module Html2rss
 
           private
 
+          # @param object [Hash, Array, Object] parsed JSON-LD value
+          # @param index [Hash{String => Hash}] accumulator of nodes (more than just `@id`) keyed by `@id`
+          # @return [Hash{String => Hash}] nodes keyed by `@id`
+          def node_index(object, index = {})
+            case object
+            when Hash
+              index[object[:@id]] ||= object if object[:@id].is_a?(String) && object.size > 1
+              object.each_value { node_index(_1, index) }
+            when Array then object.each { node_index(_1, index) }
+            end
+            index
+          end
+
+          # @param object [Hash{Symbol => Object}] schema object
+          # @param index [Hash{String => Hash}] nodes keyed by `@id`
+          # @return [Hash{Symbol => Object}] object with reference stubs replaced where a sibling node exists
+          def resolve_references(object, index)
+            object.to_h { |key, value| [key, REFERENCE_KEYS.include?(key) ? resolve_stub(value, index) : value] }
+          end
+
+          # @param value [Object] author, publisher or image value
+          # @param index [Hash{String => Hash}] nodes keyed by `@id`
+          # @return [Object] the referenced node for an `{"@id": ...}` stub, else the value unchanged
+          def resolve_stub(value, index)
+            case value
+            when Array then value.map { resolve_stub(_1, index) }
+            when Hash then value.keys == [:@id] ? index.fetch(value[:@id], value) : value
+            else value
+            end
+          end
+
           # @param hash [Hash] candidate schema object
           # @return [Array<Hash>] the object itself or nested supported objects
           def from_hash(hash)
@@ -167,12 +214,9 @@ module Html2rss
 
         # @param parsed_body [Nokogiri::HTML::Document] parsed HTML document
         # @param url [String, Html2rss::Url] base page URL
-        # @param opts [Hash] scraper-specific options
-        # @option opts [Object] :_reserved reserved for future scraper-specific options
-        def initialize(parsed_body, url:, **opts)
+        def initialize(parsed_body, url:)
           @parsed_body = parsed_body
           @url = url
-          @opts = opts
         end
 
         # @return [Boolean]
@@ -200,7 +244,7 @@ module Html2rss
 
         def schema_objects
           ::Html2rss::Html::Probe.scripts(@parsed_body, ::Html2rss::Html::Probe::APPLICATION_LD_JSON)
-                                 .flat_map { |tag| Schema.from(tag) }
+                                 .flat_map { |tag| Schema.from_script(tag) }
         end
 
         attr_reader :parsed_body, :url

data/lib/html2rss/auto_source/scraper/semantic_html/entry_deduplicator.rb

@@ -33,12 +33,8 @@ module Html2rss
           # @return [Html2rss::Article, nil]
           def article_for(entry)
             @article_cache.fetch(entry) do
-              @article_cache[entry] = ::Html2rss::Html::SstArticleExtractor.call(
-                entry,
-                base_url: @url,
-                scraper: @scraper,
-                fallback_anchorless: true
-              )
+              @article_cache[entry] =
+                ::Html2rss::Html::SstArticleExtractor.call(entry, base_url: @url, scraper: @scraper)
             end
           end
 
@@ -83,9 +79,7 @@ module Html2rss
             ]
           end
 
-          def word_count(text)
-            (@word_counts ||= {})[text] ||= text.to_s.scan(/\p{Alnum}+/).size
-          end
+          def word_count(text) = (@word_counts ||= {})[text] ||= WordCount.count(text)
         end
       end
     end

data/lib/html2rss/auto_source/scraper/semantic_html.rb

@@ -26,22 +26,17 @@ module Html2rss
           return false unless parsed_body
 
           new(parsed_body, url: 'https://example.com').extractable?
-        rescue ArgumentError
-          false
         end
 
         # @param parsed_body [Nokogiri::HTML::Document, nil] parsed HTML (when +document:+ omitted)
         # @param url [String, Html2rss::Url] base url
         # @param document [SST::Document, nil] memoized SST document from AutoSource
         # @param link_resolver [Scoring::LinkResolver, nil] shared page-scoped resolver
-        # @param opts [Hash] scraper-specific options
-        # @option opts [Boolean] :fallback_anchorless whether to keep containers without a primary anchor
-        def initialize(parsed_body = nil, url:, document: nil, link_resolver: nil, **opts)
+        def initialize(parsed_body = nil, url:, document: nil, link_resolver: nil)
           @parsed_body = parsed_body
           @provided_document = document
           @provided_link_resolver = link_resolver
           @url = url
-          @permit_unanchored = opts.fetch(:fallback_anchorless, false)
         end
 
         ##
@@ -57,8 +52,6 @@ module Html2rss
         # @return [Boolean]
         def extractable?
           ranked_segments.any?
-        rescue ArgumentError
-          false
         end
 
         private
@@ -69,21 +62,20 @@ module Html2rss
             entries = deduplicator.call(ranked_segments)
             entries.filter_map { |ranked| deduplicator.article_for(ranked) }
           end
-        rescue ArgumentError
-          []
         end
 
         def ranked_segments
-          @ranked_segments ||= begin
-            segments = Segmenter.call(
-              document,
-              base_url: @url,
-              strategy: :semantic,
-              permit_unanchored: @permit_unanchored,
-              link_resolver:
-            )
-            Scoring::Engine.new(link_resolver:).rank_top(segments, limit: TOP_K)
-          end
+          @ranked_segments ||= document ? rank_document : []
+        end
+
+        def rank_document
+          segments = Segmenter.call(
+            document,
+            base_url: @url,
+            strategy: :semantic,
+            link_resolver:
+          )
+          Scoring::Engine.new(link_resolver:).rank_top(segments, limit: TOP_K)
         end
 
         def link_resolver

data/lib/html2rss/auto_source/scraper/sitemap/parser.rb

@@ -149,10 +149,7 @@ module Html2rss
           def stale_date?(published_at)
             return false unless published_at
 
-            time = Time.parse(published_at)
-            (Time.now.utc - time) > (max_age_days * 86_400)
-          rescue ArgumentError
-            false
+            (Time.now.utc - Time.parse(published_at)) > (max_age_days * 86_400)
           end
         end
       end