html2rss 0.33.0 → 0.34.0

data/lib/html2rss/auto_source.rb

@@ -22,7 +22,13 @@ module Html2rss
     # Default max articles to keep (also the short-circuit floor across scraper tiers).
     DEFAULT_LIMIT = 25
 
-    # Operational scraper failures quarantined within a tier; programmer errors propagate.
+    # Raised when a scraper yields an item that is neither an Article nor a Hash (a scraper bug).
+    class ScraperContractError < Html2rss::ContractError; end
+
+    # Operational scraper failures quarantined within a tier; programmer errors (including any
+    # {Html2rss::ContractError}) propagate.
+    # +ArgumentError+ stays for stdlib raises on publisher bytes (e.g. +invalid byte sequence+ after
+    # +CGI.unescapeHTML+ of a surrogate entity); lib raises are {Html2rss::Error} (CONTEXT.md § Error handling).
     OPERATIONAL_ERRORS = [
       Html2rss::Error,
       ArgumentError,
@@ -71,14 +77,12 @@ module Html2rss
           enabled: true
         },
         semantic_html: {
-          enabled: true,
-          fallback_anchorless: true
+          enabled: true
         },
         html: {
           enabled: true,
           minimum_selector_frequency: Scraper::Html::DEFAULT_MINIMUM_SELECTOR_FREQUENCY,
-          use_top_selectors: Scraper::Html::DEFAULT_USE_TOP_SELECTORS,
-          fallback_anchorless: true
+          use_top_selectors: Scraper::Html::DEFAULT_USE_TOP_SELECTORS
         }
       },
       cleanup: Cleanup::DEFAULT_CONFIG
@@ -120,14 +124,18 @@ module Html2rss
       @captured_responses = response.captured_responses
       @opts = opts
       @request_session = request_session
+      @cards_read = false
     end
 
     ##
     # Extracts articles by running scraper tiers until a sufficient set is found.
     #
     # Tiers: in-page structured → follow-up IO → SemanticHtml → Html.
-    # SST is built only when a heuristic tier runs. Later tiers are skipped once
-    # +limit+ articles with url+title remain after Cleanup; the result is capped to +limit+.
+    # SST is built at most once, lazily, by the first consumer: a heuristic tier or the
+    # {Corroboration} page check of structured-tier items. Later tiers are skipped once
+    # +limit+ counting articles with url+title remain after Cleanup ({#counting}); the result
+    # is capped to +limit+, counting articles first. A native feed's items come first, and
+    # Cleanup merges a page copy of the same article into them.
     # Html is skipped when earlier tiers already admitted at least one clean article
     # below +limit+ — quality over padding with weaker heuristic junk.
     #
@@ -153,6 +161,13 @@ module Html2rss
       @admission_drops || {}
     end
 
+    ##
+    # What the native feed this page advertises states about its channel ({Scraper::NativeFeed#channel}):
+    # nil until {#articles} runs, or when no feed was read.
+    #
+    # @return [Html2rss::Syndication::ChannelFacts, nil]
+    def feed_channel = @native_feed&.channel
+
     private
 
     attr_reader :url, :parsed_body, :body, :request_session, :captured_responses
@@ -163,21 +178,13 @@ module Html2rss
 
       articles = []
       matched = false
-      document = nil
-      link_resolver = nil
       scraper_opts = @opts.fetch(:scraper, {})
 
       Scraper::SCRAPER_TIERS.each do |tier|
         break if enough_articles?(articles)
         break if skip_html_padding?(tier, articles)
 
-        if Scraper.heuristic_tier?(tier)
-          document ||= SST::Normalizer.call(parsed_body)
-          next unless document
-
-          link_resolver ||= ::Html2rss::Scoring::LinkResolver.new(url)
-        end
-
+        @cards_read = true if Scraper.heuristic_tier?(tier)
         tier.each do |scraper_class|
           instance = Scraper.build_instance(
             scraper_class,
@@ -186,7 +193,7 @@ module Html2rss
             url:,
             request_session:,
             body:,
-            document:,
+            document: (page_document if Scraper.heuristic_tier?(tier)),
             link_resolver:,
             captured_responses:
           )
@@ -195,6 +202,7 @@ module Html2rss
 
           matched = true
           articles.concat(run_scraper(instance))
+          @native_feed = instance if instance.is_a?(Scraper::NativeFeed)
         end
       end
 
@@ -231,10 +239,11 @@ module Html2rss
 
     def finish_articles(articles)
       captured_count = articles.size
-      result = Cleanup.call(articles, url:, **cleanup_options)
+      result = cleanup_result(articles)
       @admission_drops = result.drop_tallies
       log_admission_emptied(captured_count, result) if captured_count.positive? && result.articles.empty?
-      result.articles.first(article_limit)
+      counting = counting(result.articles)
+      (counting + (result.articles - counting)).first(article_limit)
     end
 
     def log_admission_emptied(captured_count, result)
@@ -252,7 +261,19 @@ module Html2rss
     def enough_articles?(articles)
       return false if articles.size < article_limit
 
-      admitted_articles(articles).size >= article_limit
+      counting(admitted_articles(articles)).size >= article_limit
+    end
+
+    # The articles that count toward +limit+, in order: those the page shows in its content
+    # ({Corroboration#shown?}), media items (audio, video) only on a media page
+    # ({Corroboration#media_page?}). The others stay admitted after the counting ones. A page is
+    # judged a media page only once a heuristic tier has read its cards: structured items alone
+    # cannot tell a podcast page from a news page with an audio list. A media page's counting
+    # items are in page order ({Corroboration#page_order}), not tier order.
+    def counting(articles)
+      media_page = @cards_read && corroboration.media_page?(articles)
+      counted = articles.select { (media_page || !_1.media?) && corroboration.shown?(_1) }
+      media_page ? corroboration.page_order(counted) : counted
     end
 
     # Prefer fewer clean items over refilling +limit+ from the Html heuristic tier.
@@ -264,8 +285,39 @@ module Html2rss
       tier == [Scraper::Html]
     end
 
+    # Cleanup is deterministic and +articles+ only grows, so one result per size is reused
+    # by the tier checks and by {#finish_articles}.
+    def cleanup_result(articles)
+      return @cleanup_memo.last if @cleanup_memo&.first == articles.size
+
+      Cleanup.call(articles.dup, url:, corroboration:, page_links: -> { page_links },
+                                 feed_url: @native_feed&.feed_url, **cleanup_options)
+             .tap { @cleanup_memo = [articles.size, _1] }
+    end
+
+    # Page SST shared by the heuristic tiers and the page check; nil for a JSON response
+    # or a page without SST nodes.
+    def page_document
+      return @page_document if defined?(@page_document)
+
+      @page_document = (SST::Normalizer.call(parsed_body) unless json_response?)
+    end
+
+    def link_resolver
+      @link_resolver ||= Scoring::LinkResolver.new(url)
+    end
+
+    def corroboration
+      @corroboration ||= Corroboration.new { page_links }
+    end
+
+    # Rendered links of the page, built once for the page check and sponsorship.
+    def page_links
+      @page_links ||= PageLinks.build(page_document, link_resolver:)
+    end
+
     def admitted_articles(articles)
-      Cleanup.call(articles.dup, url:, **cleanup_options).articles.select do |article|
+      cleanup_result(articles).articles.select do |article|
         article.url && !article.title.to_s.empty?
       end
     end
@@ -276,11 +328,11 @@ module Html2rss
 
     def run_scraper(instance)
       instance.map { |item| article_from_scraper_item(item, instance) }
+    rescue ContractError
+      raise
     rescue *OPERATIONAL_ERRORS => error
       Log.warn({
-        event_name: 'auto_source.quarantined',
-        host: url.host,
-        error_class: error.class.name,
+        event_name: 'auto_source.quarantined', host: url.host, error_class: error.class.name,
         message: "#{self.class}: #{instance.class} quarantined #{error.class}: #{error.message}"
       }.to_json)
       []
@@ -291,9 +343,12 @@ module Html2rss
       when Article
         item
       when Hash
-        Article.new(**item, scraper: instance.class)
+        # Hash items carry publisher-declared or structured titles: verbatim (collapse only), except an item
+        # with an explicit +kicker+, which composes +Kicker: Headline+. Heuristic scrapers yield Articles
+        # composed by Html::SstArticleExtractor.
+        Article.new(**item, compose_title: item.key?(:kicker), scraper: instance.class)
       else
-        raise TypeError, "#{instance.class} yielded #{item.class}; expected Article or Hash"
+        raise ScraperContractError, "#{instance.class} yielded #{item.class}; expected Article or Hash"
       end
     end
 

data/lib/html2rss/batch.rb

@@ -125,6 +125,8 @@ module Html2rss
           url: url.to_s, ok: true, items_count: wire[:items].size,
           items: wire[:items], channel_title: wire[:channel_title]
         }
+      rescue Html2rss::ContractError
+        raise
       rescue Html2rss::Error => error
         { url: url.to_s, ok: false, error: error.message }
       end

data/lib/html2rss/capture/README.md

@@ -29,19 +29,22 @@ rss = Html2rss.apply(result.config)
 
 `Html2rss.capture` and `Capture.build` both return a `CaptureResult` with YAML (`#yaml`, includes the schema modeline) and quality meta (`has_selectors`, `segment_strategy`, `admission_drops`, `selected_strategy`, `native_feed`, `candidates`).
 
-`#candidates` is ranked discovery evidence: `{ items:, title:, link:, published: }` arrays (best-first, deduplicated). Items entries are `{ selector:, enhance:, sample: }`; field entries are `{ selector:, sample: }`. `sample` stays on `#candidates` only — emitted YAML is still `items` + `enhance`. Buckets may be empty when evidence is missing. The default `a[href]` YAML fallback is not listed as discovered evidence.
+`#candidates` is ranked discovery evidence: `{ items:, title:, link:, published: }` arrays (best-first, deduplicated). Items entries are `{ selector:, enhance:, sample: }`; field entries are `{ selector:, sample: }`. `sample` stays on `#candidates` only — emitted YAML is still `items` + `enhance`. `#candidates[:candidates_by_item]` is additive: for each of the top 5 item candidates (same order as `items`) it holds `{ selector:, title:, link:, published: }`, where the field buckets are computed from that item candidate's own matched nodes (reusing the discovery evidence, no refetch). Use it when the top-level `title` / `link` / `published` (which describe the first item candidate only) do not match the item selector you choose. Its first entry equals the top-level buckets. Buckets may be empty when evidence is missing. The default `a[href]` YAML fallback is not listed as discovered evidence.
 
 **Wire vs internal:** user-facing **`Html2rss.apply`** ships RSS from a config Hash. Pipeline internals **`Html2rss.feed` / `feed_result`** stay unchanged — use `apply` in CLI/MCP/docs examples, not `feed`.
 
 ### Options
 
 ```ruby
-Html2rss.capture('https://spa-site.com', strategy: :botasaurus)
+Html2rss.capture('https://example.com', strategy: :botasaurus)
 Html2rss.capture('https://example.com', items_selector: '.article-card')
 Html2rss.capture('https://example.com', strategy: :local_file, local_file_path: './page.html')
 Html2rss.capture('https://example.com', max_redirects: 8, max_requests: 4)
+Html2rss.capture('https://example.com', total_timeout_seconds: 20)
 ```
 
+`total_timeout_seconds:` (positive Integer, default 30) bounds the **whole** request chain: the `:auto` fallback hops (`default` → `botasaurus`) share one budget, so the bound is not per hop. When it elapses, capture raises `RequestService::RequestTimedOut`. Use it to stay under a host's own request deadline.
+
 `strategy: :auto` uses the same AutoFallback chain as scrape (`default` → `botasaurus`). When AutoFallback selects a concrete strategy (or you pin one), Capture **stamps** `strategy:` into the emitted config so later `Html2rss.apply(config)` replays the same transport.
 
 ## CLI

data/lib/html2rss/capture.rb

@@ -89,6 +89,8 @@ module Html2rss
       # @option options [Boolean, nil] :enhance whether to force enhance: true
       # @option options [Integer, nil] :max_redirects optional redirect limit override
       # @option options [Integer, nil] :max_requests optional request budget override
+      # @option options [Integer, nil] :total_timeout_seconds optional wall-clock bound (seconds) for the whole
+      #   request chain, including every +:auto+ fallback hop (shared budget, not per hop)
       # @option options [Integer, nil] :limit max articles to keep
       # @option options [String, nil] :local_file_path optional local HTML file path
       # @return [CaptureResult]
@@ -108,6 +110,8 @@ module Html2rss
     # @option options [Boolean, nil] :enhance whether to force enhance: true
     # @option options [Integer, nil] :max_redirects optional redirect limit override
     # @option options [Integer, nil] :max_requests optional request budget override
+    # @option options [Integer, nil] :total_timeout_seconds optional wall-clock bound (seconds) for the whole
+    #   request chain, including every +:auto+ fallback hop (shared budget, not per hop)
     # @option options [Integer, nil] :limit max articles to keep
     # @option options [String, nil] :local_file_path optional local HTML file path
     def initialize(url, strategy: :auto, **options) # rubocop:disable Metrics/MethodLength
@@ -119,8 +123,7 @@ module Html2rss
       @summary = options.delete(:summary)
       @force = options.delete(:force) || false
       @enhance_option = options.delete(:enhance)
-      @max_redirects = options.delete(:max_redirects)
-      @max_requests = options.delete(:max_requests)
+      @request_options = options.slice(:max_redirects, :max_requests, :total_timeout_seconds)
       @limit = options.delete(:limit)
       @local_file_path = options.delete(:local_file_path)
     end
@@ -172,14 +175,9 @@ module Html2rss
     end
 
     def probe_native_feed(response)
-      return nil unless response && @url
-
-      Syndication::Discovery.best_feed_url(
-        page_url: @url,
-        request_session: discovery_session,
-        parsed_body: (response.parsed_body if response.html_response?),
-        html: response.body
-      )&.to_s
+      Syndication::Discovery.native_feed_url(response:, request_session: discovery_session)&.to_s
+    rescue Html2rss::ContractError
+      raise
     rescue Html2rss::Error
       nil
     end
@@ -236,11 +234,7 @@ module Html2rss
       Config.auto_source_config(
         url: @url,
         items_selector: @items_selector_hint,
-        request_controls: Config::RequestControls.from_shortcut(
-          strategy: @strategy,
-          max_redirects: @max_redirects,
-          max_requests: @max_requests
-        ),
+        request_controls: Config::RequestControls.from_shortcut(strategy: @strategy, **@request_options),
         limit: @limit
       ).tap { |config| apply_local_file_path!(config) }
     end
@@ -296,7 +290,7 @@ module Html2rss
 
       SEGMENT_STRATEGIES.flat_map do |strategy|
         segments = AutoSource::Segmenter.call(
-          sst, base_url: @url, strategy:, permit_unanchored: false, link_resolver:
+          sst, base_url: @url, strategy:, link_resolver:
         )
         matched = match_segments_to_articles(segments, articles)
         next [] if matched.size < MIN_SELECTOR_MATCHES
@@ -344,23 +338,24 @@ module Html2rss
     end
 
     def lift_heading_link_roots(roots)
-      roots.map { |root| lift_heading_link_root(root, roots) }
+      enclosed = enclosed_root_counts(roots)
+      roots.map { |root| lift_heading_link_root(root, enclosed) }
     end
 
-    def lift_heading_link_root(root, all_roots)
+    def lift_heading_link_root(root, enclosed)
       return root unless heading_or_inner_title_link?(root)
 
       index = SST::Index.for_node(root)
       return root unless index
 
-      walk_usable_card(root, index, all_roots)
+      walk_usable_card(root, index, enclosed)
     end
 
-    def walk_usable_card(root, index, all_roots)
+    def walk_usable_card(root, index, enclosed)
       candidate = root
       parent = index.parent_of(root)
-      while parent && Html::Navigator.usable_card_parent?(parent)
-        break if contains_other_root?(parent, root, all_roots)
+      while Html::Navigator.usable_card_parent?(parent)
+        break if contains_other_root?(parent, enclosed)
 
         candidate = parent
         parent = index.parent_of(parent)
@@ -368,6 +363,28 @@ module Html2rss
       candidate
     end
 
+    # Identity-keyed tally of how many distinct roots each node is, or strictly encloses, so the
+    # card walk answers "does this ancestor hold another root?" without rescanning all roots.
+    def enclosed_root_counts(roots)
+      tally = Hash.new(0).compare_by_identity
+      roots.uniq(&:object_id).each do |root|
+        tally[root] += 1
+        count_enclosing_ancestors(root, tally)
+      end
+      tally
+    end
+
+    def count_enclosing_ancestors(root, tally)
+      index = SST::Index.for_node(root)
+      return unless index
+
+      ancestor = index.parent_of(root)
+      while ancestor
+        tally[ancestor] += 1
+        ancestor = index.parent_of(ancestor)
+      end
+    end
+
     def heading_or_inner_title_link?(node)
       name = node.name.to_s
       return false if wrapping_anchor_root?(node)
@@ -382,16 +399,9 @@ module Html2rss
       node.find { |child| !child.equal?(node) && tags.include?(child.name.to_s) }
     end
 
-    def contains_other_root?(parent, root, all_roots)
-      all_roots.any? do |other|
-        next if other.equal?(root)
-
-        other.equal?(parent) || sst_descendant?(other, parent)
-      end
-    end
-
-    def sst_descendant?(child, ancestor)
-      SST::Index.for_node(child)&.descendant_of?(child, ancestor)
+    # The walked-from root is itself tallied under every ancestor, so more than one means another root.
+    def contains_other_root?(parent, enclosed)
+      enclosed[parent] > 1
     end
 
     def shared_class_items_selector(roots)
@@ -446,6 +456,8 @@ module Html2rss
 
     def channel_title_from(response)
       Channel.from_response(response).title
+    rescue Html2rss::ContractError
+      raise
     rescue Html2rss::Error
       nil
     end

data/lib/html2rss/channel.rb

@@ -3,7 +3,9 @@
 require 'time'
 
 module Html2rss
-  Channel = Data.define(:title, :url, :description, :language, :ttl, :last_build_date, :image, :author)
+  Channel = Data.define(
+    :title, :url, :description, :language, :ttl, :last_build_date, :image, :author, :categories, :explicit
+  )
 
   ##
   # Materialized channel metadata for feed builders and {FeedResult}.
@@ -12,6 +14,10 @@ module Html2rss
   # do not own channel extraction. Safe to Marshal (no Response / Nokogiri retained).
   # Strings and image URL are normalized and frozen at construction (including Marshal load).
   #
+  # The author is the config +channel.author+, else the followed native feed's author
+  # ({Syndication::ChannelFacts}), else the page's +<meta name="author">+. Categories and the explicit
+  # flag come only from that feed, when it states them.
+  #
   # @!attribute [r] title
   #   @return [String]
   # @!attribute [r] url
@@ -28,6 +34,10 @@ module Html2rss
   #   @return [Html2rss::Url, nil]
   # @!attribute [r] author
   #   @return [String, nil]
+  # @!attribute [r] categories
+  #   @return [Array<Array<String>>] each category followed by its subcategories (empty when unstated)
+  # @!attribute [r] explicit
+  #   @return [Boolean, nil] whether the content is declared explicit; nil when unstated
   class Channel # rubocop:disable Metrics/ClassLength -- Channel value object owns extract + freeze + Marshal
     class << self
       ##
@@ -35,8 +45,9 @@ module Html2rss
       #
       # @param response [Html2rss::RequestService::Response]
       # @param overrides [Hash{Symbol => Object}] optional overrides for channel attributes
+      # @param feed [Html2rss::Syndication::ChannelFacts, nil] what the followed native feed states
       # @return [Html2rss::Channel]
-      def from_response(response, overrides: {})
+      def from_response(response, overrides: {}, feed: nil) # rubocop:disable Metrics/MethodLength -- one line per member
         new(
           title: title_for(response, overrides),
           url: url_for(response),
@@ -45,7 +56,9 @@ module Html2rss
           ttl: ttl_for(response, overrides),
           last_build_date: last_build_date_for(response),
           image: image_for(response, overrides),
-          author: author_for(response, overrides)
+          author: overrides[:author] || feed&.author || meta_author(response),
+          categories: feed&.categories || [],
+          explicit: feed&.explicit
         )
       end
 
@@ -117,8 +130,7 @@ module Html2rss
         parsed_body(response)['lang'] || parsed_body(response).at_css('[lang]')&.[]('lang')
       end
 
-      def author_for(response, overrides)
-        return overrides[:author] if overrides[:author]
+      def meta_author(response)
         return unless html_response?(response)
 
         parsed_body(response).at_css('meta[name="author"]')&.[]('content')
@@ -156,8 +168,11 @@ module Html2rss
     # @param last_build_date [Time, String]
     # @param image [Html2rss::Url, String, nil]
     # @param author [String, nil]
-    # rubocop:disable-next Metrics/ParameterLists -- Data.define members are the channel contract
-    def initialize(title:, url:, description:, language:, ttl:, last_build_date:, image:, author:)
+    # @param categories [Array<Array<String>>] each category followed by its subcategories
+    # @param explicit [Boolean, nil]
+    # rubocop:disable-next Metrics/ParameterLists, Metrics/MethodLength -- Data.define members are the channel contract
+    def initialize(title:, url:, description:, language:, ttl:, last_build_date:, image:, author:,
+                   categories: [], explicit: nil)
       super(
         title: freeze_string(title),
         url: url.is_a?(Url) ? url : Url.from_absolute(url),
@@ -166,16 +181,19 @@ module Html2rss
         ttl: Integer(ttl),
         last_build_date: normalize_last_build_date(last_build_date),
         image: normalize_image(image),
-        author: author && freeze_string(author)
+        author: author && freeze_string(author),
+        categories: categories.map { |path| path.map { freeze_string(_1) }.freeze }.freeze,
+        explicit:
       )
     end
 
     private
 
-    def marshal_dump = [title, url, description, language, ttl, last_build_date, image, author]
+    def marshal_dump = [title, url, description, language, ttl, last_build_date, image, author, categories, explicit]
 
-    def marshal_load((title, url, description, language, ttl, last_build_date, image, author))
-      initialize(title:, url:, description:, language:, ttl:, last_build_date:, image:, author:)
+    def marshal_load((title, url, description, language, ttl, last_build_date, image, author, categories, explicit))
+      initialize(title:, url:, description:, language:, ttl:, last_build_date:, image:, author:, categories:,
+                 explicit:)
     end
 
     def freeze_string(value) = value.to_s.dup.freeze
@@ -189,7 +207,7 @@ module Html2rss
 
     def normalize_last_build_date(value)
       return value if value.is_a?(Time)
-      raise ArgumentError, 'last_build_date must be a Time or HTTP-date String' unless value.is_a?(String)
+      raise InvalidOption, 'last_build_date must be a Time or HTTP-date String' unless value.is_a?(String)
 
       Time.httpdate(value)
     end

data/lib/html2rss/cli.rb

@@ -136,7 +136,7 @@ module Html2rss
       end
 
       if result.native_feed? && !options[:force]
-        $stderr.puts "First-party RSS/Atom feed detected at #{result.native_feed}. Use --force to capture anyway." # rubocop:disable Style/StderrPuts
+        $stderr.puts "First-party RSS/Atom/RDF feed detected at #{result.native_feed}. Use --force to capture anyway." # rubocop:disable Style/StderrPuts
         exit(3)
       end
 
@@ -198,13 +198,8 @@ module Html2rss
     # @param source [String, nil] YAML file path or '-'
     # @param feed_name [String, nil] optional named feed
     # @return [void]
-    def apply(source = nil, feed_name = nil) # rubocop:disable Metrics/AbcSize
-      config = if File.file?(source.to_s)
-                 Html2rss.config_from_yaml_file(source.to_s, feed_name)
-               else
-                 raw_content, = read_config_input(source)
-                 Config.from_yaml(raw_content)
-               end
+    def apply(source = nil, feed_name = nil)
+      config = execute_feed { load_apply_config(source, feed_name) }
       config[:params] = options[:params] || {}
       execute_feed { apply_runtime_request_overrides!(config) }
       apply_local_file_input!(config, options[:input]) if options[:input]
@@ -220,6 +215,8 @@ module Html2rss
     method_option :keep_different_domain, type: :boolean, default: false,
                                           desc: 'Allow published items to link to any domain ' \
                                                 'supplied by the source (no affiliation/safety check)'
+    method_option :keep_sponsored, type: :boolean, default: false,
+                                   desc: 'Keep items the page marks as sponsored (rel="sponsored" or ad labels)'
     # @param url [String, nil]
     # @return [void]
     def scrape(url = nil)
@@ -257,6 +254,8 @@ module Html2rss
     # @return [void]
     def mcp
       Html2rss::MCP.start(transport: options[:transport].to_sym, port: options[:port])
+    rescue Html2rss::MCP::Server::HttpTransportUnavailable => error
+      raise Thor::Error, error.message
     end
 
     desc 'doctor [TYPE]', 'Runtime preflight checks (botasaurus)'
@@ -309,7 +308,7 @@ module Html2rss
 
       expected = Recon::Verdict[verdict_filter.downcase]
       results.select { |r| r.verdict == expected }
-    rescue ArgumentError => error
+    rescue InvalidOption => error
       raise Thor::Error, error.message
     end
 
@@ -332,7 +331,7 @@ module Html2rss
 
     def handle_capture_output(result, url) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
       if options[:output_dir]
-        dir = File.join(options[:output_dir], URI.parse(url).host.to_s.delete_prefix('www.'))
+        dir = File.join(options[:output_dir], Html2rss::Url.from_absolute(url).host.to_s.delete_prefix('www.'))
         FileUtils.mkdir_p(dir)
         file_path = File.join(dir, 'index.yml')
         File.write(file_path, result.yaml)
@@ -412,6 +411,13 @@ module Html2rss
       options[:strategy]&.to_sym || :auto
     end
 
+    def load_apply_config(source, feed_name)
+      return Html2rss.config_from_yaml_file(source.to_s, feed_name) if File.file?(source.to_s)
+
+      raw_content, = read_config_input(source)
+      Config.from_yaml(raw_content)
+    end
+
     def execute_feed # rubocop:disable Metrics/MethodLength
       yield
     rescue Html2rss::RequestService::RedirectLimitReached => error
@@ -427,6 +433,7 @@ module Html2rss
            Html2rss::RequestService::BlockedSurfaceDetected,
            Html2rss::NoFeedItemsExtracted,
            Html2rss::InvalidOption,
+           Html2rss::Config::InvalidConfig,
            Html2rss::Url::InvalidUrlError => error
       raise Thor::Error, error.message
     end
@@ -440,10 +447,14 @@ module Html2rss
         max_requests: options[:max_requests],
         local_file_path:,
         limit: options[:limit],
-        keep_different_domain: options[:keep_different_domain]
+        **cleanup_overrides
       )
     end
 
+    def cleanup_overrides
+      { keep_different_domain: options[:keep_different_domain], keep_sponsored: options[:keep_sponsored] }
+    end
+
     def explain_json!(payload)
       $stderr.puts JSON.pretty_generate(payload) # rubocop:disable Style/StderrPuts
     end