html2rss 0.25.0 → 0.26.0

data/lib/html2rss/config/selectors_validator.rb

@@ -75,6 +75,9 @@ module Html2rss
       ##
       # Validates the configuration of a single selector.
       class Selector < Dry::Validation::Contract
+        # Extractor Options members supplied at runtime, not from feed YAML.
+        RUNTIME_EXTRACTOR_FIELDS = %i[selector channel].to_set.freeze
+
         params do
           optional(:selector)
           optional(:extractor).filled(:string)
@@ -88,33 +91,69 @@ module Html2rss
         end
 
         rule(:extractor) do
-          # dependent on the extractor, validate required fields, (i.e. static, attribute)
-          case value
-          when 'attribute'
-            key(:attribute).failure('`attribute` must be a string') unless values[:attribute].is_a?(String)
-          when 'static'
-            key(:static).failure('`static` must be a string') unless values[:static].is_a?(String)
+          next unless value
+
+          klass = Selectors::Extractors::NAME_TO_CLASS[value.to_sym]
+          next key(:extractor).failure("unknown extractor: #{value}") unless klass
+
+          klass::Options.members.each do |field|
+            next if RUNTIME_EXTRACTOR_FIELDS.include?(field)
+            next if values[field]
+
+            key(field).failure("`#{field}` is required for extractor `#{value}`")
           end
         end
 
         rule(:post_process).each do
-          case (name = value[:name])
-          when 'gsub'
-            key(:pattern).failure('`pattern` must be a string') unless value[:pattern].is_a?(String)
-            key(:replacement).failure('`replacement` must be a string') unless value[:replacement].is_a?(String)
-          when 'substring'
-            key(:start).failure('`start` must be an integer') unless value[:start].is_a?(Integer)
-            key(:end).failure('`end` must be an integer or omitted') if !value[:end].nil? && !value[:end].is_a?(Integer)
-          when 'template'
-            key(:string).failure('`string` must be a string') unless value[:string].is_a?(String)
-          when 'html_to_markdown', 'markdown_to_html', 'parse_time', 'parse_uri', 'sanitize_html'
-            # nothing to validate
-          when nil
-            key(:post_process).failure('Missing post_processor `name`')
-          else
-            key(:post_process).failure("Unknown post_processor `name`: #{name}")
+          name = value[:name]
+          next key(:post_process).failure('Missing post_processor `name`') if name.nil?
+
+          klass = Selectors::PostProcessors::NAME_TO_CLASS[name.to_sym]
+          next key(:post_process).failure("Unknown post_processor `name`: #{name}") unless klass
+          next unless klass.const_defined?(:Options)
+
+          post_process_option_type_errors(klass, value).each do |field, message|
+            key(field).failure(message)
           end
         end
+
+        private
+
+        def post_process_option_type_errors(klass, value)
+          required_option_type_errors(klass, value) + optional_option_type_errors(klass, value)
+        end
+
+        def required_option_type_errors(klass, value)
+          option_types = klass.const_defined?(:OPTION_TYPES) ? klass::OPTION_TYPES : {}
+
+          klass::Options.members.filter_map do |field|
+            expected = option_types.fetch(field, String)
+            next if value[field].is_a?(expected)
+
+            [field, option_type_failure(field, expected)]
+          end
+        end
+
+        def optional_option_type_errors(klass, value)
+          return [] unless klass.const_defined?(:OPTIONAL_OPTION_TYPES)
+
+          klass::OPTIONAL_OPTION_TYPES.filter_map do |field, expected|
+            actual = value[field]
+            next if actual.nil? || actual.is_a?(expected)
+
+            [field, option_type_failure(field, expected, optional: true)]
+          end
+        end
+
+        def option_type_failure(field, expected, optional: false)
+          label = {
+            Integer => 'an integer',
+            String => 'a string',
+            Hash => 'a hash'
+          }.fetch(expected) { "a #{expected}" }
+          suffix = optional ? ' or omitted' : ''
+          "`#{field}` must be #{label}#{suffix}"
+        end
       end
 
       ##

data/lib/html2rss/config/validator.rb

@@ -15,6 +15,11 @@ module Html2rss
       LANGUAGE_FORMAT_REGEX = /\A[a-z]{2}(-[A-Z]{2})?\z/
       # Baseline strategy-plan enum (:auto plus concrete RequestService strategies).
       BASE_STRATEGY_OPTIONS = Html2rss::FeedPipeline::StrategyPlan.accepted_names.freeze
+      # Controlled vocabulary for catalog-only `directory.topics` (not RSS channel fields).
+      DIRECTORY_TOPICS = %w[
+        sports energy tech science news entertainment jobs finance
+        security travel environment consumer civic product research
+      ].freeze
 
       # Contract for the top-level `channel` section.
       ChannelConfig = Dry::Schema.Params do
@@ -28,6 +33,11 @@ module Html2rss
         optional(:image).maybe(:string, format?: URI_REGEXP)
       end
 
+      # Contract for catalog-only `directory` metadata (topics for feed directories).
+      DirectoryConfig = Dry::Schema.Params do
+        optional(:topics).value(:array, min_size?: 1).each(:string, included_in?: DIRECTORY_TOPICS)
+      end
+
       # Contract for a stylesheet entry in `stylesheets`.
       StylesheetConfig = Dry::Schema.Params do
         required(:href).filled(:string)
@@ -35,47 +45,28 @@ module Html2rss
         optional(:media).maybe(:string)
       end
 
-      # Contract for Browserless click-preload options.
-      BrowserlessPreloadClickSelectorConfig = Dry::Schema.Params do
-        required(:selector).filled(:string)
-        optional(:max_clicks).filled(:integer, gt?: 0)
-        optional(:wait_after_ms).filled(:integer, gteq?: 0)
-      end
-
-      # Contract for Browserless scroll-preload options.
-      BrowserlessPreloadScrollConfig = Dry::Schema.Params do
-        optional(:iterations).filled(:integer, gt?: 0)
-        optional(:wait_after_ms).filled(:integer, gteq?: 0)
-      end
-
-      # Contract for Browserless preload orchestration options.
-      BrowserlessPreloadConfig = Dry::Schema.Params do
-        optional(:wait_after_ms).filled(:integer, gteq?: 0)
-        optional(:click_selectors).array(BrowserlessPreloadClickSelectorConfig)
-        optional(:scroll_down).hash(BrowserlessPreloadScrollConfig)
-      end
-
-      # Contract for Browserless-specific request options.
-      BrowserlessRequestConfig = Dry::Schema.Params do
-        optional(:preload).hash(BrowserlessPreloadConfig)
-      end
-
       # Contract for Botasaurus-specific request options.
       BotasaurusRequestConfig = Dry::Schema.Params do
         config.validate_keys = true
 
-        optional(:navigation_mode).filled(:string, included_in?: %w[auto get google_get google_get_bypass])
+        optional(:execution_mode).filled(:string, included_in?: %w[auto request browser])
+        optional(:navigation_mode).filled(:string, included_in?: %w[auto get google_get google_get_bypass organic_get])
         optional(:max_retries).filled(:integer, gteq?: 0, lteq?: 3)
         optional(:wait_for_selector).maybe(:string)
         optional(:wait_timeout_seconds).filled(:integer, gt?: 0)
+        optional(:scroll).filled(:bool)
+        optional(:scroll_to_bottom).filled(:bool)
         optional(:block_images).filled(:bool)
         optional(:block_images_and_css).filled(:bool)
+        optional(:block_trackers).filled(:bool)
         optional(:wait_for_complete_page_load).filled(:bool)
         optional(:headless).filled(:bool)
         optional(:proxy).filled(:string)
         optional(:user_agent).filled(:string)
         optional(:window_size).value(:array, min_size?: 2, max_size?: 2).each(:integer, gt?: 0)
         optional(:lang).filled(:string)
+        optional(:cookies).hash
+        optional(:headers).hash
       end
 
       # Contract for the top-level `request` section.
@@ -83,7 +74,6 @@ module Html2rss
         optional(:max_redirects).filled(:integer, gteq?: 0)
         optional(:max_requests).filled(:integer, gt?: 0)
         optional(:total_timeout_seconds).filled(:integer, gt?: 0)
-        optional(:browserless).hash(BrowserlessRequestConfig)
         optional(:botasaurus).hash(BotasaurusRequestConfig)
         optional(:local_file_path).filled(:string)
       end
@@ -91,6 +81,7 @@ module Html2rss
       params do
         optional(:strategy).filled(:symbol)
         required(:channel).hash(ChannelConfig)
+        optional(:directory).hash(DirectoryConfig)
         optional(:headers).hash
         optional(:stylesheets).array(StylesheetConfig)
         optional(:auto_source).hash(Config::AutoSourceContract)

data/lib/html2rss/error.rb

@@ -6,16 +6,24 @@ module Html2rss
 
   # Raised when auto fallback exhausts all concrete tiers and extractors find no feed items.
   class NoFeedItemsExtracted < Error
+    # Categories that append shared surface guidance to the empty-feed message.
+    SURFACE_HINT_CATEGORIES = %i[app_shell blocked_surface].freeze
+
     ##
     # @param attempts [Array<Hash{Symbol => Object}>] tier attempt diagnostics
-    def initialize(attempts:)
+    # @param surface_category [Symbol, nil] optional Scraper surface classification
+    def initialize(attempts:, surface_category: nil)
       @attempts = attempts
+      @surface_category = surface_category
       super(build_message)
     end
 
     # @return [Array<Hash{Symbol => Object}>] tier attempt diagnostics
     attr_reader :attempts
 
+    # @return [Symbol, nil] surface classification when a response body was available
+    attr_reader :surface_category
+
     private
 
     def build_message
@@ -24,8 +32,12 @@ module Html2rss
         "#{attempt[:strategy]} (#{details})"
       end.join(', ')
 
-      "No feed items extracted after auto fallback across strategies: #{summaries}. " \
-        'Try a more specific listing URL or provide explicit selectors.'
+      message = "No feed items extracted after auto fallback across strategies: #{summaries}. " \
+                'Try a more specific listing URL or provide explicit selectors.'
+      return message unless SURFACE_HINT_CATEGORIES.include?(surface_category)
+
+      guidance = AutoSource::Scraper::NoScraperFound::CATEGORY_MESSAGES.fetch(surface_category)
+      "#{message} #{guidance}"
     end
   end
 end

data/lib/html2rss/feed_pipeline/auto_fallback.rb

@@ -19,18 +19,18 @@ module Html2rss
         RequestService::RequestBudgetExceeded,
         RequestService::PrivateNetworkDenied,
         RequestService::CrossOriginFollowUpDenied,
-        RequestService::ResponseTooLarge,
-        RequestService::BrowserlessConfigurationError
+        RequestService::ResponseTooLarge
       ].freeze
 
       ##
       # Mutable run state for one auto-fallback chain: attempts plus selected result.
       class AttemptState
-        attr_reader :attempts, :result
+        attr_reader :attempts, :result, :last_response
 
         def initialize
           @attempts = []
           @result = nil
+          @last_response = nil
         end
 
         # @return [Boolean] true when a strategy already yielded items
@@ -53,6 +53,12 @@ module Html2rss
           @attempts << attempt
         end
 
+        # @param response [RequestService::Response] last response that reached extraction
+        # @return [void]
+        def remember_response(response)
+          @last_response = response
+        end
+
         # @param response [RequestService::Response] successful response
         # @param articles [Array] extracted articles
         # @param dedup_dropped [Integer] articles removed by deduplication
@@ -92,7 +98,7 @@ module Html2rss
         state = run_attempts
         return state.result if state.succeeded?
 
-        finalize_failure(attempts: state.attempts)
+        finalize_failure(attempts: state.attempts, response: state.last_response)
       end
 
       private
@@ -127,6 +133,7 @@ module Html2rss
       end
 
       def process_response(response:, strategy:, next_strategy:, request_session:, state:)
+        state.remember_response(response)
         articles, dedup_dropped = articles_for(response:, request_session:)
         items_count = articles.size
         state.record_items(strategy:, items_count:, transport_meta: response.transport_meta)
@@ -152,8 +159,11 @@ module Html2rss
                  "budget_remaining=#{budget_remaining_label}")
       end
 
-      def finalize_failure(attempts:)
-        raise NoFeedItemsExtracted.new(attempts:)
+      def finalize_failure(attempts:, response:)
+        surface_category = response && AutoSource::Scraper.classify_no_scraper_surface(
+          response.parsed_body, body: response.body
+        )
+        raise NoFeedItemsExtracted.new(attempts:, surface_category:)
       end
 
       # rubocop:disable Metrics/AbcSize, Metrics/MethodLength

data/lib/html2rss/feed_pipeline/runtime_policy.rb

@@ -6,8 +6,7 @@ module Html2rss
     # Planner for the runtime request policy and budgets of a feed run.
     #
     # HTTP request slots (pagination, document GETs, strategy fallback) are planned
-    # separately from Browserless preload interaction budget so preload cannot steal
-    # pagination slots.
+    # from configuration options.
     class RuntimePolicy
       # Policy plus Budget from one config expansion (shared meters for a feed run).
       Resources = Data.define(:policy, :budget)
@@ -34,26 +33,18 @@ module Html2rss
           policy:,
           budget: RequestService::Budget.new(
             max_requests: policy.max_requests,
-            max_interactions: interaction_budget_for(config),
             total_timeout_seconds: policy.total_timeout_seconds
           )
         )
       end
 
       ##
-      # Builds a Budget with separate request and interaction slot pools.
+      # Builds a Budget with request slot pools.
       #
       # @param config [Html2rss::Config] validated feed config
       # @return [Html2rss::RequestService::Budget] shared budget for the feed build
       def self.budget_for(config) = resources_for(config).budget
 
-      ##
-      # @param config [Html2rss::Config] validated feed config
-      # @return [Integer] preload interaction slots derived from browserless preload config
-      def self.interaction_budget_for(config)
-        browserless_preload_budget_for(config)
-      end
-
       class << self
         private
 
@@ -64,44 +55,13 @@ module Html2rss
         end
 
         # Reserve enough HTTP request slots for the initial request plus predictable
-        # follow-ups. Preload interactions are planned separately.
+        # follow-ups.
         def baseline_request_budget_for(config)
           1 +
             RequestSession::Pager.request_slots_for(config.selectors&.dig(:items, :pagination)) +
             AutoSource.request_slots_for(config.auto_source) +
             StrategyPlan.resolve(config.strategy).request_slots
         end
-
-        def browserless_preload_budget_for(config)
-          preload = config.request.dig(:browserless, :preload)
-          return 0 unless preload
-
-          top_level_preload_wait_budget(preload) +
-            click_selector_preload_budget(preload) +
-            scroll_preload_budget(preload)
-        end
-
-        def top_level_preload_wait_budget(preload)
-          preload[:wait_after_ms] ? 2 : 0
-        end
-
-        def click_selector_preload_budget(preload)
-          preload.fetch(:click_selectors, []).sum { preload_action_budget(_1, :max_clicks) }
-        end
-
-        def scroll_preload_budget(preload)
-          scroll = preload[:scroll_down]
-          return 0 unless scroll
-
-          preload_action_budget(scroll, :iterations)
-        end
-
-        def preload_action_budget(config, count_key)
-          action_count = config.fetch(count_key, 1)
-          wait_budget = config[:wait_after_ms] ? action_count : 0
-
-          action_count + wait_budget
-        end
       end
     end
   end

data/lib/html2rss/feed_pipeline.rb

@@ -81,15 +81,22 @@ module Html2rss
       request_session = request_session_for(config, strategy:, resources:)
       response = request_session.fetch_initial_response
       articles, dedup_dropped = deduplicated_articles(config:, response:, request_session:)
-      if config.auto_source && articles.empty?
-        raise NoFeedItemsExtracted.new(attempts: [{ strategy:, items_count: 0, error_class: nil }])
-      end
+      raise_empty_auto_source!(strategy:, response:) if config.auto_source && articles.empty?
 
       PipelineOutcome.new(
         response:, articles:, dedup_dropped:, selected_strategy: nil, attempt_count: 0, strategy_attempts: []
       )
     end
 
+    def raise_empty_auto_source!(strategy:, response:)
+      raise NoFeedItemsExtracted.new(
+        attempts: [{ strategy:, items_count: 0, error_class: nil }],
+        surface_category: AutoSource::Scraper.classify_no_scraper_surface(
+          response.parsed_body, body: response.body
+        )
+      )
+    end
+
     def run_auto_pipeline(config, resources:)
       AutoFallback.new(
         strategies: AutoFallback::CHAIN,

data/lib/html2rss/hash_util.rb

@@ -54,6 +54,23 @@ module Html2rss
       end
     end
 
+    # Converts hash keys (and Symbol values) to strings recursively.
+    #
+    # @param object [Object] value to normalize
+    # @return [Object] normalized value with string keys
+    def deep_stringify_keys(object)
+      case object
+      when Hash
+        object.to_h { |key, value| [key.to_s, deep_stringify_keys(value)] }
+      when Array
+        object.map { deep_stringify_keys(_1) }
+      when Symbol
+        object.to_s
+      else
+        object
+      end
+    end
+
     # Validates that hash keys are symbols.
     #
     # @param value [Object] candidate hash container whose keys must be symbols

data/lib/html2rss/html/sst_article_extractor.rb

@@ -80,19 +80,48 @@ module Html2rss
         Url.from_relative("##{id}", @base_url) if id
       end
 
-      def extract_title # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
-        source = heading || @selected_anchor
-        title_text = if source
-                       source.visible_text
-                     elsif @fallback_anchorless && @selected_anchor.nil?
-                       first_nonempty_own_text
-                     end
+      def extract_title
+        title_text = title_from_in_card_sources
         return unless title_text
 
         kicker = kicker_node&.visible_text.to_s.strip
         kicker && !kicker.empty? && !title_text.include?(kicker) ? "#{kicker}: #{title_text}" : title_text
       end
 
+      # Prefer heading over credit-shaped visible text; never emit agency-only titles.
+      #
+      # @return [String, nil]
+      def title_from_in_card_sources
+        title_candidates.find { |text| !credit_shaped_title?(text) }
+      end
+
+      # @return [Array<String>] ordered in-card title candidates (heading → anchor → fallback)
+      def title_candidates
+        [].tap do |candidates|
+          add_title_candidate!(candidates, heading&.visible_text)
+          add_title_candidate!(candidates, @selected_anchor&.visible_text)
+          next unless candidates.empty? && @fallback_anchorless && @selected_anchor.nil?
+
+          add_title_candidate!(candidates, first_nonempty_own_text)
+        end
+      end
+
+      # @param candidates [Array<String>]
+      # @param text [String, nil]
+      # @return [void]
+      def add_title_candidate!(candidates, text)
+        stripped = text.to_s.strip
+        return if stripped.empty? || candidates.include?(stripped)
+
+        candidates << stripped
+      end
+
+      # @param text [String]
+      # @return [Boolean]
+      def credit_shaped_title?(text)
+        AutoSource::Cleanup::CREDIT_TITLE.match?(text)
+      end
+
       def heading
         @heading ||= begin
           tags = @root.find_all(&:heading?)

data/lib/html2rss/link_destination/noise_policy.rb

@@ -9,10 +9,8 @@ module Html2rss
       ICON_NAMES = %i[img svg].freeze
 
       # @param link_resolver [LinkResolver]
-      # @param index [SST::Index]
-      def initialize(link_resolver:, index:)
+      def initialize(link_resolver:)
         @link_resolver = link_resolver
-        @index = index
       end
 
       ##
@@ -21,8 +19,11 @@ module Html2rss
       # @param anchor [SST::Node, nil]
       # @param container [SST::Node, nil]
       # @param heading_anchor [Boolean]
+      # @param utility_landmark_ancestor [Boolean] pre-computed by Segmenter tree walk
       # @return [Boolean]
-      def noise_anchor?(text:, destination_facts:, anchor: nil, container: nil, heading_anchor: false) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+      # rubocop:disable Metrics/CyclomaticComplexity, Metrics/ParameterLists, Metrics/PerceivedComplexity, Lint/UnusedMethodArgument
+      def noise_anchor?(text:, destination_facts:, anchor: nil, container: nil,
+                        heading_anchor: false, utility_landmark_ancestor: false)
         return true unless destination_facts
 
         destination_facts.taxonomy_path ||
@@ -32,8 +33,9 @@ module Html2rss
           (@link_resolver.utility_text?(text) && destination_facts.vanity_path) ||
           utility_text_chrome?(text, destination_facts, heading_anchor:) ||
           icon_only_anchor?(anchor, text) ||
-          utility_landmark_ancestor?(anchor, container)
+          utility_landmark_ancestor
       end
+      # rubocop:enable Metrics/CyclomaticComplexity, Metrics/ParameterLists, Metrics/PerceivedComplexity, Lint/UnusedMethodArgument
 
       private
 
@@ -61,19 +63,6 @@ module Html2rss
 
         !!anchor.find { |n| ICON_NAMES.include?(n.name) }
       end
-
-      def utility_landmark_ancestor?(anchor, container) # rubocop:disable Metrics/CyclomaticComplexity
-        return false unless anchor && container
-
-        curr = @index.parent_of(anchor)
-        while curr && curr.name != :html
-          return true if curr != container && curr.utility_landmark?
-          return false if curr.equal?(container)
-
-          curr = @index.parent_of(curr)
-        end
-        false
-      end
     end
   end
 end

data/lib/html2rss/link_destination/path_classifier.rb

@@ -48,6 +48,9 @@ module Html2rss
             sinscrire inscription compte s-abonner saboner lettre-information confidentialite
             mentions-legales cgu menu sidebar widget social modal popup banner promo ad ads
             related recommendation recommendations pagination pager
+            dating jobs job career careers deals deal shopping shop trading broker
+            versicherung tierversicherung insurance vergleich comparison
+            partnerboerse singleboerse krypto crypto
           ] + SOFT_UTILITY.to_a
         ).to_set.freeze
         {