html2rss 0.25.0 → 0.27.0

data/lib/html2rss/request_service/context.rb

@@ -32,12 +32,6 @@ module Html2rss
       # @return [Hash] the request specific options
       attr_reader :request
 
-      # @return [Hash] browserless specific options
-      def browserless = request.fetch(:browserless, {})
-
-      # @return [Hash, nil] preload options for browserless requests
-      def browserless_preload = browserless[:preload]
-
       # @return [Symbol] the request relation
       attr_reader :relation
 

data/lib/html2rss/request_service/faraday_strategy.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require 'brotli'
 require 'faraday'
 require 'faraday/follow_redirects'
 require 'faraday/gzip'
@@ -9,6 +10,7 @@ module Html2rss
     ##
     # Strategy to use Faraday for the request.
     # @see https://rubygems.org/gems/faraday
+    # rubocop:disable Metrics/ClassLength -- terminal redirect retry colocated with Faraday transport
     class FaradayStrategy < Strategy
       ##
       # Restores buffered streamed bytes so response middleware can process them.
@@ -36,7 +38,8 @@ module Html2rss
       def perform_execute
         deadline = request_deadline
         @response_guard = ResponseGuard.new(policy: ctx.policy)
-        raw_response = faraday_request(response_guard, deadline:, streaming_buffer: true)
+        reset_redirect_tracking!
+        raw_response = request_with_terminal_redirect_retry(response_guard, deadline:)
         raw_response = retry_without_streaming(response_guard, deadline:) if retry_without_streaming?(raw_response)
         build_response(raw_response)
       end
@@ -47,8 +50,48 @@ module Html2rss
         monotonic_now + ctx.budget.effective_timeout_seconds(fallback: ctx.policy.total_timeout_seconds)
       end
 
+      def reset_redirect_tracking!
+        @last_redirect_to = nil
+        @terminal_redirect_retried = false
+        @request_url_override = nil
+      end
+
+      def request_with_terminal_redirect_retry(response_guard, deadline:)
+        faraday_request(response_guard, deadline:, streaming_buffer: true)
+      rescue Faraday::FollowRedirects::RedirectLimitReached => error
+        raise error unless terminal_redirect_retryable?
+
+        retry_from_terminal_redirect!(response_guard, deadline:)
+      end
+
+      def terminal_redirect_retryable?
+        return false if @terminal_redirect_retried || @last_redirect_to.nil?
+
+        @last_redirect_to.to_s != request_url.to_s
+      end
+
+      def retry_from_terminal_redirect!(response_guard, deadline:)
+        terminal_url = @last_redirect_to
+        @terminal_redirect_retried = true
+        Log.debug("#{self.class}: redirect limit reached; retrying once from #{terminal_url}")
+        begin_terminal_url_request!(terminal_url)
+        faraday_request(response_guard, deadline:, streaming_buffer: true, consume_budget: false)
+      end
+
+      def begin_terminal_url_request!(terminal_url)
+        ctx.policy.validate_request!(url: terminal_url, origin_url: ctx.origin_url, relation: ctx.relation)
+        @request_url_override = terminal_url
+        @client = nil
+        @last_redirect_to = nil
+      end
+
+      def request_url
+        @request_url_override || ctx.url
+      end
+
       def build_response(response)
-        Response.new(body: response.body, headers: response.headers, url: response_url(response),
+        Response.new(body: CompressedBody.decode(response.body, headers: response.headers),
+                     headers: response.headers, url: response_url(response),
                      status: response.status)
       end
 
@@ -105,7 +148,7 @@ module Html2rss
 
       # rubocop:disable Metrics/AbcSize
       def client
-        @client ||= Faraday.new(url: ctx.url.to_s, headers: ctx.headers) do |faraday|
+        @client ||= Faraday.new(url: request_url.to_s, headers: ctx.headers) do |faraday|
           faraday.use Faraday::FollowRedirects::Middleware, limit: ctx.policy.max_redirects, callback: redirect_callback
           faraday.request :gzip
           faraday.use StreamingBodyMiddleware
@@ -168,6 +211,7 @@ module Html2rss
         lambda do |old_env, new_env|
           from_url = normalize_url(old_env[:url])
           to_url = normalize_url(new_env[:url])
+          @last_redirect_to = to_url
           ctx.policy.validate_redirect!(from_url:, to_url:, origin_url: ctx.origin_url, relation: ctx.relation)
         end
       end
@@ -180,5 +224,6 @@ module Html2rss
         Process.clock_gettime(Process::CLOCK_MONOTONIC)
       end
     end
+    # rubocop:enable Metrics/ClassLength
   end
 end

data/lib/html2rss/request_service/policy.rb

@@ -18,7 +18,7 @@ module Html2rss
         connect_timeout_seconds: Integer(ENV.fetch('HTML2RSS_CONNECT_TIMEOUT_SECONDS', 5)),
         read_timeout_seconds: Integer(ENV.fetch('HTML2RSS_READ_TIMEOUT_SECONDS', 10)),
         total_timeout_seconds: Integer(ENV.fetch('HTML2RSS_TOTAL_TIMEOUT_SECONDS', 30)),
-        max_redirects: 3,
+        max_redirects: 5,
         max_response_bytes: 5_242_880,
         max_decompressed_bytes: 10_485_760,
         max_requests: 1,
@@ -115,7 +115,7 @@ module Html2rss
       # @raise [UnsupportedUrlScheme] if the redirect downgrades from HTTPS to HTTP
       def validate_redirect!(from_url:, to_url:, origin_url:, relation:)
         if from_url.scheme == 'https' && to_url.scheme == 'http'
-          raise UnsupportedUrlScheme, 'Redirect downgraded from https to http'
+          raise UnsupportedUrlScheme, "Redirect downgraded from #{from_url} to #{to_url}"
         end
 
         validate_request!(url: to_url, origin_url:, relation:)

data/lib/html2rss/request_service/response.rb

@@ -9,6 +9,16 @@ module Html2rss
     class Response
       # Default when a strategy does not attach transport telemetry.
       EMPTY_TRANSPORT_META = {}.freeze
+      # Default when a strategy does not capture sub-resource responses.
+      EMPTY_CAPTURED_RESPONSES = [].freeze
+
+      # Bodies that look like HTML even when Content-Type is missing, empty, or wrong.
+      HTML_BODY_SNIFF = /\A\s*(?:<!DOCTYPE\s+html|<html)/i
+      # Charset from Content-Type or a leading <meta charset>.
+      CHARSET_PARAMETER = /charset\s*=\s*["']?([\w.:-]+)/i
+      # Bytes scanned for a meta charset hint (HTML spec looks at the first 1024).
+      META_CHARSET_BYTES = 2048
+      private_constant :CHARSET_PARAMETER, :META_CHARSET_BYTES
 
       ##
       # @param body [String] the body of the response
@@ -16,7 +26,10 @@ module Html2rss
       # @param headers [Hash] the headers of the response
       # @param status [Integer, nil] the HTTP status code when available
       # @param transport_meta [Hash] allowlisted upstream telemetry (frozen when present)
-      def initialize(body:, url:, headers: {}, status: nil, transport_meta: EMPTY_TRANSPORT_META)
+      # @param captured_responses [Array<Hash>] JSON XHR/fetch bodies captured during browser scrapes
+      # rubocop:disable Metrics/MethodLength, Metrics/ParameterLists -- transport + capture fields stay co-located
+      def initialize(body:, url:, headers: {}, status: nil, transport_meta: EMPTY_TRANSPORT_META,
+                     captured_responses: EMPTY_CAPTURED_RESPONSES)
         @body = body
 
         headers = headers.dup
@@ -27,7 +40,13 @@ module Html2rss
         @status = status
         @url = url
         @transport_meta = transport_meta.nil? || transport_meta.empty? ? EMPTY_TRANSPORT_META : transport_meta.freeze
+        @captured_responses = if captured_responses.nil? || captured_responses.empty?
+                                EMPTY_CAPTURED_RESPONSES
+                              else
+                                captured_responses.freeze
+                              end
       end
+      # rubocop:enable Metrics/MethodLength, Metrics/ParameterLists
 
       # @return [String] the raw body of the response
       attr_reader :body
@@ -44,24 +63,26 @@ module Html2rss
       # @return [Hash] allowlisted upstream transport telemetry
       attr_reader :transport_meta
 
+      # @return [Array<Hash>] captured JSON XHR/fetch responses (empty when unsupported)
+      attr_reader :captured_responses
+
       # @return [String] normalized content type header value
       def content_type = header('content-type').to_s
 
       # @return [Boolean] whether response content is JSON
       def json_response? = content_type.include?('application/json')
 
-      # @return [Boolean] whether response content is HTML
-      def html_response? = content_type.include?('text/html')
+      # @return [Boolean] whether response content is HTML (header or sniffed body, never JSON)
+      def html_response?
+        content_type.include?('text/html') || (!json_response? && html_looking_body?)
+      end
 
       ##
       # @return [Nokogiri::HTML::Document, Hash] the parsed body of the response, frozen object
       # @raise [UnsupportedResponseContentType] if the content type is not supported
       def parsed_body
         @parsed_body ||= if html_response?
-                           Nokogiri::HTML(body).tap do |doc|
-                             # Remove comments from the document to avoid processing irrelevant content
-                             doc.xpath('//comment()').each(&:remove)
-                           end.freeze
+                           parse_html_document
                          elsif json_response?
                            JSON.parse(body, symbolize_names: true).freeze
                          else
@@ -78,6 +99,50 @@ module Html2rss
           headers.find { |key, _value| key.casecmp?(name) }&.last
         end
       end
+
+      def parse_html_document
+        Nokogiri::HTML(decoded_html_body).tap do |doc|
+          doc.xpath('//comment()').each(&:remove)
+        end.freeze
+      end
+
+      def decoded_html_body
+        bytes = body.to_s.b
+        transcode_html(bytes, header_charset || meta_charset(bytes))
+      end
+
+      def transcode_html(bytes, charset)
+        encoding = html_encoding_for(charset)
+        return utf8_scrub(bytes) unless encoding
+
+        bytes.dup.force_encoding(encoding).encode(Encoding::UTF_8)
+      rescue Encoding::UndefinedConversionError, Encoding::InvalidByteSequenceError
+        utf8_scrub(bytes)
+      end
+
+      def html_encoding_for(charset)
+        return if charset.nil? || utf8_charset?(charset)
+
+        Encoding.find(charset)
+      rescue ArgumentError
+        nil
+      end
+
+      def utf8_charset?(charset) = charset.to_s.downcase.gsub(/[\s_-]/, '') == 'utf8'
+
+      def utf8_scrub(bytes) = bytes.dup.force_encoding(Encoding::UTF_8).scrub
+
+      def header_charset
+        content_type[CHARSET_PARAMETER, 1]
+      end
+
+      def meta_charset(bytes)
+        bytes.byteslice(0, META_CHARSET_BYTES).to_s[CHARSET_PARAMETER, 1]
+      end
+
+      def html_looking_body?
+        body.to_s.b.match?(HTML_BODY_SNIFF)
+      end
     end
   end
 end

data/lib/html2rss/request_service/strategy.rb

@@ -137,8 +137,7 @@ module Html2rss
       # @return [Boolean]
       def timeout_error?(error)
         error.is_a?(Faraday::TimeoutError) ||
-          error.is_a?(Timeout::Error) ||
-          (defined?(Puppeteer::TimeoutError) && error.is_a?(Puppeteer::TimeoutError))
+          error.is_a?(Timeout::Error)
       end
 
       # @param error [StandardError]

data/lib/html2rss/request_service.rb

@@ -6,7 +6,7 @@ require 'forwardable'
 module Html2rss
   ##
   # Requests website URLs to retrieve their HTML for further processing.
-  # Provides concrete transport strategies (e.g. Faraday, Browserless).
+  # Provides concrete transport strategies (e.g. Faraday, Botasaurus).
   #
   # Feed-level +:auto+ is not registered here — {FeedPipeline::StrategyPlan} resolves
   # it to a concrete strategy (or {FeedPipeline::AutoFallback} chain) before execute.
@@ -23,8 +23,6 @@ module Html2rss
     class UnsupportedResponseContentType < Html2rss::Error; end
     # Raised when HTTP request slot limits are exceeded.
     class RequestBudgetExceeded < Html2rss::Error; end
-    # Raised when Browserless preload interaction limits are exceeded.
-    class InteractionBudgetExceeded < Html2rss::Error; end
     # Raised when policy denies private-network access.
     class PrivateNetworkDenied < Html2rss::Error; end
     # Raised when cross-origin follow-up requests are denied.
@@ -35,14 +33,17 @@ module Html2rss
     class BlockedSurfaceDetected < Html2rss::Error; end
     # Raised when a request times out.
     class RequestTimedOut < Html2rss::Error; end
-    # Raised when Browserless configuration is missing or invalid.
-    class BrowserlessConfigurationError < Html2rss::Error; end
-    # Raised when Browserless cannot be reached.
-    class BrowserlessConnectionFailed < Html2rss::Error; end
+
     # Raised when Botasaurus configuration is missing or invalid.
-    class BotasaurusConfigurationError < Html2rss::Error; end
-    # Raised when Botasaurus cannot be reached or returns invalid payloads.
+    class BotasaurusConfigurationError < Html2rss::Error
+      # Short empty-feed guidance owned once (composed by {Html2rss::NoFeedItemsExtracted}).
+      EMPTY_FEED_HINT = 'Configure BOTASAURUS_SCRAPER_URL to enable the Botasaurus request strategy.'
+    end
+
+    # Raised when the Botasaurus service cannot be reached (network / DNS / SSL).
     class BotasaurusConnectionFailed < Html2rss::Error; end
+    # Raised when Botasaurus responds but the scrape fails (upstream error, bad payload).
+    class BotasaurusServiceError < Html2rss::Error; end
 
     class << self
       extend Forwardable
@@ -62,7 +63,6 @@ module Html2rss
       @strategies = {
         faraday: FaradayStrategy,
         botasaurus: BotasaurusStrategy,
-        browserless: BrowserlessStrategy,
         local_file: LocalFileStrategy
       }
       @default_strategy_name = :faraday

data/lib/html2rss/scoring/container_assessor.rb

@@ -8,21 +8,15 @@ module Html2rss
       # Microdata itemprop values treated as publish/update markers.
       PUBLISH_ITEMPROPS = %w[datePublished dateModified].freeze
 
-      # Matches content-like tokens in class/id strings.
-      CONTENT_TOKEN_REGEXP = begin
-        words = LinkDestination::PathClassifier::SEGMENT_SETS.fetch(:content)
-        /(?:^|\s|[-_])(#{Regexp.union(words.to_a).source})(?:\s|[-_]|$)/i
-      end.freeze
-
-      # Matches utility/junk tokens in class/id strings.
-      JUNK_TOKEN_REGEXP = begin
-        words = LinkDestination::PathClassifier::SEGMENT_SETS.fetch(:utility)
-        /(?:^|\s|[-_])(#{Regexp.union(words.to_a).source})(?:\s|[-_]|$)/i
-      end.freeze
-
       # @param text_classifier [LinkDestination::TextClassifier]
-      def initialize(text_classifier: LinkDestination::TextClassifier.new)
+      # @param content_token_regexp [Regexp]
+      # @param junk_token_regexp [Regexp]
+      def initialize(text_classifier: LinkDestination::TextClassifier.new,
+                     content_token_regexp: LinkResolver::CONTENT_TOKEN_REGEXP,
+                     junk_token_regexp: LinkResolver::JUNK_TOKEN_REGEXP)
         @text_classifier = text_classifier
+        @content_token_regexp = content_token_regexp
+        @junk_token_regexp = junk_token_regexp
       end
 
       ##
@@ -41,8 +35,8 @@ module Html2rss
           publish_marker: publish_marker?(container),
           descriptive_context: descriptive_context?(container.visible_text, title),
           article_container: container.name == :article,
-          content_tokens: tokens.match?(CONTENT_TOKEN_REGEXP),
-          junk_tokens: tokens.match?(JUNK_TOKEN_REGEXP),
+          content_tokens: tokens.match?(@content_token_regexp),
+          junk_tokens: tokens.match?(@junk_token_regexp),
           utility_prefix_title: @text_classifier.utility_prefix?(title),
           recommended_title: @text_classifier.recommended?(title),
           utility_path: destination_facts&.utility_path,

data/lib/html2rss/scoring/engine.rb

@@ -10,7 +10,17 @@ module Html2rss
       # Minimum composite score retained by {#rank_top} (precision floor).
       SCORE_FLOOR = 0.0
 
-      # Quality feature triples: [FeatureId, predicate, weight].
+      FEATURE_IDS = %i[
+        title_word_count_ge3 title_word_count_ge7 path_length_gt6 content_path
+        publish_marker descriptive_context article_container content_tokens
+        non_content_utility_path utility_prefix_title_short shallow weak_container
+        recommended_title_non_content high_confidence_junk_path junk_tokens
+        heading_anchor heading_text_match meaningful_text content_like_destination
+        cluster_size cluster_avg_words cluster_heading cluster_time cluster_date
+      ].to_set.freeze
+      private_constant :FEATURE_IDS
+
+      # Quality feature triples: [feature_id, predicate, weight].
       QUALITY_RULES = [
         [:title_word_count_ge3, ->(o) { o.title_word_count >= 3 }, 40],
         [:title_word_count_ge7, ->(o) { o.title_word_count >= 7 }, 15],
@@ -22,7 +32,7 @@ module Html2rss
         [:content_tokens, lambda(&:content_tokens), 10]
       ].freeze
 
-      # Junk feature triples: [FeatureId, predicate, weight].
+      # Junk feature triples: [feature_id, predicate, weight].
       JUNK_RULES = [
         [:non_content_utility_path, ->(o) { o.utility_path && !o.content_path && !o.strong_post_suffix }, 25],
         [:utility_prefix_title_short, ->(o) { o.utility_prefix_title && o.title_word_count <= 6 }, 15],
@@ -33,7 +43,17 @@ module Html2rss
         [:junk_tokens, lambda(&:junk_tokens), 15]
       ].freeze
 
-      (QUALITY_RULES + JUNK_RULES).each { |feature_id, _, _| FeatureId.assert!(feature_id) }
+      ##
+      # @param id [Symbol]
+      # @return [Symbol]
+      # @raise [ArgumentError] when id is not in the closed set
+      def self.assert_feature_id!(id)
+        raise ArgumentError, "unknown feature: #{id.inspect}" unless FEATURE_IDS.include?(id)
+
+        id
+      end
+
+      (QUALITY_RULES + JUNK_RULES).each { |feature_id, _, _| assert_feature_id!(feature_id) }
 
       # @param link_resolver [LinkResolver]
       def initialize(link_resolver:)
@@ -78,9 +98,9 @@ module Html2rss
 
         quality, quality_parts = apply_rules(QUALITY_RULES, obs)
         junk, junk_parts = apply_rules(JUNK_RULES, obs)
-        RankedSegment.build(
+        ranked(
           segment:,
-          score: Score.build(
+          score: build_score(
             composite: quality - junk,
             quality:,
             junk:,
@@ -96,6 +116,30 @@ module Html2rss
         end
         [parts.values.sum, parts]
       end
+
+      def build_score(composite:, quality: nil, junk: nil, breakdown: nil)
+        raise ArgumentError, 'composite must be Numeric' unless composite.is_a?(Numeric)
+
+        Score.new(
+          composite: composite.to_f,
+          quality: (quality.nil? ? composite : quality).to_f,
+          junk: (junk || 0).to_f,
+          breakdown: normalize_breakdown(breakdown)
+        )
+      end
+
+      def normalize_breakdown(breakdown)
+        return Score::EMPTY_BREAKDOWN if breakdown.nil?
+
+        breakdown.transform_keys { |id| self.class.assert_feature_id!(id) }.freeze
+      end
+
+      def ranked(segment:, score:)
+        raise ArgumentError, 'segment must be AutoSource::Segment' unless segment.is_a?(Html2rss::AutoSource::Segment)
+        raise ArgumentError, 'score must be Scoring::Score' unless score.is_a?(Score)
+
+        RankedSegment.new(segment:, score:)
+      end
     end
   end
 end

data/lib/html2rss/scoring/link_resolver.rb

@@ -8,6 +8,18 @@ module Html2rss
       # Captures the href portion before a fragment for memoization keys.
       HREF_BASE_PATTERN = /\A([^#]*)/
 
+      # Matches content-like tokens in class/id strings (from PathClassifier vocabulary).
+      CONTENT_TOKEN_REGEXP = begin
+        words = LinkDestination::PathClassifier::SEGMENT_SETS.fetch(:content)
+        /(?:^|\s|[-_])(#{Regexp.union(words.to_a).source})(?:\s|[-_]|$)/i
+      end.freeze
+
+      # Matches utility/junk tokens in class/id strings (from PathClassifier vocabulary).
+      JUNK_TOKEN_REGEXP = begin
+        words = LinkDestination::PathClassifier::SEGMENT_SETS.fetch(:utility)
+        /(?:^|\s|[-_])(#{Regexp.union(words.to_a).source})(?:\s|[-_]|$)/i
+      end.freeze
+
       # @param base_url [String, Html2rss::Url]
       def initialize(base_url)
         @base_url = base_url

data/lib/html2rss/scoring/ranked_segment.rb

@@ -5,18 +5,6 @@ module Html2rss
     ##
     # A segment paired with its composite score.
     RankedSegment = Data.define(:segment, :score) do
-      ##
-      # @param segment [Html2rss::AutoSource::Segment]
-      # @param score [Score]
-      # @return [RankedSegment]
-      # @raise [ArgumentError] on invalid types
-      def self.build(segment:, score:)
-        raise ArgumentError, 'segment must be AutoSource::Segment' unless segment.is_a?(Html2rss::AutoSource::Segment)
-        raise ArgumentError, 'score must be Scoring::Score' unless score.is_a?(Score)
-
-        new(segment:, score:)
-      end
-
       ##
       # @return [SST::Node]
       def root_node = segment.root_node

data/lib/html2rss/scoring/score.rb

@@ -4,26 +4,7 @@ module Html2rss
   module Scoring
     ##
     # Composite score with quality/junk split and optional typed feature breakdown.
-    Score = Data.define(:composite, :quality, :junk, :breakdown) do
-      ##
-      # @param composite [Numeric]
-      # @param quality [Numeric, nil]
-      # @param junk [Numeric, nil]
-      # @param breakdown [Hash{Symbol => Numeric}, nil]
-      # @return [Score]
-      # @raise [ArgumentError] when composite is not numeric
-      def self.build(composite:, quality: nil, junk: nil, breakdown: nil)
-        raise ArgumentError, 'composite must be Numeric' unless composite.is_a?(Numeric)
-
-        parts = breakdown.nil? ? Score::EMPTY_BREAKDOWN : breakdown.transform_keys { FeatureId.assert!(_1) }.freeze
-        new(
-          composite: composite.to_f,
-          quality: (quality.nil? ? composite : quality).to_f,
-          junk: (junk || 0).to_f,
-          breakdown: parts
-        )
-      end
-    end
+    Score = Data.define(:composite, :quality, :junk, :breakdown)
     # Shared empty feature breakdown for scores without per-feature tallies.
     Score::EMPTY_BREAKDOWN = {}.freeze
   end

data/lib/html2rss/scoring.rb

@@ -4,30 +4,5 @@ module Html2rss
   ##
   # Declarative feature scoring for AutoSource SST segments.
   module Scoring
-    ##
-    # Closed set of feature identifiers used by the scoring registry.
-    module FeatureId
-      # Closed set of allowed feature identifier symbols.
-      IDS = %i[
-        title_word_count_ge3 title_word_count_ge7 path_length_gt6 content_path
-        publish_marker descriptive_context article_container content_tokens
-        non_content_utility_path utility_prefix_title_short shallow weak_container
-        recommended_title_non_content high_confidence_junk_path junk_tokens
-        heading_anchor heading_text_match meaningful_text content_like_destination
-        cluster_size cluster_avg_words cluster_heading cluster_time cluster_date
-      ].to_set.freeze
-
-      module_function
-
-      ##
-      # @param id [Symbol]
-      # @return [Symbol]
-      # @raise [ArgumentError] when id is not in the closed set
-      def assert!(id)
-        raise ArgumentError, "unknown FeatureId: #{id.inspect}" unless IDS.include?(id)
-
-        id
-      end
-    end
   end
 end

data/lib/html2rss/selectors/extractors/attribute.rb

@@ -24,9 +24,24 @@ module Html2rss
       # In case you're extracting a date or a time, consider parsing it
       # during post processing with {PostProcessors::ParseTime}.
       class Attribute
+        # Config-facing option types (excluding shared selector fields supplied at runtime).
+        OPTION_TYPES = { attribute: String }.freeze
+
         # The available options for the attribute extractor.
         Options = Struct.new('AttributeOptions', :selector, :attribute, keyword_init: true)
 
+        # JSON Schema description exported via +schema_doc+.
+        DESCRIPTION = 'Return the value of an HTML attribute on the selected element. ' \
+                      'Requires sibling selector option `attribute` (attribute name).'
+
+        # Example extractor name values for JSON Schema +examples+.
+        EXAMPLES = [
+          'attribute'
+        ].freeze
+
+        # @return [Hash{Symbol => Object}] JSON Schema fragment for this extractor name
+        def self.schema_doc = SchemaDoc.for_extractor(name: :attribute, klass: self)
+
         ##
         # Initializes the Attribute extractor.
         #

data/lib/html2rss/selectors/extractors/href.rb

@@ -27,6 +27,18 @@ module Html2rss
         # The available options for the href (attribute) extractor.
         Options = Struct.new('HrefOptions', :selector, :channel, keyword_init: true)
 
+        # JSON Schema description exported via +schema_doc+.
+        DESCRIPTION = 'Return the absolute URL from the selected element\'s `href` attribute ' \
+                      '(relative hrefs are resolved against the channel URL).'
+
+        # Example extractor name values for JSON Schema +examples+.
+        EXAMPLES = [
+          'href'
+        ].freeze
+
+        # @return [Hash{Symbol => Object}] JSON Schema fragment for this extractor name
+        def self.schema_doc = SchemaDoc.for_extractor(name: :href, klass: self)
+
         ##
         # Initializes the Href extractor.
         #

data/lib/html2rss/selectors/extractors/html.rb

@@ -26,6 +26,18 @@ module Html2rss
         # The available options for the html extractor.
         Options = Struct.new('HtmlOptions', :selector, keyword_init: true)
 
+        # JSON Schema description exported via +schema_doc+.
+        DESCRIPTION = 'Return the outer HTML of the selected element. ' \
+                      'Sanitize during post-processing (e.g. `sanitize_html`).'
+
+        # Example extractor name values for JSON Schema +examples+.
+        EXAMPLES = [
+          'html'
+        ].freeze
+
+        # @return [Hash{Symbol => Object}] JSON Schema fragment for this extractor name
+        def self.schema_doc = SchemaDoc.for_extractor(name: :html, klass: self)
+
         ##
         # Initializes the Html extractor.
         #

data/lib/html2rss/selectors/extractors/static.rb

@@ -16,9 +16,23 @@ module Html2rss
       # Would return:
       #    'Foobar'
       class Static
+        # Config-facing option types (excluding shared selector fields supplied at runtime).
+        OPTION_TYPES = { static: String }.freeze
+
         # The available option for the static extractor.
         Options = Struct.new('StaticOptions', :static, keyword_init: true)
 
+        # JSON Schema description exported via +schema_doc+.
+        DESCRIPTION = 'Return a fixed value from sibling selector option `static` (no DOM read).'
+
+        # Example extractor name values for JSON Schema +examples+.
+        EXAMPLES = [
+          'static'
+        ].freeze
+
+        # @return [Hash{Symbol => Object}] JSON Schema fragment for this extractor name
+        def self.schema_doc = SchemaDoc.for_extractor(name: :static, klass: self)
+
         ##
         # Initializes the Static extractor.
         #

data/lib/html2rss/selectors/extractors/text.rb

@@ -24,6 +24,17 @@ module Html2rss
         # The available options for the text extractor.
         Options = Struct.new('TextOptions', :selector, keyword_init: true)
 
+        # JSON Schema description exported via +schema_doc+.
+        DESCRIPTION = 'Return collapsed visible text of the selected element (default extractor).'
+
+        # Example extractor name values for JSON Schema +examples+.
+        EXAMPLES = [
+          'text'
+        ].freeze
+
+        # @return [Hash{Symbol => Object}] JSON Schema fragment for this extractor name
+        def self.schema_doc = SchemaDoc.for_extractor(name: :text, klass: self)
+
         ##
         # Initializes the Text extractor.
         #