html2rss 0.32.3 → 0.33.0

data/lib/html2rss/request_service/policy.rb

@@ -47,13 +47,13 @@ module Html2rss
                      allow_private_networks: DEFAULTS[:allow_private_networks],
                      allow_cross_origin_followups: DEFAULTS[:allow_cross_origin_followups],
                      resolver: Socket)
-        @connect_timeout_seconds = validate_positive_integer!(:connect_timeout_seconds, connect_timeout_seconds)
-        @read_timeout_seconds = validate_positive_integer!(:read_timeout_seconds, read_timeout_seconds)
-        @total_timeout_seconds = validate_positive_integer!(:total_timeout_seconds, total_timeout_seconds)
-        @max_redirects = validate_non_negative_integer!(:max_redirects, max_redirects)
-        @max_response_bytes = validate_positive_integer!(:max_response_bytes, max_response_bytes)
-        @max_decompressed_bytes = validate_positive_integer!(:max_decompressed_bytes, max_decompressed_bytes)
-        @max_requests = [validate_positive_integer!(:max_requests, max_requests), MAX_REQUESTS_CEILING].min
+        @connect_timeout_seconds = Option.integer(:connect_timeout_seconds, connect_timeout_seconds, min: 1)
+        @read_timeout_seconds = Option.integer(:read_timeout_seconds, read_timeout_seconds, min: 1)
+        @total_timeout_seconds = Option.integer(:total_timeout_seconds, total_timeout_seconds, min: 1)
+        @max_redirects = Option.integer(:max_redirects, max_redirects, min: 0)
+        @max_response_bytes = Option.integer(:max_response_bytes, max_response_bytes, min: 1)
+        @max_decompressed_bytes = Option.integer(:max_decompressed_bytes, max_decompressed_bytes, min: 1)
+        @max_requests = [Option.integer(:max_requests, max_requests, min: 1), MAX_REQUESTS_CEILING].min
         @allow_private_networks = allow_private_networks ? true : false
         @allow_cross_origin_followups = allow_cross_origin_followups ? true : false
         @network_guard = NetworkGuard.new(allow_private_networks: @allow_private_networks, resolver:)
@@ -135,18 +135,6 @@ module Html2rss
 
       attr_reader :network_guard
 
-      def validate_positive_integer!(name, value)
-        raise ArgumentError, "#{name} must be positive" unless value.is_a?(Integer) && value.positive?
-
-        value
-      end
-
-      def validate_non_negative_integer!(name, value)
-        raise ArgumentError, "#{name} must be non-negative" unless value.is_a?(Integer) && !value.negative?
-
-        value
-      end
-
       def enforce_same_origin!(url, origin_url, relation)
         return if relation == :initial || allow_cross_origin_followups?
 
@@ -163,13 +151,7 @@ module Html2rss
       end
 
       def comparable_origin(url)
-        [url.host, normalized_port(url)]
-      end
-
-      def normalized_port(url)
-        return url.port if url.port
-
-        url.scheme == 'https' ? 443 : 80
+        [url.scheme, url.host, url.port]
       end
     end
 

data/lib/html2rss/request_service/response.rb

@@ -31,15 +31,12 @@ module Html2rss
       # @param status [Integer, nil] the HTTP status code when available
       # @param transport_meta [Hash] allowlisted upstream telemetry (frozen when present)
       # @param captured_responses [Array<Hash>] JSON XHR/fetch bodies captured during browser scrapes
-      # rubocop:disable-next Metrics/MethodLength, Metrics/ParameterLists -- transport + capture fields stay co-located
+      # rubocop:disable-next Metrics/ParameterLists -- transport + capture fields stay co-located
       def initialize(body:, url:, headers: {}, status: nil, transport_meta: EMPTY_TRANSPORT_META,
                      captured_responses: EMPTY_CAPTURED_RESPONSES)
         @body = body
 
-        headers = (headers || {}).transform_keys(&:to_s)
-        HashUtil.assert_string_keys!(headers, context: 'response headers', deep: false)
-
-        @headers = headers
+        @headers = (headers || {}).transform_keys(&:to_s)
         @status = status
         @url = url
         @transport_meta = transport_meta.nil? || transport_meta.empty? ? EMPTY_TRANSPORT_META : transport_meta.freeze
@@ -165,7 +162,7 @@ module Html2rss
       end
 
       def feed_looking_body?
-        snippet = body.to_s[0, 800].downcase
+        snippet = body.to_s.b[0, 800].downcase
         FEED_BODY_MARKERS.any? { |marker| snippet.include?(marker) }
       end
     end

data/lib/html2rss/request_service/strategy.rb

@@ -133,9 +133,9 @@ module Html2rss
 
       # @param error [StandardError]
       # @return [void]
-      # @raise [StandardError]
+      # @raise [ConnectionFailed]
       def translate_connection_error(error)
-        raise error
+        raise ConnectionFailed, error.message
       end
 
       # @param error [StandardError]

data/lib/html2rss/request_service.rb

@@ -28,6 +28,8 @@ module Html2rss
     class RedirectLimitReached < Html2rss::Error; end
     # Raised when blocked content surfaces are detected.
     class BlockedSurfaceDetected < Html2rss::Error; end
+    # Raised when the target connection fails before a response document exists.
+    class ConnectionFailed < Html2rss::Error; end
 
     # Raised when a request times out.
     class RequestTimedOut < Html2rss::Error
@@ -76,6 +78,12 @@ module Html2rss
     # @return [Symbol]
     DEFAULT_STRATEGY_NAME = :default
 
+    # Maximum distinct [strategy, host] deprecation warnings tracked per process.
+    MAX_DEPRECATION_WARNINGS = 256
+    # Mutex guarding process-local deprecation warning deduplication.
+    DEPRECATION_MUTEX = Mutex.new
+    private_constant :DEPRECATION_MUTEX
+
     class << self
       # @return [Symbol] the default strategy name
       def default_strategy_name = DEFAULT_STRATEGY_NAME
@@ -107,6 +115,13 @@ module Html2rss
         strategy_class.new(ctx).execute
       end
 
+      ##
+      # Clears process-local deprecation warning state (used by specs).
+      # @return [void]
+      def reset_deprecation_warnings!
+        DEPRECATION_MUTEX.synchronize { @warned_deprecations = Set.new }
+      end
+
       private
 
       ##
@@ -125,14 +140,15 @@ module Html2rss
       end
 
       ##
-      # Emits a structured +strategy_deprecated+ warning once per session (+ :initial + only).
+      # Emits a structured +strategy_deprecated+ warning once per +(strategy, host)+ per process
+      # on initial session requests (+relation == :initial+). Multi-worker servers may emit once per worker.
       # @param strategy [Symbol] deprecated strategy alias
       # @param replacement [Symbol] preferred strategy name
       # @param relation [Symbol] request relation
       # @param host [String, nil] request host
       # @return [void]
       def warn_migration_strategy(strategy, replacement, relation:, host:)
-        return unless relation == :initial
+        return unless relation == :initial && claim_deprecation_warning?(strategy, host)
 
         Log.warn({
           event_name: 'strategy_deprecated',
@@ -142,6 +158,14 @@ module Html2rss
           message: "#{name}: strategy=#{strategy.inspect} is deprecated; using #{replacement.inspect} instead."
         }.to_json)
       end
+
+      def claim_deprecation_warning?(strategy, host)
+        DEPRECATION_MUTEX.synchronize do
+          @warned_deprecations ||= Set.new
+          @warned_deprecations.clear if @warned_deprecations.size >= MAX_DEPRECATION_WARNINGS
+          !@warned_deprecations.add?([strategy, host]).nil?
+        end
+      end
     end
   end
 end

data/lib/html2rss/request_session/pager/base.rb

@@ -26,7 +26,7 @@ module Html2rss
         # @yield [RequestService::Response] each page response
         # @return [Enumerator] enumerator when no block is given
         def each
-          return enum_for(:each) unless block_given?
+          return enum_for unless block_given?
 
           yield initial_response
 
@@ -48,14 +48,14 @@ module Html2rss
 
         # @return [Integer] configured maximum pages (always a positive Integer)
         def max_pages
-          value = config.fetch(:max_pages, DEFAULT_MAX_PAGES)
+          value = config[:max_pages]
           value.is_a?(Integer) && value.positive? ? value : DEFAULT_MAX_PAGES
         end
 
         # @param next_url [Html2rss::Url, String, nil]
         # @return [Boolean]
         def follow_up_allowed?(next_url)
-          !next_url.nil? && !next_url.to_s.empty? && !session.visited?(next_url)
+          !next_url.to_s.empty? && !session.visited?(next_url)
         end
 
         # @param next_url [Html2rss::Url, String]

data/lib/html2rss/request_session/pager/custom_selector.rb

@@ -11,7 +11,7 @@ module Html2rss
         # @param page_response [RequestService::Response]
         # @param page_number [Integer]
         # @return [String, nil]
-        def next_page_url(page_response, page_number: nil) # rubocop:disable Lint/UnusedMethodArgument
+        def next_page_url(page_response, page_number:) # rubocop:disable Lint/UnusedMethodArgument
           selector = config[:selector]
           return nil if selector.nil? || selector.empty?
 
@@ -19,16 +19,16 @@ module Html2rss
           return nil unless node
 
           href = node['href'] || node.text
-          return nil if href.nil? || href.strip.empty?
+          return nil if href.strip.empty?
 
-          Html2rss::Url.from_relative(href.strip, page_response.url)
+          Html2rss::Url.from_relative(href, page_response.url)
         end
 
         # @param parsed_body [Nokogiri::HTML::Document]
         # @param selector [String]
         # @return [Nokogiri::XML::Node, nil]
         def extract_node(parsed_body, selector)
-          parsed_body.at_css(selector) || parsed_body.at_xpath(selector)
+          parsed_body.at_css(selector)
         rescue Nokogiri::CSS::SyntaxError, Nokogiri::XML::XPath::SyntaxError
           # Pure XPath like `descendant::a` fails LOOKS_LIKE_XPATH / at_css; fall back explicitly.
           parsed_body.at_xpath(selector)

data/lib/html2rss/request_session/pager/offset.rb

@@ -25,14 +25,8 @@ module Html2rss
         # @return [Html2rss::Url]
         def next_page_url(page_response, page_number:)
           param = config.fetch(:param, DEFAULT_PARAM)
-          target_offset_val = target_offset_for(page_number)
           url = Html2rss::Url.from_absolute(page_response.url)
-
-          if url.to_s.include?('{offset}')
-            Html2rss::Url.from_absolute(url.to_s.gsub('{offset}', target_offset_val.to_s))
-          else
-            url.with_query_values(url.query_values.merge(param.to_s => target_offset_val.to_s))
-          end
+          url.with_query_values(url.query_values.merge(param.to_s => target_offset_for(page_number).to_s))
         end
 
         # @param page_number [Integer]

data/lib/html2rss/request_session/pager/rel_next.rb

@@ -11,7 +11,7 @@ module Html2rss
         # @param page_response [RequestService::Response]
         # @param page_number [Integer]
         # @return [String, nil]
-        def next_page_url(page_response, page_number: nil) # rubocop:disable Lint/UnusedMethodArgument
+        def next_page_url(page_response, page_number:) # rubocop:disable Lint/UnusedMethodArgument
           href = page_response.parsed_body.at_css('link[rel~="next"][href], a[rel~="next"][href]')&.[]('href')
           return nil if href.nil? || href.empty?
 

data/lib/html2rss/request_session/pager/url_template.rb

@@ -25,14 +25,8 @@ module Html2rss
         # @return [Html2rss::Url]
         def next_page_url(page_response, page_number:)
           param = config.fetch(:param, DEFAULT_PARAM)
-          target_page_val = target_page_for(page_number)
           url = Html2rss::Url.from_absolute(page_response.url)
-
-          if url.to_s.include?('{page}')
-            Html2rss::Url.from_absolute(url.to_s.gsub('{page}', target_page_val.to_s))
-          else
-            url.with_query_values(url.query_values.merge(param.to_s => target_page_val.to_s))
-          end
+          url.with_query_values(url.query_values.merge(param.to_s => target_page_for(page_number).to_s))
         end
 
         # @param page_number [Integer]

data/lib/html2rss/scoring/engine.rb

@@ -91,21 +91,17 @@ module Html2rss
 
       private
 
-      def rank_one(segment, hard_junk_mode: :strict) # rubocop:disable Metrics/MethodLength
+      def rank_one(segment, hard_junk_mode: :strict) # rubocop:disable Metrics/AbcSize
         facts = segment.primary_link ? @link_resolver.destination_facts(segment.primary_link) : nil
         obs = @assessor.call(segment.root_node, segment.primary_link, destination_facts: facts)
         return if obs.hard_junk?(mode: hard_junk_mode)
 
         quality, quality_parts = apply_rules(QUALITY_RULES, obs)
         junk, junk_parts = apply_rules(JUNK_RULES, obs)
-        ranked(
+        RankedSegment.new(
           segment:,
-          score: build_score(
-            composite: quality - junk,
-            quality:,
-            junk:,
-            breakdown: quality_parts.merge(junk_parts)
-          )
+          score: Score.new(composite: (quality - junk).to_f, quality: quality.to_f, junk: junk.to_f,
+                           breakdown: quality_parts.merge(junk_parts).freeze)
         )
       end
 
@@ -116,30 +112,6 @@ module Html2rss
         end
         [parts.values.sum, parts]
       end
-
-      def build_score(composite:, quality: nil, junk: nil, breakdown: nil)
-        raise ArgumentError, 'composite must be Numeric' unless composite.is_a?(Numeric)
-
-        Score.new(
-          composite: composite.to_f,
-          quality: (quality.nil? ? composite : quality).to_f,
-          junk: (junk || 0).to_f,
-          breakdown: normalize_breakdown(breakdown)
-        )
-      end
-
-      def normalize_breakdown(breakdown)
-        return Score::EMPTY_BREAKDOWN if breakdown.nil?
-
-        breakdown.transform_keys { |id| self.class.assert_feature_id!(id) }.freeze
-      end
-
-      def ranked(segment:, score:)
-        raise ArgumentError, 'segment must be AutoSource::Segment' unless segment.is_a?(Html2rss::AutoSource::Segment)
-        raise ArgumentError, 'score must be Scoring::Score' unless score.is_a?(Score)
-
-        RankedSegment.new(segment:, score:)
-      end
     end
   end
 end

data/lib/html2rss/scoring/link_resolver.rb

@@ -24,7 +24,6 @@ module Html2rss
       def initialize(base_url)
         @base_url = base_url
         @by_href = {}
-        @by_node = {}.compare_by_identity
         @text_classifier = LinkDestination::TextClassifier.new
       end
 
@@ -35,15 +34,11 @@ module Html2rss
       # @param node_or_href [SST::Node, String, #to_s]
       # @return [LinkDestination::DestinationFacts, nil]
       def destination_facts(node_or_href)
-        return @by_node[node_or_href] if node_or_href.is_a?(SST::Node) && @by_node.key?(node_or_href)
-
         href = extract_href(node_or_href)
         return unless href
 
-        facts = (@by_href[href] ||= build_facts(href))
-        @by_node[node_or_href] = facts if node_or_href.is_a?(SST::Node)
-        facts
-      rescue ArgumentError
+        @by_href[href] ||= build_facts(href)
+      rescue Html2rss::Url::InvalidUrlError
         nil
       end
 

data/lib/html2rss/selectors/extractors/href.rb

@@ -54,11 +54,9 @@ module Html2rss
         ##
         # Retrieves and returns the normalized absolute URL.
         #
-        # @return [Html2rss::Url, nil] The absolute URL.
+        # @return [Html2rss::Url, nil] The absolute URL, or nil when nothing matched or the +href+ is blank.
         def call
-          return nil unless @href
-
-          Url.from_relative(@href, @base_url)
+          Url.from_relative(@href, @base_url) unless @href.strip.empty?
         end
       end
     end

data/lib/html2rss/selectors/json_xml.rb

@@ -14,48 +14,47 @@ module Html2rss
         array: ['<array>', '</array>']
       }.freeze
 
-      module_function
-
-      ##
-      # Converts the object to XML format.
-      #
-      # @param object [Object] any Ruby object (Hash, Array, String, Symbol, etc.)
-      # @return [String] representing the object in XML
-      def call(object)
-        object_to_xml(object).tap do |converted|
-          Html2rss::Log.debug("#{self}: converted object to XML (#{converted.bytesize} bytes)")
+      class << self
+        ##
+        # Converts the object to XML format.
+        #
+        # @param object [Object] any Ruby object (Hash, Array, String, Symbol, etc.)
+        # @return [String] representing the object in XML
+        def call(object)
+          object_to_xml(object).tap do |converted|
+            Html2rss::Log.debug("#{self}: converted object to XML (#{converted.bytesize} bytes)")
+          end
         end
-      end
 
-      def object_to_xml(object)
-        case object
-        when Hash
-          hash_to_xml(object)
-        when Array
-          array_to_xml(object)
-        else
-          CGI.escapeHTML(object.to_s)
+        private
+
+        def object_to_xml(object)
+          case object
+          when Hash
+            hash_to_xml(object)
+          when Array
+            array_to_xml(object)
+          else
+            CGI.escapeHTML(object.to_s)
+          end
         end
-      end
-      private_class_method :object_to_xml
 
-      def hash_to_xml(object)
-        prefix, suffix = OBJECT_TO_XML_TAGS[:hash]
-        inner_xml = object.each_with_object(+'') do |(key, value), str|
-          str << "<#{key}>#{object_to_xml(value)}</#{key}>"
-        end
+        def hash_to_xml(object)
+          prefix, suffix = OBJECT_TO_XML_TAGS[:hash]
+          inner_xml = object.each_with_object(+'') do |(key, value), str|
+            str << "<#{key}>#{object_to_xml(value)}</#{key}>"
+          end
 
-        "#{prefix}#{inner_xml}#{suffix}"
-      end
-      private_class_method :hash_to_xml
+          "#{prefix}#{inner_xml}#{suffix}"
+        end
 
-      def array_to_xml(object)
-        prefix, suffix = OBJECT_TO_XML_TAGS[:array]
-        inner_xml = object.each_with_object(+'') { |value, str| str << object_to_xml(value) }
+        def array_to_xml(object)
+          prefix, suffix = OBJECT_TO_XML_TAGS[:array]
+          inner_xml = object.each_with_object(+'') { |value, str| str << object_to_xml(value) }
 
-        "#{prefix}#{inner_xml}#{suffix}"
+          "#{prefix}#{inner_xml}#{suffix}"
+        end
       end
-      private_class_method :array_to_xml
     end
   end
 end

data/lib/html2rss/selectors/post_processors/gsub.rb

@@ -101,7 +101,7 @@ module Html2rss
 
           source = regexp_source(string)
 
-          expression = Regexp::Parser.parse(source, options: ::Regexp::EXTENDED | ::Regexp::IGNORECASE)
+          expression = Regexp::Parser.parse(source)
           raise ArgumentError, 'pattern contains nested quantifiers' if nested_quantifiers?(expression)
 
           expression.to_re
@@ -161,8 +161,7 @@ module Html2rss
         ##
         # @return [Regexp]
         def pattern
-          raw = @pattern
-          raw.is_a?(String) ? self.class.compiled_pattern(raw) : raw
+          self.class.compiled_pattern(@pattern)
         end
       end
     end

data/lib/html2rss/selectors/post_processors/parse_time.rb

@@ -67,11 +67,10 @@ module Html2rss
         def self.validate_args!(_value, context)
           time_zone_value = context.time_zone
 
-          if time_zone_value.nil? || time_zone_value.empty?
-            raise ArgumentError, 'time_zone cannot be nil or empty', [], cause: nil
-          end
+          raise ArgumentError, 'time_zone cannot be nil or empty', [], cause: nil if time_zone_value.nil?
 
           assert_type(time_zone_value, String, :time_zone, context:)
+          raise ArgumentError, 'time_zone cannot be nil or empty', [], cause: nil if time_zone_value.empty?
         end
 
         ##
@@ -89,7 +88,7 @@ module Html2rss
         private
 
         def parse_time(string, time_zone)
-          components = Date._parse(string, false)
+          components = Date._parse(string)
           if components[:zone] || components.key?(:offset)
             Time.parse(string)
           else
@@ -104,7 +103,7 @@ module Html2rss
             components[:mday],
             components[:hour] || 0,
             components[:min] || 0,
-            (components[:sec] || 0) + (components[:sec_fraction] || 0)
+            components[:sec] || 0
           )
         end
       end

data/lib/html2rss/selectors/schema_export.rb

@@ -8,72 +8,72 @@ module Html2rss
     # Each registry class owns +DESCRIPTION+, +EXAMPLES+, and optional +OPTIONS+;
     # this module is the export adapter only.
     module SchemaExport
-      module_function
-
-      ##
-      # @param name [Symbol, String] registry key
-      # @param klass [Class] post-processor class
-      # @return [Hash{Symbol => Object}] JSON Schema object for one post-processor
-      def for_post_processor(name:, klass:)
-        name = name.to_s
-        {
-          type: 'object',
-          title: name,
-          description: klass::DESCRIPTION,
-          examples: klass::EXAMPLES,
-          properties: post_processor_properties(name, klass),
-          required: post_processor_required(klass),
-          additionalProperties: true
-        }
-      end
+      class << self
+        ##
+        # @param name [Symbol, String] registry key
+        # @param klass [Class] post-processor class
+        # @return [Hash{Symbol => Object}] JSON Schema object for one post-processor
+        def for_post_processor(name:, klass:)
+          name = name.to_s
+          {
+            type: 'object',
+            title: name,
+            description: klass::DESCRIPTION,
+            examples: klass::EXAMPLES,
+            properties: post_processor_properties(name, klass),
+            required: post_processor_required(klass),
+            additionalProperties: true
+          }
+        end
 
-      ##
-      # @param name [Symbol, String] registry key
-      # @param klass [Class] extractor class
-      # @return [Hash{Symbol => Object}] JSON Schema for one extractor name
-      def for_extractor(name:, klass:)
-        name = name.to_s
-        {
-          type: 'string',
-          const: name,
-          title: name,
-          description: klass::DESCRIPTION,
-          examples: klass::EXAMPLES
-        }
-      end
+        ##
+        # @param name [Symbol, String] registry key
+        # @param klass [Class] extractor class
+        # @return [Hash{Symbol => Object}] JSON Schema for one extractor name
+        def for_extractor(name:, klass:)
+          name = name.to_s
+          {
+            type: 'string',
+            const: name,
+            title: name,
+            description: klass::DESCRIPTION,
+            examples: klass::EXAMPLES
+          }
+        end
 
-      ##
-      # @param name [String] registry key
-      # @param klass [Class]
-      # @return [Hash{Symbol => Hash}]
-      def post_processor_properties(name, klass)
-        properties = { name: { type: 'string', const: name } }
-        OptionSpec.for(klass).each do |spec|
-          properties[spec.name] = property_schema_for(spec.type)
+        ##
+        # @param name [String] registry key
+        # @param klass [Class]
+        # @return [Hash{Symbol => Hash}]
+        def post_processor_properties(name, klass)
+          properties = { name: { type: 'string', const: name } }
+          OptionSpec.for(klass).each do |spec|
+            properties[spec.name] = property_schema_for(spec.type)
+          end
+          properties
         end
-        properties
-      end
 
-      ##
-      # @param klass [Class]
-      # @return [Array<String>]
-      def post_processor_required(klass)
-        required = ['name']
-        required + OptionSpec.for(klass).select(&:required).map { |spec| spec.name.to_s }
-      end
+        ##
+        # @param klass [Class]
+        # @return [Array<String>]
+        def post_processor_required(klass)
+          required = ['name']
+          required + OptionSpec.for(klass).select(&:required).map { |spec| spec.name.to_s }
+        end
 
-      ##
-      # @param ruby_type [Class, Array<Class>]
-      # @return [Hash{Symbol => String, Array<String>}]
-      def property_schema_for(ruby_type)
-        { type: json_type_for(ruby_type) }
-      end
+        ##
+        # @param ruby_type [Class, Array<Class>]
+        # @return [Hash{Symbol => String, Array<String>}]
+        def property_schema_for(ruby_type)
+          { type: json_type_for(ruby_type) }
+        end
 
-      ##
-      # @param ruby_type [Class, Array<Class>]
-      # @return [String, Array<String>]
-      def json_type_for(ruby_type)
-        OptionSpec.json_type_for(ruby_type)
+        ##
+        # @param ruby_type [Class, Array<Class>]
+        # @return [String, Array<String>]
+        def json_type_for(ruby_type)
+          OptionSpec.json_type_for(ruby_type)
+        end
       end
     end
   end