html2rss 0.23.0 → 0.25.0

data/lib/html2rss/request_service/botasaurus_contract.rb

@@ -10,7 +10,7 @@ module Html2rss
       # Default Botasaurus scrape options when no explicit config is provided.
       DEFAULT_OPTIONS = {
         navigation_mode: 'auto',
-        max_retries: 2,
+        max_retries: 1,
         headless: false
       }.freeze
 
@@ -30,6 +30,17 @@ module Html2rss
         lang
       ].freeze
 
+      # Allowlisted upstream response keys exposed as Response#transport_meta.
+      META_KEYS = %w[
+        request_id strategy_used render_ms challenge_detected blocked_detected attempts error_category
+      ].freeze
+
+      # Remaining seconds at or below which Botasaurus retries are disabled.
+      TIGHT_BUDGET_SECONDS = 12
+
+      # Seconds reserved from remaining budget before setting wait_timeout_seconds.
+      BUDGET_WAIT_RESERVE_SECONDS = 2
+
       # Parsed Botasaurus response wrapper.
       class ParsedResponse
         # Fallback headers when upstream omits response headers.
@@ -90,6 +101,9 @@ module Html2rss
         # @return [String, nil] final URL reported by upstream
         def final_url = payload['final_url']
 
+        # @return [Hash{String => Object}] allowlisted upstream telemetry (frozen)
+        def transport_meta = payload.slice(*META_KEYS).compact.freeze
+
         private
 
         attr_reader :payload, :transport_status
@@ -109,6 +123,7 @@ module Html2rss
       ##
       # @param url [Html2rss::Url] canonical URL to scrape
       # @param options [Hash] validated request.botasaurus options
+      # @param remaining_timeout_seconds [Numeric, nil] shared request budget remainder for clamps
       # @option options [String] :navigation_mode
       # @option options [Integer] :max_retries
       # @option options [String] :wait_for_selector
@@ -121,14 +136,15 @@ module Html2rss
       # @option options [String] :user_agent
       # @option options [Array<Integer>] :window_size
       # @option options [String] :lang
-      def initialize(url:, options: {})
+      def initialize(url:, options: {}, remaining_timeout_seconds: nil)
         @url = url
         @options = options
+        @remaining_timeout_seconds = remaining_timeout_seconds
       end
 
-      # @return [Hash] payload for POST /scrape
+      # @return [Hash] payload for POST /scrape (budget-clamped when remaining is known)
       def request_payload
-        DEFAULT_OPTIONS.merge(filtered_options).merge(url: url.to_s)
+        DEFAULT_OPTIONS.merge(filtered_options).merge(url: url.to_s).then { clamp_for_budget(_1) }
       end
 
       # @param transport_response [Faraday::Response] upstream HTTP response
@@ -145,13 +161,25 @@ module Html2rss
 
       private
 
-      attr_reader :url, :options
+      attr_reader :url, :options, :remaining_timeout_seconds
 
       def filtered_options
         OPTION_KEYS.each_with_object({}) do |key, normalized|
           normalized[key] = options[key] if options.key?(key)
         end
       end
+
+      def clamp_for_budget(payload)
+        remaining = remaining_timeout_seconds
+        return payload if remaining.nil?
+
+        clamped = payload.dup
+        clamped[:max_retries] = 0 if remaining <= TIGHT_BUDGET_SECONDS
+        budget_wait = [1, (remaining - BUDGET_WAIT_RESERVE_SECONDS).floor].max
+        configured = clamped[:wait_timeout_seconds]
+        clamped[:wait_timeout_seconds] = configured ? [configured, budget_wait].min : budget_wait
+        clamped
+      end
     end
   end
 end

data/lib/html2rss/request_service/botasaurus_strategy.rb

@@ -27,7 +27,8 @@ module Html2rss
           body: parsed_response.html,
           headers: parsed_response.headers,
           url: response_url(parsed_response.final_url),
-          status: parsed_response.status
+          status: parsed_response.status,
+          transport_meta: parsed_response.transport_meta
         )
       end
 
@@ -52,7 +53,11 @@ module Html2rss
       end
 
       def contract
-        @contract ||= BotasaurusContract.new(url: ctx.url, options: ctx.request.fetch(:botasaurus, {}))
+        @contract ||= BotasaurusContract.new(
+          url: ctx.url,
+          options: ctx.request.fetch(:botasaurus, {}),
+          remaining_timeout_seconds: attempt_timeout_seconds
+        )
       end
 
       def client
@@ -60,9 +65,13 @@ module Html2rss
       end
 
       def request_options
-        timeout = ctx.budget.effective_timeout_seconds(fallback: ctx.policy.total_timeout_seconds)
+        { timeout: attempt_timeout_seconds.to_i }
+      end
 
-        { timeout: timeout.to_i }
+      def attempt_timeout_seconds
+        @attempt_timeout_seconds ||= ctx.budget.effective_timeout_seconds(
+          fallback: ctx.policy.total_timeout_seconds
+        )
       end
 
       def content_type_header

data/lib/html2rss/request_service/budget.rb

@@ -82,11 +82,16 @@ module Html2rss
       def remaining_timeout_seconds
         return unless @total_timeout_seconds
 
-        elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - @start_time
-        remaining = @total_timeout_seconds - elapsed
+        remaining = @total_timeout_seconds - elapsed_seconds
         [remaining, 0.0].max
       end
 
+      ##
+      # @return [Float] seconds since this budget was created
+      def elapsed_seconds
+        Process.clock_gettime(Process::CLOCK_MONOTONIC) - @start_time
+      end
+
       ##
       # Resolves wall-clock seconds for the next adapter attempt.
       #

data/lib/html2rss/request_service/faraday_strategy.rb

@@ -66,14 +66,55 @@ module Html2rss
         faraday_request(response_guard, deadline:, streaming_buffer: false, consume_budget: false)
       end
 
+      ##
+      # Validates the remote socket peer IP during Net::HTTP start.
+      module PeerIpValidator
+        module_function
+
+        # @param http [Net::HTTP] connection to configure
+        # @param policy [Policy] request policy
+        # @return [void]
+        def install!(http, policy:)
+          orig_start = http.method(:start)
+          http.define_singleton_method(:start) do |&block|
+            orig_start.call do |opened_http|
+              PeerIpValidator.validate!(opened_http, policy:)
+              block.call(opened_http)
+            end
+          end
+        end
+
+        # @param opened_http [Net::HTTP] active connection
+        # @param policy [Policy] request policy
+        # @return [void]
+        def validate!(opened_http, policy:)
+          scheme = opened_http.use_ssl? ? 'https' : 'http'
+          url = Html2rss::Url.from_absolute("#{scheme}://#{opened_http.address}:#{opened_http.port}")
+          policy.validate_remote_ip!(ip: peer_ip_for(opened_http), url:)
+        end
+
+        # @param opened_http [Net::HTTP]
+        # @return [String, nil]
+        def peer_ip_for(opened_http)
+          sock = opened_http.instance_variable_get(:@socket)
+          io = sock.respond_to?(:io) ? sock.io : sock
+          peeraddr = io.respond_to?(:peeraddr) ? io.peeraddr : nil
+          peeraddr&.[](3) || peeraddr&.[](2)
+        end
+      end
+
+      # rubocop:disable Metrics/AbcSize
       def client
         @client ||= Faraday.new(url: ctx.url.to_s, headers: ctx.headers) do |faraday|
           faraday.use Faraday::FollowRedirects::Middleware, limit: ctx.policy.max_redirects, callback: redirect_callback
           faraday.request :gzip
           faraday.use StreamingBodyMiddleware
-          faraday.adapter Faraday.default_adapter
+          faraday.adapter Faraday.default_adapter do |http|
+            PeerIpValidator.install!(http, policy: ctx.policy)
+          end
         end
       end
+      # rubocop:enable Metrics/AbcSize
 
       def apply_timeouts(request, deadline:)
         remaining_timeout = remaining_timeout_seconds(deadline)

data/lib/html2rss/request_service/network_guard.rb

@@ -48,7 +48,10 @@ module Html2rss
       def enforce_public_network!(url)
         host = url.host
         return if allow_private_networks?
-        return unless blocked_host?(host) || resolved_ip_addresses(host).any? { |address| blocked_ip?(address) }
+        raise PrivateNetworkDenied, "Private network target denied for #{url}" if blocked_host?(host)
+
+        addresses = resolved_ip_addresses(host)
+        return unless addresses.empty? || addresses.any? { |address| blocked_ip?(address) }
 
         raise PrivateNetworkDenied, "Private network target denied for #{url}"
       end
@@ -62,9 +65,8 @@ module Html2rss
       # @raise [PrivateNetworkDenied] if the response came from a blocked address
       def validate_remote_ip!(ip:, url:)
         return if allow_private_networks?
-        return if ip.nil? || ip.empty?
 
-        parsed_ip = parse_ip(ip)
+        parsed_ip = ip && !ip.to_s.empty? ? parse_ip(ip) : nil
         raise PrivateNetworkDenied, "Remote IP could not be validated for #{url}" unless parsed_ip
         return unless blocked_ip?(parsed_ip)
 

data/lib/html2rss/request_service/response.rb

@@ -7,12 +7,16 @@ module Html2rss
     ##
     # To be used by strategies to provide their response.
     class Response
+      # Default when a strategy does not attach transport telemetry.
+      EMPTY_TRANSPORT_META = {}.freeze
+
       ##
       # @param body [String] the body of the response
       # @param url [Html2rss::Url] the final request URL
       # @param headers [Hash] the headers of the response
       # @param status [Integer, nil] the HTTP status code when available
-      def initialize(body:, url:, headers: {}, status: nil)
+      # @param transport_meta [Hash] allowlisted upstream telemetry (frozen when present)
+      def initialize(body:, url:, headers: {}, status: nil, transport_meta: EMPTY_TRANSPORT_META)
         @body = body
 
         headers = headers.dup
@@ -22,6 +26,7 @@ module Html2rss
         @headers = headers
         @status = status
         @url = url
+        @transport_meta = transport_meta.nil? || transport_meta.empty? ? EMPTY_TRANSPORT_META : transport_meta.freeze
       end
 
       # @return [String] the raw body of the response
@@ -36,6 +41,9 @@ module Html2rss
       # @return [Html2rss::Url] the URL of the response
       attr_reader :url
 
+      # @return [Hash] allowlisted upstream transport telemetry
+      attr_reader :transport_meta
+
       # @return [String] normalized content type header value
       def content_type = header('content-type').to_s
 

data/lib/html2rss/request_service/strategy.rb

@@ -86,6 +86,9 @@ module Html2rss
 
       def check_timeout!
         ctx.budget.effective_timeout_seconds(fallback: ctx.policy.total_timeout_seconds)
+      rescue RequestTimedOut
+        log_timeout!(reason: 'budget_exhausted')
+        raise
       end
 
       # @return [ResponseGuard, nil]
@@ -96,6 +99,8 @@ module Html2rss
       # @raise [StandardError]
       def handle_error(error)
         if timeout_error?(error)
+          log_timeout!(reason: 'transport')
+          Log.debug("#{self.class}: transport timeout message=#{error.message}")
           raise RequestTimedOut, error.message
         elsif connection_error?(error)
           translate_connection_error(error)
@@ -104,6 +109,23 @@ module Html2rss
         end
       end
 
+      # @param reason [String] timeout classification (budget_exhausted / transport)
+      # @return [void]
+      # rubocop:disable Metrics/AbcSize -- structured timeout fields stay in one log line
+      def log_timeout!(reason:)
+        remaining = ctx.budget.remaining_timeout_seconds
+        remaining_label = remaining.nil? ? 'untracked' : format('%.3f', remaining)
+        detail = [
+          "strategy=#{self.class.name}",
+          "host=#{ctx.url.host}",
+          "elapsed=#{format('%.3f', ctx.budget.elapsed_seconds)}s",
+          "budget_remaining=#{remaining_label}",
+          "reason=#{reason}"
+        ]
+        Log.info("#{self.class}: request timeout #{detail.join(' ')}")
+      end
+      # rubocop:enable Metrics/AbcSize
+
       # @param error [StandardError]
       # @return [void]
       # @raise [StandardError]

data/lib/html2rss/request_session.rb

@@ -109,6 +109,10 @@ module Html2rss
       visited_urls.add(normalize_url(url))
     end
 
+    ##
+    # @return [Html2rss::Url] the session's current request URL
+    def url = context.url
+
     private
 
     attr_reader :context, :strategy, :logger, :visited_urls

data/lib/html2rss/scoring/anchor_score.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Scoring
+    ##
+    # Primary-link ranking weights (sole home; FeatureId-aligned).
+    module AnchorScore
+      # Signal → integer weight map for primary-link ranking.
+      WEIGHTS = {
+        heading_anchor: 100,
+        heading_text_match: 20,
+        meaningful_text: 10,
+        content_like_destination: 10
+      }.freeze
+
+      module_function
+
+      ##
+      # @param heading_anchor [Boolean]
+      # @param heading_text_match [Boolean]
+      # @param meaningful_text [Boolean]
+      # @param content_like_destination [Boolean]
+      # @return [Integer]
+      def score(heading_anchor:, heading_text_match:, meaningful_text:, content_like_destination:)
+        total = 0
+        total += WEIGHTS[:heading_anchor] if heading_anchor
+        total += WEIGHTS[:heading_text_match] if heading_text_match
+        total += WEIGHTS[:meaningful_text] if meaningful_text
+        total += WEIGHTS[:content_like_destination] if content_like_destination
+        total
+      end
+    end
+  end
+end

data/lib/html2rss/scoring/cluster_scorer.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Scoring
+    ##
+    # Scores DomClustering-style node groups (heading/time/date/word density).
+    # Sole home for cluster-group ranking weights.
+    class ClusterScorer
+      # Minimum average visible words before a group is eligible.
+      MIN_AVG_WORDS = 5
+      # Bonus when any node has a heading or bold title class.
+      HEADING_BONUS = 20
+      # Bonus when any node has a time/datetime marker.
+      TIME_BONUS = 20
+      # Bonus when any node has date-like visible text.
+      DATE_BONUS = 40
+
+      def initialize
+        @text_words = {}.compare_by_identity
+        @has_date = {}.compare_by_identity
+      end
+
+      ##
+      # @param groups [Hash{String => Array<SST::Node>}]
+      # @return [Array<SST::Node>]
+      def select_best_group(groups)
+        best_nodes = []
+        best_score = -1
+
+        groups.each_value do |nodes|
+          score = score_group(nodes)
+          next if score.negative?
+
+          (best_nodes = nodes) && (best_score = score) if score > best_score
+        end
+        best_nodes
+      end
+
+      ##
+      # @param nodes [Array<SST::Node>]
+      # @return [Float]
+      def avg_words(nodes)
+        nodes.sum { |n| text_words(n) } / nodes.size.to_f
+      end
+
+      private
+
+      def score_group(nodes)
+        avg_w = avg_words(nodes)
+        return -1 if avg_w < MIN_AVG_WORDS
+
+        score = nodes.size + (avg_w / 5.0)
+        score += HEADING_BONUS if nodes_heading?(nodes)
+        score += TIME_BONUS if nodes_time?(nodes)
+        score += DATE_BONUS if nodes_date?(nodes)
+        score
+      end
+
+      def nodes_heading?(nodes)
+        nodes.any? do |n|
+          n.find(&:heading?) ||
+            n.attrs.class_names.any? { |c| c.match?(/font-bold|font-semibold/) }
+        end
+      end
+
+      def nodes_time?(nodes)
+        nodes.any? { |n| n.find { |d| d.name == :time || d.attrs.datetime } }
+      end
+
+      def nodes_date?(nodes)
+        nodes.any? { |n| date?(n) }
+      end
+
+      def text_words(node)
+        @text_words[node] ||= node.word_count
+      end
+
+      def date?(node)
+        @has_date[node] ||= begin
+          text = node.visible_text.to_s
+          text.match?(%r{\b\d{4}[-/]\d{2}[-/]\d{2}\b}) ||
+            text.match?(/\b(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\b/i)
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/scoring/container_assessor.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Scoring
+    ##
+    # Observes an SST container and builds typed {Observation} score inputs.
+    class ContainerAssessor
+      # Microdata itemprop values treated as publish/update markers.
+      PUBLISH_ITEMPROPS = %w[datePublished dateModified].freeze
+
+      # Matches content-like tokens in class/id strings.
+      CONTENT_TOKEN_REGEXP = begin
+        words = LinkDestination::PathClassifier::SEGMENT_SETS.fetch(:content)
+        /(?:^|\s|[-_])(#{Regexp.union(words.to_a).source})(?:\s|[-_]|$)/i
+      end.freeze
+
+      # Matches utility/junk tokens in class/id strings.
+      JUNK_TOKEN_REGEXP = begin
+        words = LinkDestination::PathClassifier::SEGMENT_SETS.fetch(:utility)
+        /(?:^|\s|[-_])(#{Regexp.union(words.to_a).source})(?:\s|[-_]|$)/i
+      end.freeze
+
+      # @param text_classifier [LinkDestination::TextClassifier]
+      def initialize(text_classifier: LinkDestination::TextClassifier.new)
+        @text_classifier = text_classifier
+      end
+
+      ##
+      # @param container [SST::Node]
+      # @param selected_anchor [SST::Node, nil]
+      # @param destination_facts [LinkDestination::DestinationFacts, nil]
+      # @return [Observation]
+      def call(container, selected_anchor, destination_facts:) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+        title = entry_title(container, selected_anchor)
+        tokens = "#{container.attrs.class_attr} #{container.attrs.id}"
+
+        Observation.new(
+          title_word_count: word_count(title),
+          path_length: destination_facts&.url&.path.to_s.length,
+          content_path: destination_facts&.content_path,
+          publish_marker: publish_marker?(container),
+          descriptive_context: descriptive_context?(container.visible_text, title),
+          article_container: container.name == :article,
+          content_tokens: tokens.match?(CONTENT_TOKEN_REGEXP),
+          junk_tokens: tokens.match?(JUNK_TOKEN_REGEXP),
+          utility_prefix_title: @text_classifier.utility_prefix?(title),
+          recommended_title: @text_classifier.recommended?(title),
+          utility_path: destination_facts&.utility_path,
+          strong_post_suffix: destination_facts&.strong_post_suffix,
+          shallow: destination_facts&.shallow,
+          high_confidence_junk_path: destination_facts&.high_confidence_junk_path,
+          high_confidence_utility_destination: destination_facts&.high_confidence_utility_destination,
+          selected_anchor_present: !selected_anchor.nil?
+        )
+      end
+
+      private
+
+      def publish_marker?(container)
+        !!container.find do |n|
+          n.name == :time ||
+            n.attrs.datetime ||
+            PUBLISH_ITEMPROPS.include?(n.attrs.itemprop.to_s)
+        end
+      end
+
+      def descriptive_context?(container_text, title)
+        snippet = container_text.to_s.sub(/\A#{Regexp.escape(title.to_s)}/i, '')
+        snippet.length > 30 && word_count(snippet) >= 8
+      end
+
+      def entry_title(container, selected_anchor)
+        heading = container.find(&:heading?)
+        source = heading || selected_anchor
+        source ? source.visible_text.to_s.strip : ''
+      end
+
+      def word_count(text)
+        text.to_s.scan(/\p{Alnum}+/).size
+      end
+    end
+  end
+end

data/lib/html2rss/scoring/engine.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Scoring
+    ##
+    # Ranks AutoSource::Segment instances with an explicit feature registry (no send).
+    class Engine
+      # Soft cap after score-floor filtering (lazy extraction guard; breadth > blind top-50).
+      TOP_K = 99
+      # Minimum composite score retained by {#rank_top} (precision floor).
+      SCORE_FLOOR = 0.0
+
+      # Quality feature triples: [FeatureId, predicate, weight].
+      QUALITY_RULES = [
+        [:title_word_count_ge3, ->(o) { o.title_word_count >= 3 }, 40],
+        [:title_word_count_ge7, ->(o) { o.title_word_count >= 7 }, 15],
+        [:path_length_gt6, ->(o) { o.path_length > 6 }, 20],
+        [:content_path, lambda(&:content_path), 15],
+        [:publish_marker, lambda(&:publish_marker), 15],
+        [:descriptive_context, lambda(&:descriptive_context), 10],
+        [:article_container, lambda(&:article_container), 10],
+        [:content_tokens, lambda(&:content_tokens), 10]
+      ].freeze
+
+      # Junk feature triples: [FeatureId, predicate, weight].
+      JUNK_RULES = [
+        [:non_content_utility_path, ->(o) { o.utility_path && !o.content_path && !o.strong_post_suffix }, 25],
+        [:utility_prefix_title_short, ->(o) { o.utility_prefix_title && o.title_word_count <= 6 }, 15],
+        [:shallow, lambda(&:shallow), 10],
+        [:weak_container, ->(o) { !o.publish_marker && !o.descriptive_context }, 10],
+        [:recommended_title_non_content, ->(o) { o.recommended_title && !o.content_path }, 10],
+        [:high_confidence_junk_path, lambda(&:high_confidence_junk_path), 5],
+        [:junk_tokens, lambda(&:junk_tokens), 15]
+      ].freeze
+
+      (QUALITY_RULES + JUNK_RULES).each { |feature_id, _, _| FeatureId.assert!(feature_id) }
+
+      # @param link_resolver [LinkResolver]
+      def initialize(link_resolver:)
+        @link_resolver = link_resolver
+        @assessor = ContainerAssessor.new(text_classifier: link_resolver.text_classifier)
+      end
+
+      ##
+      # @param segments [Array<Html2rss::AutoSource::Segment>]
+      # @return [Array<RankedSegment>] sorted by composite desc, position asc; hard-junk dropped
+      def rank(segments)
+        segments.filter_map { |segment| rank_one(segment) }
+                .sort_by { |ranked| [-ranked.final_score, ranked.position] }
+      end
+
+      ##
+      # Filters hard-junk but preserves discovery order (list/cluster scrapers).
+      #
+      # @param segments [Array<Html2rss::AutoSource::Segment>]
+      # @param limit [Integer]
+      # @return [Array<RankedSegment>]
+      def select_eligible(segments, limit: TOP_K)
+        segments.filter_map { |segment| rank_one(segment, hard_junk_mode: :lenient) }
+                .sort_by(&:position)
+                .first(limit)
+      end
+
+      ##
+      # @param segments [Array<Html2rss::AutoSource::Segment>]
+      # @param limit [Integer]
+      # @return [Array<RankedSegment>] score-floor filtered, then capped
+      def rank_top(segments, limit: TOP_K)
+        rank(segments).select { |ranked| ranked.final_score >= SCORE_FLOOR }.first(limit)
+      end
+
+      private
+
+      def rank_one(segment, hard_junk_mode: :strict) # rubocop:disable Metrics/MethodLength
+        facts = segment.primary_link ? @link_resolver.destination_facts(segment.primary_link) : nil
+        obs = @assessor.call(segment.root_node, segment.primary_link, destination_facts: facts)
+        return if obs.hard_junk?(mode: hard_junk_mode)
+
+        quality, quality_parts = apply_rules(QUALITY_RULES, obs)
+        junk, junk_parts = apply_rules(JUNK_RULES, obs)
+        RankedSegment.build(
+          segment:,
+          score: Score.build(
+            composite: quality - junk,
+            quality:,
+            junk:,
+            breakdown: quality_parts.merge(junk_parts)
+          )
+        )
+      end
+
+      def apply_rules(rules, obs)
+        parts = {}
+        rules.each do |feature_id, predicate, weight|
+          parts[feature_id] = weight if predicate.call(obs)
+        end
+        [parts.values.sum, parts]
+      end
+    end
+  end
+end