html2rss 0.27.2 → 0.29.1

data/lib/html2rss/feed_pipeline.rb

@@ -12,7 +12,7 @@ module Html2rss
     # strategy_attempts: auto attempt hashes (with optional transport_meta); empty outside :auto.
     PipelineOutcome = Data.define(
       :response, :articles, :dedup_dropped, :selected_strategy, :attempt_count, :strategy_attempts,
-      :admission_drops
+      :admission_drops, :scrape_target, :entry_resolution
     )
 
     ##
@@ -31,38 +31,39 @@ module Html2rss
       pipeline_outcome_for(config)
     end
 
+    ##
+    # Runs the pipeline once and returns scrape outcome plus {FeedResult} without a second fetch.
+    #
+    # @return [Array(PipelineOutcome, FeedResult)]
+    def to_outcome_and_result
+      config = Config.from_hash(raw_config, params: raw_config[:params])
+      outcome = pipeline_outcome_for(config)
+      [outcome, feed_result_for(config, outcome)]
+    end
+
     ##
     # Runs the pipeline once and returns an opaque, Marshal-cacheable result.
     #
     # @return [Html2rss::FeedResult]
-    # rubocop:disable Metrics/AbcSize, Metrics/MethodLength -- Status kwargs stay co-located with Channel
     def to_result
       config = Config.from_hash(raw_config, params: raw_config[:params])
       outcome = pipeline_outcome_for(config)
-      channel = Channel.from_response(outcome.response, overrides: config.channel)
-      status = Status.build(
-        articles: outcome.articles,
-        dedup_dropped: outcome.dedup_dropped,
-        selected_strategy: outcome.selected_strategy,
-        attempt_count: outcome.attempt_count,
-        strategy_attempts: outcome.strategy_attempts,
-        admission_drops: outcome.admission_drops
-      )
-      FeedResult.new(channel:, articles: outcome.articles, status:, stylesheets: config.stylesheets)
+      feed_result_for(config, outcome)
     end
-    # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
 
     # @api private Host seam for {AutoFallback} (and single-strategy path).
     # @param config [Html2rss::Config]
     # @param strategy [Symbol]
     # @param resources [Html2rss::FeedPipeline::RuntimePolicy::Resources]
+    # @param scrape_url [String, nil] effective fetch URL override for resolved targets
     # @return [Html2rss::RequestSession]
-    def request_session_for(config, strategy:, resources:)
+    def request_session_for(config, strategy:, resources:, scrape_url: nil)
       RequestSession.build(
         config:,
         strategy:,
         budget: resources.budget,
-        policy: resources.policy
+        policy: resources.policy,
+        scrape_url:
       )
     end
 
@@ -99,19 +100,22 @@ module Html2rss
 
       PipelineOutcome.new(
         response:, articles:, dedup_dropped:, selected_strategy: nil, attempt_count: 0,
-        strategy_attempts: [], admission_drops:
+        strategy_attempts: [], admission_drops:,
+        scrape_target: ScrapeTarget.from_config(config), entry_resolution: nil
       )
     end
 
     def raise_empty_auto_source!(strategy:, response:)
       raise NoFeedItemsExtracted.new(
         attempts: [{ strategy:, items_count: 0, error_class: nil }],
-        surface_category: AutoSource::Scraper.classify_no_scraper_surface(
-          response.parsed_body, body: response.body
-        )
+        surface_category: empty_auto_source_surface(response)
       )
     end
 
+    def empty_auto_source_surface(response)
+      PageRecon.surface_category_for(response:, url: response.url)
+    end
+
     def run_auto_pipeline(config, resources:)
       AutoFallback.new(
         strategies: AutoFallback::CHAIN,
@@ -128,9 +132,10 @@ module Html2rss
       [selector + auto, admission_drops]
     end
 
-    # rubocop:disable Metrics/MethodLength
+    # rubocop:disable-next Metrics/MethodLength
     def selector_articles(config:, response:, request_session:)
       return [] unless (selectors = config.selectors)
+      return [] if response.feed_response?
 
       page_responses = request_session.page_responses(
         response,
@@ -146,14 +151,36 @@ module Html2rss
       end
       articles
     end
-    # rubocop:enable Metrics/MethodLength
 
     # @return [Array(Array<Html2rss::Article>, Hash{String => Integer})]
     def auto_source_articles(config:, response:, request_session:)
       return [[], {}] unless (auto_source = config.auto_source)
+      return syndication_articles(response) if response.feed_response?
 
       source = AutoSource.new(response, auto_source, request_session:)
       [source.articles, source.admission_drops]
     end
+
+    def syndication_articles(response)
+      articles = Syndication::Parser.parse_response(response).map do |hash|
+        Article.new(**hash, scraper: AutoSource::Scraper::NativeFeed)
+      end
+      [articles, {}]
+    end
+
+    def feed_result_for(config, outcome) # rubocop:disable Metrics/MethodLength
+      channel = Channel.from_response(outcome.response, overrides: config.channel)
+      status = Status.build(
+        articles: outcome.articles,
+        dedup_dropped: outcome.dedup_dropped,
+        selected_strategy: outcome.selected_strategy,
+        attempt_count: outcome.attempt_count,
+        strategy_attempts: outcome.strategy_attempts,
+        admission_drops: outcome.admission_drops,
+        scrape_target: outcome.scrape_target,
+        entry_resolution: outcome.entry_resolution
+      )
+      FeedResult.new(channel:, articles: outcome.articles, status:, stylesheets: config.stylesheets)
+    end
   end
 end

data/lib/html2rss/feed_resolution/README.md

@@ -0,0 +1,52 @@
+# FeedResolution
+
+Backend-only entry URL tournament: when the pasted URL is a weak homepage/hub
+(or NativeFeed-majority extract), probe same-origin listing/feed candidates and
+sticky-rewrite the scrape URL only when the retry extract yields items
+(`try_apply!` `:succeeded`) before AutoFallback escalates strategies.
+
+Goal: best **article listing** for AutoSource. A native feed may still win the
+tournament on item count (feed-as-means) — intentional.
+
+## Ownership
+
+| Concern                                                                                    | Owner                                                      |
+| ------------------------------------------------------------------------------------------ | ---------------------------------------------------------- |
+| When to resolve (`articles:` required; floor / weak / NativeFeed ≥50%)                     | `FeedResolution::Policy`                                   |
+| Candidate mix (1 feed + up to 4 listing; taxonomy nav; segment first-wins)                 | `FeedResolution::CandidateGenerator`                       |
+| Listing path lexicon (`LISTING_PATHS`)                                                     | `Syndication::CandidateCatalog` (consumed, not owned here) |
+| Cheap probe + score + winner pick                                                          | `FeedResolution::Probe` + `FeedResolution::Scorer`         |
+| Typed entry_resolution options                                                             | `FeedResolution::Options`                                  |
+| Wire-safe resolution diag (`applied` = tournament win, not sticky URL)                     | `FeedResolution::Diag`                                     |
+| Public `call` → `Result`; retry orchestration (sticky `ScrapeTarget` only on `:succeeded`) | `FeedResolution` (`try_apply!`)                            |
+| Page surface for policy/scoring                                                            | `PageRecon::Assessment` + `Html2rss::SurfaceCategory`      |
+| Native feed discover/parse                                                                 | `Syndication` (not this module)                            |
+
+### CandidateGenerator mix
+
+- **Feed slot:** first `Syndication::Discovery.candidate_urls` only (alternates precede path guesses).
+- **Listing fill** (cap = `max - 1` when `max > 1`; `max == 1` → feed only): nav (`taxonomy_path +3`, `content_path +1`) → segment first-wins (`:list` → `:cluster` → `:semantic`, ≥2 same-origin primary links) → `LISTING_PATHS`.
+- Same-page / same-registrable-domain filters; feed never pads into listing slots.
+
+### Policy gates
+
+Resolve when eligible config **and** not blocked **and** any of:
+
+- `articles.size < ARTICLE_FLOOR` (3), or
+- surface `weak?`, or
+- ≥50% of articles have `scraper == AutoSource::Scraper::NativeFeed` (`native * 2 >= size`; empty → false).
+
+`FeedResolution.call` / `Runner` take **`articles:`** (Array) — not `articles_count:`.
+
+## Non-goals
+
+- Scorer weight overhaul / ranked multi-feed bake-off
+- Cross-origin probes
+- JSON Feed ingest; Sitemap/WP claim-gates
+- Create UI / Site Explorer
+- Changing `PageRecon` `:list`-only `discover_segments` (CandidateGenerator owns its own first-wins)
+- Syndication parse inside `FeedBuilder`
+
+## Config
+
+`auto_source.entry_resolution: { enabled: true, max_probes: 5 }`

data/lib/html2rss/feed_resolution/candidate_generator.rb

@@ -0,0 +1,139 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module FeedResolution
+    ##
+    # Builds same-origin entry-resolution candidates: one feed slot plus listing seeds.
+    #
+    # Mix is 1 feed + up to +listing_cap+ listing URLs (nav → segment first-wins →
+    # {Syndication::CandidateCatalog::LISTING_PATHS}). Feed never pads into listing slots.
+    class CandidateGenerator
+      # Max candidates returned to the probe stage.
+      DEFAULT_MAX = 5
+      # Nav/header/footer anchor sources for DestinationFacts ranking.
+      NAV_SELECTORS = 'header a[href], nav a[href], footer a[href]'
+      # Segment strategies tried first-wins (≥2 same-origin primary links).
+      SEGMENT_STRATEGIES = %i[list cluster semantic].freeze
+
+      ##
+      # @param entry_url [String, Html2rss::Url]
+      # @param response [Html2rss::RequestService::Response]
+      # @param max [Integer]
+      # @return [Array<Html2rss::Url>]
+      def self.call(entry_url:, response:, max: DEFAULT_MAX)
+        new(entry_url:, response:, max:).call
+      end
+
+      ##
+      # @param entry_url [String, Html2rss::Url]
+      # @param response [Html2rss::RequestService::Response]
+      # @param max [Integer]
+      def initialize(entry_url:, response:, max: DEFAULT_MAX)
+        @entry_url = Html2rss::Url.from_absolute(entry_url)
+        @response = response
+        @max = max
+      end
+
+      ##
+      # @return [Array<Html2rss::Url>]
+      def call
+        [feed_slot, *listing_urls.first(listing_cap)].compact.uniq
+      end
+
+      private
+
+      attr_reader :entry_url, :response, :max
+
+      def listing_cap = max > 1 ? max - 1 : 0
+
+      def feed_slot
+        return unless parsed_html
+
+        url = Syndication::Discovery.candidate_urls(page_url: entry_url, parsed_body: parsed_html).first
+        return unless url
+        return unless eligible_url?(url)
+
+        url
+      end
+
+      def listing_urls
+        (nav_candidates + segment_seed_urls + listing_path_candidates)
+          .uniq
+          .select { eligible_url?(_1) }
+      end
+
+      def nav_candidates
+        return [] unless parsed_html
+
+        parsed_html.css(NAV_SELECTORS)
+                   .filter_map { |anchor| ranked_anchor_url(anchor) }
+                   .sort_by { |(_url, rank)| -rank }
+                   .map(&:first)
+      end
+
+      def ranked_anchor_url(anchor)
+        url = absolute_same_origin(anchor['href'])
+        return unless url
+
+        facts = LinkDestination::DestinationFacts.build(url)
+        rank = 0
+        rank += 3 if facts.taxonomy_path
+        rank += 1 if facts.content_path
+        [url, rank] if rank.positive?
+      end
+
+      def segment_seed_urls
+        return [] unless parsed_html
+
+        sst = SST::Normalizer.call(parsed_html)
+        link_resolver = Scoring::LinkResolver.new(entry_url)
+
+        SEGMENT_STRATEGIES.each do |strategy|
+          urls = segment_urls_for(sst, strategy, link_resolver)
+          return urls if urls.size >= 2
+        end
+        []
+      rescue StandardError
+        []
+      end
+
+      def segment_urls_for(sst, strategy, link_resolver)
+        segmenter = AutoSource::Segmenter.new(sst, base_url: entry_url, strategy:, link_resolver:)
+        primary = AutoSource::Segmenter::PrimaryLink.new(segmenter)
+
+        segmenter.call.filter_map do |segment|
+          link = segment.primary_link || primary.select(segment.root_node)
+          absolute_same_origin(link&.attrs&.href)
+        end.uniq
+      end
+
+      def listing_path_candidates
+        Syndication::CandidateCatalog::LISTING_PATHS.filter_map { |path| absolute_same_origin(path) }
+      end
+
+      def parsed_html
+        return @parsed_html if defined?(@parsed_html)
+
+        @parsed_html = response.html_response? ? response.parsed_body : nil
+      rescue RequestService::UnsupportedResponseContentType
+        @parsed_html = nil
+      end
+
+      def absolute_same_origin(href)
+        return if href.nil? || href.empty? || href.start_with?('#', 'mailto:', 'javascript:')
+
+        Html2rss::Url.from_relative(href, entry_url)
+      rescue ArgumentError
+        nil
+      end
+
+      def eligible_url?(url)
+        !entry_url.same_document?(url) && same_registrable_domain?(url)
+      end
+
+      def same_registrable_domain?(url)
+        url.domain == entry_url.domain
+      end
+    end
+  end
+end

data/lib/html2rss/feed_resolution/diag.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module FeedResolution
+    ##
+    # Wire-safe diagnostics for one entry-resolution attempt (Status / Marshal edge).
+    Diag = Data.define(:applied, :probe_count, :reason, :winner_score) do
+      ##
+      # @param result [Html2rss::FeedResolution::Result]
+      # @return [Diag]
+      def self.from_result(result)
+        new(
+          applied: result.applied,
+          probe_count: result.probe_count,
+          reason: result.reason,
+          winner_score: result.winner_score
+        )
+      end
+
+      ##
+      # @return [Hash{Symbol => Object}]
+      def to_h
+        {
+          applied:,
+          probe_count:,
+          reason:,
+          **(winner_score.nil? ? {} : { winner_score: })
+        }
+      end
+    end
+  end
+end

data/lib/html2rss/feed_resolution/options.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module FeedResolution
+    ##
+    # Typed +auto_source.entry_resolution+ options (one expansion from config Hash).
+    Options = Data.define(:enabled, :max_probes) do
+      ##
+      # @param auto_source [Hash, nil]
+      # @return [Options]
+      def self.from_auto_source(auto_source)
+        raw = auto_source.is_a?(Hash) ? auto_source[:entry_resolution] : nil
+        raw = {} unless raw.is_a?(Hash)
+        new(
+          enabled: raw.fetch(:enabled, FeedResolution::DEFAULT_CONFIG[:enabled]) != false,
+          max_probes: Integer(raw.fetch(:max_probes, FeedResolution::DEFAULT_CONFIG[:max_probes]))
+        )
+      end
+
+      ##
+      # @return [Boolean]
+      def enabled? = enabled
+
+      ##
+      # Probe slots plus one Orchestrator retry GET when enabled.
+      #
+      # @return [Integer]
+      def request_slots = enabled? ? max_probes + 1 : 0
+    end
+  end
+end

data/lib/html2rss/feed_resolution/policy.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module FeedResolution
+    ##
+    # Decides whether an entry URL tournament should run after a weak extract.
+    module Policy
+      # Minimum admitted articles before the entry URL is considered strong enough.
+      ARTICLE_FLOOR = 3
+
+      module_function
+
+      ##
+      # @param config [Html2rss::Config]
+      # @param articles [Array<Html2rss::Article>] typed extract results (count derived)
+      # @param surface_category [Html2rss::SurfaceCategory, Symbol, nil]
+      # @return [Boolean]
+      def resolve?(config:, articles:, surface_category:)
+        raise ArgumentError, 'articles must be an Array' unless articles.is_a?(Array)
+        return false unless eligible_config?(config)
+
+        category = SurfaceCategory.coerce(surface_category)
+        return false if category.blocked?
+
+        articles.size < ARTICLE_FLOOR || category.weak? || native_feed_majority?(articles)
+      end
+
+      def eligible_config?(config)
+        return false unless config.auto_source
+        return false if config.selectors
+
+        Options.from_auto_source(config.auto_source).enabled?
+      end
+      module_function :eligible_config?
+      private_class_method :eligible_config?
+
+      def native_feed_majority?(articles)
+        return false if articles.empty?
+
+        native = articles.count { |article| article.scraper == AutoSource::Scraper::NativeFeed }
+        native * 2 >= articles.size
+      end
+      module_function :native_feed_majority?
+      private_class_method :native_feed_majority?
+    end
+  end
+end

data/lib/html2rss/feed_resolution/probe.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module FeedResolution
+    ##
+    # Cheap follow-up probe that scores a candidate via {PageRecon.assess} or syndication parse.
+    class Probe
+      ##
+      # @param url [Html2rss::Url]
+      # @param score [Numeric]
+      # @param articles_count [Integer]
+      Scored = Data.define(:url, :score, :articles_count)
+
+      ##
+      # @param request_session [Html2rss::RequestSession]
+      # @param origin_url [String, Html2rss::Url]
+      def initialize(request_session:, origin_url:)
+        @request_session = request_session
+        @origin_url = Html2rss::Url.from_absolute(origin_url)
+      end
+
+      ##
+      # @param url [Html2rss::Url]
+      # @return [Scored, nil]
+      def call(url)
+        response = request_session.follow_up(
+          url:,
+          relation: :entry_resolution,
+          origin_url:
+        )
+        score_response(url, response)
+      rescue Html2rss::Error
+        nil
+      end
+
+      private
+
+      attr_reader :request_session, :origin_url
+
+      def score_response(_url, response) # rubocop:disable Metrics/MethodLength -- feed vs HTML branches
+        if response.feed_response?
+          count = Syndication::Parser.parse_response(response).size
+          return Scored.new(
+            url: response.url,
+            score: Scorer.score_feed(articles_count: count),
+            articles_count: count
+          )
+        end
+
+        assessment = PageRecon.assess(response:, url: response.url)
+        Scored.new(
+          url: response.url,
+          score: Scorer.score_assessment(assessment),
+          articles_count: assessment.articles_count
+        )
+      end
+    end
+  end
+end

data/lib/html2rss/feed_resolution/scorer.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module FeedResolution
+    ##
+    # Scores probe targets and picks winners from cheap {PageRecon::Assessment} facts
+    # or syndication item counts.
+    module Scorer
+      # Weight for admitted article counts on HTML probes.
+      ARTICLE_WEIGHT = 10
+      # Bonus when the surface looks like a listing rather than a hub/shell.
+      LISTING_SURFACE_BONUS = 5
+
+      module_function
+
+      ##
+      # @param articles_count [Integer]
+      # @return [Integer]
+      def score_feed(articles_count:)
+        articles_count.to_i * ARTICLE_WEIGHT
+      end
+
+      ##
+      # @param assessment [Html2rss::PageRecon::Assessment]
+      # @return [Integer]
+      def score_assessment(assessment)
+        score = assessment.articles_count * ARTICLE_WEIGHT
+        score += LISTING_SURFACE_BONUS if assessment.listing_bonus?
+        drops = assessment.admission_drops.values.sum
+        total = assessment.articles_count + drops
+        score -= ((drops.to_f / total) * ARTICLE_WEIGHT).round if total.positive?
+        score
+      end
+
+      ##
+      # @param scored [Array<Probe::Scored>]
+      # @param entry_articles_count [Integer]
+      # @return [Probe::Scored, nil]
+      def pick_winner(scored:, entry_articles_count:)
+        winner = scored.max_by(&:score)
+        return unless winner
+        return if winner.articles_count <= entry_articles_count.to_i
+        return if winner.score <= 0
+
+        winner
+      end
+    end
+  end
+end