html2rss 0.26.0 → 0.27.0

data/lib/html2rss/link_destination/path_classifier.rb

@@ -13,6 +13,7 @@ module Html2rss
       ].to_set.freeze
 
       # Segment groups used to classify article, taxonomy, utility, and vanity routes.
+      # Utility = taxonomy ∪ chrome ∪ soft utility (do not re-list taxonomy tokens).
       SEGMENT_SETS = begin
         content = %w[
           article articles blog blogs changelog changelogs insight insights
@@ -31,28 +32,23 @@ module Html2rss
           join membership plus premium pricing plans subscribe signup
           abonnieren abo suscribirse boletin s-abonner saboner
         ].to_set.freeze
-        utility = (
-          taxonomy.to_a + %w[
-            about account archive archives author authors comment comments
-            contact feedback help login logout notification notifications
-            preference preferences profile register search settings share signup subscribe
-            feed feeds comment-feed comments-feed privacy terms cookie cookies user users
-            kategorie kategorien schlagwort schlagworte thema themen autor autoren archiv
-            ueber-uns ueber ueberuns profil kontakt impressum suche hilfe anmelden registrieren
-            konto registrierung anmeldung abonnieren abo datenschutz nutzungsbedingungen agb
-            categoria categorias etiqueta etiquetas tema temas autores archivos
-            sobre-nosotros sobre quienes-somos buscar busqueda ayuda entrar ingresar
-            registrarse registro cuenta suscribirse boletin privacidad condiciones
-            categorie etiquette etiquettes sujet sujets theme themes auteur auteurs
-            a-propos apropos recherche rechercher aide connexion s-inscrire
-            sinscrire inscription compte s-abonner saboner lettre-information confidentialite
-            mentions-legales cgu menu sidebar widget social modal popup banner promo ad ads
-            related recommendation recommendations pagination pager
-            dating jobs job career careers deals deal shopping shop trading broker
-            versicherung tierversicherung insurance vergleich comparison
-            partnerboerse singleboerse krypto crypto
-          ] + SOFT_UTILITY.to_a
-        ).to_set.freeze
+        chrome = %w[
+          about account archive archives author authors comment comments contact feedback
+          help login logout notification notifications preference preferences profile register
+          search settings share signup subscribe feed feeds comment-feed comments-feed privacy
+          terms cookie cookies user users autor autoren archiv ueber-uns ueber ueberuns profil
+          kontakt impressum suche hilfe anmelden registrieren konto registrierung anmeldung
+          abonnieren abo datenschutz nutzungsbedingungen agb autores archivos sobre-nosotros
+          sobre quienes-somos buscar busqueda ayuda entrar ingresar registrarse registro cuenta
+          suscribirse boletin privacidad condiciones auteur auteurs a-propos apropos recherche
+          rechercher aide connexion s-inscrire sinscrire inscription compte s-abonner saboner
+          lettre-information confidentialite mentions-legales cgu menu sidebar widget social
+          modal popup banner promo ad ads related recommendation recommendations pagination pager
+          dating jobs job career careers deals deal shopping shop trading broker versicherung
+          tierversicherung insurance vergleich comparison partnerboerse singleboerse krypto crypto
+          casinos casino kreditkarten kreditkarte kredit echtgeld vpn games kaufberater leasing
+        ].to_set.freeze
+        utility = (taxonomy | chrome | SOFT_UTILITY).freeze
         {
           content:,
           utility:,
@@ -66,6 +62,11 @@ module Html2rss
       YEARISH_SEGMENT = /\A\d{4,}[\w-]*\z/
       # Hyphenated slug shape common to article permalinks.
       POST_SLUG_SEGMENT = /\A[a-z0-9]+(?:-[a-z0-9]+){2,}\z/i
+      # Multi-label host mistaken for a path segment (affiliate/outlink chrome).
+      # Final label must be alphabetic (TLD-like); exclude common file extensions.
+      HOST_SHAPED_SEGMENT = /\A(?:www\.)?(?:[\w-]+\.)+[a-z]{2,24}\z/i
+      # Common file-extension suffixes excluded from host-shaped junk matching.
+      FILE_EXTENSION_SEGMENT = /\.(?:pdf|jpe?g|png|gif|svg|webp|css|js|mjs|html?|xml|json|mp4|webm|zip|gz)\z/i
 
       # @param segments [Array<String>] normalized URL path segments
       def initialize(segments)
@@ -74,8 +75,8 @@ module Html2rss
 
       # @return [Boolean] true when the route has article-like path evidence
       def content_path?
-        @content_path ||= SEGMENT_SETS[:content].intersect?(segments) ||
-                          yearish_content_context?
+        @content_path ||= !leading_high_confidence_junk? &&
+                          (SEGMENT_SETS[:content].intersect?(segments) || yearish_content_context?)
       end
 
       # @return [Boolean] true when the route includes utility/navigation evidence
@@ -96,9 +97,8 @@ module Html2rss
       # @return [Boolean] true when the route is too shallow to strongly indicate an article
       def shallow?
         segment_count = segments.size
-        junk_segments = SEGMENT_SETS.fetch(:high_confidence_junk)
 
-        segment_count <= 1 || (segment_count == 2 && junk_segments.include?(segments.last))
+        segment_count <= 1 || (segment_count == 2 && high_confidence_junk_segment?(segments.last))
       end
 
       # @return [Boolean] true when the final path segment looks like a post slug
@@ -110,18 +110,15 @@ module Html2rss
 
       # @return [Boolean] true when every path segment is utility chrome
       def utility_only_route?
-        junk_segments = SEGMENT_SETS.fetch(:high_confidence_junk)
-
-        segments.all? { |segment| junk_segments.include?(segment) }
+        segments.all? { |segment| high_confidence_junk_segment?(segment) }
       end
 
       # @return [Boolean] true when the route is shallow and contains high-confidence noise
       def shallow_high_confidence_route?
-        junk_segments = SEGMENT_SETS.fetch(:high_confidence_junk)
         vanity_segments = SEGMENT_SETS.fetch(:vanity)
 
         shallow? && segments.any? do |segment|
-          junk_segments.include?(segment) || vanity_segments.include?(segment)
+          high_confidence_junk_segment?(segment) || vanity_segments.include?(segment)
         end
       end
 
@@ -134,7 +131,8 @@ module Html2rss
       def junk_path?
         return false if excluded_content_route?
 
-        taxonomy_path? ||
+        any_high_confidence_junk_segment? ||
+          taxonomy_path? ||
           utility_only_route? ||
           deep_utility_context_route? ||
           shallow_high_confidence_route?
@@ -149,6 +147,14 @@ module Html2rss
 
       private
 
+      def leading_high_confidence_junk?
+        segments.any? && high_confidence_junk_segment?(segments.first)
+      end
+
+      def any_high_confidence_junk_segment?
+        segments.any? { |segment| high_confidence_junk_segment?(segment) }
+      end
+
       def yearish_content_context?
         segments.any? { |segment| segment.match?(YEARISH_SEGMENT) } &&
           (strong_post_suffix? || trusted_post_context?(segments.size - 1))
@@ -172,12 +178,20 @@ module Html2rss
       def all_junk?(limit)
         return false if limit <= 0
 
-        junk_segments = SEGMENT_SETS.fetch(:high_confidence_junk)
-        (0...limit).all? { |i| junk_segments.include?(segments[i]) }
+        (0...limit).all? { |i| high_confidence_junk_segment?(segments[i]) }
+      end
+
+      def high_confidence_junk_segment?(segment)
+        SEGMENT_SETS.fetch(:high_confidence_junk).include?(segment) || host_shaped_segment?(segment)
+      end
+
+      def host_shaped_segment?(segment)
+        segment.match?(HOST_SHAPED_SEGMENT) && !segment.match?(FILE_EXTENSION_SEGMENT)
       end
 
       def trusted_post_context?(limit)
         return false if limit <= 0
+        return false if leading_high_confidence_junk?
 
         content_segments = SEGMENT_SETS.fetch(:content)
         context_segments = SEGMENT_SETS.fetch(:deep_post_context)
@@ -196,7 +210,7 @@ module Html2rss
 
       def excluded_last_segment?
         last = segments.last
-        [SEGMENT_SETS[:high_confidence_junk], SEGMENT_SETS[:vanity]].any? { |set| set.include?(last) }
+        high_confidence_junk_segment?(last) || SEGMENT_SETS[:vanity].include?(last)
       end
 
       def slug_last_segment?

data/lib/html2rss/mcp/config_argument.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module MCP
+    # Feed config supplied over MCP as a hash XOR a YAML string.
+    #
+    # Config does not know MCP. YAML parsing stays on {Html2rss::Config.from_yaml}.
+    ConfigArgument = Data.define(:config) do
+      class << self
+        ##
+        # @param config [Hash, nil] feed configuration hash
+        # @param yaml [String, nil] feed configuration YAML
+        # @return [ConfigArgument]
+        # @raise [ArgumentError] unless exactly one of +config+ or +yaml+ is present
+        # @raise [Contract::UnpublishedRequestError] when the parsed config uses
+        #   an unpublished MCP strategy or +request.local_file_path+
+        def parse(config: nil, yaml: nil)
+          parsed = case [present?(config), present?(yaml)]
+                   in [true, false]
+                     HashUtil.deep_symbolize_keys(config, context: 'config')
+                   in [false, true]
+                     Config.from_yaml(yaml)
+                   else
+                     raise ArgumentError, 'Provide exactly one of config or yaml'
+                   end
+          Contract.assert_published_request!(parsed)
+          new(config: parsed)
+        end
+
+        private
+
+        def present?(value)
+          case value
+          in nil then false
+          in String then !value.strip.empty?
+          else true
+          end
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/mcp/contract.rb

@@ -0,0 +1,173 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module MCP
+    ##
+    # Published MCP contract: strategy enum, input/output schemas, listing
+    # annotations, and the single compact JSON envelope response.
+    module Contract # rubocop:disable Metrics/ModuleLength -- published listing constants stay co-located
+      # Published MCP request strategies (excludes +local_file+).
+      STRATEGIES = %w[auto faraday botasaurus].freeze
+
+      # Raised when apply/validate config uses an unpublished MCP request adapter.
+      class UnpublishedRequestError < ArgumentError; end
+
+      # JSON Schema property for a source page URL.
+      URL_PROPERTY = {
+        type: 'string',
+        format: 'uri',
+        description: 'Source page URL'
+      }.freeze
+
+      # JSON Schema property for scrape/capture +strategy+.
+      STRATEGY_PROPERTY = {
+        type: 'string',
+        enum: STRATEGIES,
+        default: 'auto',
+        description: 'Request strategy (auto runs faraday → botasaurus fallback chain)'
+      }.freeze
+
+      # JSON Schema property for inspect +strategy+ (auto stays on Faraday).
+      INSPECT_STRATEGY_PROPERTY = STRATEGY_PROPERTY.merge(
+        description: 'Request strategy (auto uses Faraday for cheap diagnostics; pin botasaurus when needed)'
+      ).freeze
+
+      # JSON Schema +oneOf+ requiring exactly one of +config+ or +yaml+.
+      XOR_ONE_OF = [
+        { required: %w[config], not: { required: %w[yaml] } }.freeze,
+        { required: %w[yaml], not: { required: %w[config] } }.freeze
+      ].freeze
+
+      # JSON Schema properties for the config/yaml XOR pair.
+      CONFIG_XOR_PROPERTIES = {
+        config: {
+          type: 'object',
+          description: 'Feed configuration hash with channel and selectors (XOR yaml)'
+        }.freeze,
+        yaml: {
+          type: 'string',
+          pattern: '\\S',
+          description: 'Feed configuration YAML string (XOR config)'
+        }.freeze
+      }.freeze
+
+      # Input schema for +validate_config+ (config XOR yaml).
+      CONFIG_XOR_SCHEMA = {
+        type: 'object',
+        properties: CONFIG_XOR_PROPERTIES,
+        oneOf: XOR_ONE_OF
+      }.freeze
+
+      # Input schema for +apply_config+ (required URL plus config XOR yaml).
+      APPLY_INPUT_SCHEMA = {
+        type: 'object',
+        properties: { url: URL_PROPERTY, **CONFIG_XOR_PROPERTIES }.freeze,
+        required: %w[url],
+        oneOf: XOR_ONE_OF
+      }.freeze
+
+      # Input schema for +scrape_url+.
+      SCRAPE_INPUT_SCHEMA = {
+        type: 'object',
+        properties: {
+          url: URL_PROPERTY,
+          strategy: STRATEGY_PROPERTY,
+          limit: { type: 'integer', description: 'Max articles to keep (default 25)', default: 25 },
+          items_selector: { type: 'string', description: 'Optional CSS selector hint for items' }
+        }.freeze,
+        required: %w[url]
+      }.freeze
+
+      # Input schema for +inspect_url+.
+      INSPECT_INPUT_SCHEMA = {
+        type: 'object',
+        properties: { url: URL_PROPERTY, strategy: INSPECT_STRATEGY_PROPERTY }.freeze,
+        required: %w[url]
+      }.freeze
+
+      # Input schema for +capture_config+.
+      CAPTURE_INPUT_SCHEMA = {
+        type: 'object',
+        properties: {
+          url: URL_PROPERTY,
+          strategy: STRATEGY_PROPERTY,
+          items_selector: { type: 'string', description: 'Optional CSS selector hint for items' }
+        }.freeze,
+        required: %w[url]
+      }.freeze
+
+      # Tool annotations for open-world read-only tools.
+      ANNOTATIONS_OPEN_WORLD = {
+        read_only_hint: true,
+        destructive_hint: false,
+        idempotent_hint: true,
+        open_world_hint: true
+      }.freeze
+
+      # Tool annotations for +validate_config+ (closed world).
+      ANNOTATIONS_VALIDATE = ANNOTATIONS_OPEN_WORLD.merge(open_world_hint: false).freeze
+
+      # Human titles for +tools/list+.
+      TITLES = {
+        scrape_url: 'Scrape URL',
+        inspect_url: 'Inspect URL',
+        capture_config: 'Capture feed config',
+        validate_config: 'Validate feed config',
+        apply_config: 'Apply feed config'
+      }.freeze
+
+      class << self
+        ##
+        # Envelope JSON Schema. Built lazily so Zeitwerk can load Contract before Outcome.
+        #
+        # @return [Hash]
+        def output_schema # rubocop:disable Metrics/MethodLength -- schema document is one hash
+          {
+            type: 'object',
+            additionalProperties: false,
+            required: %w[ok next_step guidance payload],
+            properties: {
+              ok: { type: 'boolean' },
+              next_step: { type: 'string', enum: Outcome::NextStep::NAMES.map(&:to_s) },
+              guidance: { type: 'string' },
+              payload: { type: 'object' }
+            }
+          }
+        end
+
+        ##
+        # One envelope Hash, one compact JSON body, no +_meta+.
+        #
+        # @param outcome [Outcome]
+        # @return [::MCP::Tool::Response]
+        def response(outcome)
+          wire = outcome.to_h
+          ::MCP::Tool::Response.new(
+            [{ type: 'text', text: JSON.generate(wire) }],
+            error: !outcome.ok,
+            structured_content: wire
+          )
+        end
+
+        ##
+        # Rejects unpublished MCP request adapters so apply/validate cannot
+        # {File.read} arbitrary paths. CLI and Config still allow +local_file+.
+        #
+        # @param config [Hash]
+        # @return [void]
+        # @raise [UnpublishedRequestError] when +strategy+ is outside {STRATEGIES}
+        #   or +request.local_file_path+ is present
+        def assert_published_request!(config)
+          strategy = config[:strategy]
+          unless strategy.nil? || STRATEGIES.include?(strategy.to_s)
+            raise UnpublishedRequestError,
+                  "MCP does not accept strategy #{strategy} (published: #{STRATEGIES.join(', ')})"
+          end
+          return unless config.dig(:request, :local_file_path)
+
+          raise UnpublishedRequestError, 'MCP does not accept request.local_file_path'
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/mcp/inspect.rb

@@ -0,0 +1,241 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module MCP
+    ##
+    # Diagnostic inspect path (not Capture ownership). Fetches once for SST/scraper stats
+    # and recon facts (final URL, status, scheme downgrade, native feed hints).
+    module Inspect # rubocop:disable Metrics/ModuleLength -- diagnostic helpers stay co-located
+      module_function
+
+      ##
+      # @param url [String]
+      # @param strategy [String, Symbol]
+      # @return [Hash]
+      def call(url:, strategy: :auto) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+        resolved = FeedPipeline::StrategyPlan.concrete_for_diagnostic(strategy)
+        response = fetch_response(url, resolved)
+        parsed = response.parsed_body
+
+        result = recon_fields(url, response, parsed).merge(
+          strategy: resolved,
+          content_type: response.content_type,
+          html_response: response.html_response?,
+          scraper_eligibility: scraper_info(parsed),
+          sst_stats: sst_stats_from(response)
+        )
+
+        if response.html_response?
+          sst = sst_document(response)
+          if sst
+            result[:sst] = {
+              node_count: sst.node_count,
+              degraded: sst.degraded,
+              segment_stats: segment_stats(sst, url)
+            }
+          end
+        end
+
+        blocked = Html2rss::RequestService::BlockedSurface.interstitial_signature_for(response.body)
+        result[:blocked_surface] = blocked[:key].to_s if blocked
+        result[:xhr_capture] = xhr_capture_info(response) if resolved == :botasaurus
+        merge_admission_diagnostics!(result, response)
+
+        result
+      end
+
+      ##
+      # Surfaces Cleanup admission_drops without re-running full AutoSource discovery.
+      # Uses articles already extractable from a cheap AutoSource pass only when HTML.
+      #
+      # @param result [Hash]
+      # @param response [Html2rss::RequestService::Response]
+      # @return [void]
+      def merge_admission_diagnostics!(result, response)
+        return unless response.html_response?
+
+        source = AutoSource.new(response, AutoSource::DEFAULT_CONFIG.merge(limit: 10))
+        articles = source.articles
+        result[:articles_count] = articles.size
+        drops = source.admission_drops
+        result[:admission_drops] = drops if drops.any?
+      end
+      module_function :merge_admission_diagnostics!
+
+      ##
+      # @param response [Html2rss::RequestService::Response]
+      # @return [Hash] redacted XHR capture diagnostics (no query strings)
+      def xhr_capture_info(response)
+        captured = response.captured_responses
+        {
+          count: captured.size,
+          sample_endpoints: captured.first(5).filter_map { |entry| redacted_endpoint(entry) },
+          candidate_articles: captured.any? { |entry| xhr_candidate_articles?(entry) }
+        }
+      end
+      module_function :xhr_capture_info
+
+      ##
+      # @param entry [Hash] captured response hash
+      # @return [String, nil] scheme+host+path only
+      def redacted_endpoint(entry)
+        raw = entry['url'] || entry[:url]
+        return unless raw
+
+        uri = URI.parse(raw.to_s)
+        return unless uri.scheme && uri.host
+
+        "#{uri.scheme}://#{uri.host}#{uri.path}"
+      rescue URI::InvalidURIError
+        nil
+      end
+      module_function :redacted_endpoint
+
+      ##
+      # @param entry [Hash] captured response hash
+      # @return [Boolean]
+      def xhr_candidate_articles?(entry)
+        body = entry['body'] || entry[:body]
+        return false unless body.is_a?(String)
+
+        document = JSON.parse(body, symbolize_names: true)
+        AutoSource::Scraper::JsonState::CandidateDetector.candidate_array?(document)
+      rescue JSON::ParserError
+        false
+      end
+      module_function :xhr_candidate_articles?
+
+      ##
+      # @param url [String]
+      # @param strategy [Symbol]
+      # @return [Html2rss::RequestService::Response]
+      def fetch_response(url, strategy) # rubocop:disable Metrics/MethodLength -- session construction
+        raw_config = Config.auto_source_config(
+          url:,
+          request_controls: Config::RequestControls.from_shortcut(strategy:)
+        )
+        raw_config[:strategy] = strategy
+        config = Config.from_hash(raw_config)
+        resources = FeedPipeline::RuntimePolicy.resources_for(config)
+        session = RequestSession.build(
+          config:,
+          strategy: config.strategy,
+          budget: resources.budget,
+          policy: resources.policy
+        )
+        session.fetch_initial_response
+      end
+      module_function :fetch_response
+
+      ##
+      # @param parsed [Object] parsed response body
+      # @return [Array<String>, Hash]
+      def scraper_info(parsed)
+        return { error: 'Response is not HTML' } unless parsed.is_a?(Nokogiri::HTML::Document)
+
+        begin
+          Html2rss::AutoSource::Scraper.from(parsed).map(&:name)
+        rescue Html2rss::AutoSource::Scraper::NoScraperFound => error
+          { none_found: error.category.to_s }
+        end
+      end
+      module_function :scraper_info
+
+      ##
+      # @param response [Html2rss::RequestService::Response]
+      # @return [Hash, nil]
+      def sst_stats_from(response)
+        return nil unless response.html_response?
+
+        doc = sst_document(response)
+        return nil unless doc
+
+        { node_count: doc.node_count, degraded: doc.degraded }
+      rescue StandardError
+        nil
+      end
+      module_function :sst_stats_from
+
+      ##
+      # @param response [Html2rss::RequestService::Response]
+      # @return [Html2rss::SST::Document, nil]
+      def sst_document(response)
+        Html2rss::SST::Normalizer.call(response.body)
+      rescue ArgumentError
+        nil
+      end
+      module_function :sst_document
+
+      ##
+      # @param sst [Html2rss::SST::Document]
+      # @param url [String]
+      # @return [Hash]
+      def segment_stats(sst, url)
+        segments = discover_segments(sst, url)
+        return { found: 0 } if segments.empty?
+
+        {
+          found: segments.size,
+          strategies: segments.map(&:strategy).uniq,
+          sample_paths: segments.first(5).map { |s| s.root_node.tag_path }
+        }
+      end
+      module_function :segment_stats
+
+      ##
+      # @param sst [Html2rss::SST::Document]
+      # @param url [String]
+      # @return [Array]
+      def discover_segments(sst, url)
+        link_resolver = Scoring::LinkResolver.new(url)
+        AutoSource::Segmenter.call(
+          sst,
+          base_url: url,
+          strategy: :list,
+          link_resolver:
+        )
+      rescue StandardError
+        []
+      end
+      module_function :discover_segments
+
+      ##
+      # @param requested_url [String]
+      # @param response [Html2rss::RequestService::Response]
+      # @param parsed [Object]
+      # @return [Hash]
+      def recon_fields(requested_url, response, parsed)
+        requested = Url.from_absolute(requested_url)
+        final = response.url
+
+        {
+          requested_url: requested.to_s,
+          final_url: final.to_s,
+          status: response.status,
+          scheme_downgrade: scheme_downgrade?(requested, final),
+          alternate_feeds: alternate_feeds_from(parsed)
+        }
+      end
+      module_function :recon_fields
+
+      ##
+      # @param requested [Html2rss::Url]
+      # @param final [Html2rss::Url]
+      # @return [Boolean] true when the fetch downgraded https to http
+      def scheme_downgrade?(requested, final)
+        requested.scheme == 'https' && final.scheme == 'http'
+      end
+      module_function :scheme_downgrade?
+
+      ##
+      # @param parsed [Object]
+      # @return [Array<Hash{Symbol => String}>]
+      def alternate_feeds_from(parsed)
+        return [] unless parsed.is_a?(Nokogiri::HTML::Document)
+
+        Html::FeedLink.from_document(parsed).map { |link| { href: link.href, mime_type: link.mime_type } }
+      end
+      module_function :alternate_feeds_from
+    end
+  end
+end