html2rss 0.25.0 → 0.26.0

data/lib/html2rss/auto_source/segmenter.rb

@@ -44,7 +44,7 @@ module Html2rss
         @minimum_selector_frequency = opts.fetch(:minimum_selector_frequency, 2)
         @use_top_selectors = opts.fetch(:use_top_selectors, 5)
         @link_resolver = opts[:link_resolver] || Scoring::LinkResolver.new(base_url)
-        @noise_policy = LinkDestination::NoisePolicy.new(link_resolver: @link_resolver, index: document.index)
+        @noise_policy = LinkDestination::NoisePolicy.new(link_resolver: @link_resolver)
       end
 
       # @return [Array<Segment>]
@@ -61,6 +61,35 @@ module Html2rss
 
       # @return [SST::Index]
       def index = document.index
+
+      ##
+      # Whether +anchor+ has a utility-landmark ancestor outside +container+.
+      #
+      # @param anchor [SST::Node]
+      # @param container [SST::Node]
+      # @return [Boolean]
+      def landmark_ancestor?(anchor, container)
+        cache = (@landmark_cache ||= {})
+        key = [anchor.object_id, container.object_id]
+        return cache[key] if cache.key?(key)
+
+        cache[key] = compute_landmark_ancestor?(anchor:, container:)
+      end
+
+      private
+
+      def compute_landmark_ancestor?(anchor:, container:) # rubocop:disable Metrics/CyclomaticComplexity
+        return false unless anchor && container
+
+        curr = index.parent_of(anchor)
+        while curr && curr.name != :html
+          return true if curr != container && curr.utility_landmark?
+          return false if curr.equal?(container)
+
+          curr = index.parent_of(curr)
+        end
+        false
+      end
     end
   end
 end

data/lib/html2rss/auto_source.rb

@@ -44,6 +44,9 @@ module Html2rss
         json_state: {
           enabled: true
         },
+        xhr_articles: {
+          enabled: true
+        },
         meta_oembed: {
           enabled: true
         },
@@ -93,6 +96,7 @@ module Html2rss
       @parsed_body = response.parsed_body
       @body = response.body
       @url = response.url
+      @captured_responses = response.captured_responses
       @opts = opts
       @request_session = request_session
     end
@@ -114,7 +118,7 @@ module Html2rss
 
     private
 
-    attr_reader :url, :parsed_body, :body, :request_session
+    attr_reader :url, :parsed_body, :body, :request_session, :captured_responses
 
     # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
     def extract_articles
@@ -143,7 +147,8 @@ module Html2rss
             request_session:,
             body:,
             document:,
-            link_resolver:
+            link_resolver:,
+            captured_responses:
           )
           next unless instance
           next unless Scraper.extractable_instance?(instance, parsed_body)

data/lib/html2rss/capture.rb

@@ -0,0 +1,315 @@
+# frozen_string_literal: true
+
+module Html2rss
+  ##
+  # Analyzes a URL and produces a reusable feed config hash with derived CSS selectors.
+  #
+  # Uses the auto-source pipeline to extract articles, then traces back through
+  # SST segments to build items + attribute selectors suitable for a static feed config.
+  class Capture # rubocop:disable Metrics/ClassLength
+    LEADING_TRIM_TAGS = %w[html body].freeze
+    private_constant :LEADING_TRIM_TAGS
+
+    ##
+    # Result of a capture operation.
+    # @!attribute config [Hash] feed config hash with +:channel+ and +:selectors+
+    # @!attribute articles_count [Integer] number of articles extracted
+    # @!attribute channel_title [String] derived channel title
+    CaptureResult = Data.define(:config, :articles_count, :channel_title)
+
+    class << self
+      ##
+      # Analyzes a URL and builds a reusable feed config.
+      #
+      # @param url [String] source page URL
+      # @param strategy [Symbol] request strategy (+:auto+, +:faraday+, +:botasaurus+)
+      # @option options [String, nil] :items_selector optional selector hint
+      # @option options [Integer, nil] :max_redirects optional redirect limit override
+      # @option options [Integer, nil] :max_requests optional request budget override
+      # @option options [Integer, nil] :limit max articles to keep
+      # @option options [String, nil] :local_file_path optional local HTML file path
+      # @return [CaptureResult]
+      def build(url, strategy: :auto, **)
+        new(url, strategy:, **).build
+      end
+    end
+
+    ##
+    # @param url [String] source page URL
+    # @param strategy [Symbol] request strategy
+    # @param options [Hash] additional options
+    # @option options [String, nil] :items_selector optional selector hint
+    # @option options [Integer, nil] :max_redirects optional redirect limit override
+    # @option options [Integer, nil] :max_requests optional request budget override
+    # @option options [Integer, nil] :limit max articles to keep
+    # @option options [String, nil] :local_file_path optional local HTML file path
+    def initialize(url, strategy: :auto, **options)
+      @url = url
+      @strategy = strategy
+      @items_selector_hint = options.delete(:items_selector)
+      @max_redirects = options.delete(:max_redirects)
+      @max_requests = options.delete(:max_requests)
+      @limit = options.delete(:limit)
+      @local_file_path = options.delete(:local_file_path)
+    end
+
+    ##
+    # Runs the capture pipeline.
+    #
+    # @return [CaptureResult]
+    def build # rubocop:disable Metrics/MethodLength
+      response = fetch_response
+      articles = extract_articles(response)
+      selectors = derive_selectors(response, articles)
+
+      config = {
+        channel: build_channel(response),
+        selectors: selectors.empty? ? nil : selectors,
+        **local_file_request_overlay
+      }.compact
+
+      CaptureResult.new(
+        config:,
+        articles_count: articles.size,
+        channel_title: channel_title_from(response)
+      )
+    end
+
+    private
+
+    def raw_config
+      @raw_config ||= build_raw_config
+    end
+
+    def build_raw_config # rubocop:disable Metrics/MethodLength
+      Config.auto_source_config(
+        url: @url,
+        items_selector: @items_selector_hint,
+        request_controls: Config::RequestControls.from_shortcut(
+          strategy: @strategy,
+          max_redirects: @max_redirects,
+          max_requests: @max_requests
+        ),
+        limit: @limit
+      ).tap do |config|
+        config[:strategy] = resolve_strategy(config[:strategy] || @strategy)
+        apply_local_file_path!(config)
+      end
+    end
+
+    def apply_local_file_path!(config)
+      return unless @local_file_path
+
+      config[:strategy] = :local_file
+      config[:request] ||= {}
+      config[:request][:local_file_path] = @local_file_path
+    end
+
+    def local_file_request_overlay
+      return {} unless @local_file_path
+
+      { strategy: :local_file, request: { local_file_path: @local_file_path } }
+    end
+
+    def resolve_strategy(strategy)
+      plan = FeedPipeline::StrategyPlan.resolve(strategy)
+      plan.is_a?(FeedPipeline::StrategyPlan::Auto) ? :faraday : plan.strategy
+    end
+
+    def fetch_response
+      config = Config.from_hash(raw_config)
+      resources = FeedPipeline::RuntimePolicy.resources_for(config)
+      session = RequestSession.build(
+        config:,
+        strategy: config.strategy,
+        budget: resources.budget,
+        policy: resources.policy
+      )
+      session.fetch_initial_response
+    end
+
+    def extract_articles(response)
+      auto_source_opts = raw_config[:auto_source] || AutoSource::DEFAULT_CONFIG
+      AutoSource.new(response, auto_source_opts).articles
+    rescue AutoSource::Scraper::NoScraperFound
+      []
+    end
+
+    def derive_selectors(response, articles) # rubocop:disable Metrics/MethodLength
+      return {} if articles.empty? || !response.html_response?
+
+      sst = normalize_sst(response)
+      return {} unless sst
+
+      segments = discover_segments(sst)
+      return {} if segments.empty?
+
+      matched = match_segments_to_articles(segments, articles)
+      return {} if matched.empty?
+
+      build_selector_hash(matched, sst)
+    rescue ArgumentError => error
+      Log.warn("Capture selector derivation failed: #{error.message}")
+      {}
+    end
+
+    def normalize_sst(response)
+      SST::Normalizer.call(response.body)
+    end
+
+    def discover_segments(sst)
+      link_resolver = Scoring::LinkResolver.new(@url)
+
+      AutoSource::Segmenter.call(
+        sst,
+        base_url: @url,
+        strategy: :list,
+        permit_unanchored: false,
+        link_resolver:
+      )
+    end
+
+    def match_segments_to_articles(segments, articles)
+      articles_by_url = articles.each_with_object({}) do |article, hash|
+        url_str = article.url.to_s
+        hash[url_str] = article unless url_str.empty?
+      end
+
+      segments.filter_map do |segment|
+        next unless segment.primary_link
+
+        article = find_matching_article(segment, articles_by_url)
+        { segment:, article: } if article
+      end
+    end
+
+    def find_matching_article(segment, articles_by_url) # rubocop:disable Metrics/AbcSize
+      href = segment.primary_link.attrs.href.to_s
+      return articles_by_url.values.find { |a| a.title == title_from_segment(segment) } if href.empty?
+
+      resolved = Url.from_relative(href, @url)
+      articles_by_url[resolved.to_s] || articles_by_url.values.find { |a| fuzzy_url_match?(a.url, resolved) }
+    end
+
+    def fuzzy_url_match?(article_url, resolved)
+      article_url.to_s.end_with?(resolved.path.to_s)
+    rescue StandardError
+      false
+    end
+
+    def title_from_segment(segment)
+      segment.root_node.visible_text.to_s.strip
+    end
+
+    def build_selector_hash(matched, _sst)
+      items_sel = items_selector(matched)
+      return {} unless items_sel
+
+      first = matched.first
+      root = first[:segment].root_node
+
+      attrs = {}.tap do |a|
+        a[:title] = title_selector(root)
+        a[:url] = url_selector(first[:segment])
+        a[:description] = description_selector(root, first[:segment])
+      end.compact
+
+      { items: { selector: items_sel }, **attrs }
+    end
+
+    def items_selector(matched)
+      roots = matched.map { |m| m[:segment].root_node }
+      shared = shared_class_items_selector(roots)
+      return shared if shared
+
+      paths = roots.map(&:tag_path)
+      common = common_path_prefix(paths)
+      tag_path = common.empty? ? paths.first.to_s : common
+      return nil if tag_path.empty?
+
+      css_from_trimmed_tag_path(tag_path)
+    end
+
+    def shared_class_items_selector(roots)
+      shared = roots.map { |root| root.attrs.class_names }.reduce { |left, right| left & right }
+      return nil if shared.nil? || shared.empty?
+
+      "#{roots.first.name}.#{shared.min}"
+    end
+
+    def css_from_trimmed_tag_path(tag_path)
+      segments = tag_path.split('/').reject(&:empty?)
+      trimmed = trim_leading_items_segments(segments)
+      trimmed.join(' > ')
+    end
+
+    def trim_leading_items_segments(segments)
+      return segments if segments.empty?
+
+      trimmed = segments.dup
+      trimmed.shift while trimmed.any? && LEADING_TRIM_TAGS.include?(trimmed.first)
+      trimmed.shift while trimmed.length > 1 && trimmed.first == 'div'
+      trimmed.empty? ? [segments.last] : trimmed
+    end
+
+    def common_path_prefix(paths) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity
+      return '' if paths.empty?
+
+      segments = paths.map { |p| p.split('/').reject(&:empty?) }
+      prefix = segments.first
+      segments.each do |seg|
+        i = 0
+        i += 1 while i < prefix.length && i < seg.length && prefix[i] == seg[i]
+        prefix = prefix[0...i]
+      end
+      "/#{prefix.join('/')}"
+    end
+
+    def title_selector(root) # rubocop:disable Metrics/CyclomaticComplexity
+      heading = root.find(&:heading?)
+      relative = css_for_path(heading.tag_path, root.tag_path) if heading
+      relative ||= begin
+        link = root.find { |n| n.link? && n.visible_text.to_s.strip.length > 3 }
+        css_for_path(link.tag_path, root.tag_path) if link
+      end
+      return nil unless relative
+
+      { selector: relative }
+    end
+
+    def url_selector(segment)
+      return nil unless segment.primary_link
+
+      relative = css_for_path(segment.primary_link.tag_path, segment.root_node.tag_path)
+      { selector: relative, extractor: 'href' }
+    end
+
+    def description_selector(root, segment)
+      relative = css_for_path(root.tag_path, segment.root_node.tag_path)
+      return nil if relative.nil? || relative.empty? || relative == '.'
+
+      { selector: relative }
+    end
+
+    def css_for_path(full_path, root_path)
+      relative = full_path.delete_prefix(root_path)
+      return '.' if relative.empty?
+
+      relative.split('/').reject(&:empty?).join(' > ')
+    end
+
+    def build_channel(response)
+      {
+        url: @url,
+        title: channel_title_from(response),
+        time_zone: 'UTC'
+      }.compact
+    end
+
+    def channel_title_from(response)
+      Channel.from_response(response).title
+    rescue StandardError
+      nil
+    end
+  end
+end

data/lib/html2rss/cli.rb

@@ -2,6 +2,7 @@
 
 require 'fileutils'
 require 'json'
+require 'yaml'
 require 'thor'
 
 module Html2rss
@@ -98,6 +99,43 @@ module Html2rss
       puts(format == 'jsonfeed' ? JSON.pretty_generate(result) : result)
     end
 
+    desc 'capture URL', 'Analyze a URL and print a reusable YAML feed config'
+    method_option :strategy,
+                  type: :string,
+                  desc: STRATEGY_OPTION_DESC,
+                  enum: STRATEGY_OPTION_ENUM
+    method_option :items_selector, type: :string, desc: 'CSS selector hint for items (optional)'
+    method_option :max_redirects,
+                  type: :numeric,
+                  desc: 'Maximum redirects to follow per request'
+    method_option :max_requests,
+                  type: :numeric,
+                  desc: 'Maximum requests to allow for this feed build'
+    method_option :limit,
+                  type: :numeric,
+                  desc: 'Maximum number of articles to keep (default: 25)'
+    method_option :input,
+                  type: :string,
+                  desc: 'Local HTML file path to read input from'
+    ##
+    # Captures a URL and prints a reusable YAML config.
+    #
+    # @param url [String, nil] source page URL for capture
+    # @return [void]
+    def capture(url = nil) # rubocop:disable Metrics/MethodLength
+      strategy, local_file_path, url = prepare_auto_inputs(url, options[:input])
+      config = Html2rss.capture(
+        url,
+        strategy:,
+        items_selector: options[:items_selector],
+        limit: options[:limit]&.to_i,
+        max_redirects: options[:max_redirects],
+        max_requests: options[:max_requests],
+        local_file_path:
+      )
+      puts YAML.dump(HashUtil.deep_stringify_keys(config))
+    end
+
     desc 'schema', 'Print the exported config JSON Schema'
     method_option :pretty,
                   type: :boolean,
@@ -123,6 +161,27 @@ module Html2rss
       puts schema_json
     end
 
+    desc 'mcp', 'Start the MCP server for AI client consumption'
+    method_option :transport,
+                  type: :string,
+                  desc: 'MCP transport protocol',
+                  enum: %w[stdio http],
+                  default: 'stdio'
+    method_option :port,
+                  type: :numeric,
+                  desc: 'Port for HTTP transport (binds 127.0.0.1)',
+                  default: 8080
+    ##
+    # Starts the MCP server for AI client consumption.
+    #
+    # @return [void]
+    def mcp
+      Html2rss::MCP.start(
+        transport: options[:transport].to_sym,
+        port: options[:port]
+      )
+    end
+
     desc 'validate YAML_FILE [feed_name]', 'Validate a YAML config with the runtime validator'
     method_option :params,
                   type: :hash,
@@ -183,7 +242,7 @@ module Html2rss
 
       file_path = check_file_exists!(input_option)
       detected_url = url || detect_base_url!(
-        file_path, 'Please specify a URL: html2rss auto [URL] --input <file>'
+        file_path, 'Please specify a URL: html2rss <command> [URL] --input <file>'
       )
 
       [:local_file, file_path, detected_url]
@@ -221,15 +280,15 @@ module Html2rss
       yield
     rescue Faraday::FollowRedirects::RedirectLimitReached => error
       raise Thor::Error,
-            "#{error.message}. retry with --max-redirects #{suggested_max_redirects} or use the final URL directly."
+            "#{error.message}. already retried the last redirect hop once; " \
+            "retry with --max-redirects #{suggested_max_redirects} or use the final URL directly."
     rescue Html2rss::RequestService::RequestBudgetExceeded => error
       raise Thor::Error,
             "#{error.message}. retry with --max-requests #{suggested_max_requests} " \
             'or increase request.max_requests in the config.'
-    rescue Html2rss::RequestService::BrowserlessConfigurationError,
-           Html2rss::RequestService::BrowserlessConnectionFailed,
-           Html2rss::RequestService::BotasaurusConfigurationError,
+    rescue Html2rss::RequestService::BotasaurusConfigurationError,
            Html2rss::RequestService::BotasaurusConnectionFailed,
+           Html2rss::RequestService::BotasaurusServiceError,
            Html2rss::RequestService::BlockedSurfaceDetected,
            Html2rss::NoFeedItemsExtracted => error
       raise Thor::Error, error.message

data/lib/html2rss/config/auto_source_contract.rb

@@ -29,6 +29,9 @@ module Html2rss
         optional(:json_state).hash do
           optional(:enabled).filled(:bool)
         end
+        optional(:xhr_articles).hash do
+          optional(:enabled).filled(:bool)
+        end
         optional(:meta_oembed).hash do
           optional(:enabled).filled(:bool)
         end
@@ -46,7 +49,6 @@ module Html2rss
 
       optional(:cleanup).hash do
         optional(:keep_different_domain).filled(:bool)
-        optional(:min_words_title).filled(:integer, gt?: 0)
       end
     end
   end

data/lib/html2rss/config/schema.rb

@@ -48,6 +48,7 @@ module Html2rss
       ##
       # Orchestrates schema assembly from runtime validator contracts plus
       # client-facing overlays.
+      # rubocop:disable Metrics/ClassLength -- overlay assembly stays in one builder
       class Builder
         class << self
           # @return [Hash{String => Object}] fully assembled JSON schema hash
@@ -60,9 +61,10 @@ module Html2rss
         def call
           schema = validator_schema
           apply_top_level(schema)
+          schema[:$defs] = registry_catalog_defs
           schema.fetch(:properties).merge!(overlay)
           schema.fetch(:properties).delete(:dynamic_params_error)
-          DeepStringifier.call(schema)
+          HashUtil.deep_stringify_keys(schema)
         end
 
         private
@@ -77,6 +79,23 @@ module Html2rss
             { 'required' => ['selectors'] },
             { 'required' => ['auto_source'] }
           ]
+
+          topics = schema.dig(:properties, :directory, :properties, :topics)
+          topics[:minItems] = 1 if topics.is_a?(Hash)
+        end
+
+        # @return [Hash{Symbol => Hash}] catalog under $defs.post_processors / $defs.extractors
+        def registry_catalog_defs
+          {
+            post_processors: catalog_from_registry(Selectors::PostProcessors::NAME_TO_CLASS),
+            extractors: catalog_from_registry(Selectors::Extractors::NAME_TO_CLASS)
+          }
+        end
+
+        # @param registry [Hash{Symbol => Class}]
+        # @return [Hash{String => Hash}]
+        def catalog_from_registry(registry)
+          registry.keys.sort.to_h { |name| [name.to_s, registry.fetch(name).schema_doc] }
         end
 
         # rubocop:disable Metrics/MethodLength, Metrics/AbcSize, Layout/LineLength
@@ -105,6 +124,13 @@ module Html2rss
             ]
           }
 
+          selector_schema = Schema.apply_selector_registry_refs!(
+            Html2rss::Config::SelectorsValidator::Selector.new.schema.json_schema(loose: true)
+          )
+          enclosure_schema = Schema.apply_selector_registry_refs!(
+            Html2rss::Config::SelectorsValidator::Enclosure.new.schema.json_schema(loose: true)
+          )
+
           {
             strategy: {
               type: 'string',
@@ -121,7 +147,7 @@ module Html2rss
               items: Html2rss::Config::Validator::StylesheetConfig.json_schema(loose: true)
             },
             auto_source: Html2rss::Config::AutoSourceContract.json_schema(loose: true).merge(
-              default: DeepStringifier.call(Html2rss::AutoSource::DEFAULT_CONFIG)
+              default: HashUtil.deep_stringify_keys(Html2rss::AutoSource::DEFAULT_CONFIG)
             ),
             selectors: {
               type: 'object',
@@ -130,14 +156,14 @@ module Html2rss
                 items: items_schema.merge(
                   description: 'Defines the items selector and optional enhancement settings.'
                 ),
-                enclosure: Html2rss::Config::SelectorsValidator::Enclosure.new.schema.json_schema(loose: true).merge(
+                enclosure: enclosure_schema.merge(
                   description: 'Describes enclosure extraction settings.'
                 ),
                 guid: reference_array('List of selector keys used to build the GUID. Each entry must reference a sibling selector key; runtime validation enforces those references.'),
                 categories: reference_array('List of selector keys whose values will be used as categories. Each entry must reference a sibling selector key; runtime validation enforces those references.')
               },
               patternProperties: {
-                '^(?!items$|enclosure$|guid$|categories$).+$' => Html2rss::Config::SelectorsValidator::Selector.new.schema.json_schema(loose: true).merge(
+                '^(?!items$|enclosure$|guid$|categories$).+$' => selector_schema.merge(
                   description: 'Dynamic selector definition keyed by attribute name.'
                 )
               },
@@ -159,31 +185,46 @@ module Html2rss
           }
         end
       end
+      # rubocop:enable Metrics/ClassLength
 
       ##
-      # Converts nested hash keys to strings so the resulting schema serializes cleanly.
-      module DeepStringifier
-        module_function
-
-        # @param object [Hash, Array, Object] nested data to normalize
-        # @return [Hash, Array, Object] deep copy with stringified hash keys
-        def call(object)
-          case object
-          when Hash
-            stringify_hash(object)
-          when Array
-            object.map { |value| call(value) }
-          when Symbol
-            object.to_s
-          else
-            object
-          end
-        end
+      # Wires extractor / post_process to thin oneOf $refs into the registry catalog.
+      #
+      # @param schema [Hash] dry-schema JSON schema fragment for a selector
+      # @return [Hash] same schema with registry $ref wiring applied
+      def apply_selector_registry_refs!(schema)
+        properties = schema.fetch(:properties)
+        properties[:extractor] = extractor_wire_schema
+        properties[:post_process] = post_process_wire_schema
+        schema
+      end
+
+      # @return [Hash{Symbol => Object}]
+      def extractor_wire_schema
+        {
+          description: 'Extractor used to pull a value from the selected element.',
+          oneOf: registry_ref_list('extractors', Selectors::Extractors::NAME_TO_CLASS)
+        }
+      end
 
-        # @param object [Hash{Object => Object}] hash whose keys should become strings
-        # @return [Hash{String => Object}] hash with recursively normalized values
-        def stringify_hash(object)
-          object.to_h { |key, value| [key.to_s, call(value)] }
+      # @return [Hash{Symbol => Object}]
+      def post_process_wire_schema
+        {
+          type: 'array',
+          description: 'Ordered transforms applied to the extracted value.',
+          items: {
+            oneOf: registry_ref_list('post_processors', Selectors::PostProcessors::NAME_TO_CLASS)
+          }
+        }
+      end
+
+      ##
+      # @param catalog [String] $defs catalog name (`extractors` or `post_processors`)
+      # @param registry [Hash{Symbol => Class}]
+      # @return [Array<Hash>] oneOf entries of `{ '$ref' => ... }`
+      def registry_ref_list(catalog, registry)
+        registry.keys.sort.map do |name|
+          { '$ref' => "#/$defs/#{catalog}/#{name}" }
         end
       end
     end