html2rss 0.25.0 → 0.26.0

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: html2rss
 version: !ruby/object:Gem::Version
-  version: 0.25.0
+  version: 0.26.0
 platform: ruby
 authors:
 - Gil Desmarais
@@ -113,6 +113,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: mcp
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: mime-types
   requirement: !ruby/object:Gem::Requirement
@@ -148,19 +162,33 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '2.0'
 - !ruby/object:Gem::Dependency
-  name: puppeteer-ruby
+  name: rack
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rackup
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: regexp_parser
   requirement: !ruby/object:Gem::Requirement
@@ -245,6 +273,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: webrick
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.9'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.9'
 - !ruby/object:Gem::Dependency
   name: zeitwerk
   requirement: !ruby/object:Gem::Requirement
@@ -300,12 +342,14 @@ files:
 - lib/html2rss/auto_source/scraper/wordpress_api/page_scope.rb
 - lib/html2rss/auto_source/scraper/wordpress_api/page_scope/date_archive_range.rb
 - lib/html2rss/auto_source/scraper/wordpress_api/posts_endpoint.rb
+- lib/html2rss/auto_source/scraper/xhr_articles.rb
 - lib/html2rss/auto_source/segment.rb
 - lib/html2rss/auto_source/segmenter.rb
 - lib/html2rss/auto_source/segmenter/cluster.rb
 - lib/html2rss/auto_source/segmenter/list.rb
 - lib/html2rss/auto_source/segmenter/primary_link.rb
 - lib/html2rss/auto_source/segmenter/semantic.rb
+- lib/html2rss/capture.rb
 - lib/html2rss/channel.rb
 - lib/html2rss/cli.rb
 - lib/html2rss/config.rb
@@ -360,20 +404,18 @@ files:
 - lib/html2rss/link_destination/noise_policy.rb
 - lib/html2rss/link_destination/path_classifier.rb
 - lib/html2rss/link_destination/text_classifier.rb
+- lib/html2rss/mcp.rb
+- lib/html2rss/mcp/server.rb
 - lib/html2rss/request_service.rb
 - lib/html2rss/request_service/blocked_surface.rb
 - lib/html2rss/request_service/botasaurus_contract.rb
 - lib/html2rss/request_service/botasaurus_strategy.rb
-- lib/html2rss/request_service/browserless_strategy.rb
 - lib/html2rss/request_service/budget.rb
 - lib/html2rss/request_service/context.rb
 - lib/html2rss/request_service/faraday_strategy.rb
 - lib/html2rss/request_service/local_file_strategy.rb
 - lib/html2rss/request_service/network_guard.rb
 - lib/html2rss/request_service/policy.rb
-- lib/html2rss/request_service/puppet_commander.rb
-- lib/html2rss/request_service/puppet_commander/navigation_guards.rb
-- lib/html2rss/request_service/puppet_commander/preload_runner.rb
 - lib/html2rss/request_service/response.rb
 - lib/html2rss/request_service/response_guard.rb
 - lib/html2rss/request_service/strategy.rb
@@ -386,7 +428,6 @@ files:
 - lib/html2rss/request_session/pager/rel_next.rb
 - lib/html2rss/request_session/pager/url_template.rb
 - lib/html2rss/scoring.rb
-- lib/html2rss/scoring/anchor_score.rb
 - lib/html2rss/scoring/cluster_scorer.rb
 - lib/html2rss/scoring/container_assessor.rb
 - lib/html2rss/scoring/engine.rb
@@ -415,6 +456,7 @@ files:
 - lib/html2rss/selectors/post_processors/sanitize_html.rb
 - lib/html2rss/selectors/post_processors/substring.rb
 - lib/html2rss/selectors/post_processors/template.rb
+- lib/html2rss/selectors/schema_doc.rb
 - lib/html2rss/sst.rb
 - lib/html2rss/sst/attrs.rb
 - lib/html2rss/sst/document.rb
@@ -433,7 +475,7 @@ licenses:
 - MIT
 metadata:
   allowed_push_host: https://rubygems.org
-  changelog_uri: https://github.com/html2rss/html2rss/releases/tag/v0.25.0
+  changelog_uri: https://github.com/html2rss/html2rss/releases/tag/v0.26.0
   rubygems_mfa_required: 'true'
 rdoc_options: []
 require_paths:

data/lib/html2rss/request_service/browserless_strategy.rb

@@ -1,132 +0,0 @@
-# frozen_string_literal: true
-
-require 'puppeteer'
-
-module Html2rss
-  class RequestService
-    ##
-    # Browserless.io strategy to request websites.
-    #
-    # Provide the WebSocket URL and your API token via environment variables:
-    # - BROWSERLESS_IO_WEBSOCKET_URL
-    # - BROWSERLESS_IO_API_TOKEN
-    #
-    # To use this strategy, you need to have a Browserless.io account or run a
-    # local Browserless.io instance.
-    #
-    # @see https://www.browserless.io/
-    #
-    # To run a local Browserless.io instance, you can use the following Docker command:
-    #
-    # ```sh
-    # docker run \
-    #   --rm \
-    #   -p 3000:3000 \
-    #   -e "CONCURRENT=10" \
-    #   -e "TOKEN=6R0W53R135510" \
-    #   ghcr.io/browserless/chromium
-    # ```
-    #
-    # When running locally, you can skip setting the environment variables, as above commands
-    # are aligned with the default values.
-    # @see https://github.com/browserless/browserless/pkgs/container/chromium
-    class BrowserlessStrategy < Strategy
-      ##
-      # @return [String] the Browserless websocket endpoint with token query param
-      # @raise [ArgumentError] if a custom endpoint is configured without an API token
-      def browser_ws_endpoint
-        @browser_ws_endpoint ||= begin
-          ws_url = ENV.fetch('BROWSERLESS_IO_WEBSOCKET_URL', 'ws://127.0.0.1:3000')
-          api_token = browserless_api_token(ws_url)
-
-          "#{ws_url}?token=#{api_token}"
-        end
-      end
-
-      private
-
-      def fetch
-        connect_with_timeout_support do |browser|
-          PuppetCommander.new(ctx, browser).call
-        ensure
-          browser.disconnect
-        end
-      end
-
-      def protocol_timeout_ms
-        ctx.budget.effective_timeout_ms(fallback: ctx.policy.total_timeout_seconds)
-      end
-
-      def connect_with_timeout_support(&)
-        connect_browserless(protocol_timeout: protocol_timeout_ms, &)
-      rescue ArgumentError => error
-        raise unless unsupported_protocol_timeout?(error)
-
-        connect_browserless(&)
-      end
-
-      def unsupported_protocol_timeout?(error)
-        error.message.include?('unknown keyword: :protocol_timeout')
-      end
-
-      def connect_browserless(protocol_timeout: nil, &)
-        connected = false
-
-        Puppeteer.connect(**browserless_connect_options(protocol_timeout)) do |browser|
-          connected = true
-          yield browser
-        end
-      rescue ArgumentError => error
-        handle_connection_error(error, connected:, protocol_timeout:)
-      rescue StandardError => error
-        handle_connection_error(error, connected:)
-      end
-
-      def browserless_connect_options(protocol_timeout)
-        { browser_ws_endpoint:, protocol_timeout: }.compact
-      end
-
-      def handle_connection_error(error, connected:, protocol_timeout: nil)
-        raise if connected || compatibility_timeout_error?(error, protocol_timeout:)
-
-        raise BrowserlessConnectionFailed, browserless_connection_message(error), cause: error
-      end
-
-      def compatibility_timeout_error?(error, protocol_timeout:)
-        protocol_timeout && unsupported_protocol_timeout?(error)
-      end
-
-      def browserless_connection_message(error)
-        base = "Browserless connection failed (#{error.class}: #{error.message})."
-        endpoint_hint = "Check BROWSERLESS_IO_WEBSOCKET_URL (currently #{browserless_websocket_url})."
-        token_hint = 'Check BROWSERLESS_IO_API_TOKEN and ensure it matches your Browserless TOKEN.'
-        local_hint = 'For local Browserless, confirm the service is running and reachable.'
-
-        if likely_authentication_error?(error)
-          "#{base} #{token_hint} #{endpoint_hint}"
-        else
-          "#{base} #{endpoint_hint} #{token_hint} #{local_hint}"
-        end
-      end
-
-      def likely_authentication_error?(error)
-        message = error.message.downcase
-        message.include?('unauthorized') || message.include?('forbidden') || message.include?('401')
-      end
-
-      def browserless_websocket_url
-        ENV.fetch('BROWSERLESS_IO_WEBSOCKET_URL', 'ws://127.0.0.1:3000')
-      end
-
-      def browserless_api_token(ws_url)
-        ENV.fetch('BROWSERLESS_IO_API_TOKEN') do
-          return '6R0W53R135510' if ws_url == 'ws://127.0.0.1:3000'
-
-          raise BrowserlessConfigurationError,
-                'BROWSERLESS_IO_API_TOKEN is required for custom Browserless endpoints. ' \
-                'Set BROWSERLESS_IO_API_TOKEN or use ws://127.0.0.1:3000 for local defaults.'
-        end
-      end
-    end
-  end
-end

data/lib/html2rss/request_service/puppet_commander/navigation_guards.rb

@@ -1,148 +0,0 @@
-# frozen_string_literal: true
-
-module Html2rss
-  class RequestService
-    class PuppetCommander
-      ##
-      # Owns Puppeteer request interception, deferred navigation errors, main-frame
-      # filtering, and redirect-chain validation via {Policy}.
-      #
-      # Does not re-own Policy rules — only calls +validate_*!+ on +ctx.policy+.
-      class NavigationGuards
-        class << self
-          ##
-          # Resolves the final response URL for Response building and IP validation.
-          #
-          # @param navigation_response [Puppeteer::HTTPResponse, nil]
-          # @param fallback_url [String, Html2rss::Url]
-          # @return [Html2rss::Url]
-          def response_url(navigation_response, fallback_url)
-            raw_url = navigation_response&.url || fallback_url.to_s
-            Html2rss::Url.from_absolute(raw_url)
-          end
-        end
-
-        ##
-        # @param ctx [Context] request context providing policy and origin
-        # @param skip_request_resources [Set<String>] resource types to abort
-        def initialize(ctx:, skip_request_resources:)
-          @ctx = ctx
-          @skip_request_resources = skip_request_resources
-          @navigation_error = nil
-          @latest_navigation_response = nil
-          @main_frame = nil
-        end
-
-        ##
-        # Captures the main frame and wires request/response interceptors.
-        #
-        # @param page [Puppeteer::Page] browser page
-        # @return [void]
-        def install!(page)
-          @main_frame = page.main_frame if page.respond_to?(:main_frame)
-          page.request_interception = true
-          page.on('request') { |request| handle_request(request) }
-          page.on('response') { |response| handle_response(response) }
-        end
-
-        ##
-        # Clears deferred error and latest navigation response before goto.
-        #
-        # @return [void]
-        def begin_navigation!
-          @navigation_error = nil
-          @latest_navigation_response = nil
-        end
-
-        ##
-        # Re-raises a deferred navigation error when one was captured.
-        #
-        # @return [void]
-        # @raise [Html2rss::Error] when a navigation request or response validation failed
-        def raise_deferred_error!
-          raise @navigation_error if @navigation_error
-        end
-
-        ##
-        # @return [Puppeteer::HTTPResponse, nil] latest main-frame navigation response
-        attr_reader :latest_navigation_response
-
-        ##
-        # Validates the remote IP of a navigation response via Policy.
-        #
-        # @param navigation_response [Puppeteer::HTTPResponse, nil]
-        # @return [void]
-        def validate_final!(navigation_response)
-          final_url = self.class.response_url(navigation_response, ctx.url)
-          ctx.policy.validate_remote_ip!(ip: remote_ip(navigation_response), url: final_url)
-        end
-
-        private
-
-        attr_reader :ctx, :skip_request_resources, :main_frame
-
-        def handle_request(request)
-          validate_request!(request)
-
-          skip_request_resources.member?(request.resource_type) ? request.abort : request.continue
-        rescue Html2rss::Error => error
-          store_navigation_error(error, navigation_request: request.navigation_request?)
-          request.abort
-        end
-
-        def handle_response(response)
-          @latest_navigation_response = response if main_frame_navigation_response?(response)
-          validate_final!(response)
-        rescue Html2rss::Error => error
-          store_navigation_error(error, navigation_request: response.request.navigation_request?)
-        end
-
-        def validate_request!(request)
-          validate_navigation_redirect_chain!(request)
-          validate_navigation_target!(request)
-        end
-
-        def main_frame_navigation_response?(response)
-          request = response.request
-          return false unless request.navigation_request?
-          return true unless request.respond_to?(:frame)
-
-          frame = request.frame
-          return true if frame.nil?
-          return frame == main_frame unless main_frame.nil?
-          return true unless frame.respond_to?(:parent_frame)
-
-          frame.parent_frame.nil?
-        end
-
-        def remote_ip(navigation_response)
-          navigation_response.remote_address&.ip
-        end
-
-        def request_chain(request)
-          (request.redirect_chain + [request]).map { |entry| request_url(entry) }
-        end
-
-        def request_url(request)
-          Html2rss::Url.from_absolute(request.url)
-        end
-
-        def validate_navigation_redirect_chain!(request)
-          request_chain(request).each_cons(2) do |from_url, to_url|
-            ctx.policy.validate_redirect!(from_url:, to_url:, origin_url: ctx.origin_url, relation: ctx.relation)
-          end
-        end
-
-        def validate_navigation_target!(request)
-          ctx.policy.validate_request!(url: request_url(request), origin_url: ctx.origin_url, relation: ctx.relation)
-        end
-
-        def store_navigation_error(error, navigation_request:)
-          return unless navigation_request
-
-          @navigation_error = error if @navigation_error.nil?
-        end
-      end
-    end
-  end
-end

data/lib/html2rss/request_service/puppet_commander/preload_runner.rb

@@ -1,86 +0,0 @@
-# frozen_string_literal: true
-
-module Html2rss
-  class RequestService
-    class PuppetCommander
-      ##
-      # Runs Browserless preload choreography: wait, click, and scroll against
-      # the interaction budget on {Context#budget}.
-      class PreloadRunner
-        ##
-        # @param ctx [Context] request context providing preload config and budget
-        def initialize(ctx:)
-          @ctx = ctx
-        end
-
-        ##
-        # Executes configured preload actions on the page, if any.
-        #
-        # @param page [Puppeteer::Page] browser page after initial navigation
-        # @return [void]
-        def call(page)
-          preload_config = ctx.browserless_preload
-          return unless preload_config
-
-          wait_after(page, preload_config[:wait_after_ms])
-          click_selectors(page, preload_config[:click_selectors]) if preload_config[:click_selectors]
-          scroll_down(page, preload_config[:scroll_down]) if preload_config[:scroll_down]
-          wait_after(page, preload_config[:wait_after_ms])
-        end
-
-        private
-
-        attr_reader :ctx
-
-        def wait_after(page, timeout_ms)
-          return unless timeout_ms
-
-          ctx.budget.consume_interaction!
-          page.wait_for_timeout(timeout_ms)
-        end
-
-        def click_selectors(page, selectors)
-          selectors.each { |selector_config| click_selector(page, selector_config) }
-        end
-
-        def scroll_down(page, config)
-          iterations = config.fetch(:iterations, 1)
-          wait_after_ms = config[:wait_after_ms]
-          previous_height = nil
-
-          iterations.times do
-            updated_height = perform_scroll_iteration(page, wait_after_ms, previous_height)
-            break unless updated_height
-
-            previous_height = updated_height
-          end
-        end
-
-        def click_selector(page, config)
-          selector = config.fetch(:selector)
-          max_clicks = config.fetch(:max_clicks, 1)
-          wait_after_ms = config[:wait_after_ms]
-
-          max_clicks.times do
-            break unless (element = page.query_selector(selector))
-
-            ctx.budget.consume_interaction!
-            element.click
-            wait_after(page, wait_after_ms)
-          end
-        end
-
-        def perform_scroll_iteration(page, wait_after_ms, previous_height)
-          ctx.budget.consume_interaction!
-          page.evaluate('() => window.scrollTo(0, document.body.scrollHeight)')
-          wait_after(page, wait_after_ms)
-
-          current_height = page.evaluate('() => document.body.scrollHeight')
-          return if previous_height && current_height <= previous_height
-
-          current_height
-        end
-      end
-    end
-  end
-end

data/lib/html2rss/request_service/puppet_commander.rb

@@ -1,95 +0,0 @@
-# frozen_string_literal: true
-
-module Html2rss
-  class RequestService
-    ##
-    # Commands the Puppeteer Browser to the website and builds the Response.
-    #
-    # Public interface is {#call}; page setup, navigation, and body collection
-    # are private orchestration steps.
-    class PuppetCommander
-      # Request header names that must not be forwarded to the browser session.
-      BROWSER_UNSAFE_HEADERS = %w[
-        host connection content-length transfer-encoding
-        sec-fetch-dest sec-fetch-mode sec-fetch-site sec-fetch-user
-        upgrade-insecure-requests
-      ].to_set.freeze
-
-      # @param ctx [Context]
-      # @param browser [Puppeteer::Browser]
-      # @param skip_request_resources [Set<String>] the resource types not to request
-      # @param referer [String] the referer to use for the request
-      def initialize(ctx,
-                     browser,
-                     skip_request_resources: %w[stylesheet image media font].to_set,
-                     referer: [ctx.url.scheme, ctx.url.host].join('://'))
-        @ctx = ctx
-        @browser = browser
-        @referer = referer
-        @navigation_guards = NavigationGuards.new(ctx:, skip_request_resources:)
-        @preload_runner = PreloadRunner.new(ctx:)
-      end
-
-      ##
-      # Visits the request URL and normalizes the page into a response object.
-      #
-      # @return [Response] rendered page response
-      def call
-        page = new_page
-        navigation_response = navigate_to_destination(page, ctx.url)
-        preload_runner.call(page)
-        navigation_guards.raise_deferred_error!
-        final_navigation_response = navigation_guards.latest_navigation_response || navigation_response
-        navigation_guards.validate_final!(final_navigation_response)
-        build_response(page, final_navigation_response)
-      ensure
-        page&.close
-      end
-
-      private
-
-      attr_reader :ctx, :browser, :referer, :navigation_guards, :preload_runner
-
-      def new_page
-        page = browser.new_page
-        configure_page(page)
-        navigation_guards.install!(page)
-        page
-      end
-
-      def configure_page(page)
-        page.extra_http_headers = browser_headers
-        page.default_navigation_timeout = navigation_timeout_ms
-        page.default_timeout = navigation_timeout_ms
-      end
-
-      def navigate_to_destination(page, url)
-        navigation_guards.begin_navigation!
-        page.goto(url, wait_until: 'networkidle0', referer:, timeout: navigation_timeout_ms).tap do
-          navigation_guards.raise_deferred_error!
-        end
-      rescue StandardError
-        navigation_guards.raise_deferred_error!
-
-        raise
-      end
-
-      def navigation_timeout_ms
-        ctx.budget.effective_timeout_ms(fallback: ctx.policy.total_timeout_seconds)
-      end
-
-      def browser_headers
-        ctx.headers.reject { |key, _| BROWSER_UNSAFE_HEADERS.include?(key.to_s.downcase) }
-      end
-
-      def build_response(page, navigation_response)
-        Response.new(
-          body: page.content,
-          headers: navigation_response&.headers || {},
-          url: NavigationGuards.response_url(navigation_response, ctx.url),
-          status: navigation_response&.status
-        )
-      end
-    end
-  end
-end

data/lib/html2rss/scoring/anchor_score.rb

@@ -1,34 +0,0 @@
-# frozen_string_literal: true
-
-module Html2rss
-  module Scoring
-    ##
-    # Primary-link ranking weights (sole home; FeatureId-aligned).
-    module AnchorScore
-      # Signal → integer weight map for primary-link ranking.
-      WEIGHTS = {
-        heading_anchor: 100,
-        heading_text_match: 20,
-        meaningful_text: 10,
-        content_like_destination: 10
-      }.freeze
-
-      module_function
-
-      ##
-      # @param heading_anchor [Boolean]
-      # @param heading_text_match [Boolean]
-      # @param meaningful_text [Boolean]
-      # @param content_like_destination [Boolean]
-      # @return [Integer]
-      def score(heading_anchor:, heading_text_match:, meaningful_text:, content_like_destination:)
-        total = 0
-        total += WEIGHTS[:heading_anchor] if heading_anchor
-        total += WEIGHTS[:heading_text_match] if heading_text_match
-        total += WEIGHTS[:meaningful_text] if meaningful_text
-        total += WEIGHTS[:content_like_destination] if content_like_destination
-        total
-      end
-    end
-  end
-end