html2rss 0.24.0 → 0.26.0

data/lib/html2rss/request_service/puppet_commander/navigation_guards.rb

@@ -1,148 +0,0 @@
-# frozen_string_literal: true
-
-module Html2rss
-  class RequestService
-    class PuppetCommander
-      ##
-      # Owns Puppeteer request interception, deferred navigation errors, main-frame
-      # filtering, and redirect-chain validation via {Policy}.
-      #
-      # Does not re-own Policy rules — only calls +validate_*!+ on +ctx.policy+.
-      class NavigationGuards
-        class << self
-          ##
-          # Resolves the final response URL for Response building and IP validation.
-          #
-          # @param navigation_response [Puppeteer::HTTPResponse, nil]
-          # @param fallback_url [String, Html2rss::Url]
-          # @return [Html2rss::Url]
-          def response_url(navigation_response, fallback_url)
-            raw_url = navigation_response&.url || fallback_url.to_s
-            Html2rss::Url.from_absolute(raw_url)
-          end
-        end
-
-        ##
-        # @param ctx [Context] request context providing policy and origin
-        # @param skip_request_resources [Set<String>] resource types to abort
-        def initialize(ctx:, skip_request_resources:)
-          @ctx = ctx
-          @skip_request_resources = skip_request_resources
-          @navigation_error = nil
-          @latest_navigation_response = nil
-          @main_frame = nil
-        end
-
-        ##
-        # Captures the main frame and wires request/response interceptors.
-        #
-        # @param page [Puppeteer::Page] browser page
-        # @return [void]
-        def install!(page)
-          @main_frame = page.main_frame if page.respond_to?(:main_frame)
-          page.request_interception = true
-          page.on('request') { |request| handle_request(request) }
-          page.on('response') { |response| handle_response(response) }
-        end
-
-        ##
-        # Clears deferred error and latest navigation response before goto.
-        #
-        # @return [void]
-        def begin_navigation!
-          @navigation_error = nil
-          @latest_navigation_response = nil
-        end
-
-        ##
-        # Re-raises a deferred navigation error when one was captured.
-        #
-        # @return [void]
-        # @raise [Html2rss::Error] when a navigation request or response validation failed
-        def raise_deferred_error!
-          raise @navigation_error if @navigation_error
-        end
-
-        ##
-        # @return [Puppeteer::HTTPResponse, nil] latest main-frame navigation response
-        attr_reader :latest_navigation_response
-
-        ##
-        # Validates the remote IP of a navigation response via Policy.
-        #
-        # @param navigation_response [Puppeteer::HTTPResponse, nil]
-        # @return [void]
-        def validate_final!(navigation_response)
-          final_url = self.class.response_url(navigation_response, ctx.url)
-          ctx.policy.validate_remote_ip!(ip: remote_ip(navigation_response), url: final_url)
-        end
-
-        private
-
-        attr_reader :ctx, :skip_request_resources, :main_frame
-
-        def handle_request(request)
-          validate_request!(request)
-
-          skip_request_resources.member?(request.resource_type) ? request.abort : request.continue
-        rescue Html2rss::Error => error
-          store_navigation_error(error, navigation_request: request.navigation_request?)
-          request.abort
-        end
-
-        def handle_response(response)
-          @latest_navigation_response = response if main_frame_navigation_response?(response)
-          validate_final!(response)
-        rescue Html2rss::Error => error
-          store_navigation_error(error, navigation_request: response.request.navigation_request?)
-        end
-
-        def validate_request!(request)
-          validate_navigation_redirect_chain!(request)
-          validate_navigation_target!(request)
-        end
-
-        def main_frame_navigation_response?(response)
-          request = response.request
-          return false unless request.navigation_request?
-          return true unless request.respond_to?(:frame)
-
-          frame = request.frame
-          return true if frame.nil?
-          return frame == main_frame unless main_frame.nil?
-          return true unless frame.respond_to?(:parent_frame)
-
-          frame.parent_frame.nil?
-        end
-
-        def remote_ip(navigation_response)
-          navigation_response.remote_address&.ip
-        end
-
-        def request_chain(request)
-          (request.redirect_chain + [request]).map { |entry| request_url(entry) }
-        end
-
-        def request_url(request)
-          Html2rss::Url.from_absolute(request.url)
-        end
-
-        def validate_navigation_redirect_chain!(request)
-          request_chain(request).each_cons(2) do |from_url, to_url|
-            ctx.policy.validate_redirect!(from_url:, to_url:, origin_url: ctx.origin_url, relation: ctx.relation)
-          end
-        end
-
-        def validate_navigation_target!(request)
-          ctx.policy.validate_request!(url: request_url(request), origin_url: ctx.origin_url, relation: ctx.relation)
-        end
-
-        def store_navigation_error(error, navigation_request:)
-          return unless navigation_request
-
-          @navigation_error = error if @navigation_error.nil?
-        end
-      end
-    end
-  end
-end

data/lib/html2rss/request_service/puppet_commander/preload_runner.rb

@@ -1,86 +0,0 @@
-# frozen_string_literal: true
-
-module Html2rss
-  class RequestService
-    class PuppetCommander
-      ##
-      # Runs Browserless preload choreography: wait, click, and scroll against
-      # the interaction budget on {Context#budget}.
-      class PreloadRunner
-        ##
-        # @param ctx [Context] request context providing preload config and budget
-        def initialize(ctx:)
-          @ctx = ctx
-        end
-
-        ##
-        # Executes configured preload actions on the page, if any.
-        #
-        # @param page [Puppeteer::Page] browser page after initial navigation
-        # @return [void]
-        def call(page)
-          preload_config = ctx.browserless_preload
-          return unless preload_config
-
-          wait_after(page, preload_config[:wait_after_ms])
-          click_selectors(page, preload_config[:click_selectors]) if preload_config[:click_selectors]
-          scroll_down(page, preload_config[:scroll_down]) if preload_config[:scroll_down]
-          wait_after(page, preload_config[:wait_after_ms])
-        end
-
-        private
-
-        attr_reader :ctx
-
-        def wait_after(page, timeout_ms)
-          return unless timeout_ms
-
-          ctx.budget.consume_interaction!
-          page.wait_for_timeout(timeout_ms)
-        end
-
-        def click_selectors(page, selectors)
-          selectors.each { |selector_config| click_selector(page, selector_config) }
-        end
-
-        def scroll_down(page, config)
-          iterations = config.fetch(:iterations, 1)
-          wait_after_ms = config[:wait_after_ms]
-          previous_height = nil
-
-          iterations.times do
-            updated_height = perform_scroll_iteration(page, wait_after_ms, previous_height)
-            break unless updated_height
-
-            previous_height = updated_height
-          end
-        end
-
-        def click_selector(page, config)
-          selector = config.fetch(:selector)
-          max_clicks = config.fetch(:max_clicks, 1)
-          wait_after_ms = config[:wait_after_ms]
-
-          max_clicks.times do
-            break unless (element = page.query_selector(selector))
-
-            ctx.budget.consume_interaction!
-            element.click
-            wait_after(page, wait_after_ms)
-          end
-        end
-
-        def perform_scroll_iteration(page, wait_after_ms, previous_height)
-          ctx.budget.consume_interaction!
-          page.evaluate('() => window.scrollTo(0, document.body.scrollHeight)')
-          wait_after(page, wait_after_ms)
-
-          current_height = page.evaluate('() => document.body.scrollHeight')
-          return if previous_height && current_height <= previous_height
-
-          current_height
-        end
-      end
-    end
-  end
-end

data/lib/html2rss/request_service/puppet_commander.rb

@@ -1,95 +0,0 @@
-# frozen_string_literal: true
-
-module Html2rss
-  class RequestService
-    ##
-    # Commands the Puppeteer Browser to the website and builds the Response.
-    #
-    # Public interface is {#call}; page setup, navigation, and body collection
-    # are private orchestration steps.
-    class PuppetCommander
-      # Request header names that must not be forwarded to the browser session.
-      BROWSER_UNSAFE_HEADERS = %w[
-        host connection content-length transfer-encoding
-        sec-fetch-dest sec-fetch-mode sec-fetch-site sec-fetch-user
-        upgrade-insecure-requests
-      ].to_set.freeze
-
-      # @param ctx [Context]
-      # @param browser [Puppeteer::Browser]
-      # @param skip_request_resources [Set<String>] the resource types not to request
-      # @param referer [String] the referer to use for the request
-      def initialize(ctx,
-                     browser,
-                     skip_request_resources: %w[stylesheet image media font].to_set,
-                     referer: [ctx.url.scheme, ctx.url.host].join('://'))
-        @ctx = ctx
-        @browser = browser
-        @referer = referer
-        @navigation_guards = NavigationGuards.new(ctx:, skip_request_resources:)
-        @preload_runner = PreloadRunner.new(ctx:)
-      end
-
-      ##
-      # Visits the request URL and normalizes the page into a response object.
-      #
-      # @return [Response] rendered page response
-      def call
-        page = new_page
-        navigation_response = navigate_to_destination(page, ctx.url)
-        preload_runner.call(page)
-        navigation_guards.raise_deferred_error!
-        final_navigation_response = navigation_guards.latest_navigation_response || navigation_response
-        navigation_guards.validate_final!(final_navigation_response)
-        build_response(page, final_navigation_response)
-      ensure
-        page&.close
-      end
-
-      private
-
-      attr_reader :ctx, :browser, :referer, :navigation_guards, :preload_runner
-
-      def new_page
-        page = browser.new_page
-        configure_page(page)
-        navigation_guards.install!(page)
-        page
-      end
-
-      def configure_page(page)
-        page.extra_http_headers = browser_headers
-        page.default_navigation_timeout = navigation_timeout_ms
-        page.default_timeout = navigation_timeout_ms
-      end
-
-      def navigate_to_destination(page, url)
-        navigation_guards.begin_navigation!
-        page.goto(url, wait_until: 'networkidle0', referer:, timeout: navigation_timeout_ms).tap do
-          navigation_guards.raise_deferred_error!
-        end
-      rescue StandardError
-        navigation_guards.raise_deferred_error!
-
-        raise
-      end
-
-      def navigation_timeout_ms
-        ctx.budget.effective_timeout_ms(fallback: ctx.policy.total_timeout_seconds)
-      end
-
-      def browser_headers
-        ctx.headers.reject { |key, _| BROWSER_UNSAFE_HEADERS.include?(key.to_s.downcase) }
-      end
-
-      def build_response(page, navigation_response)
-        Response.new(
-          body: page.content,
-          headers: navigation_response&.headers || {},
-          url: NavigationGuards.response_url(navigation_response, ctx.url),
-          status: navigation_response&.status
-        )
-      end
-    end
-  end
-end