html2rss 0.29.1 → 0.31.0

data/schema/html2rss-config.schema.json

@@ -434,7 +434,8 @@
           "properties": {
             "selector": {
               "type": "string",
-              "minLength": 1
+              "minLength": 1,
+              "default": "a[href]"
             },
             "order": {
               "enum": [
@@ -507,9 +508,7 @@
               ]
             }
           },
-          "required": [
-            "selector"
-          ],
+          "required": [],
           "description": "Defines the items selector and list-card enhance settings."
         },
         "enclosure": {
@@ -848,7 +847,7 @@
       "gsub": {
         "type": "object",
         "title": "gsub",
-        "description": "Replace matches of `pattern` in the extracted string with `replacement` (Ruby String#gsub; pattern may be a regexp-like string).",
+        "description": "Replace matches of `pattern` in the extracted string with `replacement` (Ruby String#gsub; pattern may be a regexp-like string). Patterns over 256 characters as written, or with nested quantifiers, are rejected.",
         "examples": [
           {
             "name": "gsub",
@@ -862,10 +861,14 @@
             "const": "gsub"
           },
           "pattern": {
-            "type": "string"
+            "type": "string",
+            "maxLength": 256
           },
           "replacement": {
-            "type": "string"
+            "type": [
+              "string",
+              "object"
+            ]
           }
         },
         "required": [
@@ -1024,7 +1027,7 @@
         "type": "string",
         "const": "href",
         "title": "href",
-        "description": "Return the absolute URL from the selected element's `href` attribute (relative hrefs are resolved against the channel URL).",
+        "description": "Return the absolute URL from the selected element's `href` attribute (relative hrefs are resolved against the page base URL).",
         "examples": [
           "href"
         ]

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: html2rss
 version: !ruby/object:Gem::Version
-  version: 0.29.1
+  version: 0.31.0
 platform: ruby
 authors:
 - Gil Desmarais
@@ -52,53 +52,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: faraday
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">"
-      - !ruby/object:Gem::Version
-        version: 2.0.1
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '3.0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">"
-      - !ruby/object:Gem::Version
-        version: 2.0.1
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '3.0'
-- !ruby/object:Gem::Dependency
-  name: faraday-follow_redirects
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: faraday-gzip
+  name: httpx
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3'
+        version: '1.8'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3'
+        version: '1.8'
 - !ruby/object:Gem::Dependency
   name: kramdown
   requirement: !ruby/object:Gem::Requirement
@@ -348,13 +314,17 @@ files:
 - lib/html2rss/config.rb
 - lib/html2rss/config/auto_source_contract.rb
 - lib/html2rss/config/dynamic_params.rb
+- lib/html2rss/config/issue_mapper.rb
 - lib/html2rss/config/multiple_feeds_config.rb
 - lib/html2rss/config/request_controls.rb
 - lib/html2rss/config/request_headers.rb
 - lib/html2rss/config/schema.rb
 - lib/html2rss/config/selectors_validator.rb
+- lib/html2rss/config/validation_issue.rb
+- lib/html2rss/config/validation_report.rb
 - lib/html2rss/config/validator.rb
 - lib/html2rss/defaults.rb
+- lib/html2rss/doctor.rb
 - lib/html2rss/doctor/botasaurus.rb
 - lib/html2rss/error.rb
 - lib/html2rss/feed_builder.rb
@@ -431,7 +401,7 @@ files:
 - lib/html2rss/request_service/budget.rb
 - lib/html2rss/request_service/compressed_body.rb
 - lib/html2rss/request_service/context.rb
-- lib/html2rss/request_service/faraday_strategy.rb
+- lib/html2rss/request_service/httpx_strategy.rb
 - lib/html2rss/request_service/local_file_strategy.rb
 - lib/html2rss/request_service/network_guard.rb
 - lib/html2rss/request_service/policy.rb
@@ -455,15 +425,18 @@ files:
 - lib/html2rss/scoring/ranked_segment.rb
 - lib/html2rss/scoring/score.rb
 - lib/html2rss/scrape_target.rb
+- lib/html2rss/selector_candidates.rb
 - lib/html2rss/selectors.rb
+- lib/html2rss/selectors/README.md
 - lib/html2rss/selectors/extractors.rb
 - lib/html2rss/selectors/extractors/attribute.rb
 - lib/html2rss/selectors/extractors/href.rb
 - lib/html2rss/selectors/extractors/html.rb
 - lib/html2rss/selectors/extractors/static.rb
 - lib/html2rss/selectors/extractors/text.rb
-- lib/html2rss/selectors/item_scope.rb
-- lib/html2rss/selectors/object_to_xml_converter.rb
+- lib/html2rss/selectors/item_env.rb
+- lib/html2rss/selectors/json_xml.rb
+- lib/html2rss/selectors/option_spec.rb
 - lib/html2rss/selectors/post_processors.rb
 - lib/html2rss/selectors/post_processors/base.rb
 - lib/html2rss/selectors/post_processors/gsub.rb
@@ -475,7 +448,8 @@ files:
 - lib/html2rss/selectors/post_processors/sanitize_html.rb
 - lib/html2rss/selectors/post_processors/substring.rb
 - lib/html2rss/selectors/post_processors/template.rb
-- lib/html2rss/selectors/schema_doc.rb
+- lib/html2rss/selectors/schema_export.rb
+- lib/html2rss/selectors/step_env.rb
 - lib/html2rss/sst.rb
 - lib/html2rss/sst/attrs.rb
 - lib/html2rss/sst/document.rb
@@ -493,6 +467,7 @@ files:
 - lib/html2rss/syndication/parser.rb
 - lib/html2rss/test.rb
 - lib/html2rss/test/enhance_audit.rb
+- lib/html2rss/test/policy.rb
 - lib/html2rss/url.rb
 - lib/html2rss/version.rb
 - lib/tasks/config_schema.rake
@@ -502,7 +477,7 @@ licenses:
 - MIT
 metadata:
   allowed_push_host: https://rubygems.org
-  changelog_uri: https://github.com/html2rss/html2rss/releases/tag/v0.29.1
+  changelog_uri: https://github.com/html2rss/html2rss/releases/tag/v0.31.0
   rubygems_mfa_required: 'true'
 rdoc_options: []
 require_paths:
@@ -518,7 +493,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.16
+rubygems_version: 4.0.20
 specification_version: 4
 summary: Generates RSS feeds from websites by scraping a URL and using CSS selectors
   to extract item.

data/lib/html2rss/request_service/faraday_strategy.rb

@@ -1,233 +0,0 @@
-# frozen_string_literal: true
-
-require 'brotli'
-require 'faraday'
-require 'faraday/follow_redirects'
-require 'faraday/gzip'
-
-module Html2rss
-  class RequestService
-    ##
-    # Strategy to use Faraday for the request.
-    # @see https://rubygems.org/gems/faraday
-    # rubocop:disable-next Metrics/ClassLength -- terminal redirect retry colocated with Faraday transport
-    class FaradayStrategy < Strategy
-      ##
-      # Restores buffered streamed bytes so response middleware can process them.
-      class StreamingBodyMiddleware < Faraday::Middleware
-        # Request-context key used to store streamed chunks before middleware completion.
-        STREAM_BUFFER_KEY = :html2rss_stream_buffer
-
-        # @param env [Faraday::Env] completed response environment
-        # @return [void]
-        def on_complete(env)
-          buffer = env.request.context&.delete(STREAM_BUFFER_KEY)
-          return if buffer.nil? || buffer.empty?
-
-          env.body = buffer
-        end
-      end
-
-      ##
-      # @return [ResponseGuard]
-      attr_reader :response_guard
-
-      # Executes the request with runtime policy enforcement, returning the normalized response.
-      #
-      # @return [Response] normalized response
-      def perform_execute
-        deadline = request_deadline
-        @response_guard = ResponseGuard.new(policy: ctx.policy)
-        reset_redirect_tracking!
-        raw_response = request_with_terminal_redirect_retry(response_guard, deadline:)
-        raw_response = retry_without_streaming(response_guard, deadline:) if retry_without_streaming?(raw_response)
-        build_response(raw_response)
-      end
-
-      private
-
-      def request_deadline
-        monotonic_now + ctx.budget.effective_timeout_seconds(fallback: ctx.policy.total_timeout_seconds)
-      end
-
-      def reset_redirect_tracking!
-        @last_redirect_to = nil
-        @terminal_redirect_retried = false
-        @request_url_override = nil
-      end
-
-      def request_with_terminal_redirect_retry(response_guard, deadline:)
-        faraday_request(response_guard, deadline:, streaming_buffer: true)
-      rescue Faraday::FollowRedirects::RedirectLimitReached => error
-        raise RedirectLimitReached, error.message unless terminal_redirect_retryable?
-
-        begin
-          retry_from_terminal_redirect!(response_guard, deadline:)
-        rescue Faraday::FollowRedirects::RedirectLimitReached => retry_error
-          raise RedirectLimitReached, retry_error.message
-        end
-      end
-
-      def terminal_redirect_retryable?
-        return false if @terminal_redirect_retried || @last_redirect_to.nil?
-
-        @last_redirect_to.to_s != request_url.to_s
-      end
-
-      def retry_from_terminal_redirect!(response_guard, deadline:)
-        terminal_url = @last_redirect_to
-        @terminal_redirect_retried = true
-        Log.debug("#{self.class}: redirect limit reached; retrying once from #{terminal_url}")
-        begin_terminal_url_request!(terminal_url)
-        faraday_request(response_guard, deadline:, streaming_buffer: true, consume_budget: false)
-      end
-
-      def begin_terminal_url_request!(terminal_url)
-        ctx.policy.validate_request!(url: terminal_url, origin_url: ctx.origin_url, relation: ctx.relation)
-        @request_url_override = terminal_url
-        @client = nil
-        @last_redirect_to = nil
-      end
-
-      def request_url
-        @request_url_override || ctx.url
-      end
-
-      def build_response(response)
-        Response.new(body: CompressedBody.decode(response.body, headers: response.headers),
-                     headers: response.headers, url: response_url(response),
-                     status: response.status)
-      end
-
-      def faraday_request(response_guard, deadline:, streaming_buffer:, consume_budget: true)
-        preflight!(consume_budget:)
-
-        client.get do |req|
-          apply_timeouts(req, deadline:)
-          next unless streaming_buffer
-
-          buffer = prepare_stream_buffer(req)
-          req.options.on_data = on_data_callback(response_guard, buffer)
-        end
-      end
-
-      def retry_without_streaming(response_guard, deadline:)
-        faraday_request(response_guard, deadline:, streaming_buffer: false, consume_budget: false)
-      end
-
-      ##
-      # Validates the remote socket peer IP during Net::HTTP start.
-      module PeerIpValidator
-        module_function
-
-        # @param http [Net::HTTP] connection to configure
-        # @param policy [Policy] request policy
-        # @return [void]
-        def install!(http, policy:)
-          orig_start = http.method(:start)
-          http.define_singleton_method(:start) do |&block|
-            orig_start.call do |opened_http|
-              PeerIpValidator.validate!(opened_http, policy:)
-              block.call(opened_http)
-            end
-          end
-        end
-
-        # @param opened_http [Net::HTTP] active connection
-        # @param policy [Policy] request policy
-        # @return [void]
-        def validate!(opened_http, policy:)
-          scheme = opened_http.use_ssl? ? 'https' : 'http'
-          url = Html2rss::Url.from_absolute("#{scheme}://#{opened_http.address}:#{opened_http.port}")
-          policy.validate_remote_ip!(ip: peer_ip_for(opened_http), url:)
-        end
-
-        # @param opened_http [Net::HTTP]
-        # @return [String, nil]
-        def peer_ip_for(opened_http)
-          sock = opened_http.instance_variable_get(:@socket)
-          io = sock.respond_to?(:io) ? sock.io : sock
-          peeraddr = io.respond_to?(:peeraddr) ? io.peeraddr : nil
-          peeraddr&.[](3) || peeraddr&.[](2)
-        end
-      end
-
-      # rubocop:disable-next Metrics/AbcSize
-      def client
-        @client ||= Faraday.new(url: request_url.to_s, headers: ctx.headers) do |faraday|
-          faraday.use Faraday::FollowRedirects::Middleware, limit: ctx.policy.max_redirects, callback: redirect_callback
-          faraday.request :gzip
-          faraday.use StreamingBodyMiddleware
-          faraday.adapter Faraday.default_adapter do |http|
-            PeerIpValidator.install!(http, policy: ctx.policy)
-          end
-        end
-      end
-
-      def apply_timeouts(request, deadline:)
-        remaining_timeout = remaining_timeout_seconds(deadline)
-        request.options.timeout = remaining_timeout
-        request.options.open_timeout = [ctx.policy.connect_timeout_seconds, remaining_timeout].min
-        request.options.read_timeout = [ctx.policy.read_timeout_seconds, remaining_timeout].min
-      end
-
-      def prepare_stream_buffer(request)
-        request.options.context ||= {}
-        request.options.context[StreamingBodyMiddleware::STREAM_BUFFER_KEY] = +''
-      end
-
-      def on_data_callback(response_guard, buffer)
-        proc do |chunk, total_bytes, env|
-          response_guard.inspect_chunk!(total_bytes:, headers: env&.response_headers)
-          buffer&.<< chunk
-        end
-      end
-
-      def remaining_timeout_seconds(deadline)
-        remaining = deadline - monotonic_now
-        raise RequestTimedOut, 'Request timed out' if remaining <= 0
-
-        remaining
-      end
-
-      def retry_without_streaming?(response)
-        return false if response.body.to_s.empty? == false
-        return false unless response_success?(response)
-
-        final_url = response.env&.url
-        return false unless final_url
-
-        final_url.to_s != ctx.url.to_s
-      end
-
-      def response_success?(response)
-        return true if response.status.nil?
-
-        response.status >= 200 && response.status < 300
-      end
-
-      def response_url(response)
-        return ctx.url unless (url = response.env&.url)
-
-        Html2rss::Url.from_absolute(url.to_s)
-      end
-
-      def redirect_callback
-        lambda do |old_env, new_env|
-          from_url = normalize_url(old_env[:url])
-          to_url = normalize_url(new_env[:url])
-          @last_redirect_to = to_url
-          ctx.policy.validate_redirect!(from_url:, to_url:, origin_url: ctx.origin_url, relation: ctx.relation)
-        end
-      end
-
-      def normalize_url(url)
-        Html2rss::Url.from_absolute(url.to_s)
-      end
-
-      def monotonic_now
-        Process.clock_gettime(Process::CLOCK_MONOTONIC)
-      end
-    end
-  end
-end

data/lib/html2rss/selectors/item_scope.rb

@@ -1,30 +0,0 @@
-# frozen_string_literal: true
-
-module Html2rss
-  class Selectors
-    ##
-    # Per-item extraction scope: owns the article node and page +base_url+ for one
-    # extraction pass. Channel hashes are injected (and cached) by {Selectors}.
-    #
-    # Distinct from {Context}, which is the post-processor invocation bag (+options+).
-    # Post-processor +config+ is derived from {#channel} — not a parallel bag.
-    ItemScope = Data.define(:item, :base_url, :scraper, :channel) do
-      ##
-      # Selects an attribute using this scope's item and base_url.
-      #
-      # @param name [Symbol, String]
-      # @return [Object, Array<Object>]
-      def select(name) = scraper.select_in_scope(name, self)
-
-      ##
-      # Builds a post-processor {Context} carrying this scope for nested selects.
-      #
-      # @param options [Hash] post-processor options from the selector config
-      # @option options [String] :name post-processor name
-      # @return [Context]
-      def context_for(options:)
-        Context.new(options:, item:, config: { channel: }, scraper:, item_scope: self)
-      end
-    end
-  end
-end