html2rss 0.22.2 → 0.24.0

data/lib/html2rss/json_feed_builder/item.rb

@@ -1,94 +0,0 @@
-# frozen_string_literal: true
-
-module Html2rss
-  class JsonFeedBuilder
-    ##
-    # Maps an {Html2rss::RssBuilder::Article} to a JSONFeed 1.1 item hash.
-    class Item
-      ##
-      # @param article [Html2rss::RssBuilder::Article]
-      def initialize(article)
-        @article = article
-      end
-
-      ##
-      # @return [Hash, nil] the JSONFeed-compliant item hash
-      def to_h
-        content = content_fields
-        return if content.empty?
-
-        item_payload.merge(content).compact
-      end
-
-      private
-
-      attr_reader :article
-
-      ##
-      # @return [Hash]
-      def item_payload
-        {
-          id: article.guid,
-          url: article.url&.to_s,
-          title: article.title,
-          image: article.image&.to_s,
-          date_published: article.published_at&.iso8601,
-          authors: author_array,
-          tags:,
-          attachments:
-        }
-      end
-
-      ##
-      # @return [Array<Hash>, nil]
-      def author_array
-        return unless (name = article.author)
-
-        [{ name: }]
-      end
-
-      ##
-      # JSON Feed items must include content_html or content_text.
-      # @return [Hash]
-      def content_fields
-        description = article.description
-        return { content_html: description } if description
-
-        title = article.title
-        return { content_text: title } if title
-
-        {}
-      end
-
-      ##
-      # @return [Array<String>, nil]
-      def tags
-        cats = article.categories
-        cats.empty? ? nil : cats
-      end
-
-      ##
-      # Maps enclosures to JSONFeed attachment objects.
-      # @return [Array<Hash>, nil]
-      def attachments
-        enclosures = article.enclosures
-        return nil if enclosures.empty?
-
-        enclosures.map { |enc| attachment_hash(enc) }
-      end
-
-      ##
-      # @param enclosure [Html2rss::RssBuilder::Article::Enclosure]
-      # @return [Hash]
-      def attachment_hash(enclosure)
-        size = enclosure.bits_length
-
-        {
-          url: enclosure.url.to_s,
-          mime_type: enclosure.type,
-          size_in_bytes: size&.positive? ? size : nil
-        }.compact
-      end
-    end
-  end
-end

data/lib/html2rss/json_feed_builder.rb

@@ -1,59 +0,0 @@
-# frozen_string_literal: true
-
-module Html2rss
-  ##
-  # Builds a JSONFeed 1.1 hash from channel metadata and articles.
-  #
-  # @see https://www.jsonfeed.org/version/1.1/
-  class JsonFeedBuilder
-    # Official JSON Feed 1.1 schema version URL.
-    VERSION_URL = 'https://jsonfeed.org/version/1.1'
-
-    ##
-    # @param channel [Html2rss::RssBuilder::Channel]
-    # @param articles [Array<Html2rss::RssBuilder::Article>]
-    def initialize(channel:, articles:)
-      @channel = channel
-      @articles = articles
-    end
-
-    ##
-    # Builds and returns the JSONFeed hash.
-    #
-    # @return [Hash] the JSONFeed-compliant hash
-    def call
-      base_payload.merge(authors: author_array, items: item_hashes).compact
-    end
-
-    private
-
-    attr_reader :channel, :articles
-
-    ##
-    # @return [Hash]
-    def base_payload
-      {
-        version: VERSION_URL,
-        title: channel.title,
-        home_page_url: channel.url.to_s,
-        description: channel.description,
-        language: channel.language,
-        icon: channel.image&.to_s
-      }
-    end
-
-    ##
-    # @return [Array<Hash>]
-    def item_hashes
-      articles.filter_map { |article| Item.new(article).to_h }
-    end
-
-    ##
-    # @return [Array<Hash>, nil]
-    def author_array
-      return unless (name = channel.author)
-
-      [{ name: }]
-    end
-  end
-end

data/lib/html2rss/rendering/audio_renderer.rb

@@ -1,36 +0,0 @@
-# frozen_string_literal: true
-
-require 'cgi'
-
-module Html2rss
-  module Rendering
-    # Renders an HTML <audio> tag from a URL and type.
-    class AudioRenderer
-      # @param url [String, Html2rss::Url] media URL for the audio source
-      # @param type [String] MIME type for the audio source
-      def initialize(url:, type:)
-        @url = url
-        @type = type
-      end
-
-      # @return [String] HTML audio snippet for article rendering
-      def to_html
-        [
-          '<audio controls preload="none" referrerpolicy="no-referrer" crossorigin="anonymous">',
-          %(<source src="#{escaped_url}" type="#{escaped_type}">),
-          '</audio>'
-        ].join
-      end
-
-      private
-
-      def escaped_url
-        CGI.escapeHTML(@url.to_s)
-      end
-
-      def escaped_type
-        CGI.escapeHTML(@type.to_s)
-      end
-    end
-  end
-end

data/lib/html2rss/rendering/description_builder.rb

@@ -1,87 +0,0 @@
-# frozen_string_literal: true
-
-module Html2rss
-  module Rendering
-    # Builds a sanitized article description from the base text, title, and optional media.
-    #
-    # Combines media elements (images, audio, video, PDFs) with sanitized text content
-    # to create rich RSS descriptions that reveal more scraped information.
-    #
-    # @example Basic usage
-    #   builder = DescriptionBuilder.new(
-    #     base: "Article content",
-    #     title: "Article Title",
-    #     url: "https://example.com",
-    #     enclosures: [enclosure_object],
-    #     image: "https://example.com/image.jpg"
-    #   )
-    #   description = builder.call
-    class DescriptionBuilder
-      # Removes the specified pattern from the beginning of the text
-      # within a given range if the pattern occurs before the range's end.
-      #
-      # @param text [String]
-      # @param pattern [String]
-      # @param end_of_range [Integer] Optional, defaults to half the text length
-      # @return [String]
-      def self.remove_pattern_from_start(text, pattern, end_of_range: (text.size * 0.5).to_i)
-        return text unless text.is_a?(String) && pattern.is_a?(String) && !pattern.empty?
-
-        index = text.index(pattern)
-        return text if index.nil? || index > end_of_range
-
-        "#{text[0, index]}#{text[(index + pattern.size)..]}"
-      end
-
-      # @param base [String] The base text content for the description
-      # @param title [String] The article title (used for alt text and title removal)
-      # @param url [String, Html2rss::Url] The article URL (used for sanitization)
-      # @param enclosures [Array<Html2rss::RssBuilder::Enclosure>, nil] Media enclosures
-      # @param image [String, Html2rss::Url, nil] Fallback image URL
-      def initialize(base:, title:, url:, enclosures:, image:)
-        @base = base.to_s
-        @title = title
-        @url = url
-        @enclosures = Array(enclosures)
-        @image = image
-      end
-
-      # Generates the complete description with media and sanitized text.
-      #
-      # @return [String, nil] The complete description or nil if empty
-      def call
-        fragments = []
-        fragments.concat(Array(rendered_media))
-        fragments << processed_base_description
-
-        result = fragments.compact.join("\n\n").strip
-        result.empty? ? nil : result
-      end
-
-      private
-
-      def rendered_media
-        rendered = render_enclosures
-        return rendered if rendered.any?
-        return render_fallback_image if @image
-
-        []
-      end
-
-      def render_enclosures
-        @enclosures.filter_map do |enclosure|
-          MediaRenderer.for(enclosure:, image: @image, title: @title)&.to_html
-        end
-      end
-
-      def render_fallback_image
-        [MediaRenderer.for(enclosure: nil, image: @image, title: @title)&.to_html]
-      end
-
-      def processed_base_description
-        text = self.class.remove_pattern_from_start(@base, @title)
-        Html2rss::Selectors::PostProcessors::SanitizeHtml.get(text, @url)
-      end
-    end
-  end
-end

data/lib/html2rss/rendering/image_renderer.rb

@@ -1,41 +0,0 @@
-# frozen_string_literal: true
-
-require 'cgi'
-
-module Html2rss
-  module Rendering
-    # Renders an HTML <img> tag from a URL and title.
-    class ImageRenderer
-      # @param url [String, Html2rss::Url] image URL for the src attribute
-      # @param title [String, nil] title/alt text for the image
-      def initialize(url:, title:)
-        @url = url
-        @title = title
-      end
-
-      # @return [String] HTML image snippet for article rendering
-      def to_html
-        attributes = [
-          %(src="#{escaped_url}"),
-          %(alt="#{escaped_title}"),
-          %(title="#{escaped_title}"),
-          'loading="lazy"',
-          'referrerpolicy="no-referrer"',
-          'decoding="async"',
-          'crossorigin="anonymous"'
-        ]
-        "<img #{attributes.join(' ')}>"
-      end
-
-      private
-
-      def escaped_url
-        CGI.escapeHTML(@url.to_s)
-      end
-
-      def escaped_title
-        CGI.escapeHTML(@title.to_s)
-      end
-    end
-  end
-end

data/lib/html2rss/rendering/media_renderer.rb

@@ -1,37 +0,0 @@
-# frozen_string_literal: true
-
-module Html2rss
-  module Rendering
-    # Factory: picks the appropriate renderer for a given enclosure or fallback image.
-    class MediaRenderer
-      # @param enclosure [Html2rss::RssBuilder::Enclosure, nil]
-      # @param image [String, Html2rss::Url, nil] Fallback image URL
-      # @param title [String]
-      # @return [ImageRenderer, VideoRenderer, AudioRenderer, PdfRenderer, nil]
-      def self.for(enclosure:, image:, title:)
-        return ImageRenderer.new(url: image, title:) if enclosure.nil? && image
-        return nil unless enclosure
-
-        create_renderer_for_type(enclosure.type, url: enclosure.url, title:)
-      end
-
-      # @private
-      # @param type [String, nil] enclosure MIME type
-      # @param url [String, Html2rss::Url] enclosure URL
-      # @param title [String, nil] title used by image renderer
-      # @return [ImageRenderer, VideoRenderer, AudioRenderer, PdfRenderer, nil]
-      def self.create_renderer_for_type(type, url:, title:)
-        case type
-        when %r{^image/}
-          ImageRenderer.new(url:, title:)
-        when %r{^video/}
-          VideoRenderer.new(url:, type:)
-        when %r{^audio/}
-          AudioRenderer.new(url:, type:)
-        when 'application/pdf'
-          PdfRenderer.new(url:)
-        end
-      end
-    end
-  end
-end

data/lib/html2rss/rendering/pdf_renderer.rb

@@ -1,34 +0,0 @@
-# frozen_string_literal: true
-
-require 'cgi'
-
-module Html2rss
-  module Rendering
-    # Renders an HTML <iframe> for PDF documents.
-    class PdfRenderer
-      # @param url [String, Html2rss::Url] PDF URL rendered in the iframe
-      def initialize(url:)
-        @url = url
-      end
-
-      # @return [String] HTML iframe snippet for PDF rendering
-      def to_html
-        attributes = [
-          %(src="#{escaped_url}"),
-          'width="100%"',
-          'height="75vh"',
-          'sandbox=""',
-          'referrerpolicy="no-referrer"',
-          'loading="lazy"'
-        ]
-        "<iframe #{attributes.join(' ')}></iframe>"
-      end
-
-      private
-
-      def escaped_url
-        CGI.escapeHTML(@url.to_s)
-      end
-    end
-  end
-end

data/lib/html2rss/rendering/video_renderer.rb

@@ -1,36 +0,0 @@
-# frozen_string_literal: true
-
-require 'cgi'
-
-module Html2rss
-  module Rendering
-    # Renders an HTML <video> tag from a URL and type.
-    class VideoRenderer
-      # @param url [String, Html2rss::Url] media URL for the video source
-      # @param type [String] MIME type for the video source
-      def initialize(url:, type:)
-        @url = url
-        @type = type
-      end
-
-      # @return [String] HTML video snippet for article rendering
-      def to_html
-        [
-          '<video controls preload="none" referrerpolicy="no-referrer" crossorigin="anonymous" playsinline>',
-          %(<source src="#{escaped_url}" type="#{escaped_type}">),
-          '</video>'
-        ].join
-      end
-
-      private
-
-      def escaped_url
-        CGI.escapeHTML(@url.to_s)
-      end
-
-      def escaped_type
-        CGI.escapeHTML(@type.to_s)
-      end
-    end
-  end
-end

data/lib/html2rss/rendering.rb

@@ -1,23 +0,0 @@
-# frozen_string_literal: true
-
-module Html2rss
-  # Namespace for HTML rendering logic, used to generate rich content such as
-  # images, audio, video, or embedded documents for feed descriptions.
-  #
-  # @see Html2rss::Rendering::DescriptionBuilder
-  #
-  # @example
-  #   Html2rss::Rendering::ImageRenderer.new(
-  #     url: "https://example.com/image.jpg",
-  #     title: "Example"
-  #   ).to_html
-  #
-  # @example
-  #   Html2rss::Rendering::MediaRenderer.for(
-  #     enclosure: nil,
-  #     image: "https://example.com/image.jpg",
-  #     title: "Example"
-  #   )
-  module Rendering
-  end
-end

data/lib/html2rss/request_controls.rb

@@ -1,133 +0,0 @@
-# frozen_string_literal: true
-
-module Html2rss
-  ##
-  # Tracks runtime request controls together with whether each value was explicitly set.
-  class RequestControls
-    # Request-control keys accepted at the top level of feed config.
-    TOP_LEVEL_KEYS = %i[strategy].freeze
-    # Request-control keys accepted under the nested `request` config.
-    REQUEST_KEYS = %i[max_redirects max_requests total_timeout_seconds].freeze
-
-    ##
-    # @param config [Hash{Symbol => Object}] raw config input
-    # @return [RequestControls] request controls extracted from the config hash
-    def self.from_config(config)
-      HashUtil.assert_symbol_keys!(config, context: 'config', deep: false)
-      HashUtil.assert_symbol_keys!(config[:request], context: 'config[:request]') if config[:request].is_a?(Hash)
-
-      new(
-        strategy: config[:strategy],
-        max_redirects: request_value_for(config, :max_redirects),
-        max_requests: request_value_for(config, :max_requests),
-        total_timeout_seconds: request_value_for(config, :total_timeout_seconds),
-        explicit_keys: explicit_keys_for(config)
-      )
-    end
-
-    def self.explicit_keys_for(config)
-      TOP_LEVEL_KEYS.filter { config.key?(_1) } +
-        REQUEST_KEYS.filter { request_key?(config, _1) }
-    end
-
-    def self.request_value_for(config, key)
-      request_config = config[:request]
-      return nil unless request_config.is_a?(Hash)
-
-      request_config[key]
-    end
-
-    def self.request_key?(config, key)
-      request_config = config[:request]
-      request_config.is_a?(Hash) && request_config.key?(key)
-    end
-
-    private_class_method :explicit_keys_for, :request_value_for, :request_key?
-
-    ##
-    # @param strategy [Symbol, nil] effective request strategy
-    # @param max_redirects [Integer, nil] effective redirect limit
-    # @param max_requests [Integer, nil] effective request budget
-    # @param total_timeout_seconds [Integer, nil] effective request timeout
-    # @param explicit_keys [Array<Symbol>] controls explicitly supplied by the caller
-    def initialize(strategy: nil, max_redirects: nil, max_requests: nil, total_timeout_seconds: nil, explicit_keys: [])
-      @strategy = strategy
-      @max_redirects = max_redirects
-      @max_requests = max_requests
-      @total_timeout_seconds = total_timeout_seconds
-      @explicit_keys = explicit_keys.map(&:to_sym).uniq.freeze
-      freeze
-    end
-
-    ##
-    # @return [Symbol, nil] effective request strategy
-    attr_reader :strategy
-
-    ##
-    # @return [Integer, nil] effective redirect limit
-    attr_reader :max_redirects
-
-    ##
-    # @return [Integer, nil] effective request budget
-    attr_reader :max_requests
-
-    ##
-    # @return [Integer, nil] effective request timeout
-    attr_reader :total_timeout_seconds
-
-    ##
-    # @param name [Symbol, String] request control name
-    # @return [Boolean] whether the control was explicitly supplied
-    def explicit?(name)
-      explicit_keys.include?(name.to_sym)
-    end
-
-    ##
-    # @param strategy [Symbol, nil] validated request strategy
-    # @param max_redirects [Integer, nil] validated redirect limit
-    # @param max_requests [Integer, nil] validated request budget
-    # @param total_timeout_seconds [Integer, nil] validated request timeout
-    # @return [RequestControls] controls updated with validated effective values
-    def with_effective_values(strategy:, max_redirects:, max_requests:, total_timeout_seconds:)
-      self.class.new(
-        strategy:,
-        max_redirects:,
-        max_requests:,
-        total_timeout_seconds:,
-        explicit_keys:
-      )
-    end
-
-    ##
-    # Applies only explicitly set controls to the provided config hash.
-    #
-    # @param config [Hash{Symbol => Object}] mutable config hash
-    # @return [Hash{Symbol => Object}] the same hash with explicit controls written
-    def apply_to(config)
-      config[:strategy] = strategy if explicit?(:strategy)
-      apply_request_value(config, :max_redirects, max_redirects)
-      apply_request_value(config, :max_requests, max_requests)
-      apply_request_value(config, :total_timeout_seconds, total_timeout_seconds)
-      config
-    end
-
-    private
-
-    attr_reader :explicit_keys
-
-    def apply_request_value(config, key, value)
-      return unless explicit?(key)
-
-      ensure_request_config!(config)
-      config[:request][key] = value
-    end
-
-    def ensure_request_config!(config)
-      request_config = config[:request]
-      return config[:request] = {} if request_config.nil?
-      return if request_config.is_a?(Hash)
-
-      raise ArgumentError, 'request config must be a hash'
-    end
-  end
-end

data/lib/html2rss/request_session/rel_next_pager.rb

@@ -1,70 +0,0 @@
-# frozen_string_literal: true
-
-module Html2rss
-  class RequestSession
-    ##
-    # Traverses a rel=next pagination chain for selector-driven extraction.
-    class RelNextPager
-      include Enumerable
-
-      ##
-      # @param session [RequestSession] request session used to execute follow-ups
-      # @param initial_response [RequestService::Response] first page response
-      # @param max_pages [Integer] configured page budget, including the initial page
-      # @param logger [Logger] logger used for pagination stop reasons
-      def initialize(session:, initial_response:, max_pages:, logger: Html2rss::Log)
-        @session = session
-        @initial_response = initial_response
-        @max_pages = max_pages
-        @logger = logger
-      end
-
-      ##
-      # Iterates over all paginated responses, beginning with the initial response.
-      #
-      # @yield [RequestService::Response] each page response
-      # @return [Enumerator] enumerator when no block is given
-      def each
-        return enum_for(:each) unless block_given?
-
-        yield initial_response
-
-        current_response = initial_response
-        session.effective_page_budget(max_pages).pred.times do
-          next_url = next_page_url(current_response)
-          break unless follow_up_allowed?(next_url)
-
-          current_response = fetch_follow_up_response_or_stop(next_url, current_response.url)
-          break unless current_response
-
-          yield current_response
-        end
-      end
-
-      private
-
-      attr_reader :session, :initial_response, :max_pages, :logger
-
-      def next_page_url(page_response)
-        href = page_response.parsed_body.at_css('link[rel~="next"][href], a[rel~="next"][href]')&.[]('href')
-        return nil if href.nil? || href.empty?
-
-        Html2rss::Url.from_relative(href, page_response.url)
-      end
-
-      def follow_up_allowed?(next_url)
-        next_url && !session.visited?(next_url)
-      end
-
-      def fetch_follow_up_response_or_stop(next_url, origin_url)
-        session.follow_up(url: next_url, relation: :pagination, origin_url:)
-      rescue RequestService::RequestBudgetExceeded => error
-        logger.warn(
-          "#{self.class}: pagination stopped at #{next_url} - #{error.message}. " \
-          "Retry with --max-requests #{session.max_requests + 1} or increase request.max_requests in the config."
-        )
-        nil
-      end
-    end
-  end
-end