html2rss 0.16.0 → 0.18.0

data/lib/html2rss/url.rb

@@ -0,0 +1,262 @@
+# frozen_string_literal: true
+
+require 'addressable/uri'
+require 'cgi'
+
+module Html2rss
+  ##
+  # A value object representing a resolved, absolute URL with built-in operations.
+  # Provides URL resolution, sanitization, and titleization capabilities.
+  #
+  # @example Creating a URL from a relative path
+  #   url = Url.from_relative('/path/to/article', 'https://example.com')
+  #   url.to_s # => "https://example.com/path/to/article"
+  #
+  # @example Sanitizing a raw URL string
+  #   url = Url.sanitize('https://example.com/  ')
+  #   url.to_s # => "https://example.com/"
+  #
+  # @example Getting titleized versions
+  #   url = Url.from_relative('/foo-bar/baz.txt', 'https://example.com')
+  #   url.titleized # => "Foo Bar Baz"
+  #   url.channel_titleized # => "example.com: Foo Bar Baz"
+  class Url
+    include Comparable
+
+    # Regular expression for basic URI format validation
+    URI_REGEXP = Addressable::URI::URIREGEX
+    SUPPORTED_SCHEMES = %w[http https].to_set.freeze
+
+    ##
+    # Creates a URL from a relative path and base URL.
+    #
+    # @param relative_url [String, Html2rss::Url] the relative URL to resolve
+    # @param base_url [String, Html2rss::Url] the base URL to resolve against
+    # @return [Url] the resolved absolute URL
+    # @raise [ArgumentError] if the URL cannot be parsed
+    def self.from_relative(relative_url, base_url)
+      url = Addressable::URI.parse(relative_url.to_s.strip)
+      return new(url) if url.absolute?
+
+      base_uri = Addressable::URI.parse(base_url.to_s)
+      base_uri.path = '/' if base_uri.path.empty?
+
+      new(base_uri.join(url).normalize)
+    end
+
+    ##
+    # Creates a URL by sanitizing a raw URL string.
+    # Removes spaces and extracts the first valid URL from the string.
+    #
+    # @param raw_url [String] the raw URL string to sanitize
+    # @return [Url, nil] the sanitized URL, or nil if no valid URL found
+    def self.sanitize(raw_url)
+      matched_urls = raw_url.to_s.scan(%r{(?:(?:https?|ftp|mailto)://|mailto:)[^\s<>"]+})
+      url = matched_urls.first.to_s.strip
+      return nil if url.empty?
+
+      new(Addressable::URI.parse(url).normalize)
+    end
+
+    ##
+    # Creates a URL from an already-absolute URL string.
+    #
+    # @param url_string [String, Html2rss::Url] the absolute URL to parse
+    # @return [Url] the parsed and normalized URL
+    # @raise [ArgumentError] if the URL is not absolute or cannot be parsed
+    def self.from_absolute(url_string)
+      return url_string if url_string.is_a?(self)
+
+      url = new(Addressable::URI.parse(url_string.to_s.strip).normalize)
+      raise ArgumentError, 'URL must be absolute' unless url.absolute?
+
+      url
+    rescue Addressable::URI::InvalidURIError
+      raise ArgumentError, 'URL must be absolute'
+    end
+
+    ##
+    # Creates a URL for channel use with validation.
+    # Validates that the URL meets channel requirements (absolute, no @, supported schemes).
+    #
+    # @param url_string [String] the URL string to validate and parse
+    # @return [Url] the validated and parsed URL
+    # @raise [ArgumentError] if the URL doesn't meet channel requirements
+    # @example Creating a channel URL
+    #   Url.for_channel('https://example.com')
+    #   # => #<Html2rss::Url:... @uri=#<Addressable::URI:... URI:https://example.com>>
+    # @example Invalid channel URL
+    #   Url.for_channel('/relative/path')
+    #   # => raises ArgumentError: "URL must be absolute"
+    def self.for_channel(url_string)
+      return nil if url_string.nil? || url_string.empty?
+
+      stripped = url_string.strip
+      return nil if stripped.empty?
+
+      url = from_absolute(stripped)
+      validate_channel_url(url)
+      url
+    end
+
+    ##
+    # Validates that a URL meets channel requirements.
+    #
+    # @param url [Url] the URL to validate
+    # @raise [ArgumentError] if the URL doesn't meet channel requirements
+    def self.validate_channel_url(url)
+      raise ArgumentError, 'URL must be absolute' unless url.absolute?
+
+      raise ArgumentError, 'URL must not contain an @ character' if url.to_s.include?('@')
+
+      scheme = url.scheme
+      raise ArgumentError, "URL scheme '#{scheme}' is not supported" unless SUPPORTED_SCHEMES.include?(scheme)
+    end
+
+    private_class_method :validate_channel_url
+
+    ##
+    # @param uri [Addressable::URI] the underlying Addressable::URI object (internal use only)
+    def initialize(uri)
+      @uri = uri.freeze
+      freeze
+    end
+
+    # Delegate common URI operations to the underlying URI
+    def to_s = @uri.to_s
+    def scheme = @uri.scheme
+    def host = @uri.host
+    def port = @uri.port
+    def path = @uri.path
+    def query = @uri.query
+    def fragment = @uri.fragment
+    def absolute? = @uri.absolute?
+
+    ##
+    # Returns the URL query string as a hash of string keys and values.
+    #
+    # @return [Hash{String => String}] normalized query parameters
+    def query_values
+      @uri.query_values(Hash) || {}
+    end
+
+    ##
+    # Returns the URL path split into non-empty segments.
+    #
+    # @return [Array<String>] normalized path segments
+    def path_segments
+      @uri.path.to_s.split('/').reject(&:empty?)
+    end
+
+    ##
+    # Returns a copy of the URL with the provided path.
+    #
+    # @param path [String] normalized absolute path
+    # @return [Url] a new URL with the updated path
+    def with_path(path)
+      uri = @uri.dup
+      uri.path = path
+      self.class.from_absolute(uri.normalize.to_s)
+    end
+
+    ##
+    # Returns a copy of the URL with the provided query values.
+    #
+    # @param values [Hash{String, Symbol => #to_s}] query parameters to assign
+    # @return [Url] a new URL with the updated query string
+    def with_query_values(values)
+      uri = @uri.dup
+      uri.query_values = values.transform_keys(&:to_s).transform_values(&:to_s)
+      self.class.from_absolute(uri.normalize.to_s)
+    end
+
+    ##
+    # Returns a titleized representation of the URL path.
+    # Converts the path to a human-readable title by cleaning and capitalizing words.
+    # Removes file extensions and special characters, then capitalizes each word.
+    #
+    # @return [String] the titleized path, or empty string if path is empty
+    # @example Basic titleization
+    #   url = Url.from_absolute('https://example.com/foo-bar/baz.txt')
+    #   url.titleized # => "Foo Bar Baz"
+    # @example With URL encoding
+    #   url = Url.from_absolute('https://example.com/hello%20world/article.html')
+    #   url.titleized # => "Hello World Article"
+    def titleized
+      path = @uri.path
+      return '' if path.empty?
+
+      nicer_path = CGI.unescapeURIComponent(path)
+                      .split('/')
+                      .flat_map do |part|
+                        part.gsub(/[^a-zA-Z0-9.]/, ' ').gsub(/\s+/, ' ').split
+                      end
+
+      nicer_path.map!(&:capitalize)
+      File.basename(nicer_path.join(' '), '.*')
+    end
+
+    ##
+    # Returns a titleized representation of the URL with prefixed host.
+    # Creates a channel title by combining host and path information.
+    # Useful for RSS channel titles that need to identify the source.
+    #
+    # @return [String] the titleized channel URL
+    # @example With path
+    #   url = Url.from_absolute('https://example.com/foo-bar/baz')
+    #   url.channel_titleized # => "example.com: Foo Bar Baz"
+    # @example Without path (root URL)
+    #   url = Url.from_absolute('https://example.com')
+    #   url.channel_titleized # => "example.com"
+    def channel_titleized
+      nicer_path = CGI.unescapeURIComponent(@uri.path).split('/').reject(&:empty?)
+      host = @uri.host
+
+      nicer_path.any? ? "#{host}: #{nicer_path.map(&:capitalize).join(' ')}" : host
+    end
+
+    ##
+    # Compares this URL with another URL for equality.
+    # URLs are considered equal if their string representations are the same.
+    #
+    # @param other [Url] the other URL to compare with
+    # @return [Integer] -1, 0, or 1 for less than, equal, or greater than
+    def <=>(other)
+      to_s <=> other.to_s
+    end
+
+    ##
+    # Returns true if this URL is equal to another URL.
+    #
+    # @param other [Object] the other object to compare with
+    # @return [Boolean] true if the URLs are equal
+    def ==(other)
+      other.is_a?(Url) && to_s == other.to_s
+    end
+
+    ##
+    # Supports hash-based comparisons by ensuring equality semantics match `hash`.
+    #
+    # @param other [Object] the other object to compare with
+    # @return [Boolean] true if the URLs are considered equal
+    def eql?(other)
+      other.is_a?(Url) && to_s == other.to_s
+    end
+
+    ##
+    # Returns the hash code for this URL.
+    #
+    # @return [Integer] the hash code
+    def hash
+      to_s.hash
+    end
+
+    ##
+    # Returns a string representation of the URL for debugging.
+    #
+    # @return [String] the debug representation
+    def inspect
+      "#<#{self.class}:#{object_id} @uri=#{@uri.inspect}>"
+    end
+  end
+end

data/lib/html2rss/version.rb

@@ -3,6 +3,6 @@
 ##
 # The Html2rss namespace.
 module Html2rss
-  VERSION = '0.16.0'
+  VERSION = '0.18.0'
   public_constant :VERSION
 end

data/lib/html2rss.rb

@@ -3,11 +3,10 @@
 require 'zeitwerk'
 
 loader = Zeitwerk::Loader.for_gem
+loader.inflector.inflect('cli' => 'CLI')
 loader.setup
 
-require 'addressable'
 require 'logger'
-require 'yaml'
 
 ##
 # The Html2rss namespace.
@@ -23,90 +22,150 @@ module Html2rss
   end
 
   ##
-  # The Html2rss::Error base class.
-  class Error < StandardError; end
-
-  ##
-  # Key for the feeds configuration in the YAML file.
-  CONFIG_KEY_FEEDS = :feeds
-
-  ##
-  # Returns an RSS object generated from the provided YAML file configuration.
-  #
-  # Example:
+  # Loads a feed configuration from YAML.
   #
-  #    feed = Html2rss.feed_from_yaml_config(File.join(['spec', 'config.test.yml']), 'nuxt-releases')
-  #    # => #<RSS::Rss:0x00007fb2f6331228
-  #
-  # @param file [String] Path to the YAML file.
-  # @param name [String, Symbol, nil] Name of the feed in the YAML file.
-  # @param global_config [Hash] Global options (e.g., HTTP headers).
-  # @param params [Hash] Dynamic parameters for the feed configuration.
-  # @return [RSS::Rss] RSS object generated from the configuration.
-  def self.feed_from_yaml_config(file, name = nil, global_config: {}, params: {})
-    yaml = YAML.safe_load_file(file, symbolize_names: true)
-    feeds = yaml[CONFIG_KEY_FEEDS] || {}
-
-    feed_config = find_feed_config(yaml, feeds, name, global_config)
-
-    feed(Config.new(feed_config, global_config, params))
+  # @param file [String] path to the YAML file
+  # @param feed_name [String, nil] optional feed name inside a multi-feed config
+  # @return [Hash<Symbol, Object>] loaded configuration hash
+  def self.config_from_yaml_file(file, feed_name = nil)
+    Config.load_yaml(file, feed_name)
   end
 
   ##
   # Returns an RSS object generated from the provided configuration.
   #
-  # Example:
-  #
-  #    feed = Html2rss.feed(
-  #      channel: { name: 'StackOverflow: Hot Network Questions', url: 'https://stackoverflow.com' },
-  #      selectors: {
-  #        items: { selector: '#hot-network-questions > ul > li' },
-  #        title: { selector: 'a' },
-  #        link: { selector: 'a', extractor: 'href' }
-  #      }
-  #    )
-  #    # => #<RSS::Rss:0x00007fb2f48d14a0 ...>
-  #
-  # @param config [Hash<Symbol, Object>, Html2rss::Config] Feed configuration.
-  # @return [RSS::Rss] RSS object generated from the configuration.
-  def self.feed(config)
-    config = Config.new(config) unless config.is_a?(Config)
-    RssBuilder.build(config)
+  # @param raw_config [Hash<Symbol, Object>] feed configuration
+  # @return [RSS::Rss] generated RSS feed
+  def self.feed(raw_config)
+    run_pipeline(raw_config) do |response:, config:, articles:|
+      build_rss_feed(response:, config:, articles:)
+    end
   end
 
   ##
-  # Builds the feed configuration based on the provided parameters.
+  # Returns a JSONFeed 1.1 hash generated from the provided configuration.
   #
-  # @param yaml [Hash] Parsed YAML content.
-  # @param feeds [Hash] Feeds from the YAML content.
-  # @param feed_name [String, Symbol, nil] Name of the feed in the YAML file.
-  # @param global_config [Hash] Global options (e.g., HTTP headers).
-  # @return [Hash] Feed configuration.
-  def self.find_feed_config(yaml, feeds, feed_name, global_config)
-    return yaml unless feed_name
-
-    feed_name = feed_name.to_sym
-    if feeds.key?(feed_name)
-      global_config.merge!(yaml.reject { |key| key == CONFIG_KEY_FEEDS })
-      feeds[feed_name]
-    else
-      yaml
+  # @param raw_config [Hash<Symbol, Object>] feed configuration
+  # @return [Hash] JSONFeed-compliant hash
+  def self.json_feed(raw_config)
+    run_pipeline(raw_config) do |response:, config:, articles:|
+      build_json_feed(response:, config:, articles:)
     end
   end
 
   ##
   # Scrapes the provided URL and returns an RSS object.
-  # No need for a "feed config".
   #
-  # @param url [String] the URL to automatically source the feed from
-  # @param strategy [Symbol] the request strategy to use
-  # @return [RSS::Rss]
-  def self.auto_source(url, strategy: :faraday)
-    ctx = RequestService::Context.new(url:, headers: {})
-    response = RequestService.execute(ctx, strategy:)
-
-    Html2rss::AutoSource.new(ctx.url, body: response.body, headers: response.headers).build
+  # @param url [String] source page URL
+  # @param strategy [Symbol] request strategy to use
+  # @param items_selector [String, nil] optional selector hint for item extraction
+  # @param max_redirects [Integer, nil] optional redirect limit override
+  # @param max_requests [Integer, nil] optional request budget override
+  # @return [RSS::Rss] generated RSS feed
+  def self.auto_source(url, strategy: :faraday, items_selector: nil, max_redirects: nil, max_requests: nil)
+    feed(build_auto_source_config(url:, strategy:, items_selector:, max_redirects:, max_requests:))
+  end
+
+  ##
+  # Scrapes the provided URL and returns a JSONFeed 1.1 hash.
+  #
+  # @param url [String] source page URL
+  # @param strategy [Symbol] request strategy to use
+  # @param items_selector [String, nil] optional selector hint for item extraction
+  # @param max_redirects [Integer, nil] optional redirect limit override
+  # @param max_requests [Integer, nil] optional request budget override
+  # @return [Hash] JSONFeed-compliant hash
+  def self.auto_json_feed(url, strategy: :faraday, items_selector: nil, max_redirects: nil, max_requests: nil)
+    json_feed(build_auto_source_config(url:, strategy:, items_selector:, max_redirects:, max_requests:))
   end
 
-  private_class_method :find_feed_config
+  class << self
+    private
+
+    def run_pipeline(raw_config)
+      # 1. Normalize and validate the user-facing feed config.
+      config = Config.from_hash(raw_config, params: raw_config[:params])
+      runtime_input = RequestSession::RuntimeInput.from_config(config)
+
+      # 2. Fetch the initial page using a shared request session.
+      request_session = RequestSession.from_runtime_input(runtime_input)
+      response = request_session.fetch_initial_response
+
+      # 3. Collect articles from configured selectors and auto-source scrapers.
+      articles = Articles::Deduplicator.new(
+        collect_articles(response:, config:, request_session:)
+      ).call
+
+      # 4. Render the final output format chosen by the public entrypoint.
+      yield response:, config:, articles:
+    end
+
+    def collect_articles(response:, config:, request_session:)
+      selector_articles(response:, config:, request_session:) +
+        auto_source_articles(response:, config:, request_session:)
+    end
+
+    def selector_articles(response:, config:, request_session:) # rubocop:disable Metrics/MethodLength
+      return [] unless (selectors = config.selectors)
+
+      page_responses = if (max_pages = selectors.dig(:items, :pagination, :max_pages))
+                         RequestSession::RelNextPager.new(
+                           session: request_session,
+                           initial_response: response,
+                           max_pages:
+                         ).to_a
+                       else
+                         [response]
+                       end
+
+      page_responses.flat_map do |page_response|
+        Selectors.new(page_response, selectors:, time_zone: config.time_zone).articles
+      end
+    end
+
+    def auto_source_articles(response:, config:, request_session:)
+      return [] unless (auto_source = config.auto_source)
+
+      AutoSource.new(response, auto_source, request_session:).articles
+    end
+
+    def build_rss_feed(response:, config:, articles:)
+      channel = RssBuilder::Channel.new(response, overrides: config.channel)
+
+      RssBuilder.new(channel:, articles:, stylesheets: config.stylesheets).call
+    end
+
+    def build_json_feed(response:, config:, articles:)
+      channel = RssBuilder::Channel.new(response, overrides: config.channel)
+
+      JsonFeedBuilder.new(channel:, articles:).call
+    end
+
+    def explicit_request_control_keys(strategy:, max_redirects:, max_requests:)
+      keys = []
+      keys << :strategy unless strategy == :faraday
+      keys << :max_redirects unless max_redirects.nil?
+      keys << :max_requests unless max_requests.nil?
+      keys
+    end
+
+    def build_auto_source_config(url:, strategy:, items_selector:, max_redirects:, max_requests:)
+      Config.auto_source_config(
+        url:,
+        items_selector:,
+        request_controls: shortcut_request_controls(strategy:, max_redirects:, max_requests:)
+      )
+    end
+
+    def shortcut_request_controls(strategy:, max_redirects:, max_requests:)
+      RequestControls.new(
+        strategy:,
+        max_redirects:,
+        max_requests:,
+        explicit_keys: explicit_request_control_keys(strategy:, max_redirects:, max_requests:)
+      )
+    end
+  end
 end
+
+loader.eager_load

data/lib/tasks/config_schema.rake

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'fileutils'
+require_relative '../html2rss'
+
+namespace :config do
+  desc 'Generate config JSON schema'
+  task :schema do
+    destination = Html2rss::Config.schema_path
+
+    FileUtils.mkdir_p(File.dirname(destination))
+    File.write(destination, "#{Html2rss::Config.json_schema_json}\n")
+
+    puts "Generated config schema at #{destination}"
+  end
+end