html2rss 0.12.0 → 0.14.0

data/lib/html2rss/item.rb

@@ -19,11 +19,12 @@ module Html2rss
     ##
     # Fetches items from a given URL using configuration settings.
     #
-    # @param url [String] URL to fetch items from.
+    # @param url [Addressable::URI] URL to fetch items from.
     # @param config [Html2rss::Config] Configuration object.
     # @return [Array<Html2rss::Item>] list of items fetched.
     def self.from_url(url, config)
-      body = Utils.request_body_from_url(url, convert_json_to_xml: config.json?, headers: config.headers)
+      body = Utils.request_url(url, headers: config.headers).body
+      body = ObjectToXmlConverter.new(JSON.parse(body)).call if config.json?
 
       Nokogiri.HTML(body)
               .css(config.selector_string(Config::Selectors::ITEMS_SELECTOR_NAME))
@@ -47,6 +48,7 @@ module Html2rss
     # @param method_name [Symbol]
     # @param _include_private [true, false]
     # @return [true, false]
+    # :reek:BooleanParameter { enabled: false }
     def respond_to_missing?(method_name, _include_private = false)
       config.selector?(method_name) || super
     end
@@ -110,7 +112,11 @@ module Html2rss
     #
     # @return [Array<String>] list of categories.
     def categories
-      config.category_selector_names.map { |method_name| public_send(method_name) }
+      config.category_selector_names
+            .filter_map do |method_name|
+        category = public_send(method_name)
+        category.strip unless category.to_s.empty?
+      end.uniq
     end
 
     ##

data/lib/html2rss/rss_builder/stylesheet.rb

@@ -3,35 +3,50 @@
 module Html2rss
   module RssBuilder
     ##
-    # Adds XML stylesheet tags (with the provided maker).
+    # Represents a stylesheet.
     class Stylesheet
-      ##
-      # Adds the stylesheet XML tags to the RSS.
-      #
-      # @param maker [RSS::Maker::RSS20] RSS maker object.
-      # @param stylesheets [Array<Html2rss::Config::Stylesheet>] Array of stylesheet configurations.
-      # @return [nil]
-      def self.add(maker, stylesheets)
-        stylesheets.each do |stylesheet|
-          add_stylesheet(maker, stylesheet)
+      class << self
+        ##
+        # Adds the stylesheet XML tags to the RSS.
+        #
+        # @param maker [RSS::Maker::RSS20] RSS maker object.
+        # @param stylesheets [Array<Html2rss::Config::Stylesheet>] Array of stylesheet configurations.
+        # @return [nil]
+        def add(maker, stylesheets)
+          stylesheets.each do |stylesheet|
+            add_stylesheet(maker, stylesheet)
+          end
         end
-      end
 
-      ##
-      # Adds a single Stylesheet to the RSS.
-      #
-      # @param maker [RSS::Maker::RSS20] RSS maker object.
-      # @param stylesheet [Html2rss::Config::Stylesheet] Stylesheet configuration.
-      # @return [nil]
-      def self.add_stylesheet(maker, stylesheet)
-        maker.xml_stylesheets.new_xml_stylesheet do |xss|
-          xss.href = stylesheet.href
-          xss.type = stylesheet.type
-          xss.media = stylesheet.media
+        private
+
+        ##
+        # Adds a single Stylesheet to the RSS.
+        #
+        # @param maker [RSS::Maker::RSS20] RSS maker object.
+        # @param stylesheet [Html2rss::Config::Stylesheet] Stylesheet configuration.
+        # @return [nil]
+        def add_stylesheet(maker, stylesheet)
+          maker.xml_stylesheets.new_xml_stylesheet do |xss|
+            xss.href = stylesheet.href
+            xss.type = stylesheet.type
+            xss.media = stylesheet.media
+          end
         end
       end
 
-      private_class_method :add_stylesheet
+      TYPES = ['text/css', 'text/xsl'].freeze
+
+      def initialize(href:, type:, media: 'all')
+        raise ArgumentError, 'stylesheet.href must be a String' unless href.is_a?(String)
+        raise ArgumentError, 'stylesheet.type invalid' unless TYPES.include?(type)
+        raise ArgumentError, 'stylesheet.media must be a String' unless media.is_a?(String)
+
+        @href = href
+        @type = type
+        @media = media
+      end
+      attr_reader :href, :type, :media
     end
   end
 end

data/lib/html2rss/utils.rb

@@ -31,12 +31,12 @@ module Html2rss
     ##
     # Removes any space, parses and normalizes the given url.
     # @param url [String]
-    # @return [String, nil] sanitized and normalized URL, or nil if input is empty
+    # @return [Addressable::URI, nil] normalized URL, or nil if input is empty
     def self.sanitize_url(url)
       url = url.to_s.gsub(/\s+/, ' ').strip
       return if url.empty?
 
-      Addressable::URI.parse(url).normalize.to_s
+      Addressable::URI.parse(url).normalize
     end
 
     ##
@@ -44,6 +44,7 @@ module Html2rss
     #
     # @param time_zone [String]
     # @param default_time_zone [String]
+    # @yield block to execute with the given time zone
     # @return [Object] whatever the given block returns
     def self.use_zone(time_zone, default_time_zone: Time.now.getlocal.zone)
       raise ArgumentError, 'a block is required' unless block_given?
@@ -71,18 +72,18 @@ module Html2rss
 
     ##
     # @param url [String, Addressable::URI]
-    # @param convert_json_to_xml [true, false] Should JSON be converted to XML
     # @param headers [Hash] additional HTTP request headers to use for the request
-    # @return [String] body of the HTTP response
-    def self.request_body_from_url(url, convert_json_to_xml: false, headers: {})
-      response = Faraday.new(url:, headers:) do |faraday|
+    # @return [Faraday::Response] body of the HTTP response
+    def self.request_url(url, headers: {})
+      url = Addressable::URI.parse(url.to_s) unless url.is_a?(Addressable::URI)
+
+      raise ArgumentError, 'URL must be absolute' unless url.absolute?
+      raise ArgumentError, 'URL must not contain an @ characater' if url.to_s.include?('@')
+
+      Faraday.new(url:, headers:) do |faraday|
         faraday.use Faraday::FollowRedirects::Middleware
         faraday.adapter Faraday.default_adapter
       end.get
-
-      body = response.body
-
-      convert_json_to_xml ? ObjectToXmlConverter.new(JSON.parse(body)).call : body
     end
 
     ##

data/lib/html2rss/version.rb

@@ -3,6 +3,6 @@
 ##
 # The Html2rss namespace.
 module Html2rss
-  VERSION = '0.12.0'
+  VERSION = '0.14.0'
   public_constant :VERSION
 end

data/lib/html2rss.rb

@@ -6,10 +6,21 @@ loader = Zeitwerk::Loader.for_gem
 loader.setup
 
 require 'yaml'
+require 'logger'
 
 ##
 # The Html2rss namespace.
 module Html2rss
+  ##
+  # The logger instance.
+  Log = Logger.new($stdout)
+
+  Log.level = ENV.fetch('LOG_LEVEL', :warn).upcase.to_sym
+
+  Log.formatter = proc do |severity, datetime, _progname, msg|
+    "#{datetime} [#{severity}] #{msg}\n"
+  end
+
   ##
   # The Html2rss::Error base class.
   class Error < StandardError; end
@@ -32,7 +43,7 @@ module Html2rss
   # @param params [Hash] Dynamic parameters for the feed configuration.
   # @return [RSS::Rss] RSS object generated from the configuration.
   def self.feed_from_yaml_config(file, name = nil, global_config: {}, params: {})
-    yaml = load_yaml(file)
+    yaml = YAML.safe_load_file(file, symbolize_names: true)
     feeds = yaml[CONFIG_KEY_FEEDS] || {}
 
     feed_config = find_feed_config(yaml, feeds, name, global_config)
@@ -62,15 +73,6 @@ module Html2rss
     RssBuilder.build(config)
   end
 
-  ##
-  # Loads and parses the YAML file.
-  #
-  # @param file [String] Path to the YAML file.
-  # @return [Hash] Parsed YAML content.
-  def self.load_yaml(file)
-    YAML.safe_load_file(file, symbolize_names: true)
-  end
-
   ##
   # Builds the feed configuration based on the provided parameters.
   #
@@ -91,5 +93,19 @@ module Html2rss
     end
   end
 
-  private_class_method :load_yaml, :find_feed_config
+  ##
+  # Scrapes the provided URL and returns an RSS object.
+  # No need for a "feed config".
+  #
+  # @param url [String] the URL to automatically source the feed from
+  # @return [RSS::Rss]
+  def self.auto_source(url)
+    url = Addressable::URI.parse(url)
+
+    response = Html2rss::Utils.request_url(url)
+
+    Html2rss::AutoSource.new(url, body: response.body, headers: response.headers).build
+  end
+
+  private_class_method :find_feed_config
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: html2rss
 version: !ruby/object:Gem::Version
-  version: 0.12.0
+  version: 0.14.0
 platform: ruby
 authors:
 - Gil Desmarais
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2024-08-10 00:00:00.000000000 Z
+date: 2024-10-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -106,6 +106,20 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: parallel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: regexp_parser
   requirement: !ruby/object:Gem::Requirement
@@ -230,6 +244,18 @@ files:
 - lib/html2rss/attribute_post_processors/sanitize_html.rb
 - lib/html2rss/attribute_post_processors/substring.rb
 - lib/html2rss/attribute_post_processors/template.rb
+- lib/html2rss/auto_source.rb
+- lib/html2rss/auto_source/article.rb
+- lib/html2rss/auto_source/channel.rb
+- lib/html2rss/auto_source/cleanup.rb
+- lib/html2rss/auto_source/reducer.rb
+- lib/html2rss/auto_source/rss_builder.rb
+- lib/html2rss/auto_source/scraper.rb
+- lib/html2rss/auto_source/scraper/schema.rb
+- lib/html2rss/auto_source/scraper/schema/base.rb
+- lib/html2rss/auto_source/scraper/semantic_html.rb
+- lib/html2rss/auto_source/scraper/semantic_html/extractor.rb
+- lib/html2rss/auto_source/scraper/semantic_html/image.rb
 - lib/html2rss/cli.rb
 - lib/html2rss/config.rb
 - lib/html2rss/config/channel.rb
@@ -253,7 +279,7 @@ licenses:
 - MIT
 metadata:
   allowed_push_host: https://rubygems.org
-  changelog_uri: https://github.com/html2rss/html2rss/releases/tag/v0.12.0
+  changelog_uri: https://github.com/html2rss/html2rss/releases/tag/v0.14.0
   rubygems_mfa_required: 'true'
 post_install_message: 
 rdoc_options: []
@@ -270,7 +296,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.11
+rubygems_version: 3.5.16
 signing_key: 
 specification_version: 4
 summary: Generates RSS feeds from websites by scraping a URL and using CSS selectors