html2rss 0.13.0 → 0.14.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7a2bf557dd65533533e07b4581e195f2d2b32ff906831526a4d7aed27a558d71
-  data.tar.gz: f42e5f03649a08219d310a2545413c371f851530c4d323fd68ef783b4b3b5e13
+  metadata.gz: a2ce9bbe8640372b5e98672760d76aee5f6f23373dd4b22ca067d2cdaa6f2b15
+  data.tar.gz: ff280d9466ee6b15b1149f582dadf9b209f0e99e4fb02e6b82f91b25a7ca0b7a
 SHA512:
-  metadata.gz: 724a1fa8ab15ae140278eb9b055f22e7aad12e94627795f7a2f13c78f5421607e39d6ba040821b4c47b69f963cc0180bf8e964ff0b896403cb6305ed1d67dbb5
-  data.tar.gz: a06c2e16b0b51c6b6d2184430efc2a4e8b2812fee413163aa2991567e7608141f1c18189fdded58c8c3383940c4790478cd631abc6a1470ad648b2030fdefaab
+  metadata.gz: d516b897253374425ccd3b26d21362df46c18c26694fe1a8aaddc06b956f93e36111d3310dc635b84d7626a2072014d27dacb075cd98d15a79d39aed40991bcb
+  data.tar.gz: 91ae4190d04967c1bc9d3f46b0a0bdbbd23f3dbda6559c3ff10391ab0d63d4b984a44789c8427dbace345af617a33e86c2f441c0a0d58dbcf2b734bd78b73b87

data/README.md

@@ -2,7 +2,7 @@
 
 [![Gem Version](https://badge.fury.io/rb/html2rss.svg)](http://rubygems.org/gems/html2rss/) [![Yard Docs](http://img.shields.io/badge/yard-docs-blue.svg)](https://www.rubydoc.info/gems/html2rss) ![Retro Badge: valid RSS](https://validator.w3.org/feed/images/valid-rss-rogers.png)
 
-`html2rss` is a Ruby gem that generates RSS 2.0 feeds from a _feed config_.
+`html2rss` is a Ruby gem that generates RSS 2.0 feeds from websites automatically, and as a fallback via _feed config_.
 
 With the _feed config_, you provide a URL to scrape and CSS selectors for extracting information (like title, URL, etc.). The gem builds the RSS feed accordingly. [Extractors](#using-extractors) and chainable [post processors](#using-post-processors) make information extraction, processing, and sanitizing a breeze. The gem also supports [scraping JSON](#scraping-and-handling-json-responses) responses and [setting HTTP request headers](#set-any-http-header-in-the-request).
 
@@ -26,9 +26,9 @@ You can also install it as a dependency in your Ruby project:
 
 ## Generating a feed on the CLI
 
-### using automatic scraping
+### using automatic generation
 
-html2rss offers an automatic scrapting feature. Try it with:
+html2rss offers an automatic RSS generation feature. Try it with:
 
 `html2rss auto https://unmatchedstyle.com/`
 

data/lib/html2rss/auto_source/channel.rb

@@ -10,21 +10,27 @@ module Html2rss
       ##
       #
       # @param parsed_body [Nokogiri::HTML::Document] The parsed HTML document.
-      # @param response [Faraday::Response] The URL of the HTML document.
-      def initialize(parsed_body, url:, response:, articles: [])
+      # @param url [Addressable::URI] The URL of the channel.
+      # @param headers [Hash<String, String>] the http headers
+      # @param articles [Array<Html2rss::AutoSource::Article>] The articles.
+      def initialize(parsed_body, url:, headers:, articles: [], stylesheets: [])
         @parsed_body = parsed_body
         @url = url
-        @response = response
+        @headers = headers
         @articles = articles
+        @stylesheets = stylesheets
       end
 
+      attr_writer :articles
+      attr_reader :stylesheets
+
       def url = extract_url
       def title = extract_title
       def language = extract_language
       def description = extract_description
       def image = extract_image
       def ttl = extract_ttl
-      def last_build_date = response.headers['last-modified']
+      def last_build_date = headers['last-modified']
 
       def generator
         "html2rss V. #{::Html2rss::VERSION} (using auto_source scrapers: #{scraper_counts})"
@@ -32,7 +38,7 @@ module Html2rss
 
       private
 
-      attr_reader :parsed_body, :response
+      attr_reader :parsed_body, :headers
 
       def extract_url
         @url.normalize.to_s
@@ -58,7 +64,7 @@ module Html2rss
       end
 
       def extract_ttl
-        ttl = response.headers['cache-control']&.match(/max-age=(\d+)/)&.[](1)
+        ttl = headers['cache-control']&.match(/max-age=(\d+)/)&.[](1)
         return unless ttl
 
         ttl.to_i.fdiv(60).ceil

data/lib/html2rss/auto_source/rss_builder.rb

@@ -31,6 +31,8 @@ module Html2rss
 
       def call
         RSS::Maker.make('2.0') do |maker|
+          Html2rss::RssBuilder::Stylesheet.add(maker, channel.stylesheets)
+
           make_channel(maker.channel)
           make_items(maker)
         end

data/lib/html2rss/auto_source/scraper/schema.rb

@@ -1,5 +1,9 @@
 # frozen_string_literal: true
 
+require 'json'
+require 'nokogiri'
+require 'set'
+
 module Html2rss
   class AutoSource
     module Scraper
@@ -99,6 +103,8 @@ module Html2rss
         # @yield [Hash] Each scraped article_hash
         # @return [Array<Hash>] the scraped article_hashes
         def each(&)
+          return enum_for(:each) unless block_given?
+
           schema_objects.filter_map do |schema_object|
             next unless (klass = self.class.scraper_for_schema_object(schema_object))
             next unless (article_hash = klass.new(schema_object, url:).call)

data/lib/html2rss/auto_source/scraper/semantic_html/extractor.rb

@@ -74,7 +74,7 @@ module Html2rss
           def find_heading
             heading_tags = article_tag.css(HEADING_TAGS.join(',')).group_by(&:name)
             smallest_heading = heading_tags.keys.min
-            heading_tags[smallest_heading]&.max_by { |tag| tag.text.size }
+            heading_tags[smallest_heading]&.max_by { |tag| visible_text_from_tag(tag)&.size }
           end
 
           def extract_title

data/lib/html2rss/auto_source.rb

@@ -16,16 +16,18 @@ module Html2rss
 
     SUPPORTED_URL_SCHEMES = %w[http https].to_set.freeze
 
-    def initialize(url)
-      unless url.is_a?(String) || url.is_a?(Addressable::URI)
-        raise ArgumentError,
-              'URL must be a String or Addressable::URI'
-      end
-
-      @url = Addressable::URI.parse(url)
-
-      raise ArgumentError, 'URL must be absolute' unless @url.absolute?
-      raise UnsupportedUrlScheme, "#{@url.scheme} not supported" unless SUPPORTED_URL_SCHEMES.include?(@url.scheme)
+    ##
+    # @param url [Addressable::URI] The URL to extract articles from.
+    # @param body [String] The body of the response.
+    # @param headers [Hash] The headers of the response.
+    def initialize(url, body:, headers: {})
+      raise ArgumentError, 'URL must be a Addressable::URI' unless url.is_a?(Addressable::URI)
+      raise ArgumentError, 'URL must be absolute' unless url.absolute?
+      raise UnsupportedUrlScheme, "#{url.scheme} not supported" unless SUPPORTED_URL_SCHEMES.include?(url.scheme)
+
+      @url = url
+      @body = body
+      @headers = headers
     end
 
     def build
@@ -34,6 +36,8 @@ module Html2rss
       Reducer.call(articles, url:)
       Cleanup.call(articles, url:, keep_different_domain: true)
 
+      channel.articles = articles
+
       Html2rss::AutoSource::RssBuilder.new(
         channel:,
         articles:
@@ -57,21 +61,20 @@ module Html2rss
     end
 
     def channel
-      Channel.new(parsed_body, response:, url:, articles:)
+      @channel ||= Channel.new(parsed_body, headers: @headers, url:)
     end
 
     private
 
     attr_reader :url
 
-    def response
-      @response ||= Html2rss::Utils.request_url(url)
-    end
-
-    # Parses the HTML body of the response using Nokogiri.
     # @return [Nokogiri::HTML::Document]
     def parsed_body
-      @parsed_body ||= Nokogiri.HTML(response.body).freeze
+      @parsed_body ||= Nokogiri.HTML(@body)
+                               .tap do |doc|
+        # Remove comments from the document
+        doc.xpath('//comment()').each(&:remove)
+      end.freeze
     end
   end
 end

data/lib/html2rss/config.rb

@@ -18,9 +18,6 @@ module Html2rss
     # Thrown when the feed config does not contain a value at `:channel`.
     class ChannelMissing < Html2rss::Error; end
 
-    # Struct to store XML Stylesheet attributes
-    Stylesheet = Struct.new(:href, :type, :media, keyword_init: true)
-
     def_delegator :@channel, :author
     def_delegator :@channel, :ttl
     def_delegator :@channel, :title
@@ -75,7 +72,7 @@ module Html2rss
     #
     # @return [Array<Stylesheet>] Array of Stylesheet structs.
     def stylesheets
-      @global.fetch(:stylesheets, []).map { |attributes| Stylesheet.new(attributes) }
+      @global.fetch(:stylesheets, []).map { |attributes| Html2rss::RssBuilder::Stylesheet.new(**attributes) }
     end
 
     # Provides read-only access to the channel object.

data/lib/html2rss/item.rb

@@ -19,7 +19,7 @@ module Html2rss
     ##
     # Fetches items from a given URL using configuration settings.
     #
-    # @param url [String] URL to fetch items from.
+    # @param url [Addressable::URI] URL to fetch items from.
     # @param config [Html2rss::Config] Configuration object.
     # @return [Array<Html2rss::Item>] list of items fetched.
     def self.from_url(url, config)

data/lib/html2rss/rss_builder/stylesheet.rb

@@ -3,35 +3,50 @@
 module Html2rss
   module RssBuilder
     ##
-    # Adds XML stylesheet tags (with the provided maker).
+    # Represents a stylesheet.
     class Stylesheet
-      ##
-      # Adds the stylesheet XML tags to the RSS.
-      #
-      # @param maker [RSS::Maker::RSS20] RSS maker object.
-      # @param stylesheets [Array<Html2rss::Config::Stylesheet>] Array of stylesheet configurations.
-      # @return [nil]
-      def self.add(maker, stylesheets)
-        stylesheets.each do |stylesheet|
-          add_stylesheet(maker, stylesheet)
+      class << self
+        ##
+        # Adds the stylesheet XML tags to the RSS.
+        #
+        # @param maker [RSS::Maker::RSS20] RSS maker object.
+        # @param stylesheets [Array<Html2rss::Config::Stylesheet>] Array of stylesheet configurations.
+        # @return [nil]
+        def add(maker, stylesheets)
+          stylesheets.each do |stylesheet|
+            add_stylesheet(maker, stylesheet)
+          end
         end
-      end
 
-      ##
-      # Adds a single Stylesheet to the RSS.
-      #
-      # @param maker [RSS::Maker::RSS20] RSS maker object.
-      # @param stylesheet [Html2rss::Config::Stylesheet] Stylesheet configuration.
-      # @return [nil]
-      def self.add_stylesheet(maker, stylesheet)
-        maker.xml_stylesheets.new_xml_stylesheet do |xss|
-          xss.href = stylesheet.href
-          xss.type = stylesheet.type
-          xss.media = stylesheet.media
+        private
+
+        ##
+        # Adds a single Stylesheet to the RSS.
+        #
+        # @param maker [RSS::Maker::RSS20] RSS maker object.
+        # @param stylesheet [Html2rss::Config::Stylesheet] Stylesheet configuration.
+        # @return [nil]
+        def add_stylesheet(maker, stylesheet)
+          maker.xml_stylesheets.new_xml_stylesheet do |xss|
+            xss.href = stylesheet.href
+            xss.type = stylesheet.type
+            xss.media = stylesheet.media
+          end
         end
       end
 
-      private_class_method :add_stylesheet
+      TYPES = ['text/css', 'text/xsl'].freeze
+
+      def initialize(href:, type:, media: 'all')
+        raise ArgumentError, 'stylesheet.href must be a String' unless href.is_a?(String)
+        raise ArgumentError, 'stylesheet.type invalid' unless TYPES.include?(type)
+        raise ArgumentError, 'stylesheet.media must be a String' unless media.is_a?(String)
+
+        @href = href
+        @type = type
+        @media = media
+      end
+      attr_reader :href, :type, :media
     end
   end
 end

data/lib/html2rss/utils.rb

@@ -44,6 +44,7 @@ module Html2rss
     #
     # @param time_zone [String]
     # @param default_time_zone [String]
+    # @yield block to execute with the given time zone
     # @return [Object] whatever the given block returns
     def self.use_zone(time_zone, default_time_zone: Time.now.getlocal.zone)
       raise ArgumentError, 'a block is required' unless block_given?
@@ -74,6 +75,11 @@ module Html2rss
     # @param headers [Hash] additional HTTP request headers to use for the request
     # @return [Faraday::Response] body of the HTTP response
     def self.request_url(url, headers: {})
+      url = Addressable::URI.parse(url.to_s) unless url.is_a?(Addressable::URI)
+
+      raise ArgumentError, 'URL must be absolute' unless url.absolute?
+      raise ArgumentError, 'URL must not contain an @ characater' if url.to_s.include?('@')
+
       Faraday.new(url:, headers:) do |faraday|
         faraday.use Faraday::FollowRedirects::Middleware
         faraday.adapter Faraday.default_adapter

data/lib/html2rss/version.rb

@@ -3,6 +3,6 @@
 ##
 # The Html2rss namespace.
 module Html2rss
-  VERSION = '0.13.0'
+  VERSION = '0.14.0'
   public_constant :VERSION
 end

data/lib/html2rss.rb

@@ -43,7 +43,7 @@ module Html2rss
   # @param params [Hash] Dynamic parameters for the feed configuration.
   # @return [RSS::Rss] RSS object generated from the configuration.
   def self.feed_from_yaml_config(file, name = nil, global_config: {}, params: {})
-    yaml = load_yaml(file)
+    yaml = YAML.safe_load_file(file, symbolize_names: true)
     feeds = yaml[CONFIG_KEY_FEEDS] || {}
 
     feed_config = find_feed_config(yaml, feeds, name, global_config)
@@ -73,15 +73,6 @@ module Html2rss
     RssBuilder.build(config)
   end
 
-  ##
-  # Loads and parses the YAML file.
-  #
-  # @param file [String] Path to the YAML file.
-  # @return [Hash] Parsed YAML content.
-  def self.load_yaml(file)
-    YAML.safe_load_file(file, symbolize_names: true)
-  end
-
   ##
   # Builds the feed configuration based on the provided parameters.
   #
@@ -109,8 +100,12 @@ module Html2rss
   # @param url [String] the URL to automatically source the feed from
   # @return [RSS::Rss]
   def self.auto_source(url)
-    Html2rss::AutoSource.new(url).build
+    url = Addressable::URI.parse(url)
+
+    response = Html2rss::Utils.request_url(url)
+
+    Html2rss::AutoSource.new(url, body: response.body, headers: response.headers).build
   end
 
-  private_class_method :load_yaml, :find_feed_config
+  private_class_method :find_feed_config
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: html2rss
 version: !ruby/object:Gem::Version
-  version: 0.13.0
+  version: 0.14.0
 platform: ruby
 authors:
 - Gil Desmarais
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2024-08-16 00:00:00.000000000 Z
+date: 2024-10-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -279,7 +279,7 @@ licenses:
 - MIT
 metadata:
   allowed_push_host: https://rubygems.org
-  changelog_uri: https://github.com/html2rss/html2rss/releases/tag/v0.13.0
+  changelog_uri: https://github.com/html2rss/html2rss/releases/tag/v0.14.0
   rubygems_mfa_required: 'true'
 post_install_message: 
 rdoc_options: []
@@ -296,7 +296,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.11
+rubygems_version: 3.5.16
 signing_key: 
 specification_version: 4
 summary: Generates RSS feeds from websites by scraping a URL and using CSS selectors