html2rss 0.12.0 → 0.13.0

data/lib/html2rss/auto_source/scraper/schema/base.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require 'date'
+
+module Html2rss
+  class AutoSource
+    module Scraper
+      class Schema
+        ##
+        # Base class for Schema.org schema_objects.
+        #
+        # @see https://schema.org/Article
+        class Base
+          DEFAULT_ATTRIBUTES = %i[id title description url image published_at].freeze
+
+          def initialize(schema_object, url:)
+            @schema_object = schema_object
+            @url = url
+          end
+
+          # @return [Hash] the scraped article hash with DEFAULT_ATTRIBUTES
+          def call
+            DEFAULT_ATTRIBUTES.to_h do |attribute|
+              [attribute, public_send(attribute)]
+            end
+          end
+
+          def id = schema_object[:@id] || url&.path || title.to_s.downcase.gsub(/\s+/, '-')
+          def title = schema_object[:title]
+
+          def description
+            [schema_object[:description], schema_object[:schema_object_body], schema_object[:abstract]]
+              .max_by { |desc| desc.to_s.size }
+          end
+
+          # @return [Addressable::URI, nil] the URL of the schema object
+          def url
+            url = schema_object[:url]
+            if url.to_s.empty?
+              Log.debug("Schema#Base.url: no url in schema_object: #{schema_object.inspect}")
+              return
+            end
+
+            Utils.build_absolute_url_from_relative(url, @url)
+          end
+
+          def image = images.first || nil
+          def published_at = schema_object[:datePublished]
+
+          private
+
+          attr_reader :schema_object
+
+          def images
+            Array(schema_object[:image]).compact
+          end
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/auto_source/scraper/schema.rb

@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class AutoSource
+    module Scraper
+      ##
+      # Scraps articles from Schema.org objects, by looking for the objects in:
+
+      #  1. <script type="application/ld+json"> "schema" tag.
+      #  2. tbd
+      #
+      # See:
+      # 1. https://schema.org/NewsArticle
+      # 2. https://developers.google.com/search/docs/appearance/structured-data/article#microdata
+      class Schema
+        include Enumerable
+
+        TAG_SELECTOR = 'script[type="application/ld+json"]'
+        SCHEMA_OBJECT_TYPES = %w[
+          AdvertiserContentArticle
+          AnalysisNewsArticle
+          APIReference
+          Article
+          AskPublicNewsArticle
+          BackgroundNewsArticle
+          BlogPosting
+          DiscussionForumPosting
+          LiveBlogPosting
+          NewsArticle
+          OpinionNewsArticle
+          Report
+          ReportageNewsArticle
+          ReviewNewsArticle
+          SatiricalArticle
+          ScholarlyArticle
+          SocialMediaPosting
+          TechArticle
+        ].to_set.freeze
+
+        class << self
+          def articles?(parsed_body)
+            parsed_body.css(TAG_SELECTOR).any? do |script|
+              SCHEMA_OBJECT_TYPES.any? { |type| script.text.match?(/"@type"\s*:\s*"#{Regexp.escape(type)}"/) }
+            end
+          end
+
+          ##
+          # Returns a flat array
+          # of all supported schema objects
+          # by recursively traversing the `from` object.
+          #
+          # @param object [Hash, Array]
+          # @return [Array<Hash>] the schema_objects, or an empty array
+          # :reek:DuplicateMethodCall
+          def from(object)
+            case object
+            when Nokogiri::XML::Element
+              from(parse_script_tag(object))
+            when Hash
+              supported_schema_object?(object) ? [object] : object.values.flat_map { |item| from(item) }
+            when Array
+              object.flat_map { |item| from(item) }
+            else
+              []
+            end
+          end
+
+          def supported_schema_object?(object)
+            scraper_for_schema_object(object) ? true : false
+          end
+
+          ##
+          # @return [Scraper::Schema::Base, Scraper::Schema::NewsArticle, nil]
+          def scraper_for_schema_object(schema_object)
+            if SCHEMA_OBJECT_TYPES.member?(schema_object[:@type])
+              Base
+            else
+              Log.warn("Schema#scraper_for_schema_object: Unsupported schema object @type: #{schema_object[:@type]}")
+              nil
+            end
+          end
+
+          private
+
+          def parse_script_tag(script_tag)
+            JSON.parse(script_tag.text, symbolize_names: true)
+          rescue JSON::ParserError => error
+            Log.warn('Schema#schema_objects: Failed to parse JSON', error: error.message)
+            []
+          end
+        end
+
+        def initialize(parsed_body, url:)
+          @parsed_body = parsed_body
+          @url = url
+        end
+
+        ##
+        # @yield [Hash] Each scraped article_hash
+        # @return [Array<Hash>] the scraped article_hashes
+        def each(&)
+          schema_objects.filter_map do |schema_object|
+            next unless (klass = self.class.scraper_for_schema_object(schema_object))
+            next unless (article_hash = klass.new(schema_object, url:).call)
+
+            yield article_hash
+          end
+        end
+
+        private
+
+        def schema_objects
+          @parsed_body.css(TAG_SELECTOR).flat_map do |tag|
+            Schema.from(tag)
+          end
+        end
+
+        attr_reader :parsed_body, :url
+      end
+    end
+  end
+end

data/lib/html2rss/auto_source/scraper/semantic_html/extractor.rb

@@ -0,0 +1,123 @@
+# frozen_string_literal: true
+
+require 'set'
+
+module Html2rss
+  class AutoSource
+    module Scraper
+      class SemanticHtml
+        ##
+        # ArticleExtractor is responsible for extracting the details of an article.
+        # It focuses on finding a headline first, and from it traverse as much as possible,
+        # to find the DOM upwards to find the other details.
+        class Extractor
+          INVISIBLE_CONTENT_TAG_SELECTORS = %w[svg script noscript style template].to_set.freeze
+          HEADING_TAGS = %w[h1 h2 h3 h4 h5 h6].freeze
+          NOT_HEADLINE_SELECTOR = (HEADING_TAGS.map { |selector| ":not(#{selector})" } +
+                                   INVISIBLE_CONTENT_TAG_SELECTORS.to_a).freeze
+
+          def self.visible_text_from_tag(tag, separator: ' ')
+            text = if (children = tag.children).empty?
+                     tag.text.strip
+                   else
+                     children.filter_map do |child|
+                       next if INVISIBLE_CONTENT_TAG_SELECTORS.include?(child.name)
+
+                       visible_text_from_tag(child)
+                     end.join(separator)
+                   end
+
+            return if (sanitized_text = text.gsub(/\s+/, ' ').strip).empty?
+
+            sanitized_text
+          end
+
+          def initialize(article_tag, url:)
+            @article_tag = article_tag
+            @url = url
+            @heading = find_heading
+            @extract_url = find_url
+          end
+
+          # @return [Hash, nil] The scraped article or nil.
+          def call
+            return unless heading
+
+            {
+              title: extract_title,
+              url: extract_url,
+              image: extract_image,
+              description: extract_description,
+              id: generate_id,
+              published_at: extract_published_at
+            }
+          end
+
+          private
+
+          attr_reader :article_tag, :url, :heading, :extract_url
+
+          def visible_text_from_tag(tag, separator: ' ') = self.class.visible_text_from_tag(tag, separator:)
+
+          # @see https://developer.mozilla.org/en-US/docs/Web/API/HTMLTimeElement/dateTime
+          def extract_published_at
+            times = article_tag.css('time[datetime]')
+                               .filter_map do |tag|
+              DateTime.parse(tag['datetime'])
+            rescue ArgumentError, TypeError
+              nil
+            end
+
+            times.min
+          end
+
+          def find_heading
+            heading_tags = article_tag.css(HEADING_TAGS.join(',')).group_by(&:name)
+            smallest_heading = heading_tags.keys.min
+            heading_tags[smallest_heading]&.max_by { |tag| tag.text.size }
+          end
+
+          def extract_title
+            @extract_title ||= if heading.children.empty? && heading.text
+                                 visible_text_from_tag(heading)
+                               else
+                                 visible_text_from_tag(
+                                   article_tag.css(HEADING_TAGS.join(','))
+                                              .max_by { |tag| tag.text.size }
+                                 )
+                               end
+          end
+
+          def extract_description
+            text = visible_text_from_tag(article_tag.css(NOT_HEADLINE_SELECTOR), separator: '<br>')
+            return text if text
+
+            description = visible_text_from_tag(article_tag)
+            return nil unless description
+
+            title_text = extract_title
+            description.gsub!(title_text, '') if title_text
+            description.strip!
+            description.empty? ? nil : description
+          end
+
+          def find_url
+            closest_anchor = SemanticHtml.find_closest_selector(heading || article_tag,
+                                                                selector: 'a[href]:not([href=""])')
+            href = closest_anchor&.[]('href')&.split('#')&.first&.strip
+            Utils.build_absolute_url_from_relative(href, url) unless href.to_s.empty?
+          end
+
+          def extract_image
+            Image.call(article_tag, url:)
+          end
+
+          def generate_id
+            [article_tag['id'], article_tag.at_css('[id]')&.attr('id'),
+             extract_url&.path].compact.reject(&:empty?).first
+          end
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/auto_source/scraper/semantic_html/image.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class AutoSource
+    module Scraper
+      class SemanticHtml
+        ##
+        # Image is responsible for extracting image URLs the article_tag.
+        class Image
+          def self.call(article_tag, url:)
+            img_src = from_source(article_tag) ||
+                      from_img(article_tag) ||
+                      from_style(article_tag)
+
+            Utils.build_absolute_url_from_relative(img_src, url) if img_src
+          end
+
+          def self.from_img(article_tag)
+            article_tag.at_css('img[src]:not([src^="data"])')&.[]('src')
+          end
+
+          ##
+          # Extracts the largest image source from the srcset attribute
+          # of an img tag or a source tag inside a picture tag.
+          #
+          # @see <https://developer.mozilla.org/en-US/docs/Learn/HTML/Multimedia_and_embedding/Responsive_images>
+          # @see <https://developer.mozilla.org/en-US/docs/Web/HTML/Element/img#srcset>
+          # @see <https://developer.mozilla.org/en-US/docs/Web/HTML/Element/picture>
+          def self.from_source(article_tag) # rubocop:disable Metrics/AbcSize
+            hash = article_tag.css('img[srcset], picture > source[srcset]')
+                              .flat_map { |source| source['srcset'].to_s.split(',') }
+                              .filter_map do |line|
+              width, url = line.split.reverse
+              next if url.nil? || url.start_with?('data:')
+
+              width_value = width.to_i.zero? ? 0 : width.scan(/\d+/).first.to_i
+
+              [width_value, url.strip]
+            end.to_h
+
+            hash[hash.keys.max]
+          end
+
+          def self.from_style(article_tag)
+            article_tag.css('[style*="url"]')
+                       .map { |tag| tag['style'][/url\(['"]?(.*?)['"]?\)/, 1] }
+                       .reject { |src| !src || src.start_with?('data:') }
+                       .max_by(&:size)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/auto_source/scraper/semantic_html.rb

@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+require 'addressable'
+require 'parallel'
+
+module Html2rss
+  class AutoSource
+    module Scraper
+      ##
+      # Scrapes articles by looking for common markup tags (article, section, li)
+      # containing an <a href> tag.
+      #
+      # See:
+      # 1. https://developer.mozilla.org/en-US/docs/Web/HTML/Element/article
+      class SemanticHtml
+        include Enumerable
+
+        ##
+        # Map of parent element names to CSS selectors for finding <a href> tags.
+        ANCHOR_TAG_SELECTORS = {
+          'section' => ['section :not(section) a[href]'],
+          'tr' => ['table tr :not(tr) a[href]'],
+          'article' => [
+            'article :not(article) a[href]',
+            'article a[href]'
+          ],
+          'li' => [
+            'ul > li :not(li) a[href]',
+            'ol > li :not(li) a[href]'
+          ]
+        }.freeze
+
+        # Check if the parsed_body contains articles
+        # @param parsed_body [Nokogiri::HTML::Document] The parsed HTML document
+        # @return [Boolean] True if articles are found, otherwise false.
+        def self.articles?(parsed_body)
+          return false unless parsed_body
+
+          ANCHOR_TAG_SELECTORS.each_value do |selectors|
+            return true if selectors.any? { |selector| parsed_body.at_css(selector) }
+          end
+          false
+        end
+
+        # Finds the closest ancestor tag matching the specified tag name
+        # @param current_tag [Nokogiri::XML::Node] The current tag to start searching from
+        # @param tag_name [String] The tag name to search for
+        # @param stop_tag [String] The tag name to stop searching at
+        # @return [Nokogiri::XML::Node] The found ancestor tag or the current tag if matched
+        def self.find_tag_in_ancestors(current_tag, tag_name, stop_tag: 'html')
+          return current_tag if current_tag.name == tag_name
+
+          stop_tags = Set[tag_name, stop_tag]
+
+          while current_tag.respond_to?(:parent) && !stop_tags.member?(current_tag.name)
+            current_tag = current_tag.parent
+          end
+
+          current_tag
+        end
+
+        # Finds the closest matching selector upwards in the DOM tree
+        # @param current_tag [Nokogiri::XML::Node] The current tag to start searching from
+        # @param selector [String] The CSS selector to search for
+        # @return [Nokogiri::XML::Node, nil] The closest matching tag or nil if not found
+        def self.find_closest_selector(current_tag, selector: 'a[href]:not([href=""])')
+          current_tag.at_css(selector) || find_closest_selector_upwards(current_tag, selector:)
+        end
+
+        # Helper method to find a matching selector upwards
+        # @param current_tag [Nokogiri::XML::Node] The current tag to start searching from
+        # @param selector [String] The CSS selector to search for
+        # @return [Nokogiri::XML::Node, nil] The closest matching tag or nil if not found
+        def self.find_closest_selector_upwards(current_tag, selector:)
+          while current_tag
+            found = current_tag.at_css(selector)
+            return found if found
+
+            return nil unless current_tag.respond_to?(:parent)
+
+            current_tag = current_tag.parent
+          end
+        end
+
+        # Returns an array of [tag_name, selector] pairs
+        # @return [Array<[String, String]>] Array of tag name and selector pairs
+        def self.anchor_tag_selector_pairs
+          ANCHOR_TAG_SELECTORS.flat_map do |tag_name, selectors|
+            selectors.map { |selector| [tag_name, selector] }
+          end
+        end
+
+        def initialize(parsed_body, url:)
+          @parsed_body = parsed_body
+          @url = url
+        end
+
+        attr_reader :parsed_body
+
+        ##
+        # @yieldparam [Hash] The scraped article hash
+        # @return [Enumerator] Enumerator for the scraped articles
+        def each
+          return enum_for(:each) unless block_given?
+
+          SemanticHtml.anchor_tag_selector_pairs.each do |tag_name, selector|
+            parsed_body.css(selector).each do |selected_tag|
+              article_tag = SemanticHtml.find_tag_in_ancestors(selected_tag, tag_name)
+              article_hash = Extractor.new(article_tag, url: @url).call
+
+              yield article_hash if article_hash
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/auto_source/scraper.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class AutoSource
+    ##
+    # The Scraper module contains all scrapers that can be used to extract articles.
+    # Each scraper should implement a `call` method that returns an array of article hashes.
+    # Each scraper should also implement an `articles?` method that returns true if the scraper
+    # can potentially be used to extract articles from the given HTML.
+    #
+    module Scraper
+      SCRAPERS = [
+        Schema,
+        SemanticHtml
+      ].freeze
+
+      ##
+      # Error raised when no suitable scraper is found.
+      class NoScraperFound < Html2rss::Error; end
+
+      ##
+      # Returns an array of scrapers that claim to find articles in the parsed body.
+      # @param parsed_body [Nokogiri::HTML::Document] The parsed HTML body.
+      # @return [Array<Class>] An array of scraper classes that can handle the parsed body.
+      def self.from(parsed_body)
+        scrapers = SCRAPERS.select { |scraper| scraper.articles?(parsed_body) }
+        raise NoScraperFound, 'No suitable scraper found for URL.' if scrapers.empty?
+
+        scrapers
+      end
+    end
+  end
+end

data/lib/html2rss/auto_source.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+require 'nokogiri'
+require 'parallel'
+require 'addressable'
+
+module Html2rss
+  ##
+  # The AutoSource class is responsible for extracting channel and articles
+  # from a given URL.
+  # It uses a set of ArticleExtractors to extract articles, utilizing popular ways of
+  # marking articles, e.g. schema, microdata, open graph, etc.
+  class AutoSource
+    class UnsupportedUrlScheme < Html2rss::Error; end
+    class NoArticlesFound < Html2rss::Error; end
+
+    SUPPORTED_URL_SCHEMES = %w[http https].to_set.freeze
+
+    def initialize(url)
+      unless url.is_a?(String) || url.is_a?(Addressable::URI)
+        raise ArgumentError,
+              'URL must be a String or Addressable::URI'
+      end
+
+      @url = Addressable::URI.parse(url)
+
+      raise ArgumentError, 'URL must be absolute' unless @url.absolute?
+      raise UnsupportedUrlScheme, "#{@url.scheme} not supported" unless SUPPORTED_URL_SCHEMES.include?(@url.scheme)
+    end
+
+    def build
+      raise NoArticlesFound if articles.empty?
+
+      Reducer.call(articles, url:)
+      Cleanup.call(articles, url:, keep_different_domain: true)
+
+      Html2rss::AutoSource::RssBuilder.new(
+        channel:,
+        articles:
+      ).call
+    end
+
+    def articles
+      @articles ||= Scraper.from(parsed_body).flat_map do |scraper|
+        instance = scraper.new(parsed_body, url:)
+
+        articles_in_thread = Parallel.map(instance.each) do |article_hash|
+          Log.debug "Scraper: #{scraper} in worker: #{Parallel.worker_number} [#{article_hash[:url]}]"
+
+          Article.new(**article_hash, scraper:)
+        end
+
+        Reducer.call(articles_in_thread, url:)
+
+        articles_in_thread
+      end
+    end
+
+    def channel
+      Channel.new(parsed_body, response:, url:, articles:)
+    end
+
+    private
+
+    attr_reader :url
+
+    def response
+      @response ||= Html2rss::Utils.request_url(url)
+    end
+
+    # Parses the HTML body of the response using Nokogiri.
+    # @return [Nokogiri::HTML::Document]
+    def parsed_body
+      @parsed_body ||= Nokogiri.HTML(response.body).freeze
+    end
+  end
+end

data/lib/html2rss/cli.rb

@@ -2,8 +2,13 @@
 
 require_relative '../html2rss'
 require 'thor'
+require 'addressable'
 
+##
+# The Html2rss namespace / command line interface.
 module Html2rss
+  Log = Logger.new($stderr)
+
   ##
   # The Html2rss command line interface.
   class CLI < Thor
@@ -25,5 +30,10 @@ module Html2rss
       params = options.to_h { |opt| opt.split('=', 2) }
       puts Html2rss.feed_from_yaml_config(yaml_file, feed_name, params:)
     end
+
+    desc 'auto URL', 'automatically sources an RSS feed from the URL'
+    def auto(url)
+      puts Html2rss.auto_source(url)
+    end
   end
 end

data/lib/html2rss/config/channel.rb

@@ -16,7 +16,7 @@ module Html2rss
       # @return [Set<String>] the required parameter names
       def self.required_params_for_config(config)
         config.each_with_object(Set.new) do |(_, value), required_params|
-          required_params.merge(value.scan(/%<([\w_\d]+)>/).flatten) if value.is_a?(String)
+          required_params.merge(value.scan(/%<(\w+)>[s|d]/).flatten) if value.is_a?(String)
         end
       end
 
@@ -25,7 +25,9 @@ module Html2rss
       # @param params [Hash]
       def initialize(channel, params: {})
         raise ArgumentError, 'channel must be a hash' unless channel.is_a?(Hash)
-        raise ArgumentError, 'missing key :url' unless channel[:url].is_a?(String)
+
+        url = channel[:url]
+        raise ArgumentError, 'missing key :url' unless url.is_a?(String) || url.is_a?(Addressable::URI)
 
         @config = process_params(channel, params.transform_keys(&:to_sym))
       end

data/lib/html2rss/config/selectors.rb

@@ -35,8 +35,8 @@ module Html2rss
 
         keywords = config[name].slice(*available_keys)
 
-        if (additional_keys = available_keys - keywords.keys).any?
-          warn "additional keys (#{additional_keys.join(', ')}) present in selector #{name}"
+        if (additional_keys = keywords.keys - available_keys).any?
+          Log.warn "additional keys (#{additional_keys.join(', ')}) present in selector #{name}"
         end
 
         Selector.new(keywords)

data/lib/html2rss/item.rb

@@ -23,7 +23,8 @@ module Html2rss
     # @param config [Html2rss::Config] Configuration object.
     # @return [Array<Html2rss::Item>] list of items fetched.
     def self.from_url(url, config)
-      body = Utils.request_body_from_url(url, convert_json_to_xml: config.json?, headers: config.headers)
+      body = Utils.request_url(url, headers: config.headers).body
+      body = ObjectToXmlConverter.new(JSON.parse(body)).call if config.json?
 
       Nokogiri.HTML(body)
               .css(config.selector_string(Config::Selectors::ITEMS_SELECTOR_NAME))
@@ -47,6 +48,7 @@ module Html2rss
     # @param method_name [Symbol]
     # @param _include_private [true, false]
     # @return [true, false]
+    # :reek:BooleanParameter { enabled: false }
     def respond_to_missing?(method_name, _include_private = false)
       config.selector?(method_name) || super
     end
@@ -110,7 +112,11 @@ module Html2rss
     #
     # @return [Array<String>] list of categories.
     def categories
-      config.category_selector_names.map { |method_name| public_send(method_name) }
+      config.category_selector_names
+            .filter_map do |method_name|
+        category = public_send(method_name)
+        category.strip unless category.to_s.empty?
+      end.uniq
     end
 
     ##