html2rss 0.16.0 → 0.18.0

data/lib/html2rss/auto_source/scraper/semantic_html/extractor.rb

@@ -1,136 +0,0 @@
-# frozen_string_literal: true
-
-require 'set'
-
-module Html2rss
-  class AutoSource
-    module Scraper
-      class SemanticHtml
-        ##
-        # ArticleExtractor is responsible for extracting the details of an article.
-        # It focuses on finding a headline first, and from it traverse as much as possible,
-        # to find the DOM upwards to find the other details.
-        class Extractor
-          INVISIBLE_CONTENT_TAG_SELECTORS = %w[svg script noscript style template].to_set.freeze
-          HEADING_TAGS = %w[h1 h2 h3 h4 h5 h6].freeze
-          NOT_HEADLINE_SELECTOR = (HEADING_TAGS.map { |selector| ":not(#{selector})" } +
-                                   INVISIBLE_CONTENT_TAG_SELECTORS.to_a).freeze
-
-          def self.visible_text_from_tag(tag, separator: ' ')
-            text = if (children = tag.children).empty?
-                     tag.text.strip
-                   else
-                     children.filter_map do |child|
-                       next if INVISIBLE_CONTENT_TAG_SELECTORS.include?(child.name)
-
-                       visible_text_from_tag(child)
-                     end.join(separator)
-                   end
-
-            return if (sanitized_text = text.gsub(/\s+/, ' ').strip).empty?
-
-            sanitized_text
-          end
-
-          def initialize(article_tag, url:)
-            @article_tag = article_tag
-            @url = url
-          end
-
-          # @return [Hash, nil] The scraped article or nil.
-          def call
-            @heading = find_heading || closest_anchor || return
-
-            @extract_url = find_url
-
-            {
-              title: extract_title,
-              url: extract_url,
-              image: extract_image,
-              description: extract_description,
-              id: generate_id,
-              published_at: extract_published_at
-            }
-          end
-
-          private
-
-          attr_reader :article_tag, :url, :heading, :extract_url
-
-          ##
-          # Find the heading of the article.
-          # @return [Nokogiri::XML::Node, nil]
-          def find_heading
-            heading_tags = article_tag.css(HEADING_TAGS.join(',')).group_by(&:name)
-
-            return if heading_tags.empty?
-
-            smallest_heading = heading_tags.keys.min
-            heading_tags[smallest_heading]&.max_by { |tag| visible_text_from_tag(tag)&.size.to_i }
-          end
-
-          def visible_text_from_tag(tag, separator: ' ') = self.class.visible_text_from_tag(tag, separator:)
-
-          def closest_anchor
-            SemanticHtml.find_closest_selector(heading || article_tag,
-                                               selector: 'a[href]:not([href=""])')
-          end
-
-          def find_url
-            href = closest_anchor&.[]('href')
-
-            return if (parts = href.to_s.split('#')).empty?
-
-            Utils.build_absolute_url_from_relative(parts.first.strip, url)
-          end
-
-          def extract_title
-            if heading && (heading.children.empty? || heading.text)
-              visible_text_from_tag(heading)
-            else
-              visible_text_from_tag(article_tag.css(HEADING_TAGS.join(','))
-                                               .max_by { |tag| tag.text.size })
-
-            end
-          end
-
-          def extract_image
-            Image.call(article_tag, url:)
-          end
-
-          def extract_description
-            text = visible_text_from_tag(article_tag.css(NOT_HEADLINE_SELECTOR), separator: '<br>')
-            return text if text
-
-            description = visible_text_from_tag(article_tag)
-            return nil unless description
-
-            description.strip!
-            description.empty? ? nil : description
-          end
-
-          def generate_id
-            [
-              article_tag['id'],
-              article_tag.at_css('[id]')&.attr('id'),
-              extract_url&.path,
-              extract_url&.query
-            ].compact.reject(&:empty?).first
-          end
-
-          # @see https://developer.mozilla.org/en-US/docs/Web/API/HTMLTimeElement/dateTime
-          def extract_published_at
-            times = article_tag.css('time[datetime]')
-                               .filter_map do |tag|
-              DateTime.parse(tag['datetime'])
-            rescue ArgumentError, TypeError
-              nil
-            end
-
-            times.min
-          end
-        end
-      end
-    end
-  end
-end

data/lib/html2rss/auto_source/scraper/semantic_html/image.rb

@@ -1,54 +0,0 @@
-# frozen_string_literal: true
-
-module Html2rss
-  class AutoSource
-    module Scraper
-      class SemanticHtml
-        ##
-        # Image is responsible for extracting image URLs the article_tag.
-        class Image
-          def self.call(article_tag, url:)
-            img_src = from_source(article_tag) ||
-                      from_img(article_tag) ||
-                      from_style(article_tag)
-
-            Utils.build_absolute_url_from_relative(img_src, url) if img_src
-          end
-
-          def self.from_img(article_tag)
-            article_tag.at_css('img[src]:not([src^="data"])')&.[]('src')
-          end
-
-          ##
-          # Extracts the largest image source from the srcset attribute
-          # of an img tag or a source tag inside a picture tag.
-          #
-          # @see <https://developer.mozilla.org/en-US/docs/Learn/HTML/Multimedia_and_embedding/Responsive_images>
-          # @see <https://developer.mozilla.org/en-US/docs/Web/HTML/Element/img#srcset>
-          # @see <https://developer.mozilla.org/en-US/docs/Web/HTML/Element/picture>
-          def self.from_source(article_tag) # rubocop:disable Metrics/AbcSize
-            hash = article_tag.css('img[srcset], picture > source[srcset]')
-                              .flat_map { |source| source['srcset'].to_s.split(',') }
-                              .filter_map do |line|
-              width, url = line.split.reverse
-              next if url.nil? || url.start_with?('data:')
-
-              width_value = width.to_i.zero? ? 0 : width.scan(/\d+/).first.to_i
-
-              [width_value, url.strip]
-            end.to_h
-
-            hash[hash.keys.max]
-          end
-
-          def self.from_style(article_tag)
-            article_tag.css('[style*="url"]')
-                       .map { |tag| tag['style'][/url\(['"]?(.*?)['"]?\)/, 1] }
-                       .reject { |src| !src || src.start_with?('data:') }
-                       .max_by(&:size)
-          end
-        end
-      end
-    end
-  end
-end

data/lib/html2rss/config/channel.rb

@@ -1,125 +0,0 @@
-# frozen_string_literal: true
-
-require 'addressable'
-
-module Html2rss
-  class Config
-    ##
-    # Holds the configuration for the feed's channel options.
-    # This contains:
-    #
-    # 1. the RSS channel attributes
-    # 2. html2rss options like json or custom HTTP-headers for the request
-    class Channel
-      ##
-      # @param config [Hash<Symbol, Object>]
-      # @return [Set<String>] the required parameter names
-      def self.required_params_for_config(config)
-        config.each_with_object(Set.new) do |(_, value), required_params|
-          required_params.merge(value.scan(/%<(\w+)>[s|d]/).flatten) if value.is_a?(String)
-        end
-      end
-
-      ##
-      # @param channel [Hash<Symbol, Object>]
-      # @param params [Hash]
-      def initialize(channel, params: {})
-        raise ArgumentError, 'channel must be a hash' unless channel.is_a?(Hash)
-
-        url = channel[:url]
-        raise ArgumentError, 'missing key :url' unless url.is_a?(String) || url.is_a?(Addressable::URI)
-
-        @config = process_params(channel, params.transform_keys(&:to_sym))
-      end
-
-      ##
-      # The HTTP headers to use for the request.
-      #
-      # @return [Hash<Symbol, String>]
-      def headers
-        config.fetch(:headers, {})
-      end
-
-      ##
-      # @return [String]
-      def author
-        config.fetch(:author, 'html2rss')
-      end
-
-      ##
-      # @return [Integer]
-      def ttl
-        config.fetch(:ttl, 360)
-      end
-
-      ##
-      # @return [String]
-      def title
-        config.fetch(:title) { Utils.titleized_channel_url(url) }
-      end
-
-      ##
-      # @return [String] language code
-      def language
-        config.fetch(:language, 'en')
-      end
-
-      ##
-      # @return [String]
-      def description
-        config.fetch(:description) { "Latest items from #{url}." }
-      end
-
-      ##
-      # @return [Addressable::URI]
-      def url
-        Addressable::URI.parse(config[:url]).normalize
-      end
-
-      ##
-      # @return [String] time_zone name
-      def time_zone
-        config.fetch(:time_zone, 'UTC')
-      end
-
-      ##
-      # @return [true, false]
-      def json?
-        config.fetch(:json, false)
-      end
-
-      ##
-      # @return [Symbol]
-      def strategy
-        config.fetch(:strategy) { RequestService.default_strategy_name }.to_sym
-      end
-
-      private
-
-      # @return [Hash<Symbol, Object>]
-      attr_reader :config
-
-      ##
-      # @param config [Hash<Symbol, Object>]
-      # @param params [Hash<Symbol, String>]
-      # @return [nil]
-      def assert_required_params_presence(config, params)
-        missing_params = self.class.required_params_for_config(config) - params.keys.map(&:to_s)
-        raise ParamsMissing, missing_params.to_a.join(', ') unless missing_params.empty?
-      end
-
-      ##
-      # Sets the variables used in the feed config's channel.
-      #
-      # @param config [Hash<Symbol, Object>]
-      # @param params [Hash<Symbol, Object>]
-      # @return [Hash<Symbol, Object>]
-      def process_params(config, params)
-        assert_required_params_presence(config, params)
-        config.transform_values do |value|
-          value.is_a?(String) ? format(value, params) : value
-        end
-      end
-    end
-  end
-end

data/lib/html2rss/config/selectors.rb

@@ -1,103 +0,0 @@
-# frozen_string_literal: true
-
-module Html2rss
-  class Config
-    ##
-    # Holds the configurations of the selectors.
-    class Selectors
-      ITEMS_SELECTOR_NAME = :items
-
-      # Struct to represent a selector with associated attributes for extraction and processing.
-      Selector = Struct.new(:selector, :attribute, :extractor, :post_process, :order, :static, :content_type,
-                            keyword_init: true)
-
-      # raised when an invalid selector name is used
-      class InvalidSelectorName < Html2rss::Error; end
-
-      ##
-      # @param config [Hash<Symbol, Object>]
-      def initialize(config)
-        validate_config(config)
-        @config = config
-      end
-
-      ##
-      # @param name [Symbol]
-      # @return [true, false]
-      def selector?(name)
-        name != ITEMS_SELECTOR_NAME && item_selector_names.include?(name)
-      end
-
-      ##
-      # @param name [Symbol]
-      # @return [Selector]
-      def selector(name)
-        raise InvalidSelectorName, "invalid selector name: #{name}" unless selector?(name)
-
-        keywords = config[name].slice(*available_keys)
-
-        if (additional_keys = keywords.keys - available_keys).any?
-          Log.warn "additional keys (#{additional_keys.join(', ')}) present in selector #{name}"
-        end
-
-        Selector.new(keywords)
-      end
-
-      ##
-      # @return [Set<Symbol>]
-      def category_selector_names
-        selector_keys_for(:categories)
-      end
-
-      ##
-      # @return [Set<Symbol>]
-      def guid_selector_names
-        selector_keys_for(:guid, default: :title_or_description)
-      end
-
-      ##
-      # Returns the CSS/XPath selector.
-      #
-      # @param name [Symbol]
-      # @return [String]
-      def selector_string(name)
-        Selector.new(config[name]).selector
-      end
-
-      ##
-      # @return [Set<Symbol>]
-      def item_selector_names
-        @item_selector_names ||= config.keys.reject { |key| key == ITEMS_SELECTOR_NAME }.to_set
-      end
-
-      ##
-      # @return [Symbol, nil]
-      def items_order
-        config.dig(ITEMS_SELECTOR_NAME, :order)&.to_sym
-      end
-
-      private
-
-      attr_reader :config
-
-      def validate_config(config)
-        raise ArgumentError, 'selector for items is required' unless config[ITEMS_SELECTOR_NAME].is_a?(Hash)
-      end
-
-      ##
-      # Returns the selector keys for the selector named `name`. If none, returns [default].
-      #
-      # @param name [Symbol]
-      # @param default [String, Symbol]
-      # @return [Set<Symbol>]
-      def selector_keys_for(name, default: nil)
-        config.fetch(name) { Array(default) }.tap do |array|
-          array.reject! { |entry| entry.to_s == '' }
-          array.map!(&:to_sym)
-        end.to_set
-      end
-
-      def available_keys = @available_keys ||= Selector.members
-    end
-  end
-end

data/lib/html2rss/item.rb

@@ -1,186 +0,0 @@
-# frozen_string_literal: true
-
-require 'nokogiri'
-
-module Html2rss
-  ##
-  # Takes the selected Nokogiri::HTML and responds to accessor names
-  # defined in the feed config.
-  #
-  # Instances can only be created via `.from_url` and
-  # each represents an internally used "RSS item".
-  # Such an item provides dynamically defined attributes as methods.
-  class Item
-    # A context instance is passed to Item Extractors.
-    Context = Struct.new('Context', :options, :item, :config, keyword_init: true)
-    # Class to keep an Item's <enclosure>.
-    Enclosure = Struct.new('Enclosure', :type, :bits_length, :url, keyword_init: true)
-
-    ##
-    # Fetches items from a given URL using configuration settings.
-    #
-    # @param url [Addressable::URI] URL to fetch items from.
-    # @param config [Html2rss::Config] Configuration object.
-    # @return [Array<Html2rss::Item>] list of items fetched.
-    def self.from_url(url, config)
-      ctx = RequestService::Context.new(url:, headers: config.headers)
-
-      body = RequestService.execute(ctx, strategy: config.strategy).body
-      body = ObjectToXmlConverter.new(JSON.parse(body)).call if config.json?
-
-      Nokogiri.HTML(body)
-              .css(config.selector_string(Config::Selectors::ITEMS_SELECTOR_NAME))
-              .map { |xml| new(xml, config) }
-              .select(&:valid?)
-    end
-
-    ##
-    # @param xml [Nokogiri::XML::Element]
-    # @param config [Html2rss::Config]
-    def initialize(xml, config)
-      @xml = xml
-      @config = config
-    end
-
-    private_class_method :new
-
-    ##
-    # Checks if the object responds to a method dynamically based on the configuration.
-    #
-    # @param method_name [Symbol]
-    # @param _include_private [true, false]
-    # @return [true, false]
-    # :reek:BooleanParameter { enabled: false }
-    def respond_to_missing?(method_name, _include_private = false)
-      config.selector?(method_name) || super
-    end
-
-    ##
-    # Dynamically extracts data based on the method name.
-    #
-    # @param method_name [Symbol]
-    # @param _args [Array]
-    # @return [String] extracted value for the selector.
-    def method_missing(method_name, *_args)
-      return super unless respond_to_missing?(method_name)
-
-      extract(method_name)
-    end
-
-    ##
-    # Selects and processes data according to the selector name.
-    #
-    # @param tag [Symbol]
-    # @return [String] the extracted value for the selector.
-    def extract(tag)
-      attribute_options = config.selector_attributes_with_channel(tag.to_sym)
-
-      post_process(
-        ItemExtractors.item_extractor_factory(attribute_options, xml).get,
-        attribute_options.fetch(:post_process, false)
-      )
-    end
-
-    ##
-    # Checks if the item is valid accordin to RSS 2.0 spec,
-    # by ensuring it has at least a title or a description.
-    #
-    # @return [true, false]
-    def valid?
-      title_or_description.to_s != ''
-    end
-
-    ##
-    # Returns either the title or the description, preferring title if available.
-    #
-    # @return [String, nil]
-    def title_or_description
-      return title if config.selector?(:title)
-
-      description if config.selector?(:description)
-    end
-
-    ##
-    #
-    # @return [String] SHA1 hashed GUID.
-    def guid
-      content = config.guid_selector_names.flat_map { |method_name| public_send(method_name) }.join
-
-      Digest::SHA1.hexdigest(content)
-    end
-
-    ##
-    # Retrieves categories for the item based on configured category selectors.
-    #
-    # @return [Array<String>] list of categories.
-    def categories
-      config.category_selector_names
-            .filter_map do |method_name|
-        category = public_send(method_name)
-        category.strip unless category.to_s.empty?
-      end.uniq
-    end
-
-    ##
-    # Checks if the item has an enclosure based on configuration.
-    #
-    # @return [true, false]
-    def enclosure?
-      config.selector?(:enclosure)
-    end
-
-    ##
-    # Retrieves enclosure details for the item.
-    #
-    # @return [Enclosure] enclosure details.
-    def enclosure
-      url = enclosure_url
-
-      raise 'An item.enclosure requires an absolute URL' unless url&.absolute?
-
-      type = config.selector_attributes_with_channel(:enclosure)[:content_type] ||
-             Html2rss::Utils.guess_content_type_from_url(url)
-
-      Enclosure.new(
-        type:,
-        bits_length: 0,
-        url: url.to_s
-      )
-    end
-
-    private
-
-    # @return [Nokogiri::XML::Element] XML element representing the item.
-    attr_reader :xml
-    # @return [Html2rss::Config] Configuration object for the item.
-    attr_reader :config
-
-    ##
-    # Processes the extracted value according to post-processing options.
-    #
-    # @param value [String] extracted value.
-    # @param post_process_options [Hash<Symbol, Object>] post-processing options.
-    # @return [String] processed value.
-    def post_process(value, post_process_options)
-      return value unless post_process_options
-
-      [post_process_options].flatten.each do |options|
-        value = AttributePostProcessors.get_processor(options[:name])
-                                       .new(value, Context.new(options:, item: self, config:))
-                                       .get
-      end
-
-      value
-    end
-
-    ##
-    # Retrieves the URL for the enclosure, sanitizing and ensuring it's absolute.
-    #
-    # @return [Addressable::URI, nil] absolute URL of the enclosure.
-    def enclosure_url
-      enclosure = Html2rss::Utils.sanitize_url(extract(:enclosure))
-
-      Html2rss::Utils.build_absolute_url_from_relative(enclosure, config.url) if enclosure
-    end
-  end
-end

data/lib/html2rss/item_extractors/attribute.rb

@@ -1,50 +0,0 @@
-# frozen_string_literal: true
-
-module Html2rss
-  module ItemExtractors
-    ##
-    # Returns the value of the attribute.
-    #
-    # Imagine this +time+ HTML tag with a +datetime+ attribute:
-    #
-    #     <time datetime="2019-07-01">...</time>
-    #
-    # YAML usage example:
-    #
-    #    selectors:
-    #      link:
-    #        selector: time
-    #        extractor: attribute
-    #        attribute: datetime
-    #
-    # Would return:
-    #    '2019-07-01'
-    #
-    # In case you're extracting a date or a time, consider parsing it
-    # during post processing with {AttributePostProcessors::ParseTime}.
-    class Attribute
-      # The available options for the attribute extractor.
-      Options = Struct.new('AttributeOptions', :selector, :attribute, keyword_init: true)
-
-      ##
-      # Initializes the Attribute extractor.
-      #
-      # @param xml [Nokogiri::XML::Element]
-      # @param options [Options]
-      def initialize(xml, options)
-        @options = options
-        @element = ItemExtractors.element(xml, options.selector)
-      end
-
-      ##
-      # Retrieves and returns the attribute's value as a string.
-      #
-      # @return [String] The value of the attribute.
-      def get
-        @element.attr(@options.attribute).to_s.freeze
-      rescue NoMethodError => error
-        raise "Failed to extract attribute: #{error.message}"
-      end
-    end
-  end
-end

data/lib/html2rss/item_extractors/href.rb

@@ -1,52 +0,0 @@
-# frozen_string_literal: true
-
-module Html2rss
-  module ItemExtractors
-    ##
-    # Returns the value of the +href+ attribute.
-    # It always returns absolute URLs. If the extracted +href+ value is a
-    # relative URL, it prepends the channel's URL.
-    #
-    # Imagine this +a+ HTML element with a +href+ attribute:
-    #
-    #     <a href="/posts/latest-findings">...</a>
-    #
-    # YAML usage example:
-    #    channel:
-    #      url: http://blog-without-a-feed.example.com
-    #      ...
-    #    selectors:
-    #      link:
-    #        selector: a
-    #        extractor: href
-    #
-    # Would return:
-    #    'http://blog-without-a-feed.example.com/posts/latest-findings'
-    class Href
-      # The available options for the href (attribute) extractor.
-      Options = Struct.new('HrefOptions', :selector, :channel, keyword_init: true)
-
-      ##
-      # Initializes the Href extractor.
-      #
-      # @param xml [Nokogiri::XML::Element]
-      # @param options [Options]
-      def initialize(xml, options)
-        @options = options
-        @element = ItemExtractors.element(xml, options.selector)
-        @href = @element.attr('href').to_s
-      end
-
-      ##
-      # Retrieves and returns the normalized absolute URL.
-      #
-      # @return [String] The absolute URL.
-      def get
-        return nil unless @href
-
-        sanitized_href = Html2rss::Utils.sanitize_url(@href)
-        Html2rss::Utils.build_absolute_url_from_relative(sanitized_href, @options.channel.url)
-      end
-    end
-  end
-end