html2rss 0.16.0 → 0.18.0

data/lib/html2rss/attribute_post_processors/sanitize_html.rb

@@ -1,108 +0,0 @@
-# frozen_string_literal: true
-
-require 'sanitize'
-require_relative 'html_transformers/transform_urls_to_absolute_ones'
-require_relative 'html_transformers/wrap_img_in_a'
-
-module Html2rss
-  module AttributePostProcessors
-    ##
-    # Returns sanitized HTML code as String.
-    #
-    # It sanitizes by using the [sanitize gem](https://github.com/rgrove/sanitize) with
-    # [Sanitize::Config::RELAXED](https://github.com/rgrove/sanitize#sanitizeconfigrelaxed).
-    #
-    # Furthermore, it adds:
-    #
-    # - `rel="nofollow noopener noreferrer"` to <a> tags
-    # - `referrer-policy='no-referrer'` to <img> tags
-    # - wraps all <img> tags, whose direct parent is not an <a>, into an <a>
-    #   linking to the <img>'s `src`.
-    #
-    # Imagine this HTML structure:
-    #
-    #     <section>
-    #       Lorem <b>ipsum</b> dolor...
-    #       <iframe src="https://evil.corp/miner"></iframe>
-    #       <script>alert();</script>
-    #     </section>
-    #
-    # YAML usage example:
-    #
-    #    selectors:
-    #      description:
-    #        selector: '.section'
-    #        extractor: html
-    #        post_process:
-    #          name: sanitize_html
-    #
-    # Would return:
-    #    '<p>Lorem <b>ipsum</b> dolor ...</p>'
-    class SanitizeHtml < Base
-      def self.validate_args!(value, context)
-        assert_type value, String, :value, context:
-      end
-
-      ##
-      # Shorthand method to get the sanitized HTML.
-      # @param html [String]
-      # @param url [String, Addressable::URI]
-      def self.get(html, url)
-        raise ArgumentError, 'url must be a String or Addressable::URI' if url.to_s.empty?
-        return nil if html.to_s.empty?
-
-        new(html, { config: Config::Channel.new({ url: }) }).get
-      end
-
-      ##
-      # @return [String]
-      def get
-        sanitized_html = Sanitize.fragment(value, sanitize_config)
-        sanitized_html.to_s.gsub(/\s+/, ' ').strip
-      end
-
-      private
-
-      ##
-      # @return [Sanitize::Config]
-      def sanitize_config
-        Sanitize::Config.merge(
-          Sanitize::Config::RELAXED,
-          attributes: { all: %w[dir lang alt title translate] },
-          add_attributes:,
-          transformers: [
-            method(:transform_urls_to_absolute_ones),
-            method(:wrap_img_in_a)
-          ]
-        )
-      end
-
-      def add_attributes
-        {
-          'a' => { 'rel' => 'nofollow noopener noreferrer', 'target' => '_blank' },
-          'img' => { 'referrer-policy' => 'no-referrer' }
-        }
-      end
-
-      def channel_url = context[:config].url
-
-      ##
-      # Wrapper for transform_urls_to_absolute_ones to pass the channel_url.
-      #
-      # @param env [Hash]
-      # @return [nil]
-      def transform_urls_to_absolute_ones(env)
-        HtmlTransformers::TransformUrlsToAbsoluteOnes.new(channel_url).call(**env)
-      end
-
-      ##
-      # Wrapper for wrap_img_in_a.
-      #
-      # @param env [Hash]
-      # @return [nil]
-      def wrap_img_in_a(env)
-        HtmlTransformers::WrapImgInA.new.call(**env)
-      end
-    end
-  end
-end

data/lib/html2rss/attribute_post_processors/substring.rb

@@ -1,72 +0,0 @@
-# frozen_string_literal: true
-
-module Html2rss
-  module AttributePostProcessors
-    ##
-    # Returns a defined part of a String.
-    #
-    # Both parameters must be an Integer and they can be negative.
-    # The +end+ parameter can be omitted, in that case it will not cut the
-    # String at the end.
-    #
-    # A Regexp or a MatchString is not supported.
-    #
-    # See the [`String#[]`](https://ruby-doc.org/core/String.html#method-i-5B-5D)
-    # documentation for more information.
-    #
-    # Imagine this HTML:
-    #    <h1>Foo bar and baz<h1>
-    #
-    # YAML usage example:
-    #    selectors:
-    #      title:
-    #        selector: h1
-    #        post_process:
-    #          name: substring
-    #          start: 4
-    #          end: 6
-    #
-    # Would return:
-    #    'bar'
-    class Substring < Base
-      def self.validate_args!(value, context)
-        assert_type value, String, :value, context:
-
-        options = context[:options]
-        assert_type options[:start], Integer, :start, context:
-
-        end_index = options[:end]
-        assert_type(end_index, Integer, :end, context:) if end_index
-      end
-
-      ##
-      # Extracts the substring from the original string based on the provided start and end indices.
-      #
-      # @return [String] The extracted substring.
-      def get
-        value[range]
-      end
-
-      ##
-      # Determines the range for the substring extraction based on the provided start and end indices.
-      #
-      # @return [Range] The range object representing the start and end/Infinity (integers).
-      def range
-        return (start_index..) unless end_index?
-
-        if start_index == end_index
-          raise ArgumentError,
-                'The `start` value must be unequal to the `end` value.'
-        end
-
-        (start_index..end_index)
-      end
-
-      private
-
-      def end_index?  = !context[:options][:end].to_s.empty?
-      def end_index   = context[:options][:end].to_i
-      def start_index = context[:options][:start].to_i
-    end
-  end
-end

data/lib/html2rss/attribute_post_processors/template.rb

@@ -1,101 +0,0 @@
-# frozen_string_literal: true
-
-module Html2rss
-  module AttributePostProcessors
-    ##
-    # Returns a formatted String according to the string pattern.
-    #
-    # If +self+ is used, the selectors extracted value will be used.
-    # It uses [Kernel#format](https://ruby-doc.org/core/Kernel.html#method-i-format)
-    #
-    # Imagine this HTML:
-    #
-    #    <li>
-    #      <h1>Product</h1>
-    #      <span class="price">23,42€</span>
-    #    </li>
-    #
-    #
-    # YAML usage example:
-    #
-    #    selectors:
-    #      items:
-    #        selector: 'li'
-    #      price:
-    #        selector: '.price'
-    #      title:
-    #        selector: h1
-    #        post_process:
-    #          name: template
-    #          string: '%{self} (%{price})'
-    #
-    # Would return:
-    #    'Product (23,42€)'
-    class Template < Base
-      def self.validate_args!(value, context)
-        assert_type value, String, :value, context:
-
-        string = context[:options]&.dig(:string).to_s
-        raise InvalidType, 'The `string` template is absent.' if string.empty?
-      end
-
-      ##
-      # @param value [String]
-      # @param context [Item::Context]
-      def initialize(value, context)
-        super
-
-        @options = context[:options] || {}
-        @item = context[:item]
-        @string = @options[:string].to_s
-      end
-
-      ##
-      # @return [String]
-      def get
-        @options[:methods] ? format_string_with_methods : format_string_with_dynamic_params
-      end
-
-      private
-
-      ##
-      # @return [String] the string containing the template
-      attr_reader :string
-
-      ##
-      # @return [Array<String>]
-      def methods
-        @methods ||= @options[:methods].map { |method_name| item_value(method_name) }
-      end
-
-      ##
-      # Formats a string using methods.
-      #
-      # @return [String]
-      # @deprecated Use %<id>s formatting instead. Will be removed in version 1.0.0. See README / Dynamic parameters.
-      def format_string_with_methods
-        Log.warn '[DEPRECATION] This method of using params is deprecated and \
-                  support for it will be removed in version 1.0.0.\
-                  Please use dynamic parameters (i.e. %<id>s, see README.md) instead.'
-
-        string % methods
-      end
-
-      ##
-      # @return [String]
-      def format_string_with_dynamic_params
-        param_names = string.scan(/%[<|{](\w*)[>|}]/)
-        param_names.flatten!
-
-        format(string, param_names.to_h { |name| [name.to_sym, item_value(name)] })
-      end
-
-      ##
-      # @param method_name [String, Symbol]
-      # @return [String]
-      def item_value(method_name)
-        method_name.to_sym == :self ? value : @item.public_send(method_name).to_s
-      end
-    end
-  end
-end

data/lib/html2rss/attribute_post_processors.rb

@@ -1,44 +0,0 @@
-# frozen_string_literal: true
-
-module Html2rss
-  ##
-  # Provides a namespace for attribute post processors.
-  module AttributePostProcessors
-    ##
-    # Error raised when an unknown post processor name is requested.
-    class UnknownPostProcessorName < Html2rss::Error; end
-
-    ##
-    # Error raised when a required option is missing.
-    class MissingOption < Html2rss::Error; end
-
-    ##
-    # Error raised when an invalid type is provided.
-    class InvalidType < Html2rss::Error; end
-
-    ##
-    # Maps the post processor name to the class implementing the post processor.
-    #
-    # The key is the name to use in the feed config.
-    NAME_TO_CLASS = {
-      gsub: Gsub,
-      html_to_markdown: HtmlToMarkdown,
-      markdown_to_html: MarkdownToHtml,
-      parse_time: ParseTime,
-      parse_uri: ParseUri,
-      sanitize_html: SanitizeHtml,
-      substring: Substring,
-      template: Template
-    }.freeze
-
-    ##
-    # Retrieves the attribute post processor class based on the given name.
-    #
-    # @param name [Symbol] The name of the post processor.
-    # @return [Class] The attribute post processor class.
-    # @raise [UnknownPostProcessorName] If the requested name is not found in NAME_TO_CLASS.
-    def self.get_processor(name)
-      NAME_TO_CLASS[name.to_sym] || raise(UnknownPostProcessorName, "Can't find a post processor named '#{name}'")
-    end
-  end
-end

data/lib/html2rss/auto_source/article.rb

@@ -1,127 +0,0 @@
-# frozen_string_literal: true
-
-require 'zlib'
-require 'sanitize'
-require 'nokogiri'
-
-module Html2rss
-  class AutoSource
-    ##
-    # Article is a simple data object representing an article extracted from a page.
-    # It is enumerable and responds to all keys specified in PROVIDED_KEYS.
-    class Article
-      include Enumerable
-      include Comparable
-
-      PROVIDED_KEYS = %i[id title description url image guid published_at scraper].freeze
-
-      ##
-      # Removes the specified pattern from the beginning of the text
-      # within a given range if the pattern occurs before the range's end.
-      #
-      # @param text [String]
-      # @param pattern [String]
-      # @param end_of_range [Integer] - Optional, defaults to half the size of the text
-      # @return [String]
-      def self.remove_pattern_from_start(text, pattern, end_of_range: (text.size * 0.5).to_i)
-        return text unless text.is_a?(String) && pattern.is_a?(String)
-
-        index = text.index(pattern)
-        return text if index.nil? || index >= end_of_range
-
-        text.gsub(/^(.{0,#{end_of_range}})#{Regexp.escape(pattern)}/, '\1')
-      end
-
-      ##
-      # Checks if the text contains HTML tags.
-      # @param text [String]
-      # @return [Boolean]
-      def self.contains_html?(text)
-        Nokogiri::HTML.fragment(text).children.any?(&:element?)
-      end
-
-      # @param options [Hash<Symbol, String>]
-      def initialize(**options)
-        @to_h = {}
-        options.each_pair { |key, value| @to_h[key] = value.freeze if value }
-        @to_h.freeze
-
-        return unless (unknown_keys = options.keys - PROVIDED_KEYS).any?
-
-        Log.warn "Article: unknown keys found: #{unknown_keys.join(', ')}"
-      end
-
-      # Checks if the article is valid based on the presence of URL, ID, and either title or description.
-      # @return [Boolean] True if the article is valid, otherwise false.
-      def valid?
-        !url.to_s.empty? && (!title.to_s.empty? || !description.to_s.empty?) && !id.to_s.empty?
-      end
-
-      # @yield [key, value]
-      # @return [Enumerator] if no block is given
-      def each
-        return enum_for(:each) unless block_given?
-
-        PROVIDED_KEYS.each { |key| yield(key, public_send(key)) }
-      end
-
-      def id
-        @to_h[:id]
-      end
-
-      def title
-        @to_h[:title]
-      end
-
-      def description
-        return @description if defined?(@description)
-
-        return if (description = @to_h[:description]).to_s.empty?
-
-        @description = self.class.remove_pattern_from_start(description, title) if title
-
-        if self.class.contains_html?(@description) && url
-          @description = Html2rss::AttributePostProcessors::SanitizeHtml.get(description, url)
-        else
-          @description
-        end
-      end
-
-      # @return [Addressable::URI, nil]
-      def url
-        @url ||= Html2rss::Utils.sanitize_url(@to_h[:url])
-      end
-
-      # @return [Addressable::URI, nil]
-      def image
-        @image ||= Html2rss::Utils.sanitize_url(@to_h[:image])
-      end
-
-      # Generates a unique identifier based on the URL and ID using CRC32.
-      # @return [String]
-      def guid
-        @guid ||= Zlib.crc32([url, id].join('#!/')).to_s(36).encode('utf-8')
-      end
-
-      # Parses and returns the published_at time.
-      # @return [DateTime, nil]
-      def published_at
-        return if (string = @to_h[:published_at].to_s.strip).empty?
-
-        @published_at ||= DateTime.parse(string)
-      rescue ArgumentError
-        nil
-      end
-
-      def scraper
-        @to_h[:scraper]
-      end
-
-      def <=>(other)
-        return nil unless other.is_a?(Article)
-
-        0 if other.all? { |key, value| value == public_send(key) ? public_send(key) <=> value : false }
-      end
-    end
-  end
-end

data/lib/html2rss/auto_source/channel.rb

@@ -1,78 +0,0 @@
-# frozen_string_literal: true
-
-module Html2rss
-  class AutoSource
-    ##
-    # Extracts channel information from
-    # 1. the HTML document's <head>.
-    # 2. the HTTP response
-    class Channel
-      ##
-      #
-      # @param parsed_body [Nokogiri::HTML::Document] The parsed HTML document.
-      # @param url [Addressable::URI] The URL of the channel.
-      # @param headers [Hash<String, String>] the http headers
-      # @param articles [Array<Html2rss::AutoSource::Article>] The articles.
-      def initialize(parsed_body, url:, headers:, articles: [], stylesheets: [])
-        @parsed_body = parsed_body
-        @url = url
-        @headers = headers
-        @articles = articles
-        @stylesheets = stylesheets
-      end
-
-      attr_writer :articles
-      attr_reader :stylesheets
-
-      def url = @url.normalize.to_s
-
-      def title
-        @title ||= if (title = parsed_body.at_css('head > title')&.text.to_s) && !title.empty?
-                     title.gsub(/\s+/, ' ').strip
-                   else
-                     Utils.titleized_channel_url(@url)
-                   end
-      end
-
-      def description = parsed_body.at_css('meta[name="description"]')&.[]('content')
-      def last_build_date = headers['last-modified']
-
-      def language
-        return parsed_body['lang'] if parsed_body.name == 'html' && parsed_body['lang']
-
-        parsed_body.at_css('[lang]')&.[]('lang')
-      end
-
-      def image
-        url = parsed_body.at_css('meta[property="og:image"]')&.[]('content')
-        Html2rss::Utils.sanitize_url(url) if url
-      end
-
-      def ttl
-        ttl = headers['cache-control']&.match(/max-age=(\d+)/)&.[](1)
-        return unless ttl
-
-        ttl.to_i.fdiv(60).ceil
-      end
-
-      def generator
-        "html2rss V. #{::Html2rss::VERSION} (using auto_source scrapers: #{scraper_counts})"
-      end
-
-      private
-
-      attr_reader :parsed_body, :headers
-
-      def scraper_counts
-        scraper_counts = +''
-
-        @articles.each_with_object(Hash.new(0)) { |article, counts| counts[article.scraper] += 1 }
-                 .each do |klass, count|
-          scraper_counts.concat("[#{klass.to_s.gsub('Html2rss::AutoSource::Scraper::', '')}=#{count}]")
-        end
-
-        scraper_counts
-      end
-    end
-  end
-end

data/lib/html2rss/auto_source/reducer.rb

@@ -1,48 +0,0 @@
-# frozen_string_literal: true
-
-module Html2rss
-  class AutoSource
-    ##
-    # Reducer is responsible for reducing the list of articles.
-    # It keeps only the longest attributes of articles with the same URL.
-    # It also filters out invalid articles.
-    class Reducer
-      class << self
-        def call(articles, **_options)
-          Log.debug "Reducer: inited with #{articles.size} articles"
-
-          reduce_by_keeping_longest_values(articles, keep: [:scraper]) { |article| article.url&.path }
-        end
-
-        private
-
-        # @param articles [Array<Article>]
-        # @return [Array<Article>] reduced articles
-        def reduce_by_keeping_longest_values(articles, keep:, &)
-          grouped_by_block = articles.group_by(&)
-          grouped_by_block.each_with_object([]) do |(_key, grouped_articles), result|
-            memo_object = {}
-            grouped_articles.each do |article_hash|
-              keep_longest_values(memo_object, article_hash, keep:)
-            end
-
-            result << Article.new(**memo_object)
-          end
-        end
-
-        def keep_longest_values(memo_object, article_hash, keep:)
-          article_hash.each do |key, value|
-            next if value.eql?(memo_object[key])
-
-            if keep.include?(key)
-              memo_object[key] ||= []
-              memo_object[key] << value
-            elsif value && value.to_s.size > memo_object[key].to_s.size
-              memo_object[key] = value
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/html2rss/auto_source/rss_builder.rb

@@ -1,70 +0,0 @@
-# frozen_string_literal: true
-
-require 'rss'
-
-module Html2rss
-  class AutoSource
-    ##
-    # Converts the autosourced channel and articles to an RSS feed.
-    class RssBuilder
-      def self.add_guid(article, maker)
-        maker.guid.tap do |guid|
-          guid.content = article.guid
-          guid.isPermaLink = false
-        end
-      end
-
-      def self.add_image(article, maker)
-        url = article.image || return
-
-        maker.enclosure.tap do |enclosure|
-          enclosure.url = url
-          enclosure.type = Html2rss::Utils.guess_content_type_from_url(url)
-          enclosure.length = 0
-        end
-      end
-
-      def initialize(channel:, articles:)
-        @channel = channel
-        @articles = articles
-      end
-
-      def call
-        RSS::Maker.make('2.0') do |maker|
-          Html2rss::RssBuilder::Stylesheet.add(maker, channel.stylesheets)
-
-          make_channel(maker.channel)
-          make_items(maker)
-        end
-      end
-
-      private
-
-      attr_reader :channel, :articles
-
-      def make_channel(maker)
-        %i[language title description ttl].each do |key|
-          maker.public_send(:"#{key}=", channel.public_send(key))
-        end
-
-        maker.link = channel.url
-        maker.generator = channel.generator
-        maker.updated = channel.last_build_date
-      end
-
-      def make_items(maker)
-        articles.each do |article|
-          maker.items.new_item do |item_maker|
-            RssBuilder.add_guid(article, item_maker)
-            RssBuilder.add_image(article, item_maker)
-
-            item_maker.title = article.title
-            item_maker.description = article.description
-            item_maker.pubDate = article.published_at&.rfc2822
-            item_maker.link = article.url
-          end
-        end
-      end
-    end
-  end
-end