html2rss 0.17.0 → 0.19.0

data/lib/html2rss/selectors/post_processors/html_to_markdown.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require 'reverse_markdown'
+
+module Html2rss
+  class Selectors
+    module PostProcessors
+      ##
+      # Returns HTML code as Markdown formatted String.
+      # Before converting to markdown, the HTML is sanitized with SanitizeHtml.
+      # Imagine this HTML structure:
+      #
+      #     <section>
+      #       Lorem <b>ipsum</b> dolor...
+      #       <iframe src="https://evil.corp/miner"></iframe>
+      #       <script>alert();</script>
+      #     </section>
+      #
+      # YAML usage example:
+      #
+      #    selectors:
+      #      description:
+      #        selector: section
+      #        extractor: html
+      #        post_process:
+      #          name: html_to_markdown
+      #
+      # Would return:
+      #    'Lorem **ipsum** dolor'
+      class HtmlToMarkdown < Base
+        # @param value [String] extracted selector value
+        # @param context [Selectors::Context] post-processor context
+        # @return [void]
+        def self.validate_args!(value, context)
+          assert_type value, String, :value, context:
+        end
+
+        ##
+        # @return [String] formatted in Markdown
+        def get
+          sanitized_value = SanitizeHtml.new(value, context).get
+
+          ReverseMarkdown.convert(sanitized_value)
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/selectors/post_processors/html_transformers/transform_urls_to_absolute_ones.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class Selectors
+    module PostProcessors
+      # HTML tree transformers used by selectors post-processing.
+      module HtmlTransformers
+        ##
+        # Transformer that converts relative URLs to absolute URLs within specified HTML elements.
+        class TransformUrlsToAbsoluteOnes
+          # HTML tags and the URL-bearing attribute that should be normalized.
+          URL_ELEMENTS_WITH_URL_ATTRIBUTE = {
+            'a' => :href, # Visible link
+            'img' => :src, # Visible image
+            'iframe' => :src, # Embedded frame (visible content)
+            'audio' => :src, # Can show controls, so potentially visible
+            'video' => :src # Video player is visible
+          }.freeze
+
+          # @param channel_url [String, Html2rss::Url] base URL used to resolve relative links
+          def initialize(channel_url)
+            @channel_url = channel_url
+          end
+
+          ##
+          # Transforms URLs to absolute ones.
+          #
+          # @param node_name [String] node name currently being transformed
+          # @param node [Nokogiri::XML::Node] node currently being transformed
+          # @param _env [Hash] transformer context
+          # @option _env [Object] :_reserved reserved for transformer pipeline context
+          # @return [void]
+          def call(node_name:, node:, **_env)
+            return unless URL_ELEMENTS_WITH_URL_ATTRIBUTE.key?(node_name)
+
+            url_attribute = URL_ELEMENTS_WITH_URL_ATTRIBUTE[node_name]
+            url = node[url_attribute]
+            node[url_attribute] = Url.from_relative(url, @channel_url).to_s
+          end
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/selectors/post_processors/html_transformers/wrap_img_in_a.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class Selectors
+    module PostProcessors
+      module HtmlTransformers
+        ##
+        # Transformer that wraps <img> tags into <a> tags linking to `img.src`.
+        class WrapImgInA
+          ##
+          # Wraps <img> tags into <a> tags that link to `img.src`.
+          #
+          # @param node_name [String]
+          # @param node [Nokogiri::XML::Node]
+          # @param _env [Hash] transformer context
+          # @option _env [Object] :_reserved reserved for transformer pipeline context
+          # @return [nil]
+          def call(node_name:, node:, **_env)
+            return unless should_process?(node_name)
+
+            wrap_image_in_anchor(node) unless already_wrapped?(node)
+          end
+
+          # @param node_name [String] node name currently being transformed
+          # @return [Boolean] whether this transformer should run for the node
+          def should_process?(node_name)
+            node_name == 'img'
+          end
+
+          # @param node [Nokogiri::XML::Node] node currently being transformed
+          # @return [Boolean] whether the image is already wrapped in a link
+          def already_wrapped?(node)
+            node.parent.name == 'a'
+          end
+
+          private
+
+          ##
+          # Wraps the <img> node in an <a> tag.
+          #
+          # @param node [Nokogiri::XML::Node]
+          # @return [nil]
+          def wrap_image_in_anchor(node)
+            anchor = Nokogiri::XML::Node.new('a', node.document)
+            anchor['href'] = node['src']
+            node.add_next_sibling(anchor)
+            anchor.add_child(node.remove)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/selectors/post_processors/markdown_to_html.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require 'kramdown'
+require_relative 'sanitize_html'
+
+module Html2rss
+  class Selectors
+    module PostProcessors
+      ##
+      # Generates HTML from Markdown.
+      #
+      # It's particularly useful in conjunction with the Template post processor
+      # to generate a description from other selectors.
+      #
+      # YAML usage example:
+      #
+      #    selectors:
+      #      description:
+      #        selector: section
+      #        post_process:
+      #          - name: template
+      #            string: |
+      #              # %s
+      #
+      #              Price: %s
+      #            methods:
+      #              - self
+      #              - price
+      #          - name: markdown_to_html
+      #
+      # Would e.g. return:
+      #
+      #    <h1>Section</h1>
+      #
+      #    <p>Price: 12.34</p>
+      class MarkdownToHtml < Base
+        # @param value [String] extracted selector value
+        # @param context [Selectors::Context] post-processor context
+        # @return [void]
+        def self.validate_args!(value, context)
+          assert_type value, String, :value, context:
+        end
+
+        ##
+        # Converts Markdown to sanitized HTML.
+        #
+        # @return [String] Sanitized HTML content
+        def get
+          html_content = Kramdown::Document.new(value).to_html
+          SanitizeHtml.new(html_content, context).get
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/selectors/post_processors/parse_time.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+require 'time'
+require 'tzinfo'
+
+module Html2rss
+  class Selectors
+    module PostProcessors
+      ##
+      # Returns the {https://www.w3.org/Protocols/rfc822/ RFC822} representation of a time.
+      #
+      # Imagine this HTML structure:
+      #
+      #     <p>Published on <span>2019-07-02</span></p>
+      #
+      # YAML usage example:
+      #
+      #    selectors:
+      #      description:
+      #        selector: span
+      #        post_process:
+      #          name: 'parse_time'
+      #          time_zone: 'Europe/Berlin'
+      #
+      # Would return:
+      #    "Tue, 02 Jul 2019 00:00:00 +0200"
+      #
+      # It uses `Time.parse`.
+      class ParseTime < Base
+        # @param value [String] extracted selector value
+        # @param context [Selectors::Context] post-processor context
+        # @return [void]
+        def self.validate_args!(value, context)
+          assert_type(value, String, :value, context:)
+          time_zone_value = time_zone(context)
+
+          if time_zone_value.nil? || time_zone_value.empty?
+            raise ArgumentError, 'time_zone cannot be nil or empty', [], cause: nil
+          end
+
+          assert_type(time_zone_value, String, :time_zone, context:)
+        end
+
+        # @param context [Selectors::Context] post-processor context
+        # @return [String, nil] configured channel time zone
+        def self.time_zone(context) = context.dig(:config, :channel, :time_zone)
+
+        ##
+        # Converts the provided time string to RFC822 format, taking into account the time_zone.
+        #
+        # @return [String] RFC822 formatted time
+        # @raise [TZInfo::InvalidTimezoneIdentifier] if the configured time zone is invalid
+        def get
+          with_timezone(time_zone) { Time.parse(value).rfc822 }
+        end
+
+        private
+
+        def time_zone
+          self.class.time_zone(context)
+        end
+
+        def with_timezone(time_zone)
+          return yield if time_zone.nil? || time_zone.empty?
+
+          # Validate timezone using TZInfo
+          TZInfo::Timezone.get(time_zone)
+
+          prev_tz = ENV.fetch('TZ', Time.now.getlocal.zone)
+          ENV['TZ'] = time_zone
+          yield
+        ensure
+          ENV['TZ'] = prev_tz if prev_tz
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/selectors/post_processors/parse_uri.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class Selectors
+    module PostProcessors
+      ##
+      # Returns the normalized URL as a String.
+      # If the URL is relative, it resolves it against the channel URL.
+      #
+      # Imagine this HTML structure:
+      #
+      #    <span>http://why-not-use-a-link.uh </span>
+      #
+      # YAML usage example:
+      #
+      #    selectors:
+      #      link:
+      #        selector: span
+      #        extractor: text
+      #        post_process:
+      #          name: parse_uri
+      #
+      # Would return:
+      #    'http://why-not-use-a-link.uh'
+      class ParseUri < Base
+        # @param value [String] extracted selector value
+        # @param _context [Selectors::Context] post-processor context
+        # @return [void]
+        def self.validate_args!(value, _context)
+          raise ArgumentError, 'The `value` option is missing or empty.' if value.to_s.empty?
+        end
+
+        ##
+        # @return [String]
+        def get
+          config_url = context.dig(:config, :channel, :url)
+
+          Url.from_relative(value, config_url).to_s
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/selectors/post_processors/sanitize_html.rb

@@ -0,0 +1,154 @@
+# frozen_string_literal: true
+
+require 'sanitize'
+require_relative 'html_transformers/transform_urls_to_absolute_ones'
+require_relative 'html_transformers/wrap_img_in_a'
+
+module Html2rss
+  class Selectors
+    module PostProcessors
+      ##
+      # Returns sanitized HTML code as String.
+      #
+      # It sanitizes by using the [sanitize gem](https://github.com/rgrove/sanitize) with
+      # [Sanitize::Config::RELAXED](https://github.com/rgrove/sanitize#sanitizeconfigrelaxed).
+      #
+      # Furthermore, it adds:
+      #
+      # - `rel="nofollow noopener noreferrer"` to <a> tags
+      # - `referrer-policy='no-referrer'` to <img> tags
+      # - wraps all <img> tags, whose direct parent is not an <a>, into an <a>
+      #   linking to the <img>'s `src`.
+      #
+      # Imagine this HTML structure:
+      #
+      #     <section>
+      #       Lorem <b>ipsum</b> dolor...
+      #       <iframe src="https://evil.corp/miner"></iframe>
+      #       <script>alert();</script>
+      #     </section>
+      #
+      # YAML usage example:
+      #
+      #    selectors:
+      #      description:
+      #        selector: '.section'
+      #        extractor: html
+      #        post_process:
+      #          name: sanitize_html
+      #
+      # Would return:
+      #    '<p>Lorem <b>ipsum</b> dolor ...</p>'
+      class SanitizeHtml < Base
+        # @see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Referrer-Policy
+        TAG_ATTRIBUTES = {
+          'a' => {
+            'rel' => 'nofollow noopener noreferrer',
+            'target' => '_blank'
+          },
+
+          'area' => {
+            'rel' => 'nofollow noopener noreferrer',
+            'target' => '_blank'
+          },
+
+          'img' => {
+            'referrerpolicy' => 'no-referrer',
+            'crossorigin' => 'anonymous',
+            'loading' => 'lazy',
+            'decoding' => 'async'
+          },
+
+          'iframe' => {
+            'referrerpolicy' => 'no-referrer',
+            'crossorigin' => 'anonymous',
+            'loading' => 'lazy',
+            'sandbox' => 'allow-same-origin',
+            'src' => true,
+            'width' => true,
+            'height' => true
+          },
+
+          'video' => {
+            'referrerpolicy' => 'no-referrer',
+            'crossorigin' => 'anonymous',
+            'preload' => 'none',
+            'playsinline' => 'true',
+            'controls' => 'true'
+          },
+
+          'audio' => {
+            'referrerpolicy' => 'no-referrer',
+            'crossorigin' => 'anonymous',
+            'preload' => 'none'
+          }
+        }.freeze
+        # @param value [String] extracted selector value
+        # @param context [Selectors::Context] post-processor context
+        # @return [void]
+        def self.validate_args!(value, context)
+          assert_type value, String, :value, context:
+        end
+
+        ##
+        # Shorthand method to get the sanitized HTML.
+        # @param html [String]
+        # @param url [String, Html2rss::Url]
+        # @return [String, nil]
+        def self.get(html, url)
+          return nil if String(html).empty?
+
+          context = Selectors::Context.new(config: { channel: { url: } }, options: {})
+          new(html, context).get
+        end
+
+        ##
+        # @return [String, nil]
+        def get
+          sanitized_html = Sanitize.fragment(value, sanitize_config).to_s
+          sanitized_html.gsub!(/\s+/, ' ')
+          sanitized_html.strip!
+          sanitized_html.empty? ? nil : sanitized_html
+        end
+
+        private
+
+        def channel_url = context.dig(:config, :channel, :url)
+
+        ##
+        # @return [Sanitize::Config]
+        def sanitize_config # rubocop:disable Metrics/MethodLength
+          config = Sanitize::Config.merge(
+            Sanitize::Config::RELAXED,
+            attributes: { all: %w[dir lang alt title translate] },
+            add_attributes: TAG_ATTRIBUTES,
+            transformers: [
+              method(:transform_urls_to_absolute_ones),
+              method(:wrap_img_in_a)
+            ]
+          )
+          config[:elements].push('audio', 'video', 'source')
+          config
+        end
+
+        ##
+        # Wrapper for transform_urls_to_absolute_ones to pass the channel_url.
+        #
+        # @param env [Hash]
+        # @return [nil]
+        def transform_urls_to_absolute_ones(env)
+          HtmlTransformers::TransformUrlsToAbsoluteOnes.new(channel_url).call(**env)
+        end
+
+        ##
+        # Wrapper for wrap_img_in_a.
+        #
+        # @param env [Hash]
+        # @return [nil]
+        def wrap_img_in_a(env)
+          HtmlTransformers::WrapImgInA.new.call(**env)
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/selectors/post_processors/substring.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class Selectors
+    module PostProcessors
+      ##
+      # Returns a defined part of a String.
+      #
+      # Both parameters must be an Integer and they can be negative.
+      # The +end+ parameter can be omitted, in that case it will not cut the
+      # String at the end.
+      #
+      # A Regexp or a MatchString is not supported.
+      #
+      # See the [`String#[]`](https://ruby-doc.org/core/String.html#method-i-5B-5D)
+      # documentation for more information.
+      #
+      # Imagine this HTML:
+      #    <h1>Foo bar and baz<h1>
+      #
+      # YAML usage example:
+      #    selectors:
+      #      title:
+      #        selector: h1
+      #        post_process:
+      #          name: substring
+      #          start: 4
+      #          end: 6
+      #
+      # Would return:
+      #    'bar'
+      class Substring < Base
+        # @param value [String] extracted selector value
+        # @param context [Selectors::Context] post-processor context
+        # @return [void]
+        def self.validate_args!(value, context)
+          assert_type value, String, :value, context:
+
+          options = context[:options]
+          assert_type options[:start], Integer, :start, context:
+
+          end_index = options[:end]
+          assert_type(end_index, Integer, :end, context:) if end_index
+        end
+
+        ##
+        # Extracts the substring from the original string based on the provided start and end indices.
+        #
+        # @return [String] The extracted substring.
+        def get
+          value[range]
+        end
+
+        ##
+        # Determines the range for the substring extraction based on the provided start and end indices.
+        #
+        # @return [Range] The range object representing the start and end/Infinity (integers).
+        def range
+          return (start_index..) unless end_index?
+
+          if start_index == end_index
+            raise ArgumentError,
+                  'The `start` value must be unequal to the `end` value.'
+          end
+
+          (start_index..end_index)
+        end
+
+        private
+
+        def end_index?  = !context[:options][:end].to_s.empty?
+        def end_index   = context[:options][:end].to_i
+        def start_index = context[:options][:start].to_i
+      end
+    end
+  end
+end

data/lib/html2rss/selectors/post_processors/template.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class Selectors
+    module PostProcessors
+      ##
+      # Returns a formatted String according to the string pattern.
+      # It uses [Kernel#format](https://ruby-doc.org/core/Kernel.html#method-i-format)
+      #
+      # It supports the format pattern `%<key>s` and `%{key}`, where `key` is the key of the selector.
+      # If `%{self}` is used, the selectors extracted value will be used.
+      #
+      # Imagine this HTML:
+      #
+      #    <li>
+      #      <h1>Product</h1>
+      #      <span class="price">23,42€</span>
+      #    </li>
+      #
+      #
+      # YAML usage example:
+      #
+      #    selectors:
+      #      items:
+      #        selector: 'li'
+      #      price:
+      #        selector: '.price'
+      #      title:
+      #        selector: h1
+      #        post_process:
+      #          name: template
+      #          string: '%{self} (%{price})'
+      #
+      # Would return:
+      #    'Product (23,42€)'
+      class Template < Base
+        # @param value [String] extracted selector value
+        # @param context [Selectors::Context] post-processor context
+        # @return [void]
+        def self.validate_args!(value, context)
+          assert_type value, String, :value, context:
+
+          string = context[:options]&.dig(:string).to_s
+          raise InvalidType, 'The `string` template is absent.' if string.empty?
+        end
+
+        ##
+        # @param value [String]
+        # @param context [Selectors::Context]
+        def initialize(value, context)
+          super
+
+          @options = context[:options] || {}
+          @scraper = context[:scraper]
+          @item = context[:item]
+          @string = @options[:string].to_s
+        end
+
+        ##
+        # @return [String]
+        def get
+          Html2rss::Config::DynamicParams.call(@string, {}, getter: method(:item_value), replace_missing_with: '')
+        end
+
+        private
+
+        # @param key [String, Symbol]
+        # @return [String]
+        def item_value(key)
+          key = key.to_sym
+          key == :self ? value : @scraper.select(key, @item)
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/selectors/post_processors.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class Selectors
+    ##
+    # Provides a namespace for attribute post processors.
+    module PostProcessors
+      ##
+      # Error raised when an unknown post processor name is requested.
+      class UnknownPostProcessorName < Html2rss::Error; end
+
+      ##
+      # Error raised when a required option is missing.
+      class MissingOption < Html2rss::Error; end
+
+      ##
+      # Error raised when an invalid type is provided.
+      class InvalidType < Html2rss::Error; end
+
+      ##
+      # Maps the post processor name to the class implementing the post processor.
+      #
+      # The key is the name to use in the feed config.
+      NAME_TO_CLASS = {
+        gsub: Gsub,
+        html_to_markdown: HtmlToMarkdown,
+        markdown_to_html: MarkdownToHtml,
+        parse_time: ParseTime,
+        parse_uri: ParseUri,
+        sanitize_html: SanitizeHtml,
+        substring: Substring,
+        template: Template
+      }.freeze
+
+      ##
+      # Shorthand method to instantiate the post processor and call `#get` on it
+      #
+      # @param name [String, Symbol] post-processor name from selector config
+      # @param value [Object] extracted selector value
+      # @param context [Selectors::Context] post-processor context
+      # @return [Object] transformed selector value
+      def self.get(name, value, context)
+        klass = NAME_TO_CLASS[name.to_sym] || raise(UnknownPostProcessorName, "Unknown name '#{name}'")
+        klass.new(value, context).get
+      end
+    end
+  end
+end