html2rss 0.17.0 → 0.19.0

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: html2rss
 version: !ruby/object:Gem::Version
-  version: 0.17.0
+  version: 0.19.0
 platform: ruby
 authors:
 - Gil Desmarais
 bindir: exe
 cert_chain: []
-date: 2025-01-18 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -23,6 +23,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.7'
+- !ruby/object:Gem::Dependency
+  name: brotli
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: dry-validation
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: faraday
   requirement: !ruby/object:Gem::Requirement
@@ -57,6 +85,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: faraday-gzip
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
 - !ruby/object:Gem::Dependency
   name: kramdown
   requirement: !ruby/object:Gem::Requirement
@@ -179,16 +221,16 @@ dependencies:
   name: sanitize
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '6.0'
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '6.0'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement
@@ -245,64 +287,104 @@ files:
 - exe/html2rss
 - html2rss.gemspec
 - lib/html2rss.rb
-- lib/html2rss/attribute_post_processors.rb
-- lib/html2rss/attribute_post_processors/base.rb
-- lib/html2rss/attribute_post_processors/gsub.rb
-- lib/html2rss/attribute_post_processors/html_to_markdown.rb
-- lib/html2rss/attribute_post_processors/html_transformers/transform_urls_to_absolute_ones.rb
-- lib/html2rss/attribute_post_processors/html_transformers/wrap_img_in_a.rb
-- lib/html2rss/attribute_post_processors/markdown_to_html.rb
-- lib/html2rss/attribute_post_processors/parse_time.rb
-- lib/html2rss/attribute_post_processors/parse_uri.rb
-- lib/html2rss/attribute_post_processors/sanitize_html.rb
-- lib/html2rss/attribute_post_processors/substring.rb
-- lib/html2rss/attribute_post_processors/template.rb
+- lib/html2rss/articles/deduplicator.rb
 - lib/html2rss/auto_source.rb
-- lib/html2rss/auto_source/article.rb
-- lib/html2rss/auto_source/channel.rb
 - lib/html2rss/auto_source/cleanup.rb
-- lib/html2rss/auto_source/reducer.rb
-- lib/html2rss/auto_source/rss_builder.rb
 - lib/html2rss/auto_source/scraper.rb
 - lib/html2rss/auto_source/scraper/html.rb
+- lib/html2rss/auto_source/scraper/json_state.rb
+- lib/html2rss/auto_source/scraper/microdata.rb
 - lib/html2rss/auto_source/scraper/schema.rb
+- lib/html2rss/auto_source/scraper/schema/category_extractor.rb
 - lib/html2rss/auto_source/scraper/schema/item_list.rb
 - lib/html2rss/auto_source/scraper/schema/list_item.rb
 - lib/html2rss/auto_source/scraper/schema/thing.rb
 - lib/html2rss/auto_source/scraper/semantic_html.rb
-- lib/html2rss/auto_source/scraper/semantic_html/extractor.rb
-- lib/html2rss/auto_source/scraper/semantic_html/image.rb
+- lib/html2rss/auto_source/scraper/semantic_html/anchor_selector.rb
+- lib/html2rss/auto_source/scraper/wordpress_api.rb
+- lib/html2rss/auto_source/scraper/wordpress_api/page_scope.rb
+- lib/html2rss/auto_source/scraper/wordpress_api/posts_endpoint.rb
+- lib/html2rss/blocked_surface.rb
+- lib/html2rss/category_extractor.rb
 - lib/html2rss/cli.rb
 - lib/html2rss/config.rb
-- lib/html2rss/config/channel.rb
-- lib/html2rss/config/selectors.rb
-- lib/html2rss/item.rb
-- lib/html2rss/item_extractors.rb
-- lib/html2rss/item_extractors/attribute.rb
-- lib/html2rss/item_extractors/href.rb
-- lib/html2rss/item_extractors/html.rb
-- lib/html2rss/item_extractors/static.rb
-- lib/html2rss/item_extractors/text.rb
-- lib/html2rss/object_to_xml_converter.rb
+- lib/html2rss/config/class_methods.rb
+- lib/html2rss/config/dynamic_params.rb
+- lib/html2rss/config/multiple_feeds_config.rb
+- lib/html2rss/config/request_headers.rb
+- lib/html2rss/config/schema.rb
+- lib/html2rss/config/validator.rb
+- lib/html2rss/error.rb
+- lib/html2rss/feed_pipeline.rb
+- lib/html2rss/feed_pipeline/auto_fallback.rb
+- lib/html2rss/hash_util.rb
+- lib/html2rss/html_extractor.rb
+- lib/html2rss/html_extractor/date_extractor.rb
+- lib/html2rss/html_extractor/enclosure_extractor.rb
+- lib/html2rss/html_extractor/image_extractor.rb
+- lib/html2rss/html_navigator.rb
+- lib/html2rss/json_feed_builder.rb
+- lib/html2rss/json_feed_builder/item.rb
+- lib/html2rss/rendering.rb
+- lib/html2rss/rendering/audio_renderer.rb
+- lib/html2rss/rendering/description_builder.rb
+- lib/html2rss/rendering/image_renderer.rb
+- lib/html2rss/rendering/media_renderer.rb
+- lib/html2rss/rendering/pdf_renderer.rb
+- lib/html2rss/rendering/video_renderer.rb
+- lib/html2rss/request_controls.rb
 - lib/html2rss/request_service.rb
+- lib/html2rss/request_service/botasaurus_contract.rb
+- lib/html2rss/request_service/botasaurus_strategy.rb
 - lib/html2rss/request_service/browserless_strategy.rb
+- lib/html2rss/request_service/budget.rb
 - lib/html2rss/request_service/context.rb
 - lib/html2rss/request_service/faraday_strategy.rb
+- lib/html2rss/request_service/policy.rb
 - lib/html2rss/request_service/puppet_commander.rb
 - lib/html2rss/request_service/response.rb
+- lib/html2rss/request_service/response_guard.rb
 - lib/html2rss/request_service/strategy.rb
+- lib/html2rss/request_session.rb
+- lib/html2rss/request_session/rel_next_pager.rb
+- lib/html2rss/request_session/runtime_input.rb
+- lib/html2rss/request_session/runtime_policy.rb
 - lib/html2rss/rss_builder.rb
+- lib/html2rss/rss_builder/article.rb
 - lib/html2rss/rss_builder/channel.rb
-- lib/html2rss/rss_builder/item.rb
+- lib/html2rss/rss_builder/enclosure.rb
 - lib/html2rss/rss_builder/stylesheet.rb
-- lib/html2rss/utils.rb
+- lib/html2rss/selectors.rb
+- lib/html2rss/selectors/config.rb
+- lib/html2rss/selectors/extractors.rb
+- lib/html2rss/selectors/extractors/attribute.rb
+- lib/html2rss/selectors/extractors/href.rb
+- lib/html2rss/selectors/extractors/html.rb
+- lib/html2rss/selectors/extractors/static.rb
+- lib/html2rss/selectors/extractors/text.rb
+- lib/html2rss/selectors/object_to_xml_converter.rb
+- lib/html2rss/selectors/post_processors.rb
+- lib/html2rss/selectors/post_processors/base.rb
+- lib/html2rss/selectors/post_processors/gsub.rb
+- lib/html2rss/selectors/post_processors/html_to_markdown.rb
+- lib/html2rss/selectors/post_processors/html_transformers/transform_urls_to_absolute_ones.rb
+- lib/html2rss/selectors/post_processors/html_transformers/wrap_img_in_a.rb
+- lib/html2rss/selectors/post_processors/markdown_to_html.rb
+- lib/html2rss/selectors/post_processors/parse_time.rb
+- lib/html2rss/selectors/post_processors/parse_uri.rb
+- lib/html2rss/selectors/post_processors/sanitize_html.rb
+- lib/html2rss/selectors/post_processors/substring.rb
+- lib/html2rss/selectors/post_processors/template.rb
+- lib/html2rss/url.rb
 - lib/html2rss/version.rb
+- lib/tasks/config_schema.rake
+- schema/html2rss-config.schema.json
 homepage: https://github.com/html2rss/html2rss
 licenses:
 - MIT
 metadata:
   allowed_push_host: https://rubygems.org
-  changelog_uri: https://github.com/html2rss/html2rss/releases/tag/v0.17.0
+  changelog_uri: https://github.com/html2rss/html2rss/releases/tag/v0.19.0
   rubygems_mfa_required: 'true'
 rdoc_options: []
 require_paths:
@@ -318,7 +400,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.2
+rubygems_version: 4.0.6
 specification_version: 4
 summary: Generates RSS feeds from websites by scraping a URL and using CSS selectors
   to extract item.

data/lib/html2rss/attribute_post_processors/base.rb

@@ -1,74 +0,0 @@
-# frozen_string_literal: true
-
-module Html2rss
-  ##
-  # Provides a namespace for attribute post processors.
-  module AttributePostProcessors
-    ##
-    # All post processors must inherit from this base class and implement `self.validate_args!` and `#get`.
-    class Base
-      # Validates the presence of required options in the context
-      #
-      # @param keys [Array<Symbol>] the keys to check for presence
-      # @param context [Hash] the context containing options
-      # @raise [MissingOption] if any key is missing
-      def self.expect_options(keys, context)
-        keys.each do |key|
-          unless (options = context[:options]).key?(key)
-            raise MissingOption, "The `#{key}` option is missing in: #{options.inspect}", [],
-                  cause: nil
-          end
-        end
-      end
-
-      # Asserts that the value is of the expected type(s)
-      #
-      # @param value [Object] the value to check
-      # @param types [Array<Class>, Class] the expected type(s)
-      # @param name [String] the name of the option being checked
-      # @param context [Item::Context] the context
-      # @raise [InvalidType] if the value is not of the expected type(s)
-      def self.assert_type(value, types = [], name, context:)
-        types = [types] unless types.is_a?(Array)
-
-        return if types.any? { |type| value.is_a?(type) }
-
-        options = context[:options] if context.is_a?(Hash)
-        options ||= { file: File.basename(caller_locations(1, 1).first.absolute_path) }
-
-        raise InvalidType, format('The type of `%<name>s` must be %<types>s, but is: %<type>s in: %<options>s',
-                                  name:, types: types.join(' or '), type: value.class, options: options.inspect),
-              [], cause: nil
-      end
-
-      ##
-      # This method validates the arguments passed to the post processor. Must be implemented by subclasses.
-      def self.validate_args!(_value, _context)
-        raise NotImplementedError, 'You must implement the `validate_args!` method in the post processor'
-      end
-
-      # Initializes the post processor
-      #
-      # @param value [Object] the value to be processed
-      # @param context [Item::Context] the context
-      def initialize(value, context)
-        klass = self.class
-        # TODO: get rid of Hash
-        klass.assert_type(context, [Item::Context, Hash], 'context', context:)
-        klass.validate_args!(value, context)
-
-        @value = value
-        @context = context
-      end
-
-      attr_reader :value, :context
-
-      # Abstract method to be implemented by subclasses
-      #
-      # @raise [NotImplementedError] if not implemented in subclass
-      def get
-        raise NotImplementedError, 'You must implement the `get` method in the post processor'
-      end
-    end
-  end
-end

data/lib/html2rss/attribute_post_processors/gsub.rb

@@ -1,64 +0,0 @@
-# frozen_string_literal: true
-
-module Html2rss
-  module AttributePostProcessors
-    ##
-    # Imagine this HTML:
-    #    <h1>Foo bar and boo<h1>
-    #
-    # YAML usage example:
-    #    selectors:
-    #      title:
-    #        selector: h1
-    #        post_process:
-    #          name: gsub
-    #          pattern: boo
-    #          replacement: baz
-    #
-    # Would return:
-    #    'Foo bar and baz'
-    #
-    # `pattern` can be a Regexp or a String. If it is a String, it will remove
-    # one pair of surrounding slashes ('/') to keep backwards compatibility
-    # and then parse it to build a Regexp.
-    #
-    # `replacement` can be a String or a Hash.
-    #
-    # See the doc on [String#gsub](https://ruby-doc.org/core/String.html#method-i-gsub) for more info.
-    class Gsub < Base
-      def self.validate_args!(value, context)
-        assert_type value, String, :value, context:
-        expect_options(%i[replacement pattern], context)
-        assert_type context.dig(:options, :replacement), [String, Hash], :replacement, context:
-      end
-
-      ##
-      # @param value [String]
-      # @param context [Item::Context]
-      def initialize(value, context)
-        super
-
-        options = context[:options]
-
-        @replacement = options[:replacement]
-        @pattern = options[:pattern]
-      end
-
-      ##
-      # @return [String]
-      def get
-        value.to_s.gsub(pattern, replacement)
-      end
-
-      private
-
-      attr_accessor :replacement
-
-      ##
-      # @return [Regexp]
-      def pattern
-        @pattern.is_a?(String) ? Utils.build_regexp_from_string(@pattern) : @pattern
-      end
-    end
-  end
-end

data/lib/html2rss/attribute_post_processors/html_to_markdown.rb

@@ -1,43 +0,0 @@
-# frozen_string_literal: true
-
-require 'reverse_markdown'
-
-module Html2rss
-  module AttributePostProcessors
-    ##
-    # Returns HTML code as Markdown formatted String.
-    # Before converting to markdown, the HTML is sanitized with SanitizeHtml.
-    # Imagine this HTML structure:
-    #
-    #     <section>
-    #       Lorem <b>ipsum</b> dolor...
-    #       <iframe src="https://evil.corp/miner"></iframe>
-    #       <script>alert();</script>
-    #     </section>
-    #
-    # YAML usage example:
-    #
-    #    selectors:
-    #      description:
-    #        selector: section
-    #        extractor: html
-    #        post_process:
-    #          name: html_to_markdown
-    #
-    # Would return:
-    #    'Lorem **ipsum** dolor'
-    class HtmlToMarkdown < Base
-      def self.validate_args!(value, context)
-        assert_type value, String, :value, context:
-      end
-
-      ##
-      # @return [String] formatted in Markdown
-      def get
-        sanitized_value = SanitizeHtml.new(value, context).get
-
-        ReverseMarkdown.convert(sanitized_value)
-      end
-    end
-  end
-end

data/lib/html2rss/attribute_post_processors/html_transformers/transform_urls_to_absolute_ones.rb

@@ -1,27 +0,0 @@
-# frozen_string_literal: true
-
-module Html2rss
-  module AttributePostProcessors
-    module HtmlTransformers
-      ##
-      # Transformer that converts relative URLs to absolute URLs within specified HTML elements.
-      class TransformUrlsToAbsoluteOnes
-        URL_ELEMENTS_WITH_URL_ATTRIBUTE = { 'a' => :href, 'img' => :src }.freeze
-
-        def initialize(channel_url)
-          @channel_url = channel_url
-        end
-
-        ##
-        # Transforms URLs to absolute ones.
-        def call(node_name:, node:, **_env)
-          return unless URL_ELEMENTS_WITH_URL_ATTRIBUTE.key?(node_name)
-
-          url_attribute = URL_ELEMENTS_WITH_URL_ATTRIBUTE[node_name]
-          url = node[url_attribute]
-          node[url_attribute] = Html2rss::Utils.build_absolute_url_from_relative(url, @channel_url)
-        end
-      end
-    end
-  end
-end

data/lib/html2rss/attribute_post_processors/html_transformers/wrap_img_in_a.rb

@@ -1,41 +0,0 @@
-# frozen_string_literal: true
-
-module Html2rss
-  module AttributePostProcessors
-    module HtmlTransformers
-      ##
-      # Transformer that wraps <img> tags into <a> tags linking to `img.src`.
-      class WrapImgInA
-        ##
-        # Wraps <img> tags into <a> tags that link to `img.src`.
-        #
-        # @param node_name [String]
-        # @param node [Nokogiri::XML::Node]
-        # @return [nil]
-        def call(node_name:, node:, **_env)
-          return unless already_wrapped?(node_name, node)
-
-          wrap_image_in_anchor(node)
-        end
-
-        def already_wrapped?(node_name, node)
-          node_name == 'img' && node.parent.name != 'a'
-        end
-
-        private
-
-        ##
-        # Wraps the <img> node in an <a> tag.
-        #
-        # @param node [Nokogiri::XML::Node]
-        # @return [nil]
-        def wrap_image_in_anchor(node)
-          anchor = Nokogiri::XML::Node.new('a', node.document)
-          anchor['href'] = node['src']
-          node.add_next_sibling(anchor)
-          anchor.add_child(node.remove)
-        end
-      end
-    end
-  end
-end

data/lib/html2rss/attribute_post_processors/markdown_to_html.rb

@@ -1,50 +0,0 @@
-# frozen_string_literal: true
-
-require 'kramdown'
-require_relative 'sanitize_html'
-
-module Html2rss
-  module AttributePostProcessors
-    ##
-    # Generates HTML from Markdown.
-    #
-    # It's particularly useful in conjunction with the Template post processor
-    # to generate a description from other selectors.
-    #
-    # YAML usage example:
-    #
-    #    selectors:
-    #      description:
-    #        selector: section
-    #        post_process:
-    #          - name: template
-    #            string: |
-    #              # %s
-    #
-    #              Price: %s
-    #            methods:
-    #              - self
-    #              - price
-    #          - name: markdown_to_html
-    #
-    # Would e.g. return:
-    #
-    #    <h1>Section</h1>
-    #
-    #    <p>Price: 12.34</p>
-    class MarkdownToHtml < Base
-      def self.validate_args!(value, context)
-        assert_type value, String, :value, context:
-      end
-
-      ##
-      # Converts Markdown to sanitized HTML.
-      #
-      # @return [String] Sanitized HTML content
-      def get
-        html_content = Kramdown::Document.new(value).to_html
-        SanitizeHtml.new(html_content, context).get
-      end
-    end
-  end
-end

data/lib/html2rss/attribute_post_processors/parse_time.rb

@@ -1,46 +0,0 @@
-# frozen_string_literal: true
-
-require 'time'
-require_relative '../utils'
-
-module Html2rss
-  module AttributePostProcessors
-    ##
-    # Returns the {https://www.w3.org/Protocols/rfc822/ RFC822} representation of a time.
-    #
-    # Imagine this HTML structure:
-    #
-    #     <p>Published on <span>2019-07-02</span></p>
-    #
-    # YAML usage example:
-    #
-    #    selectors:
-    #      description:
-    #        selector: span
-    #        post_process:
-    #          name: 'parse_time'
-    #          time_zone: 'Europe/Berlin'
-    #
-    # Would return:
-    #    "Tue, 02 Jul 2019 00:00:00 +0200"
-    #
-    # It uses `Time.parse`.
-    class ParseTime < Base
-      def self.validate_args!(value, context)
-        assert_type(value, String, :value, context:)
-        assert_type(context[:config].time_zone, String, :time_zone, context:)
-      end
-
-      ##
-      # Converts the provided time string to RFC822 format, taking into account the time_zone.
-      #
-      # @return [String] RFC822 formatted time
-      # @raise [TZInfo::InvalidTimezoneIdentifier] if the configured time zone is invalid
-      def get
-        time_zone = context[:config].time_zone
-
-        Utils.use_zone(time_zone) { Time.parse(value).rfc822 }
-      end
-    end
-  end
-end

data/lib/html2rss/attribute_post_processors/parse_uri.rb

@@ -1,46 +0,0 @@
-# frozen_string_literal: true
-
-module Html2rss
-  module AttributePostProcessors
-    ##
-    # Returns the URI as String.
-    # If the URL is relative, it builds an absolute one with the channel's URL as base.
-    #
-    # Imagine this HTML structure:
-    #
-    #    <span>http://why-not-use-a-link.uh </span>
-    #
-    # YAML usage example:
-    #
-    #    selectors:
-    #      link:
-    #        selector: span
-    #        extractor: text
-    #        post_process:
-    #          name: parse_uri
-    #
-    # Would return:
-    #    'http://why-not-use-a-link.uh'
-    class ParseUri < Base
-      def self.validate_args!(value, context)
-        url_types = [String, URI::HTTP, Addressable::URI].freeze
-
-        assert_type(value, url_types, :value, context:)
-        assert_type(context.config.url, url_types, :url, context:)
-
-        raise ArgumentError, 'The `value` option is missing or empty.' if value.to_s.empty?
-      end
-
-      ##
-      # @return [String]
-      def get
-        config_url = context.config.url
-
-        Html2rss::Utils.build_absolute_url_from_relative(
-          Html2rss::Utils.sanitize_url(value),
-          config_url
-        ).to_s
-      end
-    end
-  end
-end