html2rss 0.20.1 → 0.22.0

data/lib/html2rss/selectors/post_processors/template.rb

@@ -29,7 +29,7 @@ module Html2rss
       #        selector: h1
       #        post_process:
       #          name: template
-      #          string: '%{self} (%{price})'
+      #          string: '`%{self}` (`%{price}`)'
       #
       # Would return:
       #    'Product (23,42€)'
@@ -54,12 +54,13 @@ module Html2rss
           @scraper = context[:scraper]
           @item = context[:item]
           @string = @options[:string].to_s
+          @getter = ->(key) { item_value(key) }
         end
 
         ##
         # @return [String]
         def get
-          Html2rss::Config::DynamicParams.call(@string, {}, getter: method(:item_value), replace_missing_with: '')
+          Html2rss::Config::DynamicParams.call(@string, {}, getter: @getter, replace_missing_with: '')
         end
 
         private

data/lib/html2rss/selectors.rb

@@ -18,7 +18,7 @@ module Html2rss
     include Enumerable
 
     # A context instance passed to item extractors and post-processors.
-    Context = Struct.new('Context', :options, :item, :config, :scraper, keyword_init: true) # rubocop:disable Style/RedundantStructKeywordInit
+    Context = Struct.new('Context', :options, :item, :config, :scraper, keyword_init: true)
 
     # Default selectors options merged into user configuration.
     DEFAULT_CONFIG = { items: { enhance: true } }.freeze
@@ -199,7 +199,10 @@ module Html2rss
     end
 
     def select_regular(_name, item:, config:, base_url:)
-      value = Extractors.get(config.merge(channel: channel_context(base_url)), item)
+      @merged_configs ||= {}
+      merged_config = @merged_configs[[config.object_id, base_url]] ||=
+        config.merge(channel: channel_context(base_url)).freeze
+      value = Extractors.get(merged_config, item)
 
       if value && (post_process_steps = config[:post_process])
         steps = post_process_steps.is_a?(Array) ? post_process_steps : [post_process_steps]
@@ -210,8 +213,9 @@ module Html2rss
     end
 
     def post_process(item, value, post_process_steps, base_url:)
+      pp_context = channel_post_process_context(base_url)
       post_process_steps.each do |options|
-        context = Context.new(config: { channel: { url: base_url, time_zone: @time_zone } },
+        context = Context.new(config: pp_context,
                               item:, scraper: self, options:)
 
         value = PostProcessors.get(options[:name], value, context)
@@ -262,7 +266,11 @@ module Html2rss
     end
 
     def category_node_options(selector_config, base_url:)
-      selector_config.merge(channel: channel_context(base_url), selector: nil)
+      @category_node_configs ||= {}
+      @category_node_configs[[selector_config.object_id, base_url]] ||= selector_config.merge(
+        channel: channel_context(base_url),
+        selector: nil
+      ).freeze
     end
 
     def apply_post_process_steps(item:, value:, post_process_steps:, base_url:)
@@ -288,7 +296,13 @@ module Html2rss
     end
 
     def channel_context(base_url)
-      { url: base_url, time_zone: @time_zone }
+      @channel_contexts ||= {}
+      @channel_contexts[base_url] ||= { url: base_url, time_zone: @time_zone }.freeze
+    end
+
+    def channel_post_process_context(base_url)
+      @channel_pp_contexts ||= {}
+      @channel_pp_contexts[base_url] ||= { channel: channel_context(base_url) }.freeze
     end
 
     # @return [Hash] enclosure details.

data/lib/html2rss/url.rb

@@ -2,6 +2,7 @@
 
 require 'addressable/uri'
 require 'cgi'
+require 'nokogiri'
 
 module Html2rss
   ##
@@ -54,11 +55,10 @@ module Html2rss
     # @param raw_url [String] the raw URL string to sanitize
     # @return [Url, nil] the sanitized URL, or nil if no valid URL found
     def self.sanitize(raw_url)
-      matched_urls = raw_url.to_s.scan(%r{(?:(?:https?|ftp|mailto)://|mailto:)[^\s<>"]+})
-      url = matched_urls.first.to_s.strip
-      return nil if url.empty?
+      match = raw_url.to_s.match(%r{(?:(?:https?|ftp|mailto)://|mailto:)[^\s<>"]+})
+      return unless match
 
-      new(Addressable::URI.parse(url).normalize)
+      new(Addressable::URI.parse(match[0].strip).normalize)
     end
 
     ##
@@ -70,10 +70,9 @@ module Html2rss
     def self.from_absolute(url_string)
       return url_string if url_string.is_a?(self)
 
-      url = new(Addressable::URI.parse(url_string.to_s.strip).normalize)
-      raise ArgumentError, 'URL must be absolute' unless url.absolute?
-
-      url
+      new(Addressable::URI.parse(url_string.to_s.strip).normalize).tap do |url|
+        raise ArgumentError, 'URL must be absolute' unless url.absolute?
+      end
     rescue Addressable::URI::InvalidURIError
       raise ArgumentError, 'URL must be absolute'
     end
@@ -92,14 +91,28 @@ module Html2rss
     #   Url.for_channel('/relative/path')
     #   # => raises ArgumentError: "URL must be absolute"
     def self.for_channel(url_string)
-      return nil if url_string.nil? || url_string.empty?
-
-      stripped = url_string.strip
+      stripped = url_string.to_s.strip
       return nil if stripped.empty?
 
-      url = from_absolute(stripped)
-      validate_channel_url(url)
-      url
+      from_absolute(stripped).tap { validate_channel_url(_1) }
+    end
+
+    ##
+    # Extracts a base URL from HTML metadata tags.
+    #
+    # @param html [String] raw HTML content
+    # @return [Url, nil] the extracted absolute URL, or nil if none is found
+    def self.extract_from_html(html)
+      doc = Nokogiri::HTML(html)
+      tags = { 'link[rel="canonical"]' => 'href', 'meta[property="og:url"]' => 'content',
+               'meta[name="twitter:url"]' => 'content', 'base[href]' => 'href' }
+      tags.each do |sel, attr|
+        val = doc.at_css(sel)&.[](attr).to_s.strip
+        return from_absolute(val) unless val.empty?
+      rescue ArgumentError
+        next
+      end
+      nil
     end
 
     ##
@@ -125,6 +138,7 @@ module Html2rss
     # @param uri [Addressable::URI] the underlying Addressable::URI object (internal use only)
     def initialize(uri)
       @uri = uri.freeze
+      @path_segments = @uri.path.to_s.split('/').reject(&:empty?).freeze
       freeze
     end
 
@@ -162,7 +176,7 @@ module Html2rss
     # Returns the URL path split into non-empty segments.
     #
     # @return [Array<String>] normalized path segments
-    def path_segments = @uri.path.to_s.split('/').reject(&:empty?)
+    attr_reader :path_segments
 
     ##
     # Returns a copy of the URL with the provided path.

data/lib/html2rss/version.rb

@@ -4,6 +4,6 @@
 # The Html2rss namespace.
 module Html2rss
   # Current application version.
-  VERSION = '0.20.1'
+  VERSION = '0.22.0'
   public_constant :VERSION
 end

data/lib/html2rss.rb

@@ -52,6 +52,8 @@ module Html2rss
     FeedPipeline.new(raw_config).to_json_feed
   end
 
+  # rubocop:disable Metrics/ParameterLists
+
   ##
   # Scrapes the provided URL and returns an RSS object.
   #
@@ -60,9 +62,15 @@ module Html2rss
   # @param items_selector [String, nil] optional selector hint for item extraction
   # @param max_redirects [Integer, nil] optional redirect limit override
   # @param max_requests [Integer, nil] optional request budget override
+  # @param local_file_path [String, nil] optional local HTML file path
   # @return [RSS::Rss] generated RSS feed
-  def self.auto_source(url, strategy: :auto, items_selector: nil, max_redirects: nil, max_requests: nil)
-    feed(build_auto_source_config(url:, strategy:, items_selector:, max_redirects:, max_requests:))
+  def self.auto_source(url,
+                       strategy: :auto,
+                       items_selector: nil,
+                       max_redirects: nil,
+                       max_requests: nil,
+                       local_file_path: nil)
+    feed(build_auto_source_config(url:, strategy:, items_selector:, max_redirects:, max_requests:, local_file_path:))
   end
 
   ##
@@ -73,11 +81,20 @@ module Html2rss
   # @param items_selector [String, nil] optional selector hint for item extraction
   # @param max_redirects [Integer, nil] optional redirect limit override
   # @param max_requests [Integer, nil] optional request budget override
+  # @param local_file_path [String, nil] optional local HTML file path
   # @return [Hash] JSONFeed-compliant hash
-  def self.auto_json_feed(url, strategy: :auto, items_selector: nil, max_redirects: nil, max_requests: nil)
-    json_feed(build_auto_source_config(url:, strategy:, items_selector:, max_redirects:, max_requests:))
+  def self.auto_json_feed(url,
+                          strategy: :auto,
+                          items_selector: nil,
+                          max_redirects: nil,
+                          max_requests: nil,
+                          local_file_path: nil)
+    json_feed(build_auto_source_config(url:, strategy:, items_selector:, max_redirects:, max_requests:,
+                                       local_file_path:))
   end
 
+  # rubocop:enable Metrics/ParameterLists
+
   # rubocop:disable ThreadSafety/ClassInstanceVariable
   class << self
     ##
@@ -125,12 +142,17 @@ module Html2rss
   class << self
     private
 
-    def build_auto_source_config(url:, strategy:, items_selector:, max_redirects:, max_requests:)
-      Config.auto_source_config(
+    def build_auto_source_config(url:, strategy:, items_selector:, max_redirects:, max_requests:, local_file_path: nil) # rubocop:disable Metrics/ParameterLists
+      config = Config.auto_source_config(
         url:,
         items_selector:,
         request_controls: shortcut_request_controls(strategy:, max_redirects:, max_requests:)
       )
+      if local_file_path
+        config[:request] ||= {}
+        config[:request][:local_file_path] = local_file_path
+      end
+      config
     end
 
     def shortcut_request_controls(strategy:, max_redirects:, max_requests:)

data/schema/html2rss-config.schema.json

@@ -179,6 +179,12 @@
                     "type": "null"
                   },
                   "exclusiveMinimum": 0
+                },
+                "fallback_anchorless": {
+                  "type": "boolean",
+                  "not": {
+                    "type": "null"
+                  }
                 }
               },
               "required": []
@@ -227,7 +233,8 @@
           "html": {
             "enabled": true,
             "minimum_selector_frequency": 2,
-            "use_top_selectors": 5
+            "use_top_selectors": 5,
+            "fallback_anchorless": true
           }
         },
         "cleanup": {
@@ -535,6 +542,10 @@
             }
           },
           "required": []
+        },
+        "local_file_path": {
+          "type": "string",
+          "minLength": 1
         }
       },
       "required": []

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: html2rss
 version: !ruby/object:Gem::Version
-  version: 0.20.1
+  version: 0.22.0
 platform: ruby
 authors:
 - Gil Desmarais
@@ -147,20 +147,6 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '2.0'
-- !ruby/object:Gem::Dependency
-  name: parallel
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: puppeteer-ruby
   requirement: !ruby/object:Gem::Requirement
@@ -292,6 +278,7 @@ files:
 - lib/html2rss/auto_source/cleanup.rb
 - lib/html2rss/auto_source/scraper.rb
 - lib/html2rss/auto_source/scraper/html.rb
+- lib/html2rss/auto_source/scraper/html/class_clustering.rb
 - lib/html2rss/auto_source/scraper/json_state.rb
 - lib/html2rss/auto_source/scraper/link_heuristics.rb
 - lib/html2rss/auto_source/scraper/microdata.rb
@@ -324,10 +311,13 @@ files:
 - lib/html2rss/html_extractor.rb
 - lib/html2rss/html_extractor/date_extractor.rb
 - lib/html2rss/html_extractor/enclosure_extractor.rb
+- lib/html2rss/html_extractor/heading_extractor.rb
+- lib/html2rss/html_extractor/id_generator.rb
 - lib/html2rss/html_extractor/image_extractor.rb
 - lib/html2rss/html_extractor/list_candidates.rb
 - lib/html2rss/html_extractor/semantic_anchor_candidates.rb
 - lib/html2rss/html_extractor/semantic_containers.rb
+- lib/html2rss/html_extractor/text_extractor.rb
 - lib/html2rss/html_navigator.rb
 - lib/html2rss/json_feed_builder.rb
 - lib/html2rss/json_feed_builder/item.rb
@@ -346,6 +336,7 @@ files:
 - lib/html2rss/request_service/budget.rb
 - lib/html2rss/request_service/context.rb
 - lib/html2rss/request_service/faraday_strategy.rb
+- lib/html2rss/request_service/local_file_strategy.rb
 - lib/html2rss/request_service/policy.rb
 - lib/html2rss/request_service/puppet_commander.rb
 - lib/html2rss/request_service/response.rb
@@ -390,7 +381,7 @@ licenses:
 - MIT
 metadata:
   allowed_push_host: https://rubygems.org
-  changelog_uri: https://github.com/html2rss/html2rss/releases/tag/v0.20.1
+  changelog_uri: https://github.com/html2rss/html2rss/releases/tag/v0.22.0
   rubygems_mfa_required: 'true'
 rdoc_options: []
 require_paths:
@@ -399,7 +390,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '3.2'
+      version: '3.3'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="