html2rss 0.21.0 → 0.22.1

data/lib/html2rss/url.rb

@@ -2,6 +2,7 @@
 
 require 'addressable/uri'
 require 'cgi'
+require 'nokogiri'
 
 module Html2rss
   ##
@@ -55,10 +56,9 @@ module Html2rss
     # @return [Url, nil] the sanitized URL, or nil if no valid URL found
     def self.sanitize(raw_url)
       match = raw_url.to_s.match(%r{(?:(?:https?|ftp|mailto)://|mailto:)[^\s<>"]+})
-      url = match ? match[0].strip : ''
-      return nil if url.empty?
+      return unless match
 
-      new(Addressable::URI.parse(url).normalize)
+      new(Addressable::URI.parse(match[0].strip).normalize)
     end
 
     ##
@@ -70,10 +70,9 @@ module Html2rss
     def self.from_absolute(url_string)
       return url_string if url_string.is_a?(self)
 
-      url = new(Addressable::URI.parse(url_string.to_s.strip).normalize)
-      raise ArgumentError, 'URL must be absolute' unless url.absolute?
-
-      url
+      new(Addressable::URI.parse(url_string.to_s.strip).normalize).tap do |url|
+        raise ArgumentError, 'URL must be absolute' unless url.absolute?
+      end
     rescue Addressable::URI::InvalidURIError
       raise ArgumentError, 'URL must be absolute'
     end
@@ -92,14 +91,28 @@ module Html2rss
     #   Url.for_channel('/relative/path')
     #   # => raises ArgumentError: "URL must be absolute"
     def self.for_channel(url_string)
-      return nil if url_string.nil? || url_string.empty?
-
-      stripped = url_string.strip
+      stripped = url_string.to_s.strip
       return nil if stripped.empty?
 
-      url = from_absolute(stripped)
-      validate_channel_url(url)
-      url
+      from_absolute(stripped).tap { validate_channel_url(_1) }
+    end
+
+    ##
+    # Extracts a base URL from HTML metadata tags.
+    #
+    # @param html [String] raw HTML content
+    # @return [Url, nil] the extracted absolute URL, or nil if none is found
+    def self.extract_from_html(html)
+      doc = Nokogiri::HTML(html)
+      tags = { 'link[rel="canonical"]' => 'href', 'meta[property="og:url"]' => 'content',
+               'meta[name="twitter:url"]' => 'content', 'base[href]' => 'href' }
+      tags.each do |sel, attr|
+        val = doc.at_css(sel)&.[](attr).to_s.strip
+        return from_absolute(val) unless val.empty?
+      rescue ArgumentError
+        next
+      end
+      nil
     end
 
     ##

data/lib/html2rss/version.rb

@@ -4,6 +4,6 @@
 # The Html2rss namespace.
 module Html2rss
   # Current application version.
-  VERSION = '0.21.0'
+  VERSION = '0.22.1'
   public_constant :VERSION
 end

data/lib/html2rss.rb

@@ -52,6 +52,8 @@ module Html2rss
     FeedPipeline.new(raw_config).to_json_feed
   end
 
+  # rubocop:disable Metrics/ParameterLists
+
   ##
   # Scrapes the provided URL and returns an RSS object.
   #
@@ -60,9 +62,15 @@ module Html2rss
   # @param items_selector [String, nil] optional selector hint for item extraction
   # @param max_redirects [Integer, nil] optional redirect limit override
   # @param max_requests [Integer, nil] optional request budget override
+  # @param local_file_path [String, nil] optional local HTML file path
   # @return [RSS::Rss] generated RSS feed
-  def self.auto_source(url, strategy: :auto, items_selector: nil, max_redirects: nil, max_requests: nil)
-    feed(build_auto_source_config(url:, strategy:, items_selector:, max_redirects:, max_requests:))
+  def self.auto_source(url,
+                       strategy: :auto,
+                       items_selector: nil,
+                       max_redirects: nil,
+                       max_requests: nil,
+                       local_file_path: nil)
+    feed(build_auto_source_config(url:, strategy:, items_selector:, max_redirects:, max_requests:, local_file_path:))
   end
 
   ##
@@ -73,11 +81,20 @@ module Html2rss
   # @param items_selector [String, nil] optional selector hint for item extraction
   # @param max_redirects [Integer, nil] optional redirect limit override
   # @param max_requests [Integer, nil] optional request budget override
+  # @param local_file_path [String, nil] optional local HTML file path
   # @return [Hash] JSONFeed-compliant hash
-  def self.auto_json_feed(url, strategy: :auto, items_selector: nil, max_redirects: nil, max_requests: nil)
-    json_feed(build_auto_source_config(url:, strategy:, items_selector:, max_redirects:, max_requests:))
+  def self.auto_json_feed(url,
+                          strategy: :auto,
+                          items_selector: nil,
+                          max_redirects: nil,
+                          max_requests: nil,
+                          local_file_path: nil)
+    json_feed(build_auto_source_config(url:, strategy:, items_selector:, max_redirects:, max_requests:,
+                                       local_file_path:))
   end
 
+  # rubocop:enable Metrics/ParameterLists
+
   # rubocop:disable ThreadSafety/ClassInstanceVariable
   class << self
     ##
@@ -125,12 +142,17 @@ module Html2rss
   class << self
     private
 
-    def build_auto_source_config(url:, strategy:, items_selector:, max_redirects:, max_requests:)
-      Config.auto_source_config(
+    def build_auto_source_config(url:, strategy:, items_selector:, max_redirects:, max_requests:, local_file_path: nil) # rubocop:disable Metrics/ParameterLists
+      config = Config.auto_source_config(
         url:,
         items_selector:,
         request_controls: shortcut_request_controls(strategy:, max_redirects:, max_requests:)
       )
+      if local_file_path
+        config[:request] ||= {}
+        config[:request][:local_file_path] = local_file_path
+      end
+      config
     end
 
     def shortcut_request_controls(strategy:, max_redirects:, max_requests:)

data/schema/html2rss-config.schema.json

@@ -153,6 +153,12 @@
                   "not": {
                     "type": "null"
                   }
+                },
+                "fallback_anchorless": {
+                  "type": "boolean",
+                  "not": {
+                    "type": "null"
+                  }
                 }
               },
               "required": []
@@ -179,6 +185,12 @@
                     "type": "null"
                   },
                   "exclusiveMinimum": 0
+                },
+                "fallback_anchorless": {
+                  "type": "boolean",
+                  "not": {
+                    "type": "null"
+                  }
                 }
               },
               "required": []
@@ -222,12 +234,14 @@
             "enabled": true
           },
           "semantic_html": {
-            "enabled": true
+            "enabled": true,
+            "fallback_anchorless": true
           },
           "html": {
             "enabled": true,
             "minimum_selector_frequency": 2,
-            "use_top_selectors": 5
+            "use_top_selectors": 5,
+            "fallback_anchorless": true
           }
         },
         "cleanup": {
@@ -535,6 +549,10 @@
             }
           },
           "required": []
+        },
+        "local_file_path": {
+          "type": "string",
+          "minLength": 1
         }
       },
       "required": []

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: html2rss
 version: !ruby/object:Gem::Version
-  version: 0.21.0
+  version: 0.22.1
 platform: ruby
 authors:
 - Gil Desmarais
@@ -278,6 +278,7 @@ files:
 - lib/html2rss/auto_source/cleanup.rb
 - lib/html2rss/auto_source/scraper.rb
 - lib/html2rss/auto_source/scraper/html.rb
+- lib/html2rss/auto_source/scraper/html/class_clustering.rb
 - lib/html2rss/auto_source/scraper/json_state.rb
 - lib/html2rss/auto_source/scraper/link_heuristics.rb
 - lib/html2rss/auto_source/scraper/microdata.rb
@@ -310,10 +311,13 @@ files:
 - lib/html2rss/html_extractor.rb
 - lib/html2rss/html_extractor/date_extractor.rb
 - lib/html2rss/html_extractor/enclosure_extractor.rb
+- lib/html2rss/html_extractor/heading_extractor.rb
+- lib/html2rss/html_extractor/id_generator.rb
 - lib/html2rss/html_extractor/image_extractor.rb
 - lib/html2rss/html_extractor/list_candidates.rb
 - lib/html2rss/html_extractor/semantic_anchor_candidates.rb
 - lib/html2rss/html_extractor/semantic_containers.rb
+- lib/html2rss/html_extractor/text_extractor.rb
 - lib/html2rss/html_navigator.rb
 - lib/html2rss/json_feed_builder.rb
 - lib/html2rss/json_feed_builder/item.rb
@@ -332,6 +336,7 @@ files:
 - lib/html2rss/request_service/budget.rb
 - lib/html2rss/request_service/context.rb
 - lib/html2rss/request_service/faraday_strategy.rb
+- lib/html2rss/request_service/local_file_strategy.rb
 - lib/html2rss/request_service/policy.rb
 - lib/html2rss/request_service/puppet_commander.rb
 - lib/html2rss/request_service/response.rb
@@ -376,7 +381,7 @@ licenses:
 - MIT
 metadata:
   allowed_push_host: https://rubygems.org
-  changelog_uri: https://github.com/html2rss/html2rss/releases/tag/v0.21.0
+  changelog_uri: https://github.com/html2rss/html2rss/releases/tag/v0.22.1
   rubygems_mfa_required: 'true'
 rdoc_options: []
 require_paths: