html2rss 0.33.0 → 0.34.0

data/lib/html2rss/url.rb

@@ -26,24 +26,35 @@ module Html2rss
   class Url
     include Comparable
 
+    # Closed set of query params that carry campaign or referrer data, never page identity.
+    TRACKING_PARAM = /\A(?:utm_.*|fbclid|gclid|msclkid|mc_cid|mc_eid|ref|ref_src)\z/i
+
+    NO_PARAMS = [].freeze
+    private_constant :NO_PARAMS
+
     # Raised when a string cannot be parsed or resolved to a valid URL.
-    class InvalidUrlError < Html2rss::Error; end
+    class InvalidUrlError < Html2rss::Error
+      # @return [Symbol]
+      def kind = :invalid_url
+    end
 
     # Regular expression for basic URI format validation
     URI_REGEXP = Addressable::URI::URIREGEX
     # Schemes accepted by channel URL validation.
     SUPPORTED_SCHEMES = %w[http https].to_set.freeze
+    SCHEME_PREFIX = /\A([a-z][a-z0-9+.-]*):/i
+    private_constant :SCHEME_PREFIX
 
     ##
     # Creates a URL from a relative path and base URL.
     #
     # @param relative_url [String, Html2rss::Url] the relative URL to resolve
     # @param base_url [String, Html2rss::Url] the base URL to resolve against
-    # @return [Url] the resolved absolute URL
+    # @return [Url] the resolved, normalized absolute URL
     # @raise [InvalidUrlError] if the URL cannot be parsed
     def self.from_relative(relative_url, base_url)
-      url = Addressable::URI.parse(relative_url.to_s.strip)
-      return new(url) if url.absolute?
+      url = parse(relative_url.to_s.strip, base_url.to_s[SCHEME_PREFIX, 1])
+      return new(url.normalize) if url.absolute?
 
       join_parsed_base(base_url, url)
     rescue Addressable::URI::InvalidURIError
@@ -59,6 +70,30 @@ module Html2rss
     end
     private_class_method :join_parsed_base
 
+    ##
+    # The one parse entry. Like browsers (WHATWG URL), a backslash before the query or fragment of an
+    # http(s) URL is a path separator: +files\a.pdf+ is +files/a.pdf+ and +https:\\example.com\x+ is
+    # +https://example.com/x+. A backslash ends the authority, so +https://example.com\@host.example/+
+    # keeps +example.com+ as host. Query and fragment, and non-http(s) URLs, are left untouched.
+    #
+    # @param text [String] raw URL or reference
+    # @param base_scheme [String, nil] scheme a scheme-less +text+ resolves against
+    # @return [Addressable::URI]
+    # @raise [Addressable::URI::InvalidURIError]
+    def self.parse(text, base_scheme = nil)
+      return Addressable::URI.parse(text) unless text.include?('\\') && web_scheme?(text, base_scheme)
+
+      head, separator, tail = text.partition(/[?#]/)
+      Addressable::URI.parse("#{head.tr('\\', '/')}#{separator}#{tail}")
+    end
+    private_class_method :parse
+
+    # @return [Boolean] whether +text+, or the +base_scheme+ a scheme-less +text+ resolves against, is http(s)
+    def self.web_scheme?(text, base_scheme)
+      SUPPORTED_SCHEMES.include?((text[SCHEME_PREFIX, 1] || base_scheme)&.downcase)
+    end
+    private_class_method :web_scheme?
+
     ##
     # Creates a URL by sanitizing a raw URL string.
     # Removes spaces and extracts the first valid URL from the string.
@@ -66,10 +101,10 @@ module Html2rss
     # @param raw_url [String] the raw URL string to sanitize
     # @return [Url, nil] the sanitized URL, or nil if no valid URL found
     def self.sanitize(raw_url)
-      match = raw_url.to_s.match(%r{(?:(?:https?|ftp)://|mailto:)[^\s<>"]+})
+      match = raw_url.to_s.match(%r{(?:https?:[/\\]{2}|ftp://|mailto:)[^\s<>"]+})
       return unless match
 
-      new(Addressable::URI.parse(match[0]).normalize)
+      new(parse(match[0]).normalize)
     end
 
     ##
@@ -81,7 +116,7 @@ module Html2rss
     def self.from_absolute(url_string)
       return url_string if url_string.is_a?(self)
 
-      new(Addressable::URI.parse(url_string.to_s.lstrip).normalize).tap do |url|
+      new(parse(url_string.to_s.lstrip).normalize).tap do |url|
         raise InvalidUrlError, 'URL must be absolute' unless url.absolute?
       end
     rescue Addressable::URI::InvalidURIError
@@ -118,6 +153,23 @@ module Html2rss
       from_absolute(url_string).tap { validate_scheme(_1) }
     end
 
+    ##
+    # The one resolver for a declared web reference (a media file, an item link, a player +src+, an
+    # advertised feed): +ref+ resolved against +base_url+ when relative, kept only when http(s) ({#web?}).
+    # A +javascript:+, +data:+, +ftp:+ or +mailto:+ reference, or a relative one without a base, names none.
+    #
+    # @param ref [String, Html2rss::Url, nil] an absolute or relative URL, possibly padded with whitespace
+    # @param base_url [String, Html2rss::Url, nil] the document the reference appears in
+    # @return [Url, nil] the absolute http(s) URL, or nil when +ref+ is blank, unparseable or not http(s)
+    def self.resolve_web(ref, base_url = nil)
+      return unless ref.to_s.match?(/\S/)
+
+      url = from_relative(ref, base_url)
+      url if url.web?
+    rescue InvalidUrlError
+      nil
+    end
+
     ##
     # Extracts a base URL from HTML metadata tags.
     #
@@ -150,16 +202,18 @@ module Html2rss
     end
 
     def self.validate_scheme(url)
-      scheme = url.scheme
-      raise InvalidUrlError, "URL scheme '#{scheme}' is not supported" unless SUPPORTED_SCHEMES.include?(scheme)
+      raise InvalidUrlError, "URL scheme '#{url.scheme}' is not supported" unless url.web?
     end
 
     private_class_method :validate_channel_url, :validate_scheme
 
     ##
-    # @param uri [Addressable::URI] the underlying Addressable::URI object (internal use only)
+    # The wrapped URI stays unfrozen and private: freezing makes Addressable recompute every
+    # normalized part that +normalize+ just produced. Mutating operations dup first.
+    #
+    # @param uri [Addressable::URI] the normalized Addressable::URI (internal use only)
     def initialize(uri)
-      @uri = uri.freeze
+      @uri = uri
       @path_segments = path.split('/').reject(&:empty?).freeze
       freeze
     end
@@ -176,8 +230,8 @@ module Html2rss
     # @return [String, nil] userinfo user, without the password
     def user = @uri.user
 
-    # Registrable domain (eTLD+1). Same-site hosts like www.wp.pl and wiadomosci.wp.pl
-    # share `wp.pl`. IP literals stay the host so 1.2.3.4 is not treated as `3.4`.
+    # Registrable domain (eTLD+1). Same-site hosts like www.example.com and news.example.com
+    # share `example.com`. IP literals stay the host so 1.2.3.4 is not treated as `3.4`.
     #
     # @return [String, nil]
     def domain
@@ -201,6 +255,10 @@ module Html2rss
     # @return [Boolean] whether the URL includes scheme and host
     def absolute? = @uri.absolute?
 
+    # @return [Boolean] whether the URL is an absolute http(s) URL ({SUPPORTED_SCHEMES}), the only kind a feed
+    #   ships as a link, enclosure or player source
+    def web? = SUPPORTED_SCHEMES.include?(scheme)
+
     ##
     # Returns the URL query string as a hash of string keys and values.
     #
@@ -232,6 +290,30 @@ module Html2rss
       fragment ? self.class.from_absolute(@uri.omit(:fragment)) : self
     end
 
+    ##
+    # Returns a copy of the URL with {TRACKING_PARAM} query params removed; the remaining
+    # params keep their order.
+    #
+    # @return [Url] this URL when it carries no tracking param; otherwise a new URL
+    def without_tracking_params
+      return self if query_params.none? { tracking_param?(_1) }
+
+      uri = @uri.dup
+      uri.query = query_params.reject { tracking_param?(_1) }.join('&')
+      self.class.from_absolute(uri)
+    end
+
+    ##
+    # Returns a copy of the URL with the provided scheme.
+    #
+    # @param scheme [String] for example +https+
+    # @return [Url] a new URL with the updated scheme
+    def with_scheme(scheme)
+      uri = @uri.dup
+      uri.scheme = scheme
+      self.class.from_absolute(uri)
+    end
+
     ##
     # Returns a copy of the URL with the provided query values.
     #
@@ -328,9 +410,20 @@ module Html2rss
       self.class.new(@uri.join(relative_uri).normalize)
     end
 
+    ##
+    # "Same article" key: host, port, slash-stripped path, and the sorted query params minus
+    # {TRACKING_PARAM}. Scheme, fragment, trailing slash, tracking params, and param order are
+    # ignored, so +http://example.com/a+ and +https://example.com/a/?utm_source=x#top+ share it;
+    # every other query param stays significant. Keys page placement, self-link rejection, and
+    # feed-item dedupe, unlike {#same_document?}, which stays scheme- and query-exact.
+    #
+    # @return [Array] equal for URLs naming the same article
+    def article_identity = [host, port, path.chomp('/'), article_params]
+
     protected
 
-    # Scheme, authority, slash-stripped path, and query (fragment omitted).
+    # Same-document key: scheme, authority, slash-stripped path, and query (fragment omitted).
+    # Two URLs share it exactly when {#same_document?} holds.
     #
     # @return [String]
     def document_identity
@@ -339,6 +432,14 @@ module Html2rss
 
     private
 
+    # @return [Array<String>] raw +key=value+ query pairs, in source order
+    def query_params = query ? query.split('&') : NO_PARAMS
+
+    # @return [Array<String>] sorted non-tracking query pairs; the shared empty list without a query
+    def article_params = query ? query_params.reject { tracking_param?(_1) }.sort : NO_PARAMS
+
+    def tracking_param?(pair) = TRACKING_PARAM.match?(pair.partition('=').first)
+
     def ip_address_host?
       !IPAddr.new(host).nil?
     rescue IPAddr::Error

data/lib/html2rss/version.rb

@@ -2,6 +2,6 @@
 
 module Html2rss
   # Current application version.
-  VERSION = '0.33.0'
+  VERSION = '0.34.0'
   public_constant :VERSION
 end

data/lib/html2rss/word_count.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Html2rss
+  ##
+  # Counts text in two facts. {.count} is the one definition of a word for the length rules on text (title
+  # floor, short anchor labels, description richness): a run of letters and digits, where Han, Hiragana
+  # and Katakana text, which carries no spaces, counts one word per {CJK_CHUNK} characters
+  # (+アクメ 会員の登録情報…+ is several words, not one). {.runs} counts raw letter/digit runs, an
+  # unspaced CJK run as one: structural scoring ({Scoring::ContainerAssessor}, {SST::Node#run_count}) is
+  # calibrated on it, and counting CJK chunks there moves which container wins.
+  module WordCount
+    # A run of letters and digits holding at least one Han, Hiragana or Katakana character.
+    CJK = /[\p{Han}\p{Hiragana}\p{Katakana}]/
+    # Characters of unspaced CJK text that make up one word.
+    CJK_CHUNK = 6
+    # A run of letters and digits.
+    RUN = /\p{Alnum}+/
+
+    class << self
+      ##
+      # @param text [String, nil]
+      # @param ceiling [Integer, nil] stops counting once this many words are seen
+      # @return [Integer] words in +text+, or at least +ceiling+ when counting stopped there
+      def count(text, ceiling: nil)
+        total = 0
+        text.to_s.scan(RUN) do |run|
+          total += run.match?(CJK) ? run.length.ceildiv(CJK_CHUNK) : 1
+          return total if ceiling && total >= ceiling
+        end
+        total
+      end
+
+      ##
+      # @param text [String, nil]
+      # @param limit [Integer]
+      # @return [Boolean] whether +text+ holds at least +limit+ words
+      def at_least?(text, limit) = count(text, ceiling: limit) >= limit
+
+      ##
+      # @param text [String, nil]
+      # @param ceiling [Integer, nil] stops counting once this many runs are seen
+      # @return [Integer] letter/digit runs in +text+, or +ceiling+ when counting stopped there
+      def runs(text, ceiling: nil)
+        total = 0
+        text.to_s.scan(RUN) do
+          total += 1
+          return total if total == ceiling
+        end
+        total
+      end
+    end
+  end
+end

data/lib/html2rss.rb

@@ -106,11 +106,14 @@ module Html2rss # rubocop:disable Metrics/ModuleLength
   # @param strategy [Symbol, nil] optional strategy override
   # @param strict_quality [Boolean] when true, fail on ship-quality audit thresholds
   # @param compare_enhance [Boolean] diagnostic enhance off vs on comparison on cached HTML
+  # @param sample_limit [Integer] maximum number of +Result#items+, >= 0 (default: 3)
   # @return [Html2rss::Test::Result]
   def self.test(config_input, feed_name = nil, min_items: 1, params: {}, strategy: nil, strict_quality: false, # rubocop:disable Metrics/ParameterLists
-                compare_enhance: false)
+                compare_enhance: false, sample_limit: 3)
     Option.integer(:min_items, min_items, min: 0)
-    Test.call(config_input, feed_name, min_items:, params:, strategy:, strict_quality:, compare_enhance:)
+    Option.integer(:sample_limit, sample_limit, min: 0)
+    Test.call(config_input, feed_name, min_items:, params:, strategy:, strict_quality:, compare_enhance:,
+                                       sample_limit:)
   end
 
   ##
@@ -136,6 +139,7 @@ module Html2rss # rubocop:disable Metrics/ModuleLength
   # @param local_file_path [String, nil] optional local HTML file path
   # @param limit [Integer, nil] max articles to keep (default: {AutoSource::DEFAULT_LIMIT})
   # @param keep_different_domain [Boolean, nil] when set, overrides cleanup.keep_different_domain
+  # @param keep_sponsored [Boolean, nil] when set, overrides cleanup.keep_sponsored
   # @return [Html2rss::FeedResult]
   def self.scrape(url,
                   strategy: :auto,
@@ -144,9 +148,10 @@ module Html2rss # rubocop:disable Metrics/ModuleLength
                   max_requests: 4,
                   local_file_path: nil,
                   limit: nil,
-                  keep_different_domain: nil)
+                  keep_different_domain: nil,
+                  keep_sponsored: nil)
     auto_feed_result(url, strategy:, items_selector:, max_redirects:, max_requests:,
-                          local_file_path:, limit:, keep_different_domain:)
+                          local_file_path:, limit:, keep_different_domain:, keep_sponsored:)
   end
 
   ##
@@ -250,6 +255,7 @@ module Html2rss # rubocop:disable Metrics/ModuleLength
   # @param local_file_path [String, nil] optional local HTML file path
   # @param limit [Integer, nil] max articles to keep (default: {AutoSource::DEFAULT_LIMIT})
   # @param keep_different_domain [Boolean, nil] when set, overrides cleanup.keep_different_domain
+  # @param keep_sponsored [Boolean, nil] when set, overrides cleanup.keep_sponsored
   # @return [Html2rss::FeedResult]
   def self.auto_feed_result(url,
                             strategy: :auto,
@@ -258,9 +264,10 @@ module Html2rss # rubocop:disable Metrics/ModuleLength
                             max_requests: 4,
                             local_file_path: nil,
                             limit: nil,
-                            keep_different_domain: nil)
+                            keep_different_domain: nil,
+                            keep_sponsored: nil)
     feed_result(build_auto_source_config(url:, strategy:, items_selector:, max_redirects:, max_requests:,
-                                         local_file_path:, limit:, keep_different_domain:))
+                                         local_file_path:, limit:, keep_different_domain:, keep_sponsored:))
   end
 
   ##
@@ -360,13 +367,15 @@ module Html2rss # rubocop:disable Metrics/ModuleLength
 
     # rubocop:disable-next Metrics/ParameterLists, Metrics/MethodLength
     def build_auto_source_config(url:, strategy:, items_selector:, max_redirects:, max_requests:,
-                                 local_file_path: nil, limit: nil, keep_different_domain: nil)
+                                 local_file_path: nil, limit: nil, keep_different_domain: nil,
+                                 keep_sponsored: nil)
       config = Config.auto_source_config(
         url:,
         items_selector:,
         request_controls: Config::RequestControls.from_shortcut(strategy:, max_redirects:, max_requests:),
         limit:,
-        keep_different_domain:
+        keep_different_domain:,
+        keep_sponsored:
       )
       if local_file_path
         config[:request] ||= {}

data/schema/html2rss-config.schema.json

@@ -13,7 +13,8 @@
       "properties": {
         "url": {
           "type": "string",
-          "minLength": 1
+          "minLength": 1,
+          "pattern": "^(([^:\\/?#]+):)?(\\/\\/([^\\/?#]*))?([^?#]*)(\\?([^#]*))?(#(.*))?$"
         },
         "title": {
           "type": [
@@ -31,7 +32,8 @@
           "type": [
             "null",
             "string"
-          ]
+          ],
+          "pattern": "\\A[a-z]{2}(-[A-Z]{2})?\\z"
         },
         "ttl": {
           "type": [
@@ -56,7 +58,8 @@
           "type": [
             "null",
             "string"
-          ]
+          ],
+          "pattern": "^(([^:\\/?#]+):)?(\\/\\/([^\\/?#]*))?([^?#]*)(\\?([^#]*))?(#(.*))?$"
         }
       },
       "required": [
@@ -79,8 +82,8 @@
         },
         "topics": {
           "type": "array",
+          "minItems": 1,
           "items": {
-            "minLength": 1,
             "type": "string",
             "enum": [
               "sports",
@@ -104,8 +107,7 @@
               "rights",
               "transport"
             ]
-          },
-          "minItems": 1
+          }
         }
       },
       "required": [
@@ -194,7 +196,8 @@
                   }
                 }
               },
-              "required": []
+              "required": [],
+              "additionalProperties": false
             },
             "wordpress_api": {
               "type": "object",
@@ -206,7 +209,8 @@
                   }
                 }
               },
-              "required": []
+              "required": [],
+              "additionalProperties": false
             },
             "sitemap": {
               "type": "object",
@@ -218,7 +222,14 @@
                   }
                 },
                 "min_priority": {
-                  "type": "number",
+                  "anyOf": [
+                    {
+                      "type": "integer"
+                    },
+                    {
+                      "type": "number"
+                    }
+                  ],
                   "not": {
                     "type": "null"
                   }
@@ -231,7 +242,8 @@
                   "exclusiveMinimum": 0
                 }
               },
-              "required": []
+              "required": [],
+              "additionalProperties": false
             },
             "schema": {
               "type": "object",
@@ -243,7 +255,8 @@
                   }
                 }
               },
-              "required": []
+              "required": [],
+              "additionalProperties": false
             },
             "microdata": {
               "type": "object",
@@ -255,7 +268,8 @@
                   }
                 }
               },
-              "required": []
+              "required": [],
+              "additionalProperties": false
             },
             "microformats2": {
               "type": "object",
@@ -267,7 +281,8 @@
                   }
                 }
               },
-              "required": []
+              "required": [],
+              "additionalProperties": false
             },
             "json_state": {
               "type": "object",
@@ -279,7 +294,8 @@
                   }
                 }
               },
-              "required": []
+              "required": [],
+              "additionalProperties": false
             },
             "json_document": {
               "type": "object",
@@ -291,7 +307,8 @@
                   }
                 }
               },
-              "required": []
+              "required": [],
+              "additionalProperties": false
             },
             "xhr_articles": {
               "type": "object",
@@ -303,7 +320,8 @@
                   }
                 }
               },
-              "required": []
+              "required": [],
+              "additionalProperties": false
             },
             "meta_oembed": {
               "type": "object",
@@ -315,7 +333,8 @@
                   }
                 }
               },
-              "required": []
+              "required": [],
+              "additionalProperties": false
             },
             "semantic_html": {
               "type": "object",
@@ -325,15 +344,10 @@
                   "not": {
                     "type": "null"
                   }
-                },
-                "fallback_anchorless": {
-                  "type": "boolean",
-                  "not": {
-                    "type": "null"
-                  }
                 }
               },
-              "required": []
+              "required": [],
+              "additionalProperties": false
             },
             "html": {
               "type": "object",
@@ -357,18 +371,14 @@
                     "type": "null"
                   },
                   "exclusiveMinimum": 0
-                },
-                "fallback_anchorless": {
-                  "type": "boolean",
-                  "not": {
-                    "type": "null"
-                  }
                 }
               },
-              "required": []
+              "required": [],
+              "additionalProperties": false
             }
           },
-          "required": []
+          "required": [],
+          "additionalProperties": false
         },
         "cleanup": {
           "type": "object",
@@ -378,6 +388,12 @@
               "not": {
                 "type": "null"
               }
+            },
+            "keep_sponsored": {
+              "type": "boolean",
+              "not": {
+                "type": "null"
+              }
             }
           },
           "required": []
@@ -424,18 +440,17 @@
             "enabled": true
           },
           "semantic_html": {
-            "enabled": true,
-            "fallback_anchorless": true
+            "enabled": true
           },
           "html": {
             "enabled": true,
             "minimum_selector_frequency": 2,
-            "use_top_selectors": 5,
-            "fallback_anchorless": true
+            "use_top_selectors": 5
           }
         },
         "cleanup": {
-          "keep_different_domain": false
+          "keep_different_domain": false,
+          "keep_sponsored": false
         }
       }
     },
@@ -465,7 +480,7 @@
               "not": {
                 "type": "null"
               },
-              "description": "List-card enrichment: run Html::ArticleExtractor on each matched item node to fill missing fields from the card HTML."
+              "description": "Fills missing title, link, description, date, image and categories from each item's own markup via Html::ArticleExtractor. Explicit selectors win; no extra requests are made."
             },
             "pagination": {
               "description": "Pagination configuration or maximum page count integer.",
@@ -589,7 +604,8 @@
             },
             "content_type": {
               "type": "string",
-              "minLength": 1
+              "minLength": 1,
+              "pattern": "^[\\w-]+\\/[\\w-]+$"
             }
           },
           "required": [],
@@ -726,8 +742,7 @@
                 "auto",
                 "get",
                 "google_get",
-                "google_get_bypass",
-                "organic_get"
+                "google_get_bypass"
               ]
             },
             "max_retries": {