html2rss 0.32.4 → 0.34.0

data/lib/html2rss/url.rb

@@ -26,37 +26,74 @@ module Html2rss
   class Url
     include Comparable
 
+    # Closed set of query params that carry campaign or referrer data, never page identity.
+    TRACKING_PARAM = /\A(?:utm_.*|fbclid|gclid|msclkid|mc_cid|mc_eid|ref|ref_src)\z/i
+
+    NO_PARAMS = [].freeze
+    private_constant :NO_PARAMS
+
+    # Raised when a string cannot be parsed or resolved to a valid URL.
+    class InvalidUrlError < Html2rss::Error
+      # @return [Symbol]
+      def kind = :invalid_url
+    end
+
     # Regular expression for basic URI format validation
     URI_REGEXP = Addressable::URI::URIREGEX
     # Schemes accepted by channel URL validation.
     SUPPORTED_SCHEMES = %w[http https].to_set.freeze
+    SCHEME_PREFIX = /\A([a-z][a-z0-9+.-]*):/i
+    private_constant :SCHEME_PREFIX
 
     ##
     # Creates a URL from a relative path and base URL.
     #
     # @param relative_url [String, Html2rss::Url] the relative URL to resolve
     # @param base_url [String, Html2rss::Url] the base URL to resolve against
-    # @return [Url] the resolved absolute URL
-    # @raise [ArgumentError] if the URL cannot be parsed
+    # @return [Url] the resolved, normalized absolute URL
+    # @raise [InvalidUrlError] if the URL cannot be parsed
     def self.from_relative(relative_url, base_url)
-      url = Addressable::URI.parse(relative_url.to_s.strip)
-      return new(url) if url.absolute?
+      url = parse(relative_url.to_s.strip, base_url.to_s[SCHEME_PREFIX, 1])
+      return new(url.normalize) if url.absolute?
 
-      base_url.is_a?(self) ? base_url.join_uri(url) : join_parsed_base(base_url, url)
+      join_parsed_base(base_url, url)
     rescue Addressable::URI::InvalidURIError
-      raise ArgumentError, 'URL could not be parsed'
+      raise InvalidUrlError, 'URL could not be parsed'
     end
 
-    # @param base_url [String]
+    # @param base_url [String, Url]
     # @param url [Addressable::URI]
     # @return [Url]
     def self.join_parsed_base(base_url, url)
       uri = Addressable::URI.parse(base_url.to_s)
-      uri.path = '/' if uri.path.empty?
       new(uri.join(url).normalize)
     end
     private_class_method :join_parsed_base
 
+    ##
+    # The one parse entry. Like browsers (WHATWG URL), a backslash before the query or fragment of an
+    # http(s) URL is a path separator: +files\a.pdf+ is +files/a.pdf+ and +https:\\example.com\x+ is
+    # +https://example.com/x+. A backslash ends the authority, so +https://example.com\@host.example/+
+    # keeps +example.com+ as host. Query and fragment, and non-http(s) URLs, are left untouched.
+    #
+    # @param text [String] raw URL or reference
+    # @param base_scheme [String, nil] scheme a scheme-less +text+ resolves against
+    # @return [Addressable::URI]
+    # @raise [Addressable::URI::InvalidURIError]
+    def self.parse(text, base_scheme = nil)
+      return Addressable::URI.parse(text) unless text.include?('\\') && web_scheme?(text, base_scheme)
+
+      head, separator, tail = text.partition(/[?#]/)
+      Addressable::URI.parse("#{head.tr('\\', '/')}#{separator}#{tail}")
+    end
+    private_class_method :parse
+
+    # @return [Boolean] whether +text+, or the +base_scheme+ a scheme-less +text+ resolves against, is http(s)
+    def self.web_scheme?(text, base_scheme)
+      SUPPORTED_SCHEMES.include?((text[SCHEME_PREFIX, 1] || base_scheme)&.downcase)
+    end
+    private_class_method :web_scheme?
+
     ##
     # Creates a URL by sanitizing a raw URL string.
     # Removes spaces and extracts the first valid URL from the string.
@@ -64,10 +101,10 @@ module Html2rss
     # @param raw_url [String] the raw URL string to sanitize
     # @return [Url, nil] the sanitized URL, or nil if no valid URL found
     def self.sanitize(raw_url)
-      match = raw_url.to_s.match(%r{(?:(?:https?|ftp|mailto)://|mailto:)[^\s<>"]+})
+      match = raw_url.to_s.match(%r{(?:https?:[/\\]{2}|ftp://|mailto:)[^\s<>"]+})
       return unless match
 
-      new(Addressable::URI.parse(match[0].strip).normalize)
+      new(parse(match[0]).normalize)
     end
 
     ##
@@ -75,15 +112,15 @@ module Html2rss
     #
     # @param url_string [String, Html2rss::Url] the absolute URL to parse
     # @return [Url] the parsed and normalized URL
-    # @raise [ArgumentError] if the URL is not absolute or cannot be parsed
+    # @raise [InvalidUrlError] if the URL is not absolute or cannot be parsed
     def self.from_absolute(url_string)
       return url_string if url_string.is_a?(self)
 
-      new(Addressable::URI.parse(url_string.to_s.strip).normalize).tap do |url|
-        raise ArgumentError, 'URL must be absolute' unless url.absolute?
+      new(parse(url_string.to_s.lstrip).normalize).tap do |url|
+        raise InvalidUrlError, 'URL must be absolute' unless url.absolute?
       end
     rescue Addressable::URI::InvalidURIError
-      raise ArgumentError, 'URL must be absolute'
+      raise InvalidUrlError, 'URL must be absolute'
     end
 
     ##
@@ -92,20 +129,47 @@ module Html2rss
     #
     # @param url_string [String] the URL string to validate and parse
     # @return [Url] the validated and parsed URL
-    # @raise [ArgumentError] if the URL doesn't meet channel requirements
+    # @raise [InvalidUrlError] if the URL doesn't meet channel requirements
     # @example Creating a channel URL
     #   Url.for_channel('https://example.com')
     #   # => #<Html2rss::Url:... @uri=#<Addressable::URI:... URI:https://example.com>>
     # @example Invalid channel URL
     #   Url.for_channel('/relative/path')
-    #   # => raises ArgumentError: "URL must be absolute"
+    #   # => raises InvalidUrlError: "URL must be absolute"
     def self.for_channel(url_string)
-      stripped = url_string.to_s.strip
+      stripped = url_string.to_s.lstrip
       return nil if stripped.empty?
 
       from_absolute(stripped).tap { validate_channel_url(_1) }
     end
 
+    ##
+    # Creates an absolute http(s) URL, for example for a channel image.
+    #
+    # @param url_string [String, Html2rss::Url] the URL to validate and parse
+    # @return [Url] the validated and parsed URL
+    # @raise [InvalidUrlError] if the URL is not absolute or not http(s)
+    def self.from_absolute_web(url_string)
+      from_absolute(url_string).tap { validate_scheme(_1) }
+    end
+
+    ##
+    # The one resolver for a declared web reference (a media file, an item link, a player +src+, an
+    # advertised feed): +ref+ resolved against +base_url+ when relative, kept only when http(s) ({#web?}).
+    # A +javascript:+, +data:+, +ftp:+ or +mailto:+ reference, or a relative one without a base, names none.
+    #
+    # @param ref [String, Html2rss::Url, nil] an absolute or relative URL, possibly padded with whitespace
+    # @param base_url [String, Html2rss::Url, nil] the document the reference appears in
+    # @return [Url, nil] the absolute http(s) URL, or nil when +ref+ is blank, unparseable or not http(s)
+    def self.resolve_web(ref, base_url = nil)
+      return unless ref.to_s.match?(/\S/)
+
+      url = from_relative(ref, base_url)
+      url if url.web?
+    rescue InvalidUrlError
+      nil
+    end
+
     ##
     # Extracts a base URL from HTML metadata tags.
     #
@@ -116,10 +180,10 @@ module Html2rss
       tags = { 'link[rel="canonical"]' => 'href', 'meta[property="og:url"]' => 'content',
                'meta[name="twitter:url"]' => 'content', 'base[href]' => 'href' }
       tags.each do |sel, attr|
-        val = doc.at_css(sel)&.[](attr).to_s.strip
-        return from_absolute(val) unless val.empty?
-      rescue ArgumentError
-        next
+        val = doc.at_css(sel)&.[](attr)
+        return from_absolute(val)
+      rescue InvalidUrlError
+        nil
       end
       nil
     end
@@ -128,31 +192,34 @@ module Html2rss
     # Validates that a URL meets channel requirements.
     #
     # @param url [Url] the URL to validate
-    # @raise [ArgumentError] if the URL doesn't meet channel requirements
+    # @raise [InvalidUrlError] if the URL doesn't meet channel requirements
     def self.validate_channel_url(url)
-      raise ArgumentError, 'URL must be absolute' unless url.absolute?
+      has_forbidden_at = url.user
+      has_forbidden_at ||= [url.query, url.fragment].compact.any? { |value| value.include?('@') }
+      raise InvalidUrlError, 'URL must not contain an @ character' if has_forbidden_at
 
-      uri = Addressable::URI.parse(url.to_s)
-      has_forbidden_at = uri.user || uri.password
-      has_forbidden_at ||= [uri.query, uri.fragment].compact.any? { |value| value.include?('@') }
-      raise ArgumentError, 'URL must not contain an @ character' if has_forbidden_at
+      validate_scheme(url)
+    end
 
-      scheme = url.scheme
-      raise ArgumentError, "URL scheme '#{scheme}' is not supported" unless SUPPORTED_SCHEMES.include?(scheme)
+    def self.validate_scheme(url)
+      raise InvalidUrlError, "URL scheme '#{url.scheme}' is not supported" unless url.web?
     end
 
-    private_class_method :validate_channel_url
+    private_class_method :validate_channel_url, :validate_scheme
 
     ##
-    # @param uri [Addressable::URI] the underlying Addressable::URI object (internal use only)
+    # The wrapped URI stays unfrozen and private: freezing makes Addressable recompute every
+    # normalized part that +normalize+ just produced. Mutating operations dup first.
+    #
+    # @param uri [Addressable::URI] the normalized Addressable::URI (internal use only)
     def initialize(uri)
-      @uri = uri.freeze
-      @path_segments = @uri.path.to_s.split('/').reject(&:empty?).freeze
+      @uri = uri
+      @path_segments = path.split('/').reject(&:empty?).freeze
       freeze
     end
 
     # @return [String] normalized URL string
-    def to_s = @uri.to_s
+    def to_s = @uri.to_str
 
     # @return [String, nil] URI scheme, for example `http` or `https`
     def scheme = @uri.scheme
@@ -160,8 +227,11 @@ module Html2rss
     # @return [String, nil] URI host component
     def host = @uri.host
 
-    # Registrable domain (eTLD+1). Same-site hosts like www.wp.pl and wiadomosci.wp.pl
-    # share `wp.pl`. IP literals stay the host so 1.2.3.4 is not treated as `3.4`.
+    # @return [String, nil] userinfo user, without the password
+    def user = @uri.user
+
+    # Registrable domain (eTLD+1). Same-site hosts like www.example.com and news.example.com
+    # share `example.com`. IP literals stay the host so 1.2.3.4 is not treated as `3.4`.
     #
     # @return [String, nil]
     def domain
@@ -185,11 +255,15 @@ module Html2rss
     # @return [Boolean] whether the URL includes scheme and host
     def absolute? = @uri.absolute?
 
+    # @return [Boolean] whether the URL is an absolute http(s) URL ({SUPPORTED_SCHEMES}), the only kind a feed
+    #   ships as a link, enclosure or player source
+    def web? = SUPPORTED_SCHEMES.include?(scheme)
+
     ##
     # Returns the URL query string as a hash of string keys and values.
     #
     # @return [Hash{String => String}] normalized query parameters
-    def query_values = @uri.query_values(Hash) || {}
+    def query_values = @uri.query_values || {}
 
     ##
     # Returns the URL path split into non-empty segments.
@@ -205,7 +279,7 @@ module Html2rss
     def with_path(path)
       uri = @uri.dup
       uri.path = path
-      self.class.from_absolute(uri.normalize.to_s)
+      self.class.from_absolute(uri)
     end
 
     ##
@@ -213,7 +287,31 @@ module Html2rss
     #
     # @return [Url] this URL when already fragment-free; otherwise a new URL
     def without_fragment
-      @uri.fragment ? self.class.from_absolute(@uri.omit(:fragment).normalize.to_s) : self
+      fragment ? self.class.from_absolute(@uri.omit(:fragment)) : self
+    end
+
+    ##
+    # Returns a copy of the URL with {TRACKING_PARAM} query params removed; the remaining
+    # params keep their order.
+    #
+    # @return [Url] this URL when it carries no tracking param; otherwise a new URL
+    def without_tracking_params
+      return self if query_params.none? { tracking_param?(_1) }
+
+      uri = @uri.dup
+      uri.query = query_params.reject { tracking_param?(_1) }.join('&')
+      self.class.from_absolute(uri)
+    end
+
+    ##
+    # Returns a copy of the URL with the provided scheme.
+    #
+    # @param scheme [String] for example +https+
+    # @return [Url] a new URL with the updated scheme
+    def with_scheme(scheme)
+      uri = @uri.dup
+      uri.scheme = scheme
+      self.class.from_absolute(uri)
     end
 
     ##
@@ -224,7 +322,7 @@ module Html2rss
     def with_query_values(values)
       uri = @uri.dup
       uri.query_values = values.transform_keys(&:to_s).transform_values(&:to_s)
-      self.class.from_absolute(uri.normalize.to_s)
+      self.class.from_absolute(uri)
     end
 
     ##
@@ -240,15 +338,7 @@ module Html2rss
     #   url = Url.from_absolute('https://example.com/hello%20world/article.html')
     #   url.titleized # => "Hello World Article"
     def titleized
-      path = @uri.path
-      return '' if path.empty?
-
-      nicer_path = CGI.unescapeURIComponent(path)
-                      .split('/')
-                      .flat_map do |part|
-                        part.gsub(/[^a-zA-Z0-9.]/, ' ').gsub(/\s+/, ' ').split
-                      end
-
+      nicer_path = CGI.unescapeURIComponent(path).gsub(/[^a-zA-Z0-9.]/, ' ').split
       nicer_path.map!(&:capitalize)
       File.basename(nicer_path.join(' '), '.*')
     end
@@ -266,8 +356,7 @@ module Html2rss
     #   url = Url.from_absolute('https://example.com')
     #   url.channel_titleized # => "example.com"
     def channel_titleized
-      nicer_path = CGI.unescapeURIComponent(@uri.path).split('/').reject(&:empty?)
-      host = @uri.host
+      nicer_path = CGI.unescapeURIComponent(path).split('/').reject(&:empty?)
 
       nicer_path.any? ? "#{host}: #{nicer_path.map(&:capitalize).join(' ')}" : host
     end
@@ -318,29 +407,41 @@ module Html2rss
     # @param relative_uri [Addressable::URI]
     # @return [Url]
     def join_uri(relative_uri)
-      base = @uri
-      if base.path.empty?
-        base = base.dup
-        base.path = '/'
-      end
-      self.class.new(base.join(relative_uri).normalize)
+      self.class.new(@uri.join(relative_uri).normalize)
     end
 
+    ##
+    # "Same article" key: host, port, slash-stripped path, and the sorted query params minus
+    # {TRACKING_PARAM}. Scheme, fragment, trailing slash, tracking params, and param order are
+    # ignored, so +http://example.com/a+ and +https://example.com/a/?utm_source=x#top+ share it;
+    # every other query param stays significant. Keys page placement, self-link rejection, and
+    # feed-item dedupe, unlike {#same_document?}, which stays scheme- and query-exact.
+    #
+    # @return [Array] equal for URLs naming the same article
+    def article_identity = [host, port, path.chomp('/'), article_params]
+
     protected
 
-    # Scheme, authority, slash-stripped path, and query (fragment omitted).
+    # Same-document key: scheme, authority, slash-stripped path, and query (fragment omitted).
+    # Two URLs share it exactly when {#same_document?} holds.
     #
     # @return [String]
     def document_identity
-      uri = @uri.omit(:fragment)
-      "#{uri.scheme}\0#{uri.authority}\0#{uri.path.to_s.chomp('/')}\0#{uri.query}"
+      "#{scheme}\0#{@uri.authority}\0#{path.chomp('/')}\0#{query}"
     end
 
     private
 
+    # @return [Array<String>] raw +key=value+ query pairs, in source order
+    def query_params = query ? query.split('&') : NO_PARAMS
+
+    # @return [Array<String>] sorted non-tracking query pairs; the shared empty list without a query
+    def article_params = query ? query_params.reject { tracking_param?(_1) }.sort : NO_PARAMS
+
+    def tracking_param?(pair) = TRACKING_PARAM.match?(pair.partition('=').first)
+
     def ip_address_host?
-      IPAddr.new(host.to_s)
-      true
+      !IPAddr.new(host).nil?
     rescue IPAddr::Error
       false
     end

data/lib/html2rss/version.rb

@@ -2,6 +2,6 @@
 
 module Html2rss
   # Current application version.
-  VERSION = '0.32.4'
+  VERSION = '0.34.0'
   public_constant :VERSION
 end

data/lib/html2rss/word_count.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Html2rss
+  ##
+  # Counts text in two facts. {.count} is the one definition of a word for the length rules on text (title
+  # floor, short anchor labels, description richness): a run of letters and digits, where Han, Hiragana
+  # and Katakana text, which carries no spaces, counts one word per {CJK_CHUNK} characters
+  # (+アクメ 会員の登録情報…+ is several words, not one). {.runs} counts raw letter/digit runs, an
+  # unspaced CJK run as one: structural scoring ({Scoring::ContainerAssessor}, {SST::Node#run_count}) is
+  # calibrated on it, and counting CJK chunks there moves which container wins.
+  module WordCount
+    # A run of letters and digits holding at least one Han, Hiragana or Katakana character.
+    CJK = /[\p{Han}\p{Hiragana}\p{Katakana}]/
+    # Characters of unspaced CJK text that make up one word.
+    CJK_CHUNK = 6
+    # A run of letters and digits.
+    RUN = /\p{Alnum}+/
+
+    class << self
+      ##
+      # @param text [String, nil]
+      # @param ceiling [Integer, nil] stops counting once this many words are seen
+      # @return [Integer] words in +text+, or at least +ceiling+ when counting stopped there
+      def count(text, ceiling: nil)
+        total = 0
+        text.to_s.scan(RUN) do |run|
+          total += run.match?(CJK) ? run.length.ceildiv(CJK_CHUNK) : 1
+          return total if ceiling && total >= ceiling
+        end
+        total
+      end
+
+      ##
+      # @param text [String, nil]
+      # @param limit [Integer]
+      # @return [Boolean] whether +text+ holds at least +limit+ words
+      def at_least?(text, limit) = count(text, ceiling: limit) >= limit
+
+      ##
+      # @param text [String, nil]
+      # @param ceiling [Integer, nil] stops counting once this many runs are seen
+      # @return [Integer] letter/digit runs in +text+, or +ceiling+ when counting stopped there
+      def runs(text, ceiling: nil)
+        total = 0
+        text.to_s.scan(RUN) do
+          total += 1
+          return total if total == ceiling
+        end
+        total
+      end
+    end
+  end
+end

data/lib/html2rss.rb

@@ -101,15 +101,19 @@ module Html2rss # rubocop:disable Metrics/ModuleLength
   #
   # @param config_input [Hash, String] config hash, YAML string, or file path
   # @param feed_name [String, nil] optional feed name in multi-feed file
-  # @param min_items [Integer] minimum required items (default: 1)
+  # @param min_items [Integer] minimum required items, >= 0 (default: 1)
   # @param params [Hash] dynamic feed params
   # @param strategy [Symbol, nil] optional strategy override
   # @param strict_quality [Boolean] when true, fail on ship-quality audit thresholds
   # @param compare_enhance [Boolean] diagnostic enhance off vs on comparison on cached HTML
+  # @param sample_limit [Integer] maximum number of +Result#items+, >= 0 (default: 3)
   # @return [Html2rss::Test::Result]
   def self.test(config_input, feed_name = nil, min_items: 1, params: {}, strategy: nil, strict_quality: false, # rubocop:disable Metrics/ParameterLists
-                compare_enhance: false)
-    Test.call(config_input, feed_name, min_items:, params:, strategy:, strict_quality:, compare_enhance:)
+                compare_enhance: false, sample_limit: 3)
+    Option.integer(:min_items, min_items, min: 0)
+    Option.integer(:sample_limit, sample_limit, min: 0)
+    Test.call(config_input, feed_name, min_items:, params:, strategy:, strict_quality:, compare_enhance:,
+                                       sample_limit:)
   end
 
   ##
@@ -135,6 +139,7 @@ module Html2rss # rubocop:disable Metrics/ModuleLength
   # @param local_file_path [String, nil] optional local HTML file path
   # @param limit [Integer, nil] max articles to keep (default: {AutoSource::DEFAULT_LIMIT})
   # @param keep_different_domain [Boolean, nil] when set, overrides cleanup.keep_different_domain
+  # @param keep_sponsored [Boolean, nil] when set, overrides cleanup.keep_sponsored
   # @return [Html2rss::FeedResult]
   def self.scrape(url,
                   strategy: :auto,
@@ -143,9 +148,10 @@ module Html2rss # rubocop:disable Metrics/ModuleLength
                   max_requests: 4,
                   local_file_path: nil,
                   limit: nil,
-                  keep_different_domain: nil)
+                  keep_different_domain: nil,
+                  keep_sponsored: nil)
     auto_feed_result(url, strategy:, items_selector:, max_redirects:, max_requests:,
-                          local_file_path:, limit:, keep_different_domain:)
+                          local_file_path:, limit:, keep_different_domain:, keep_sponsored:)
   end
 
   ##
@@ -249,6 +255,7 @@ module Html2rss # rubocop:disable Metrics/ModuleLength
   # @param local_file_path [String, nil] optional local HTML file path
   # @param limit [Integer, nil] max articles to keep (default: {AutoSource::DEFAULT_LIMIT})
   # @param keep_different_domain [Boolean, nil] when set, overrides cleanup.keep_different_domain
+  # @param keep_sponsored [Boolean, nil] when set, overrides cleanup.keep_sponsored
   # @return [Html2rss::FeedResult]
   def self.auto_feed_result(url,
                             strategy: :auto,
@@ -257,9 +264,10 @@ module Html2rss # rubocop:disable Metrics/ModuleLength
                             max_requests: 4,
                             local_file_path: nil,
                             limit: nil,
-                            keep_different_domain: nil)
+                            keep_different_domain: nil,
+                            keep_sponsored: nil)
     feed_result(build_auto_source_config(url:, strategy:, items_selector:, max_redirects:, max_requests:,
-                                         local_file_path:, limit:, keep_different_domain:))
+                                         local_file_path:, limit:, keep_different_domain:, keep_sponsored:))
   end
 
   ##
@@ -359,13 +367,15 @@ module Html2rss # rubocop:disable Metrics/ModuleLength
 
     # rubocop:disable-next Metrics/ParameterLists, Metrics/MethodLength
     def build_auto_source_config(url:, strategy:, items_selector:, max_redirects:, max_requests:,
-                                 local_file_path: nil, limit: nil, keep_different_domain: nil)
+                                 local_file_path: nil, limit: nil, keep_different_domain: nil,
+                                 keep_sponsored: nil)
       config = Config.auto_source_config(
         url:,
         items_selector:,
         request_controls: Config::RequestControls.from_shortcut(strategy:, max_redirects:, max_requests:),
         limit:,
-        keep_different_domain:
+        keep_different_domain:,
+        keep_sponsored:
       )
       if local_file_path
         config[:request] ||= {}