readability-rb 0.4.0 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ef0802d2a5e51416ef279914f65666a671217644df050a35b437e8a40046c8de
-  data.tar.gz: fe5be59a935e77a94af0b1243a7b2c5457eccbfaaf18452f18a7debf4a7b0560
+  metadata.gz: c451d1e4e2e12c168b323f813a36594c25177e7a8fa4c1999cd55fd652128413
+  data.tar.gz: be5c5fe5aef9cc1186c25112f6e3ca11220aec7b4cb122682092fe3db14c4008
 SHA512:
-  metadata.gz: '0269968c8945402342b91435868227b09301303ecbffe12ee499161dc0accb231d576cef67c84e434a382dbea1895fc7a520a28c8068ba4a25b5108704f7b480'
-  data.tar.gz: 5b6a4b7647248c90a40f5316d8a9b498164e6510209c653751664470a2af8d0b9900651a3a5e37be6f3c36b15320f92684044e0279a42363f145e744c34f0dd5
+  metadata.gz: 210bd7531212e4c9ad91ec6da99e3bd0a74eb1dd62040b646bc7e90a07daadc13e19c9681d2bd6c7e2412c2ef3f30752671b52472f22b26fbc263eaf2f473e64
+  data.tar.gz: 692bcc0189b89d1b440d7a5bf0cf4bd53a313ef30cebab92ca93d2c6f6d88a81fb531d1cabe85c07487748467e17a1e591b6f32237b5f01ed89eb14f7f5ea329

data/README.md

@@ -92,6 +92,8 @@ result = Readability.parse(html, serializer: ->(el) { el.to_html })
 | `allowed_video_regex` | Regex for allowed video embed URLs | built-in |
 | `link_density_modifier` | Adjust link density calculation | `0` |
 | `serializer` | Lambda to serialize the content element | `inner_html` |
+| `max_attributes` | Max attributes per element at parse time | `1000` |
+| `max_tree_depth` | Max document tree depth at parse time (`-1` to disable) | `1000` |
 
 ## Contributing
 

data/lib/readability/document.rb

@@ -170,7 +170,7 @@ module Readability
                 nxt = get_next_node(nxt)
               end
             end
-            @article_byline = (item_prop_name_node || node).text.strip
+            @article_byline = js_trim((item_prop_name_node || node).text)
             node = remove_and_get_next(node)
             next
           end

data/lib/readability/metadata.rb

@@ -41,7 +41,7 @@ module Readability
 
           if parsed.is_a?(Array)
             parsed = parsed.find do |it|
-              Array(it["@type"]).any? { |t| t.match?(JSON_LD_ARTICLE_TYPES) }
+              Array(it["@type"]).any? { |t| t.is_a?(String) && t.match?(JSON_LD_ARTICLE_TYPES) }
             end
             next unless parsed
           end
@@ -64,6 +64,8 @@ module Readability
             end
           end
 
+          # Unlike JS, accept an array @type, which schema.org allows. JS calls
+          # @type.match, which throws for an array, so it skips the script.
           next if !parsed || !parsed["@type"] || !Array(parsed["@type"]).any? { |t| t.is_a?(String) && t.match?(JSON_LD_ARTICLE_TYPES) }
 
           metadata = {}
@@ -88,13 +90,13 @@ module Readability
 
           if parsed["author"]
             if parsed["author"].is_a?(Hash) && parsed["author"]["name"].is_a?(String)
-              metadata["byline"] = parsed["author"]["name"].strip
+              metadata["byline"] = js_trim(parsed["author"]["name"])
             elsif parsed["author"].is_a?(Array) &&
-                parsed["author"][0] &&
+                parsed["author"][0].is_a?(Hash) &&
                 parsed["author"][0]["name"].is_a?(String)
               metadata["byline"] = parsed["author"]
-                .select { |author| author && author["name"].is_a?(String) }
-                .map { |author| author["name"].strip }
+                .select { |author| author.is_a?(Hash) && author["name"].is_a?(String) }
+                .map { |author| js_trim(author["name"]) }
                 .join(", ")
             end
           end
@@ -110,8 +112,8 @@ module Readability
           if parsed["datePublished"].is_a?(String)
             metadata["date_published"] = parsed["datePublished"].strip
           end
-        rescue JSON::ParserError => e
-          # Handle malformed JSON gracefully
+        rescue => e
+          # Skip malformed JSON or JSON with an unexpected shape, as JS does
           log(e.message) if respond_to?(:log, true)
         end
       end
@@ -225,7 +227,7 @@ module Readability
           if matches
             # Convert to lowercase, and remove any whitespace
             name = matches[0].downcase.gsub(/\s/, "")
-            values[name] = content.strip
+            values[name] = js_trim(content)
           end
         end
 
@@ -234,7 +236,7 @@ module Readability
           if content
             # Convert to lowercase, remove whitespace, convert dots to colons
             name = name.downcase.gsub(/\s/, "").gsub(".", ":")
-            values[name] = content.strip
+            values[name] = js_trim(content)
           end
         end
       end

data/lib/readability/utils.rb

@@ -33,10 +33,10 @@ module Readability
     end
 
     # JS-compatible trim that also strips Unicode whitespace like \u00A0 (NBSP)
-    # JS's String.prototype.trim() strips all Unicode whitespace including NBSP.
+    # JS's String.prototype.trim() strips all Unicode whitespace and \uFEFF (BOM).
     # Ruby's String#strip only removes ASCII whitespace.
     def js_trim(str)
-      str.gsub(/\A[\s\u00A0]+|[\s\u00A0]+\z/, "")
+      str.gsub(/\A[[:space:]\uFEFF]+|[[:space:]\uFEFF]+\z/, "")
     end
 
     # Port of _getInnerText (JS line 2084)
@@ -211,9 +211,9 @@ module Readability
     end
 
     # Port of _isUrl (JS line 442)
+    # JS's URL constructor accepts any absolute URL, so any scheme counts.
     def is_url?(str)
-      uri = URI.parse(str)
-      uri.is_a?(URI::HTTP) || uri.is_a?(URI::HTTPS)
+      !URI.parse(str).scheme.nil?
     rescue URI::InvalidURIError, URI::InvalidComponentError
       false
     end

data/lib/readability/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Readability
-  VERSION = "0.4.0"
+  VERSION = "0.6.0"
 end

data/lib/readability.rb

@@ -15,13 +15,16 @@ require_relative "readability/document"
 require_relative "readability/readerable"
 
 module Readability
-  def self.parse(html, url: nil, **options)
-    doc = Nokogiri::HTML5(html)
+  DEFAULT_MAX_ATTRIBUTES = 1000
+  DEFAULT_MAX_TREE_DEPTH = 1000
+
+  def self.parse(html, url: nil, max_attributes: DEFAULT_MAX_ATTRIBUTES, max_tree_depth: DEFAULT_MAX_TREE_DEPTH, **options)
+    doc = Nokogiri::HTML5(html, max_attributes: max_attributes, max_tree_depth: max_tree_depth)
     Document.new(doc, url: url, **options).parse
   end
 
-  def self.readerable?(html, **options)
-    doc = Nokogiri::HTML5(html)
+  def self.readerable?(html, max_attributes: DEFAULT_MAX_ATTRIBUTES, max_tree_depth: DEFAULT_MAX_TREE_DEPTH, **options)
+    doc = Nokogiri::HTML5(html, max_attributes: max_attributes, max_tree_depth: max_tree_depth)
     Readerable.probably_readerable?(doc, **options)
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: readability-rb
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.6.0
 platform: ruby
 authors:
 - Andy Croll
@@ -58,14 +58,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '3.2'
+      version: '3.3'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.9
+rubygems_version: 4.0.20
 specification_version: 4
 summary: Extract readable article content from HTML pages
 test_files: []