readability_js 0.0.5 → 0.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9a5faf9c156d80158ba2ee471bb3d7ec102396c3c03ca5472162211706ac74d9
-  data.tar.gz: 16a0a329c137ecfbae089501b24f4dea0cc00395d62d5bedc123c40757018e27
+  metadata.gz: ff2c212926fdd4859f0a70a538ad28f3bc4b760b0d022b0e77f61bc21b40e859
+  data.tar.gz: 99ea17969d31e4de8d2c1718693b1c43c254ec8099a4ec85c5ba80342c80fda1
 SHA512:
-  metadata.gz: 0f3b5aa2bb14ee235f3aad9b884cdeeb801e28ea18048d33d7a2c50921e89b33210df375e7d67607f86c33c23a0c1cfa3b82fd8420951ea5008534164eede0a8
-  data.tar.gz: ae1fe7c1decc638e7da359f16c8178a6ce8f02a2632bef7b5e3e4542bc7de17b52bba16bce4dbb70692a4db87c46728c1943f13b6134c5a0b101d458fc7baf78
+  metadata.gz: 7050f66c9af71d4b420a49a322b9be44afa9777ea500c59f1bda10f350ca39806c572de0645e563d8f1d18d4ad06955c43f94c6907e8af79045184f3e5bd6112
+  data.tar.gz: 32520125bda871a44601ee83eaa653e62ef9c4104c8ccc3d1dbc0a2624c09304e0a4e63bf5e29672fcb8c6f8035081696640a3b7bac020ad627049d37c772dad

data/README.md

@@ -64,7 +64,9 @@ and includes a beautified markdown version of the content.
 ```ruby
     require 'readability_js'
     html = File.read("my_article.html")
-    result = ReadabilityJs.parse_extended(html)
+    # extend has included a DEFAULT_SELECTOR_BLACKLIST and you can add your own selectors to it as well, 
+    # that will be used to remove unwanted elements from the content before parsing at all.
+    result = ReadabilityJs.parse_extended(html, blacklist_selectors: [".advertisement", "#sponsored"])
     p result
 ```
 
@@ -81,7 +83,7 @@ data = ReadabilityJs.parse(
 # => Hash
 ```
 
-### Query response
+### Parse response
 The response object is of type `Hash`.
 It contains the data returned by readability, with hash keys transformed in snake_case.
 

data/lib/readability_js/extended.rb

@@ -1,8 +1,7 @@
-
 module ReadabilityJs
   class Extended
 
-    SELECTOR_BLACKLIST = [
+    DEFAULT_SELECTOR_BLACKLIST = [
       ".Article-Partner",
       ".Article-Partner-Text",
       ".Article-Comments-Button",
@@ -11,34 +10,26 @@ module ReadabilityJs
       "*[data-element-tracking-name]",
       "*[aria-label='Anzeige']",
       "nav[aria-label='breadcrumb']",
-      # heise
       "a-video",
       "a-gift",
       "a-collapse",
       "a-opt-in",
-      # spiegel
       "[data-area='related_articles']",
-      # welt
       "nav[aria-label='Breadcrumb']",
       ".c-inline-teaser-list",
       "[width='1'][height='1']",
-      # golem
       ".go-alink-list",
-      # faz
       "[data-external-selector='related-articles-entries']",
       ".BigBox",
-      # frankfurter rundschau
       ".id-Breadcrumb-item",
       ".id-Story-interactionBar",
       "revenue-reel",
       ".id-StoryElement-factBox",
-      # stern
       ".breadcrumb",
       ".teaser",
       ".group-teaserblock__items",
       ".title__kicker",
       "ws-adtag",
-      # taz
       "[data-for='webelement_bio']",
       "[data-for='webelement_citation']",
       "#articleTeaser",
@@ -46,10 +37,13 @@ module ReadabilityJs
       "[x-data='{}']",
       "#komune",
       ".community",
+      ".article-head__topline",
+      ".article__audioicon",
+      ".auplayer",
     ]
 
-    def self.before_cleanup(html)
-      pre_parser html
+    def self.before_cleanup(html, blacklist_selectors: [])
+      pre_parser html, blacklist_selectors: blacklist_selectors
     end
 
     def self.after_cleanup(result, html)
@@ -62,16 +56,18 @@ module ReadabilityJs
     #
     # Pre-parser to clean up HTML before passing it to Readability
     #
-    # SELECTOR_BLACKLIST contains CSS selectors of elements to be removed from the HTML
+    # DEFAULT_SELECTOR_BLACKLIST and given blacklist_selectors contains CSS selectors of elements to be removed from the HTML
     # before parsing to improve content extraction.
     #
     # @param html [String] The HTML document as a string.
     # @return [String] The cleaned HTML document as a string.
     #
-    def self.pre_parser(html)
+    def self.pre_parser(html, blacklist_selectors: [])
+      final_blacklist = DEFAULT_SELECTOR_BLACKLIST
+      final_blacklist += blacklist_selectors if blacklist_selectors.is_a?(Array) && !blacklist_selectors.empty?
       doc = Nokogiri::HTML(html)
       # Remove blacklisted elements by selector
-      SELECTOR_BLACKLIST.each do |classname|
+      final_blacklist.each do |classname|
         doc.css("#{classname}").remove
       end
       doc.to_html
@@ -148,7 +144,7 @@ module ReadabilityJs
     # @return [String] The cleaned HTML content as a string.
     #
     def self.clean_up_comments(html)
-      copy = html.dup.to_s
+      copy = html.dup || ""
       # Turn \x3C before comment start into '<'
       copy.gsub!(/\\x3C(?=!--)/, '<')
       # Decode encoded comment end --&gt; to -->
@@ -204,9 +200,9 @@ module ReadabilityJs
       end
       # Check for image and if none is found, add after title if available
       if result.key?("image_url") && !result["image_url"].to_s.strip.empty?
-        has_image = mark_down.match(/!\[.*?\]\(.*?\)/)
+        has_image = mark_down.match(/!\[.*?\]\(.*?\)/) || mark_down.match(/<img\b[^>]*>/) || mark_down.match(/<picture\b[^>]*>.*?<\/picture>/m)
         if !has_image
-          img_md = "![Lead Image](#{result['image_url']})\n\n"
+          img_md = "![image](#{result['image_url']})\n\n"
           mark_down = mark_down.sub(/^# .+?\n/, "\\0" + img_md)
         end
       end
@@ -237,7 +233,7 @@ module ReadabilityJs
         # check for img tags but also for picture tags
         has_image = !doc.css('img, picture').empty?
         if !has_image
-          img_tag = "<p><img src=\"#{result['image_url']}\" alt=\"Lead Image\"></p>\n"
+          img_tag = "<p><img src=\"#{result['image_url']}\"></p>\n"
           h1 = doc.at_css('h1')
           if h1
             h1.add_next_sibling(Nokogiri::HTML::DocumentFragment.parse(img_tag))

data/lib/readability_js/version.rb

@@ -1,3 +1,3 @@
 module ReadabilityJs
-  VERSION = '0.0.5'.freeze
+  VERSION = '0.0.7'.freeze
 end

data/lib/readability_js.rb

@@ -63,12 +63,13 @@ module ReadabilityJs
   # @param serializer [String, nil] Serializer to use for output (optional).
   # @param allow_video_regex [String, nil] Regular expression to allow video URLs (optional).
   # @param link_density_modifier [Float] Modifier for link density calculation (default: 0).
+  # @param blacklist_selectors [Array<String>] List of CSS selectors to remove from the HTML before parsing (default: []).
   # @return [Hash] A hash containing the extracted content and metadata.
   #
   # @raise [ReadabilityJs::Error] if an error occurs during execution
   #
-  def self.parse_extended(html, url: nil, debug: false, max_elems_to_parse: 0, nb_top_candidates: 5, char_threshold: 500, classes_to_preserve: [], keep_classes: false, disable_json_ld: false, serializer: nil, allow_video_regex: nil, link_density_modifier: 0)
-    result = Extended::before_cleanup html
+  def self.parse_extended(html, url: nil, debug: false, max_elems_to_parse: 0, nb_top_candidates: 5, char_threshold: 500, classes_to_preserve: [], keep_classes: false, disable_json_ld: false, serializer: nil, allow_video_regex: nil, link_density_modifier: 0, blacklist_selectors: [])
+    result = Extended::before_cleanup html, blacklist_selectors: blacklist_selectors
     result = parse result, url: url, debug: debug, max_elems_to_parse: max_elems_to_parse, nb_top_candidates: nb_top_candidates, char_threshold: char_threshold, classes_to_preserve: classes_to_preserve, keep_classes: keep_classes, disable_json_ld: disable_json_ld, serializer: serializer, allow_video_regex: allow_video_regex, link_density_modifier: link_density_modifier
     Extended::after_cleanup result, html
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: readability_js
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.7
 platform: ruby
 authors:
 - Matthäus Beyrle