readability_js 0.0.1 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 54d488477df33ef1471e6bf880cd39ba345debf2afa6d8cdc55dd2e058187373
-  data.tar.gz: 1345de9c85515fb049c3dfe4ae97349256a8fa12711265a7b55ddde394340c7b
+  metadata.gz: c536792a9e26ab4080c31b6065e02fab9a5ed5536993eeac15e53b0fd1c288fc
+  data.tar.gz: 88baef9c54969ca53e2bef99550d9da1445b72b62d8da43250d564c0ed1bc3e5
 SHA512:
-  metadata.gz: c3b20a5240e202ce12c7af97418c420dc81ff9de8cf592c415bffeb7da5eeaaa65f1204f443056aece6cf6fa709f87189355340edea0066a194b769b6e1c8d18
-  data.tar.gz: 50e9905ba059fff544b5dc3b9bc8fb21ba1f9e139c5c9b257b0de3d29b74043ce8ec8ec7424885babe994a5a1bba944c9a3641a48ee67fbfce329943895a759a
+  metadata.gz: 552063ebef5709acb6ca6a9744bf69e8c33499d24d0b9db7d5bcaef53a3429f0b72b7bc7f12350d2e0137ef20c5603489feb731de831044cb2f1cda742168d75
+  data.tar.gz: 11b0a652f07d3c722775a7f9fc401af4e20ee2abb8513e60bba44a7b5a1e0e8d9df0bf7477f8e5aa0fdfc1c147186094a610e0aea02757a70f2752845f241265

data/.gitattributes

@@ -0,0 +1,2 @@
+spec/examples/* linguist-vendored
+node_modules/* linguist-vendored

data/README.md

@@ -97,7 +97,8 @@ It contains the data returned by readability, with hash keys transformed in snak
   "dir" => "ltr",
   "site_name" => "example.com",
   "lang" => "en",
-  "published_time" => "2024-01-01T12:00:00Z"
+  "published_time" => "2024-01-01T12:00:00Z",
+  "image_url" => "https://example.com/image.jpg" # only for extended parse
 }    
 ```
 

data/lib/readability_js/extended.rb

@@ -0,0 +1,262 @@
+
+module ReadabilityJs
+  class Extended
+
+    SELECTOR_BLACKLIST = [
+      ".Article-Partner",
+      ".Article-Partner-Text",
+      ".Article-Comments-Button",
+      "#isl-5-AdCarousel",
+      "#isl-10-ArticleComments",
+      "*[data-element-tracking-name]",
+      "*[aria-label='Anzeige']",
+      "nav[aria-label='breadcrumb']",
+      # heise
+      "a-video",
+      "a-gift",
+      "a-collapse",
+      "a-opt-in",
+      # spiegel
+      "[data-area='related_articles']",
+      # welt
+      "nav[aria-label='Breadcrumb']",
+      ".c-inline-teaser-list",
+      "[width='1'][height='1']",
+      # golem
+      ".go-alink-list",
+      # faz
+      "[data-external-selector='related-articles-entries']",
+      ".BigBox",
+      # frankfurter rundschau
+      ".id-Breadcrumb-item",
+      ".id-Story-interactionBar",
+      "revenue-reel",
+      ".id-StoryElement-factBox",
+      # stern
+      ".breadcrumb",
+      ".teaser",
+      ".group-teaserblock__items",
+      ".title__kicker",
+      "ws-adtag",
+      # taz
+      "[data-for='webelement_bio']",
+      "[data-for='webelement_citation']",
+      "#articleTeaser",
+      ".article-produktteaser-container",
+      "[x-data='{}']",
+      "#komune",
+      ".community",
+    ]
+
+    def self.before_cleanup(html)
+      pre_parser html
+    end
+
+    def self.after_cleanup(result, html)
+      find_and_add_picture result, html
+      clean_up_and_enrich_result result
+    end
+
+    private
+
+    #
+    # Pre-parser to clean up HTML before passing it to Readability
+    #
+    # SELECTOR_BLACKLIST contains CSS selectors of elements to be removed from the HTML
+    # before parsing to improve content extraction.
+    #
+    # @param html [String] The HTML document as a string.
+    # @return [String] The cleaned HTML document as a string.
+    #
+    def self.pre_parser(html)
+      doc = Nokogiri::HTML(html)
+      # Remove blacklisted elements by selector
+      SELECTOR_BLACKLIST.each do |classname|
+        doc.css("#{classname}").remove
+      end
+      doc.to_html
+    end
+
+    #
+    # Post-parser to find and add lead image URL if missing.
+    #
+    # Will add a picture into the result hash under the key "image_url".
+    #
+    # Looks for Open Graph and Twitter Card meta tags to find a lead image URL.
+    # If not found, it will have a look into the markdown content for the first image.
+    #
+    # @param result [Hash] The result hash from Readability parsing.
+    # @param html [String] The original HTML document as a string.
+    # @return [Hash] The updated result hash.
+    #
+    def self.find_and_add_picture(result, html)
+      return result if result.key?("lead_image_url") && !result["lead_image_url"].to_s.strip.empty?
+      doc = Nokogiri::HTML(html)
+      # try to find og:image or twitter:image meta tags
+      meta_tags = doc.css('meta[property="og:image"], meta[name="og:image"], meta[name="twitter:image"]')
+      meta_tags.each do |meta_tag|
+        content = meta_tag['content']
+        if content && !content.strip.empty?
+          result["image_url"] = content.strip
+          break
+        end
+      end
+      # try to find first image in markdown content if no meta tag found before
+      if !result.key?("image_url") || result["image_url"].to_s.strip.empty?
+        if result.key?("markdown_content")
+          md_content = result["markdown_content"]
+          md_content.scan(/!\[.*?\]\((.*?)\)/).each do |match|
+            img_url = match[0]
+            if img_url && !img_url.strip.empty?
+              # check if img ends with common image file extensions
+              if img_url =~ /\.(jpg|jpeg|png|gif|webp|svg|tif|avif)(\?.*)?$/i
+                result["image_url"] = img_url.strip
+                break
+              end
+            end
+          end
+        end
+      end
+      result
+    end
+
+    #
+    # Post-parser to clean up extracted content after Readability processing
+    #
+    # Cleans up comment artifacts and beautifies HTML and adds beautified Markdown content.
+    #
+    # @param result [Hash] The result hash from Readability parsing.
+    # @return [Hash] The cleaned result hash.
+    #
+    def self.clean_up_and_enrich_result(result)
+      result["content"] = clean_up_comments(result["content"]) if result.key?("content")
+      result["text_content"] = clean_up_comments(result["text_content"]) if result.key?("text_content")
+      result["excerpt"] = clean_up_comments(result["excerpt"]) if result.key?("excerpt")
+      result["byline"] = clean_up_comments(result["byline"]) if result.key?("byline")
+      if result.key?("content")
+        result = beautify_html_and_text(result)
+        result["markdown_content"] = ReverseMarkdown.convert(result["content"]) if result.key?("content")
+        result = beautify_markdown(result)
+      end
+      result
+    end
+
+    #
+    # Remove/replace comment / artifact noise like <!--[--&gt;, <!----&gt; etc.
+    #
+    # @param html [String] The HTML content as a string.
+    # @return [String] The cleaned HTML content as a string.
+    #
+    def self.clean_up_comments(html)
+      copy = html.dup
+      # Turn \x3C before comment start into '<'
+      copy.gsub!(/\\x3C(?=!--)/, '<')
+      # Decode encoded comment end --&gt; to -->
+      copy.gsub!(/--&gt;/, '-->')
+      # Remove fully empty or artifact comments ([], only whitespace)
+      copy.gsub!(/<!--\s*(?:\[|\]|)*\s*-->/, '')
+      # Collapse multiple dummy comment chains
+      copy.gsub!(/(?:<!--\s*-->\s*)+/, '')
+      # Remove remaining comment artifacts like <!--[-->, <!--]-->
+      copy.gsub!(/<!--\[\]-->|<!--\[\s*-->|<!--\]\s*-->/, '')
+      # Remove any remaining regular comments
+      copy.gsub!(/<!--.*?-->/m, '')
+      # Reduce excessive whitespace / blank lines (real newlines)
+      copy.gsub!(/\n[ \t]+\n/, "\n")
+      copy.gsub!(/\n{3,}/, "\n\n")
+      # Remove any remaining script tags (including encoded variants)
+      copy.gsub!(/(?:\\x3C|<)script\b[^>]*?(?:>|\\x3E|&gt;).*?(?:\\x3C|<)\/script(?:>|\\x3E|&gt;)/im, '')
+      # Preserve blocks where whitespace/newlines matter
+      preserve_tags = %w[pre code textarea]
+      preserved = {}
+      preserve_tags.each_with_index do |tag, idx|
+        copy.scan(/<#{tag}[^>]*?>.*?<\/#{tag}>/mi).each do |block|
+          key = "__PRESERVE_BLOCK_#{tag.upcase}_#{idx}_#{preserved.size}__"
+          preserved[key] = block
+          copy.sub!(block, key)
+        end
+      end
+      # Remove literal backslash+n sequences (if they exist as textual artifacts) outside preserved blocks
+      copy.gsub!(/\\n\s*/, ' ')
+      # Collapse whitespace between tags to a single space or nothing
+      # Remove whitespace-only text nodes represented by spaces/newlines between tags
+      copy.gsub!(/>\s+</, '><')
+      # Normalize multiple spaces to a single space
+      copy.gsub!(/ {2,}/, ' ')
+      # Trim spaces directly inside tags (e.g., <p> text </p>)
+      copy.gsub!(/>\s+([^<])/) { ">#{$1}" }
+      # Restore preserved blocks
+      preserved.each { |k, v| copy.sub!(k, v) }
+      copy.strip
+    end
+
+    #
+    # Beautify Markdown content by adding title if not present and fixing link spacing
+    #
+    # @param result [Hash] The result hash from Readability parsing.
+    # @return [Hash] The beautified result hash.
+    #
+    def self.beautify_markdown(result)
+      mark_down = result["markdown_content"]
+      # add title to markdown if not present
+      if !mark_down.start_with?("# ") && result.key?("title") && !result["title"].to_s.strip.empty? && !mark_down.include?(result["title"])
+        mark_down = "# #{result['title']}\n\n" + mark_down
+      end
+      # Check for image and if none is found, add after title if available
+      if result.key?("image_url") && !result["image_url"].to_s.strip.empty?
+        has_image = mark_down.match(/!\[.*?\]\(.*?\)/)
+        if !has_image
+          img_md = "![Lead Image](#{result['image_url']})\n\n"
+          mark_down = mark_down.sub(/^# .+?\n/, "\\0" + img_md)
+        end
+      end
+      # Add a space after markdown links if immediately followed by an alphanumeric char (missing separation).
+      mark_down.gsub!(/(\[[^\]]+\]\((?:[^\)"']+|"[^"]*"|'[^']*')*\))(?=[A-Za-z0-9ÄÖÜäöüß])/, '\1 ')
+      result["markdown_content"] = mark_down
+      result
+    end
+
+    #
+    # Beautify HTML content by adding title if not present and fixing link spacing
+    #
+    # @param result [Hash] The result hash from Readability parsing.
+    # @return [String] The beautified HTML content as a string.
+    #
+    def self.beautify_html_and_text(result)
+      html = result["content"]
+      text = result["text_content"]
+      # Add title to html and text if not present
+      if (html.index(/h[1-2]/) && html.index(/h[1-2]/).to_i > 128 && result.key?("title") && !result["title"].to_s.strip.empty? && !html.include?(result["title"])) || html.index(/h[1-2]/).nil?
+        title_tag = "<h1>#{result['title']}</h1>\n"
+        html = title_tag + html
+        text = result['title'] + "\n\n" + text
+      end
+      # Check for image and if none is found, add after title if available
+      if result.key?("image_url") && !result["image_url"].to_s.strip.empty?
+        doc = Nokogiri::HTML(html)
+        # check for img tags but also for picture tags
+        has_image = !doc.css('img, picture').empty?
+        if !has_image
+          img_tag = "<p><img src=\"#{result['image_url']}\" alt=\"Lead Image\"></p>\n"
+          h1 = doc.at_css('h1')
+          if h1
+            h1.add_next_sibling(Nokogiri::HTML::DocumentFragment.parse(img_tag))
+            html = doc.to_html
+            text = result['image_url'] + "\n\n" + text
+          end
+        end
+      end
+      # Add a space after a links if immediately followed by an alphanumeric char (missing separation).
+      doc = Nokogiri::HTML(html)
+      doc.css('a').each do |link|
+        next if link.next_sibling.nil?
+        if link.next_sibling.text? && link.next_sibling.content =~ /\A[A-Za-z0-9ÄÖÜäöüß]/
+          link.add_next_sibling(Nokogiri::XML::Text.new(' ', doc))
+        end
+      end
+      result["content"] = doc.to_html
+      result["text_content"] = text
+      result
+    end
+  end
+end

data/lib/readability_js/node/readability-example.js

@@ -1,4 +1,4 @@
-const { Readability } = require('@mozilla/readability');
+const { Readability, isProbablyReaderable } = require('@mozilla/readability');
 const { JSDOM } = require('jsdom');
 
 const doc = new JSDOM("<body>Look at this cat: <img src='./cat.jpg'></body>", {
@@ -7,4 +7,10 @@ const doc = new JSDOM("<body>Look at this cat: <img src='./cat.jpg'></body>", {
 let reader = new Readability(doc.window.document);
 let article = reader.parse();
 
-console.log(article);
+console.log(article);
+
+if(isProbablyReaderable(doc.window.document)) {
+    console.log("This document is probably readerable.");
+} else {
+    console.log("This document is probably not readerable.");
+}

data/lib/readability_js/nodo.rb

@@ -14,15 +14,22 @@ module ReadabilityJs
     #
     def self.parse(html, url: nil, debug: false, max_elems_to_parse: 0, nb_top_candidates: 5, char_threshold: 500, classes_to_preserve: [], keep_classes: false, disable_json_ld: false, serializer: nil, allow_video_regex: nil, link_density_modifier: 0)
       begin
-        self.new.parse html
+        # remove style tags from html, so jsdom does not need to process css and its warnings are not shown
+        html = html.gsub(/<style[^>]*>.*?<\/style>/m, '')
+        self.new.parse html, url, debug, max_elems_to_parse, nb_top_candidates, char_threshold, classes_to_preserve, keep_classes, disable_json_ld, serializer, allow_video_regex, link_density_modifier
       rescue ::Nodo::JavaScriptError => e
         raise ReadabilityJs::Error.new "#{e.message}"
       end
     end
 
-    def self.is_probably_readerable(html, min_content_length: 140, min_score: 20, visibility_checker: 'isNodeVisible')
+    #
+    # instance wrapper method, as nodo does not support class methods
+    #
+    def self.is_probably_readerable(html, min_content_length: 140, min_score: 20, visibility_checker: nil)
       begin
-        self.new.is_probably_readerable html
+        # remove style tags from html, so jsdom does not need to process css and its warnings are not shown
+        html = html.gsub(/<style[^>]*>.*?<\/style>/m, '')
+        self.new.is_probably_readerable html, min_content_length, min_score, visibility_checker
       rescue ::Nodo::JavaScriptError => e
         raise ReadabilityJs::Error.new "#{e.message}"
       end
@@ -57,9 +64,16 @@ module ReadabilityJs
     JS
 
     function :is_probably_readerable, <<~JS
-      async (html) => {
+      async (html, minContentLength, minScore, visibilityChecker) => {
         const doc = new jsdom.JSDOM(html);
-        return readability.Readability.isProbablyReaderable(doc);
+        
+        let readability_options = {};
+        if(minContentLength !== undefined && minContentLength !== null) readability_options.minContentLength = minContentLength;
+        if(minScore !== undefined && minScore !== null) readability_options.minScore = minScore;
+        if(visibilityChecker !== undefined && visibilityChecker !== null) {
+          readability_options.visibilityChecker = eval(visibilityChecker);
+        }
+        return readability.isProbablyReaderable(doc.window.document, readability_options);
       }
     JS
 

data/lib/readability_js/version.rb

@@ -1,3 +1,3 @@
 module ReadabilityJs
-  VERSION = '0.0.1'.freeze
+  VERSION = '0.0.3'.freeze
 end

data/lib/readability_js.rb

@@ -6,6 +6,7 @@ require 'nokogiri'
 
 require_relative 'readability_js/version'
 require_relative 'readability_js/nodo'
+require_relative 'readability_js/extended'
 
 require_relative 'custom_errors/error'
 
@@ -15,55 +16,27 @@ require_relative 'custom_errors/error'
 
 module ReadabilityJs
 
-  SELECTOR_BLACKLIST = [
-    ".Article-Partner",
-    ".Article-Partner-Text",
-    ".Article-Comments-Button",
-    "#isl-5-AdCarousel",
-    "#isl-10-ArticleComments",
-    "*[data-element-tracking-name]",
-    "*[aria-label='Anzeige']",
-    "nav[aria-label='breadcrumb']",
-    # heise
-    "a-video",
-    "a-gift",
-    "a-collapse",
-    # spiegel
-    "[data-area='related_articles']",
-    # welt
-    "nav[aria-label='Breadcrumb']",
-    ".c-inline-teaser-list",
-    # golem
-    ".go-alink-list",
-    # faz
-    "[data-external-selector='related-articles-entries']",
-    ".BigBox",
-    # frankfurter rundschau
-    ".id-Breadcrumb-item",
-    ".id-Story-interactionBar",
-    "revenue-reel",
-    ".id-StoryElement-factBox",
-    # stern
-    ".breadcrumb",
-    ".teaser",
-    ".group-teaserblock__items",
-    ".title__kicker",
-    # taz
-    "[data-for='webelement_bio']",
-    "[data-for='webelement_citation']",
-    "#articleTeaser",
-    ".article-produktteaser-container",
-    "[x-data='{}']",
-    "#komune",
-    ".community",
-  ]
-
   #
   # Parse a HTML document and extract its main content using Mozilla's Readability library.
-  # Raises ReadabilityJs::Error on failure.
   #
   # 'html' is a required parameters, all others are optional.
   #
+  # @param html [String] The HTML document as a string.
+  # @param url [String, nil] The URL of the document (optional, used for resolving relative links).
+  # @param debug [Boolean] Enable debug mode (default: false).
+  # @param max_elems_to_parse [Integer] Maximum number of elements to parse (default: 0, meaning no limit).
+  # @param nb_top_candidates [Integer] Number of top candidates to consider (default: 5).
+  # @param char_threshold [Integer] Minimum number of characters for an element to be considered (default: 500).
+  # @param classes_to_preserve [Array<String>] List of CSS classes to preserve in the output (default: []).
+  # @param keep_classes [Boolean] Whether to keep the original classes in the output (default: false).
+  # @param disable_json_ld [Boolean] Disable JSON-LD parsing (default: false).
+  # @param serializer [String, nil] Serializer to use for output (optional).
+  # @param allow_video_regex [String, nil] Regular expression to allow video URLs (optional).
+  # @param link_density_modifier [Float] Modifier for link density calculation (default: 0).
+  # @return [Hash] A hash containing the extracted content and metadata.
+  #
+  # @raise [ReadabilityJs::Error] if an error occurs during execution
+  #
   def self.parse(html, url: nil, debug: false, max_elems_to_parse: 0, nb_top_candidates: 5, char_threshold: 500, classes_to_preserve: [], keep_classes: false, disable_json_ld: false, serializer: nil, allow_video_regex: nil, link_density_modifier: 0)
     begin
       result = ReadabilityJs::Nodo.parse(html, url: url, debug: debug, max_elems_to_parse: max_elems_to_parse, nb_top_candidates: nb_top_candidates, char_threshold: char_threshold, classes_to_preserve: classes_to_preserve, keep_classes: keep_classes, disable_json_ld: disable_json_ld, serializer: serializer, allow_video_regex: allow_video_regex, link_density_modifier: link_density_modifier)
@@ -73,13 +46,60 @@ module ReadabilityJs
     end
   end
 
+  #
+  # Like #parse but with additional pre- and post-processing to enhance content extraction.
+  #
+  # 'html' is a required parameters, all others are optional.
+  #
+  # @param html [String] The HTML document as a string.
+  # @param url [String, nil] The URL of the document (optional, used for resolving relative links).
+  # @param debug [Boolean] Enable debug mode (default: false).
+  # @param max_elems_to_parse [Integer] Maximum number of elements to parse (default: 0, meaning no limit).
+  # @param nb_top_candidates [Integer] Number of top candidates to consider (default: 5).
+  # @param char_threshold [Integer] Minimum number of characters for an element to be considered (default: 500).
+  # @param classes_to_preserve [Array<String>] List of CSS classes to preserve in the output (default: []).
+  # @param keep_classes [Boolean] Whether to keep the original classes in the output (default: false).
+  # @param disable_json_ld [Boolean] Disable JSON-LD parsing (default: false).
+  # @param serializer [String, nil] Serializer to use for output (optional).
+  # @param allow_video_regex [String, nil] Regular expression to allow video URLs (optional).
+  # @param link_density_modifier [Float] Modifier for link density calculation (default: 0).
+  # @return [Hash] A hash containing the extracted content and metadata.
+  #
+  # @raise [ReadabilityJs::Error] if an error occurs during execution
+  #
   def self.parse_extended(html, url: nil, debug: false, max_elems_to_parse: 0, nb_top_candidates: 5, char_threshold: 500, classes_to_preserve: [], keep_classes: false, disable_json_ld: false, serializer: nil, allow_video_regex: nil, link_density_modifier: 0)
-    result = pre_parser html
+    result = Extended::before_cleanup html
     result = parse result, url: url, debug: debug, max_elems_to_parse: max_elems_to_parse, nb_top_candidates: nb_top_candidates, char_threshold: char_threshold, classes_to_preserve: classes_to_preserve, keep_classes: keep_classes, disable_json_ld: disable_json_ld, serializer: serializer, allow_video_regex: allow_video_regex, link_density_modifier: link_density_modifier
-    clean_up_result result
+    Extended::after_cleanup result, html
   end
 
-  def self.is_probably_readerable(html, min_content_length: 140, min_score: 20, visibility_checker: 'isNodeVisible')
+  #
+  # Decides whether a document is probably readerable without parsing the whole document.
+  #
+  # Only 'html' is a required parameter, all others are optional.
+  #
+  # @param html [String] The HTML document as a string.
+  # @param min_content_length [Integer] Minimum content length to consider the document readerable
+  # @param min_score [Integer] Minimum score to consider the document readerable
+  # @param visibility_checker [String] anonymous JavaScript function definition to check node visibility as string. Uses default visibility checker if not provided.
+  # @return [Boolean] true if the document is probably readerable, false otherwise.
+  #
+  # @raise [ReadabilityJs::Error] if an error occurs during execution
+  #
+  # @example
+  #
+  # html = "<html>...</html>"
+  #
+  # visibility_checker = <<~JS
+  #   (node) => {
+  #    const style = node.ownerDocument.defaultView.getComputedStyle(node);
+  #    return (style && style.display !== 'none' && style.visibility !== 'hidden' && parseFloat(style.opacity) > 0);
+  #   }
+  # JS
+  #
+  # ReadabilityJs.is_probably_readerable(html, min_content_length: 200, min_score: 25, visibility_checker: visibility_checker)
+  #
+  def self.is_probably_readerable(html, min_content_length: 140, min_score: 20, visibility_checker: nil)
     begin
       ReadabilityJs::Nodo.is_probably_readerable(html, min_content_length: min_content_length, min_score: min_score, visibility_checker: visibility_checker)
     rescue => e
@@ -87,12 +107,45 @@ module ReadabilityJs
     end
   end
 
-  def self.probably_readerable?(html)
-    self.is_probably_readerable(html)
+
+  #
+  # Decides whether a document is probably readerable without parsing the whole document.
+  #
+  # Only 'html' is a required parameter, all others are optional.
+  #
+  # @param html [String] The HTML document as a string.
+  # @param min_content_length [Integer] Minimum content length to consider the document readerable
+  # @param min_score [Integer] Minimum score to consider the document readerable
+  # @param visibility_checker [String] anonymous JavaScript function definition to check node visibility as string. Uses default visibility checker if not provided.
+  # @return [Boolean] true if the document is probably readerable, false otherwise.
+  #
+  # @raise [ReadabilityJs::Error] if an error occurs during execution
+  #
+  # @example
+  #
+  # html = "<html>...</html>"
+  #
+  # visibility_checker = <<~JS
+  #   (node) => {
+  #    const style = node.ownerDocument.defaultView.getComputedStyle(node);
+  #    return (style && style.display !== 'none' && style.visibility !== 'hidden' && parseFloat(style.opacity) > 0);
+  #   }
+  # JS
+  #
+  # ReadabilityJs.probably_readerable?(html, min_content_length: 200, min_score: 25, visibility_checker: visibility_checker)
+  #
+  def self.probably_readerable?(html, min_content_length: 140, min_score: 20, visibility_checker: nil)
+    self.is_probably_readerable(html, min_content_length: min_content_length, min_score: min_score, visibility_checker: visibility_checker)
   end
 
   private
 
+  #
+  # Normalize result keys to snake_case for ruby style
+  #
+  # @param result [Hash] The result hash from Readability
+  # @return [Hash] The normalized result hash
+  #
   def self.normalize_result(result)
     result["text_content"] = result.delete("textContent") if result.key?("textContent")
     result["site_name"] = result.delete("siteName") if result.key?("siteName")
@@ -100,108 +153,5 @@ module ReadabilityJs
     result
   end
 
-  def self.clean_up_result(result)
-    result["content"] = clean_up_comments(result["content"]) if result.key?("content")
-    result["text_content"] = clean_up_comments(result["text_content"]) if result.key?("text_content")
-    result["excerpt"] = clean_up_comments(result["excerpt"]) if result.key?("excerpt")
-    result["byline"] = clean_up_comments(result["byline"]) if result.key?("byline")
-    if result.key?("content")
-      result["content"] = beautify_html(result["content"])
-      result["markdown_content"] = ReverseMarkdown.convert(result["content"]) if result.key?("content")
-      result = beautify_markdown(result)
-    end
-    result
-  end
-
-  # Replaces comment / artifact noise like <!--[--&gt;, <!----&gt; etc.
-  def self.clean_up_comments(html)
-    copy = html.dup
-
-    # Turn \x3C before comment start into '<'
-    copy.gsub!(/\\x3C(?=!--)/, '<')
-
-    # Decode encoded comment end --&gt; to -->
-    copy.gsub!(/--&gt;/, '-->')
-
-    # Remove fully empty or artifact comments ([], only whitespace)
-    copy.gsub!(/<!--\s*(?:\[|\]|)*\s*-->/, '')
-
-    # Collapse multiple dummy comment chains
-    copy.gsub!(/(?:<!--\s*-->\s*)+/, '')
-
-    # Remove remaining comment artifacts like <!--[-->, <!--]-->
-    copy.gsub!(/<!--\[\]-->|<!--\[\s*-->|<!--\]\s*-->/, '')
-
-    # Remove any remaining regular comments
-    copy.gsub!(/<!--.*?-->/m, '')
-
-    # Reduce excessive whitespace / blank lines (real newlines)
-    copy.gsub!(/\n[ \t]+\n/, "\n")
-    copy.gsub!(/\n{3,}/, "\n\n")
-
-    # Remove any remaining script tags (including encoded variants)
-    copy.gsub!(/(?:\\x3C|<)script\b[^>]*?(?:>|\\x3E|&gt;).*?(?:\\x3C|<)\/script(?:>|\\x3E|&gt;)/im, '')
-
-    # Preserve blocks where whitespace/newlines matter
-    preserve_tags = %w[pre code textarea]
-    preserved = {}
-    preserve_tags.each_with_index do |tag, idx|
-      copy.scan(/<#{tag}[^>]*?>.*?<\/#{tag}>/mi).each do |block|
-        key = "__PRESERVE_BLOCK_#{tag.upcase}_#{idx}_#{preserved.size}__"
-        preserved[key] = block
-        copy.sub!(block, key)
-      end
-    end
-
-    # Remove literal backslash+n sequences (if they exist as textual artifacts) outside preserved blocks
-    copy.gsub!(/\\n\s*/, ' ')
-
-    # Collapse whitespace between tags to a single space or nothing
-    # Remove whitespace-only text nodes represented by spaces/newlines between tags
-    copy.gsub!(/>\s+</, '><')
-
-    # Normalize multiple spaces to a single space
-    copy.gsub!(/ {2,}/, ' ')
-
-    # Trim spaces directly inside tags (e.g., <p> text </p>)
-    copy.gsub!(/>\s+([^<])/) { ">#{$1}" }
-
-    # Restore preserved blocks
-    preserved.each { |k, v| copy.sub!(k, v) }
-    copy.strip
-  end
-
-  def self.beautify_markdown(result)
-    mark_down = result["markdown_content"]
-    # add title to markdown if not present
-    if !mark_down.start_with?("# ") && result.key?("title") && !result["title"].to_s.strip.empty? && !mark_down.include?(result["title"])
-      mark_down = "# #{result['title']}\n\n" + mark_down
-    end
-    # Add a space after markdown links if immediately followed by an alphanumeric char (missing separation).
-    mark_down.gsub!(/(\[[^\]]+\]\((?:[^\)"']+|"[^"]*"|'[^']*')*\))(?=[A-Za-z0-9ÄÖÜäöüß])/, '\1 ')
-    result["markdown_content"] = mark_down
-    result
-  end
-
-  def self.beautify_html(html)
-    doc = Nokogiri::HTML(html)
-    # Add a space after a links if immediately followed by an alphanumeric char (missing separation).
-    doc.css('a').each do |link|
-      next if link.next_sibling.nil?
-      if link.next_sibling.text? && link.next_sibling.content =~ /\A[A-Za-z0-9ÄÖÜäöüß]/
-        link.add_next_sibling(Nokogiri::XML::Text.new(' ', doc))
-      end
-    end
-    doc.to_html
-  end
-
-  def self.pre_parser(html)
-    doc = Nokogiri::HTML(html)
-    # Remove blacklisted classes
-    SELECTOR_BLACKLIST.each do |classname|
-      doc.css("#{classname}").remove
-    end
-    doc.to_html
-  end
 
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: readability_js
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.3
 platform: ruby
 authors:
 - Matthäus Beyrle
@@ -145,6 +145,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".gitattributes"
 - ".gitignore"
 - ".rspec"
 - CHANGELOG.md
@@ -158,6 +159,7 @@ files:
 - cli/pry.rb
 - lib/custom_errors/error.rb
 - lib/readability_js.rb
+- lib/readability_js/extended.rb
 - lib/readability_js/node/node_modules/.bin/tldts
 - lib/readability_js/node/node_modules/.yarn-integrity
 - lib/readability_js/node/node_modules/@asamuzakjp/css-color/LICENSE