RubyGems - detergent - Versions diffs - 1.0.0 → 2.0.0 - Mend

detergent 1.0.0 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (10) hide show

checksums.yaml +4 -4
data/README.md +7 -7
data/lib/detergent/cleaner.rb +69 -117
data/lib/detergent/content_locator.rb +82 -0
data/lib/detergent/matchers/obvious_junk_matcher.rb +7 -6
data/lib/detergent/matchers/removable_node_matcher.rb +20 -14
data/lib/detergent/node_scorer.rb +93 -51
data/lib/detergent/version.rb +3 -1
data/lib/detergent.rb +34 -3
metadata +2 -1

checksums.yaml CHANGED Viewed

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4ad549b0acaa35ac06eeea236937b7db8df4eb4c19111b1650b7b23738d35be1
-  data.tar.gz: b3da9c4ba188b6f750adf27dbe1e9c52403371abb410dbea651a6ff387a2d6bc
+  metadata.gz: 492c13c265f4d55f83ea2d4e32352ebf0209abf4bf15524ad291d352a7ca56ff
+  data.tar.gz: cd5fcadafd69fb5d43e9db36e7b3f2aff3516d792098a982584fb69eab91b2ba
 SHA512:
-  metadata.gz: 3e88afce9a71395e045a7b7c63f204c9f9d7ff9ba64e36d0d02610ea4884d34432ca853b5721dbc4cf67b1888a63c08062e8a66b242b6288d6f6eca3737b15cd
-  data.tar.gz: cce8cdcd13d4b12f6775ead158468822c1c803b3100cacf03ceb548a937870109f82592f1aea5f045334b6327d5e975ea2d9856aea8a6182c08390c6a70322e7
+  metadata.gz: 1135a0396b72e5f1c3892befd5f9930d5a8193ca9ffd36d42b5d431e7824d0e2a62b2a2a8afbd1084942780257d6651dc066a3efd18995fed81b53b6d3b3cf8c
+  data.tar.gz: e9809a3df3521cb9cd78c8c9b54f590de968ca84693454feb0e64e62137b0f1aaecc6df7877fed74f19251ece1ccf415328e68f60e92e8569ce52167d920c881

data/README.md CHANGED Viewed

@@ -15,16 +15,16 @@ gem "detergent"
 ```ruby
 require "detergent"
-cleaner = Detergent::Cleaner.new
 # Get a cleaned, self-contained HTML document:
-clean_html = cleaner.clean(dirty_html)
-# Or just the title:
-title = cleaner.title(dirty_html)
+clean_html = Detergent.clean(dirty_html)
 # Or the title and the extracted content node (a Nokogiri node):
-title, content = cleaner.cleaned_html(dirty_html)
+title, content = Detergent.extract(dirty_html)
+# A Cleaner instance is reusable if you're processing many pages:
+cleaner = Detergent::Cleaner.new
+clean_html = cleaner.clean(dirty_html)
+title = cleaner.title(dirty_html)
 ```
 ## How it works

data/lib/detergent/cleaner.rb CHANGED Viewed

@@ -1,160 +1,101 @@
+# frozen_string_literal: true
 module Detergent
+  # Orchestrates cleaning: prunes obvious junk, locates the main content,
+  # scrubs it, and renders the result back out as a standalone document.
   class Cleaner
     def initialize
-      @node_scorer = NodeScorer.new
       @obvious_junk_matcher = Matchers::ObviousJunkMatcher.new
       @removable_node_matcher = Matchers::RemovableNodeMatcher.new
     end
+    # Returns a complete, standalone HTML document containing only the
+    # page's title and its extracted main content.
     def clean(html)
-      @html = html
-      title, body = cleaned_html(@html)
-      <<-HTML
-      <html>
-        <head>
-          <meta charset="utf-8">
-          <meta name="viewport" content="width=device-width, initial-scale=1">
-          <title>#{title}</title>
-        </head>
-        #{body}
-      </html>
-      HTML
-    end
-    def title(html)
-      doc = Nokogiri::HTML(html)
-      title_node = doc.at('title')
-      title = title_node ? title_node.text.strip : ""
+      title, content = extract(html)
+      # The content root is usually an inner element (article, main, div),
+      # so wrap it in a body tag unless it already is one.
+      body = if content.nil? || content.name.downcase == "body"
+        content.to_s
+      else
+        "<body>#{content}</body>"
+      end
-      title
+      <<~HTML
+        <!DOCTYPE html>
+        <html>
+          <head>
+            <meta charset="utf-8">
+            <meta name="viewport" content="width=device-width, initial-scale=1">
+            <title>#{CGI.escapeHTML(title.to_s)}</title>
+          </head>
+          #{body}
+        </html>
+      HTML
     end
-    def cleaned_html(html)
-      doc = Nokogiri::HTML(html)
-      title_node = doc.at('title')
-      title = title_node ? title_node.text.strip : ""
+    # Returns [title, content]: the page title and the cleaned Nokogiri
+    # node containing the main content (nil if none was found).
+    def extract(html)
+      doc = Nokogiri::HTML5(html)
+      title = extract_title(doc)
       body = doc.at('body')
       content = nil
       if body
         # First remove the most egregious crap, like obvious ads, etc.
-        prune(node: body, matcher: Matchers::ObviousJunkMatcher.new)
-        # Find the highest-scoring node in the cleaned tree
-        highest_scoring = find_highest_scoring_node(body)
-        return nil unless highest_scoring
+        prune(node: body, matcher: @obvious_junk_matcher)
-        # Find the appropriate root container for that node
-        content_root = find_content_root(highest_scoring)
+        # Score caches are only valid for a single parse of a single
+        # document, so the scorer and locator are built per extract.
+        content = ContentLocator.new(NodeScorer.new).locate(body)
         # Apply second-pass cleaning to the content
-        if content_root
-          clean_node(content_root)
+        if content
+          clean_node(content)
+          strip_junk_attributes(content)
         end
-        content = content_root
       end
       [title, content]
     end
-    private
-    def prune(node:, matcher:)
-      node.children.to_a.each do |child|
-        if child.element?
-          # Recurse first
-          prune(node: child, matcher: matcher)
-          child.remove if matcher.match?(child)
-        end
-      end
-      node
-    end
-    # Scores a node based on how likely it is to be main content
-    # Higher scores indicate more likely to be the primary article content
-    def score_node(node)
-      @node_scorer.score(node)
+    def title(html)
+      extract_title(Nokogiri::HTML5(html))
     end
-    # Finds the node with the highest content score
-    # Returns the node with the highest score in the tree
-    def find_highest_scoring_node(node)
-      return nil unless node.element?
-      best_node = node
-      best_score = score_node(node)
-      # Recursively check all child elements
-      node.children.each do |child|
-        next unless child.element?
-        candidate = find_highest_scoring_node(child)
-        if candidate
-          candidate_score = score_node(candidate)
-          if candidate_score > best_score
-            best_node = candidate
-            best_score = candidate_score
-          end
-        end
-      end
+    private
-      best_node
+    def extract_title(doc)
+      title_node = doc.at('title')
+      title_node ? title_node.text.strip : ""
     end
-    # Finds an appropriate root ancestor for the given node
-    # Traverses up the tree to find a good container for the main content
-    # Stops at body or when we find a semantic container
-    def find_content_root(node)
-      return node if node.nil? || node.name.downcase == 'body'
-      current = node
-      best_ancestor = node
-      # Traverse up the tree
-      while current.parent && current.parent.element?
-        parent = current.parent
-        parent_tag = parent.name.downcase
-        # Stop at body
-        break if parent_tag == 'body'
-        # Prefer semantic containers
-        if %w(article main section).include?(parent_tag)
-          best_ancestor = parent
-        end
-        # If parent has significantly more content than current node,
-        # it might be a better root (captures related content)
-        parent_score = score_node(parent)
-        current_score = score_node(current)
+    def prune(node:, matcher:)
+      node.children.to_a.each do |child|
+        next unless child.element?
-        # Only move up if parent score is meaningfully higher
-        # (not just marginally better due to including the child)
-        if parent_score > current_score * 1.3
-          best_ancestor = parent
+        # Match first so we never bother walking a subtree we're removing
+        if matcher.match?(child)
+          child.remove
+        else
+          prune(node: child, matcher: matcher)
         end
-        current = parent
       end
-      best_ancestor
+      node
     end
     # Recursively process an element's children and remove any that are "empty"
     def clean_node(node, within_article: false)
+      within_article ||= node.name.downcase == "article"
       # Iterate over a copy of the children to avoid modification issues
       node.children.to_a.each do |child|
         if child.element?
-          clean_node(child, within_article: within_article || node.name.downcase == "article" )  # process children first
+          clean_node(child, within_article: within_article)  # process children first
           # Remove the child if it qualifies as "empty"
           if child.name.downcase == "aside" && !within_article
@@ -162,18 +103,29 @@ module Detergent
           elsif removable?(child)
             child.remove
           else
-            child.remove_attribute("class") # Cleanup
+            strip_junk_attributes(child)
           end
         end
       end
     end
+    # Strip presentation and tracking attributes from a kept element.
+    # Must run only after removability has been decided, because the
+    # matchers read class and style. Keeps id so in-page anchors work.
+    def strip_junk_attributes(node)
+      node.attribute_nodes.each do |attr|
+        name = attr.name.downcase
+        if %w[class style].include?(name) || name.start_with?("on", "data-")
+          node.remove_attribute(attr.name)
+        end
+      end
+    end
     # An element is removable if:
     # - It is a <script> or <style> tag (non-visible content), or other element we don't want
     # - It has no descendant text nodes (ignoring whitespace)
     def removable?(node)
       @removable_node_matcher.match?(node)
     end
   end
 end

data/lib/detergent/content_locator.rb ADDED Viewed

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+module Detergent
+  # Locates the main content node within a document's body: finds the
+  # highest-scoring node in the tree, then walks up its ancestors to find
+  # the best root container for the content.
+  class ContentLocator
+    SEMANTIC_CONTAINERS = %w[article main section].freeze
+    # A parent must beat the best root found so far by this ratio to be
+    # promoted — not just score marginally higher from including the child.
+    PARENT_PROMOTION_RATIO = 1.3
+    def initialize(scorer)
+      @scorer = scorer
+    end
+    def locate(body)
+      highest_scoring = find_highest_scoring_node(body)
+      return nil unless highest_scoring
+      find_content_root(highest_scoring)
+    end
+    private
+    def score(node)
+      @scorer.score(node)
+    end
+    # Finds the node with the highest content score in the tree
+    def find_highest_scoring_node(node)
+      return nil unless node.element?
+      best_node = node
+      best_score = score(node)
+      node.children.each do |child|
+        next unless child.element?
+        candidate = find_highest_scoring_node(child)
+        if candidate
+          candidate_score = score(candidate)
+          if candidate_score > best_score
+            best_node = candidate
+            best_score = candidate_score
+          end
+        end
+      end
+      best_node
+    end
+    # Walks up from the given node to find a good container for the main
+    # content, preferring semantic containers and ancestors that score
+    # meaningfully higher than the best root found so far. Stops at body.
+    def find_content_root(node)
+      return node if node.name.downcase == 'body'
+      best_ancestor = node
+      best_score = score(node)
+      current = node
+      while current.parent&.element?
+        parent = current.parent
+        parent_tag = parent.name.downcase
+        break if parent_tag == 'body'
+        if SEMANTIC_CONTAINERS.include?(parent_tag) ||
+           score(parent) > best_score * PARENT_PROMOTION_RATIO
+          best_ancestor = parent
+          best_score = score(parent)
+        end
+        current = parent
+      end
+      best_ancestor
+    end
+  end
+end

data/lib/detergent/matchers/obvious_junk_matcher.rb CHANGED Viewed

@@ -1,21 +1,22 @@
+# frozen_string_literal: true
 module Detergent
   module Matchers
+    # First-pass matcher: junk that can be removed on sight, before any
+    # content scoring happens.
     class ObviousJunkMatcher
       def match?(node)
         tag = node.name.downcase
         # Always remove these tags
-        return true if %w(script style link iframe noscript).include?(tag)
+        return true if JUNK_TAGS.include?(tag)
         # Remove hidden elements
         return true if node['aria-hidden'] == 'true'
-        # Remove display:none elements
-        style = node['style'].to_s.downcase
-        return true if style.include?('display:none') || style.include?('display: none')
+        return true if Detergent.display_none?(node)
         # Remove structural navigation
-        return true if %w(nav header footer).include?(tag)
+        return true if CHROME_TAGS.include?(tag)
         return true if node['role'].to_s.downcase == 'navigation'
         false

data/lib/detergent/matchers/removable_node_matcher.rb CHANGED Viewed

@@ -1,35 +1,41 @@
+# frozen_string_literal: true
 module Detergent
   module Matchers
+    # Second-pass matcher: applied within the extracted content to drop
+    # leftover chrome, suspect containers, and empty elements.
     class RemovableNodeMatcher
+      REMOVABLE_TAGS = (JUNK_TAGS + %w[footer nav] + FORM_TAGS).freeze
+      SUSPECT_CLASSES = %w[comments-show comments actionbar related-stories navigation
+                           nodisplay sidebar admz hidden header footer social share].freeze
+      SUSPECT_IDS = %w[header navigation ad admz sidebar related-stories hidden].freeze
       def match?(node)
+        tag = node.name.downcase
         # Remove button elements with Share or Save aria-labels
-        if node.name.downcase == "button"
+        if tag == "button"
           aria_label = node['aria-label'].to_s.downcase
           return true if ["share", "save"].include?(aria_label)
         end
-        # Remove nav with role=navigation
-        if node.name.downcase == "nav"
-          return true if node['role'].to_s.downcase == "navigation"
-        end
         # Elements we don't care about
-        return true if %w(link iframe script style footer nav form select textarea).include?(node.name.downcase)
+        return true if REMOVABLE_TAGS.include?(tag)
-        # Get rid of elements with classnames that look suspect
+        # Get rid of elements with classnames or ids that look suspect
         class_list = node['class'].to_s.downcase.split(" ")
-        return true if class_list.any?{ %w(comments-show comments actionbar related-stories navigation nodisplay sidebar admz hidden header footer social share).include?(_1) }
+        return true if class_list.any? { SUSPECT_CLASSES.include?(_1) }
         id_list = node['id'].to_s.downcase.split(" ")
-        return true if id_list.any?{ %w(header navigation ad admz sidebar related-stories hidden).include?(_1) }
+        return true if id_list.any? { SUSPECT_IDS.include?(_1) }
-        # Get rid of elements with inline styles that look suspect
-        style_list = node['style'].to_s.downcase.split(";")
-        return true if style_list.any?{ ["display: none", "display:none"].include?(_1) }
+        # Get rid of hidden elements
+        return true if Detergent.display_none?(node)
         # Don't remove images, etc in this step, which never have text content:
-        return false if %w(img picture figure).include?(node.name.downcase)
+        return false if MEDIA_TAGS.include?(tag)
         # Don't remove if any descendants are imgs, etc:
         return false if node.xpath(".//img | .//picture | .//figure").any?

data/lib/detergent/node_scorer.rb CHANGED Viewed

@@ -1,74 +1,116 @@
-# LOL yes, I'm doing this.
-class BasicObject
-  def readify_score=(new_score)
-    @readify_score = new_score
-  end
-  def readify_score
-    @readify_score
-  end
-end
+# frozen_string_literal: true
 module Detergent
+  # Scores nodes on how likely they are to be the page's main content.
+  #
+  # Descendant statistics are computed bottom-up, with each node's stats
+  # built from its children's already-computed stats, so scoring every
+  # node in a document is O(n) instead of a full-subtree scan per node.
+  # A scorer instance is scoped to a single parse of a single document;
+  # discard it once the tree is mutated.
   class NodeScorer
+    ARTICLE_TAG_BONUS = 100
+    MAIN_TAG_BONUS = 50
+    MAIN_ROLE_BONUS = 25
+    POINTS_PER_PARAGRAPH = 5
+    CHARS_PER_TEXT_POINT = 100
+    LINK_DENSITY_PENALTY = 10
+    POINTS_PER_MEDIA = 10
+    LONG_PARAGRAPH_BONUS = 15
+    LONG_PARAGRAPH_LENGTH = 100
+    POINTS_PER_BLOCKQUOTE = 10
+    POINTS_PER_LIST = 5
+    SIDEBAR_PENALTY = 50
+    COMMENT_PENALTY = 50
+    AD_PENALTY = 70
+    SOCIAL_PENALTY = 70
+    def initialize
+      @scores = {}.compare_by_identity
+      @stats = {}.compare_by_identity
+    end
     def score(node)
-      return node.readify_score unless node.readify_score.nil?
+      cached = @scores[node]
+      return cached unless cached.nil?
       return 0 unless node.element?
+      @scores[node] = compute_score(node)
+    end
+    private
+    def compute_score(node)
+      stats = stats_for(node)
       score = 0
       tag = node.name.downcase
       # Positive indicators for main content
-      score += 100 if tag == 'article'
-      score += 50 if tag == 'main'
-      score += 25 if node['role'].to_s.downcase == 'main'
-      # Count paragraphs - strong indicator of article content
-      paragraph_count = node.xpath('.//p').count
-      score += paragraph_count * 5
-      # Calculate text length
-      begin
-        text_content = node.xpath('.//text()').map(&:text).join
-        text_length = text_content.strip.length
-        score += (text_length / 100).to_i  # 1 point per 100 characters
-      rescue StandardError
-        text_length = 0
-      end
-      # Calculate link density (high link density suggests navigation)
-      link_count = node.xpath('.//a').count
-      if text_length > 0
-        link_density = (link_count.to_f / (text_length / 100.0))
-        score -= (link_density * 10).to_i  # Penalize high link density
-      end
+      score += ARTICLE_TAG_BONUS if tag == 'article'
+      score += MAIN_TAG_BONUS if tag == 'main'
+      score += MAIN_ROLE_BONUS if node['role'].to_s.downcase == 'main'
-      # Bonus for images/figures (suggests article content)
-      media_count = node.xpath('.//img | .//picture | .//figure').count
-      score += media_count * 10
+      # Paragraphs and raw text are strong indicators of article content
+      score += stats[:paragraphs] * POINTS_PER_PARAGRAPH
+      score += stats[:text_length] / CHARS_PER_TEXT_POINT
+      score += stats[:long_paragraphs] * LONG_PARAGRAPH_BONUS
-      # Bonus for long paragraphs
-      begin
-        long_paragraphs = node.xpath('.//p').select { |p| p.text.strip.length > 100 }.count
-        score += long_paragraphs * 15
-      rescue StandardError
-        # Do nothing
+      # High link density suggests navigation
+      if stats[:text_length] > 0
+        link_density = stats[:links].to_f / (stats[:text_length] / 100.0)
+        score -= (link_density * LINK_DENSITY_PENALTY).to_i
       end
-      # Bonus for blockquotes and lists (often in articles)
-      score += node.xpath('.//blockquote').count * 10
-      score += node.xpath('.//ul | .//ol').count * 5
+      # Media, blockquotes, and lists all suggest article content
+      score += stats[:media] * POINTS_PER_MEDIA
+      score += stats[:blockquotes] * POINTS_PER_BLOCKQUOTE
+      score += stats[:lists] * POINTS_PER_LIST
       # Penalty for suspicious classes/ids
       classes = node['class'].to_s.downcase
       ids = node['id'].to_s.downcase
-      score -= 50 if classes.include?('sidebar') || ids.include?('sidebar')
-      score -= 50 if classes.include?('comment') || ids.include?('comment')
-      score -= 70 if classes.include?('ad') || ids.include?('ad')
-      score -= 70 if classes.include?('social') || ids.include?('social')
+      score -= SIDEBAR_PENALTY if classes.include?('sidebar') || ids.include?('sidebar')
+      score -= COMMENT_PENALTY if classes.include?('comment') || ids.include?('comment')
+      score -= AD_PENALTY if classes.include?('ad') || ids.include?('ad')
+      score -= SOCIAL_PENALTY if classes.include?('social') || ids.include?('social')
+      [score, 0].max # Don't return negative scores
+    end
+    # Statistics about a node's descendants (not the node itself),
+    # aggregated from its children's stats in a single pass.
+    def stats_for(node)
+      cached = @stats[node]
+      return cached if cached
+      stats = { text_length: 0, links: 0, paragraphs: 0, long_paragraphs: 0,
+                media: 0, blockquotes: 0, lists: 0 }
+      node.children.each do |child|
+        if child.text?
+          stats[:text_length] += child.text.strip.length
+        elsif child.element?
+          child_stats = stats_for(child)
+          stats.each_key { |key| stats[key] += child_stats[key] }
+          case child.name.downcase
+          when 'a'
+            stats[:links] += 1
+          when 'p'
+            stats[:paragraphs] += 1
+            stats[:long_paragraphs] += 1 if child_stats[:text_length] > LONG_PARAGRAPH_LENGTH
+          when 'img', 'picture', 'figure'
+            stats[:media] += 1
+          when 'blockquote'
+            stats[:blockquotes] += 1
+          when 'ul', 'ol'
+            stats[:lists] += 1
+          end
+        end
+      end
-      node.readify_score = [score, 0].max  # Don't return negative scores
+      @stats[node] = stats
     end
   end
 end

data/lib/detergent/version.rb CHANGED Viewed

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
 module Detergent
-  VERSION = "1.0.0"
+  VERSION = "2.0.0"
 end

data/lib/detergent.rb CHANGED Viewed

@@ -1,10 +1,41 @@
+# frozen_string_literal: true
+require "cgi"
 require "nokogiri"
+module Detergent
+  # Tags that never contain readable content.
+  JUNK_TAGS = %w[script style link iframe noscript].freeze
+  # Structural chrome that surrounds the content.
+  CHROME_TAGS = %w[nav header footer].freeze
+  # Interactive elements that don't belong in cleaned output.
+  FORM_TAGS = %w[form select textarea].freeze
+  # Media elements that legitimately contain no text.
+  MEDIA_TAGS = %w[img picture figure].freeze
+  def self.display_none?(node)
+    style = node["style"].to_s.downcase
+    style.include?("display:none") || style.include?("display: none")
+  end
+  # Returns a cleaned, standalone HTML document.
+  def self.clean(html)
+    Cleaner.new.clean(html)
+  end
+  # Returns [title, content]: the page title and the cleaned Nokogiri
+  # node containing the main content (nil if none was found).
+  def self.extract(html)
+    Cleaner.new.extract(html)
+  end
+end
 require_relative "detergent/version"
 require_relative "detergent/node_scorer"
+require_relative "detergent/content_locator"
 require_relative "detergent/matchers/obvious_junk_matcher"
 require_relative "detergent/matchers/removable_node_matcher"
 require_relative "detergent/cleaner"
-module Detergent
-end

metadata CHANGED Viewed

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: detergent
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 2.0.0
 platform: ruby
 authors:
 - Jeff McFadden
@@ -63,6 +63,7 @@ files:
 - README.md
 - lib/detergent.rb
 - lib/detergent/cleaner.rb
+- lib/detergent/content_locator.rb
 - lib/detergent/matchers/obvious_junk_matcher.rb
 - lib/detergent/matchers/removable_node_matcher.rb
 - lib/detergent/node_scorer.rb