RubyGems - detergent - Versions diffs - 1.0.0 - Mend

detergent 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (10) hide show

checksums.yaml +7 -0
data/LICENSE +13 -0
data/README.md +39 -0
data/lib/detergent/cleaner.rb +179 -0
data/lib/detergent/matchers/obvious_junk_matcher.rb +25 -0
data/lib/detergent/matchers/removable_node_matcher.rb +42 -0
data/lib/detergent/node_scorer.rb +74 -0
data/lib/detergent/version.rb +3 -0
data/lib/detergent.rb +10 -0
metadata +93 -0

checksums.yaml ADDED Viewed

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 4ad549b0acaa35ac06eeea236937b7db8df4eb4c19111b1650b7b23738d35be1
+  data.tar.gz: b3da9c4ba188b6f750adf27dbe1e9c52403371abb410dbea651a6ff387a2d6bc
+SHA512:
+  metadata.gz: 3e88afce9a71395e045a7b7c63f204c9f9d7ff9ba64e36d0d02610ea4884d34432ca853b5721dbc4cf67b1888a63c08062e8a66b242b6288d6f6eca3737b15cd
+  data.tar.gz: cce8cdcd13d4b12f6775ead158468822c1c803b3100cacf03ceb548a937870109f82592f1aea5f045334b6327d5e975ea2d9856aea8a6182c08390c6a70322e7

data/LICENSE ADDED Viewed

@@ -0,0 +1,13 @@
+            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
+                    Version 2, December 2004
+ Copyright (C) 2026 Jeff McFadden
+ Everyone is permitted to copy and distribute verbatim or modified
+ copies of this license document, and changing it is allowed as long
+ as the name is changed.
+            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+  0. You just DO WHAT THE FUCK YOU WANT TO.

data/README.md ADDED Viewed

@@ -0,0 +1,39 @@
+# Detergent
+Detergent scrubs junk out of web pages. Give it an HTML document and it removes ads, navigation, scripts, sidebars, and other cruft, then extracts the main article content.
+## Installation
+Add to your Gemfile:
+```ruby
+gem "detergent"
+```
+## Usage
+```ruby
+require "detergent"
+cleaner = Detergent::Cleaner.new
+# Get a cleaned, self-contained HTML document:
+clean_html = cleaner.clean(dirty_html)
+# Or just the title:
+title = cleaner.title(dirty_html)
+# Or the title and the extracted content node (a Nokogiri node):
+title, content = cleaner.cleaned_html(dirty_html)
+```
+## How it works
+1. A first pass prunes obvious junk (scripts, styles, iframes, nav, headers, footers, hidden elements).
+2. Each remaining node is scored on how likely it is to be the main content (paragraph count, text length, link density, media, suspicious class names, etc.).
+3. The highest-scoring node is located and walked up to an appropriate semantic container.
+4. A second pass cleans that content of empty and suspect elements.
+## License
+[WTFPL](LICENSE)

data/lib/detergent/cleaner.rb ADDED Viewed

@@ -0,0 +1,179 @@
+module Detergent
+  class Cleaner
+    def initialize
+      @node_scorer = NodeScorer.new
+      @obvious_junk_matcher = Matchers::ObviousJunkMatcher.new
+      @removable_node_matcher = Matchers::RemovableNodeMatcher.new
+    end
+    def clean(html)
+      @html = html
+      title, body = cleaned_html(@html)
+      <<-HTML
+      <html>
+        <head>
+          <meta charset="utf-8">
+          <meta name="viewport" content="width=device-width, initial-scale=1">
+          <title>#{title}</title>
+        </head>
+        #{body}
+      </html>
+      HTML
+    end
+    def title(html)
+      doc = Nokogiri::HTML(html)
+      title_node = doc.at('title')
+      title = title_node ? title_node.text.strip : ""
+      title
+    end
+    def cleaned_html(html)
+      doc = Nokogiri::HTML(html)
+      title_node = doc.at('title')
+      title = title_node ? title_node.text.strip : ""
+      body = doc.at('body')
+      content = nil
+      if body
+        # First remove the most egregious crap, like obvious ads, etc.
+        prune(node: body, matcher: Matchers::ObviousJunkMatcher.new)
+        # Find the highest-scoring node in the cleaned tree
+        highest_scoring = find_highest_scoring_node(body)
+        return nil unless highest_scoring
+        # Find the appropriate root container for that node
+        content_root = find_content_root(highest_scoring)
+        # Apply second-pass cleaning to the content
+        if content_root
+          clean_node(content_root)
+        end
+        content = content_root
+      end
+      [title, content]
+    end
+    private
+    def prune(node:, matcher:)
+      node.children.to_a.each do |child|
+        if child.element?
+          # Recurse first
+          prune(node: child, matcher: matcher)
+          child.remove if matcher.match?(child)
+        end
+      end
+      node
+    end
+    # Scores a node based on how likely it is to be main content
+    # Higher scores indicate more likely to be the primary article content
+    def score_node(node)
+      @node_scorer.score(node)
+    end
+    # Finds the node with the highest content score
+    # Returns the node with the highest score in the tree
+    def find_highest_scoring_node(node)
+      return nil unless node.element?
+      best_node = node
+      best_score = score_node(node)
+      # Recursively check all child elements
+      node.children.each do |child|
+        next unless child.element?
+        candidate = find_highest_scoring_node(child)
+        if candidate
+          candidate_score = score_node(candidate)
+          if candidate_score > best_score
+            best_node = candidate
+            best_score = candidate_score
+          end
+        end
+      end
+      best_node
+    end
+    # Finds an appropriate root ancestor for the given node
+    # Traverses up the tree to find a good container for the main content
+    # Stops at body or when we find a semantic container
+    def find_content_root(node)
+      return node if node.nil? || node.name.downcase == 'body'
+      current = node
+      best_ancestor = node
+      # Traverse up the tree
+      while current.parent && current.parent.element?
+        parent = current.parent
+        parent_tag = parent.name.downcase
+        # Stop at body
+        break if parent_tag == 'body'
+        # Prefer semantic containers
+        if %w(article main section).include?(parent_tag)
+          best_ancestor = parent
+        end
+        # If parent has significantly more content than current node,
+        # it might be a better root (captures related content)
+        parent_score = score_node(parent)
+        current_score = score_node(current)
+        # Only move up if parent score is meaningfully higher
+        # (not just marginally better due to including the child)
+        if parent_score > current_score * 1.3
+          best_ancestor = parent
+        end
+        current = parent
+      end
+      best_ancestor
+    end
+    # Recursively process an element's children and remove any that are "empty"
+    def clean_node(node, within_article: false)
+      # Iterate over a copy of the children to avoid modification issues
+      node.children.to_a.each do |child|
+        if child.element?
+          clean_node(child, within_article: within_article || node.name.downcase == "article" )  # process children first
+          # Remove the child if it qualifies as "empty"
+          if child.name.downcase == "aside" && !within_article
+            child.remove
+          elsif removable?(child)
+            child.remove
+          else
+            child.remove_attribute("class") # Cleanup
+          end
+        end
+      end
+    end
+    # An element is removable if:
+    # - It is a <script> or <style> tag (non-visible content), or other element we don't want
+    # - It has no descendant text nodes (ignoring whitespace)
+    def removable?(node)
+      @removable_node_matcher.match?(node)
+    end
+  end
+end

data/lib/detergent/matchers/obvious_junk_matcher.rb ADDED Viewed

@@ -0,0 +1,25 @@
+module Detergent
+  module Matchers
+    class ObviousJunkMatcher
+      def match?(node)
+        tag = node.name.downcase
+        # Always remove these tags
+        return true if %w(script style link iframe noscript).include?(tag)
+        # Remove hidden elements
+        return true if node['aria-hidden'] == 'true'
+        # Remove display:none elements
+        style = node['style'].to_s.downcase
+        return true if style.include?('display:none') || style.include?('display: none')
+        # Remove structural navigation
+        return true if %w(nav header footer).include?(tag)
+        return true if node['role'].to_s.downcase == 'navigation'
+        false
+      end
+    end
+  end
+end

data/lib/detergent/matchers/removable_node_matcher.rb ADDED Viewed

@@ -0,0 +1,42 @@
+module Detergent
+  module Matchers
+    class RemovableNodeMatcher
+      def match?(node)
+        # Remove button elements with Share or Save aria-labels
+        if node.name.downcase == "button"
+          aria_label = node['aria-label'].to_s.downcase
+          return true if ["share", "save"].include?(aria_label)
+        end
+        # Remove nav with role=navigation
+        if node.name.downcase == "nav"
+          return true if node['role'].to_s.downcase == "navigation"
+        end
+        # Elements we don't care about
+        return true if %w(link iframe script style footer nav form select textarea).include?(node.name.downcase)
+        # Get rid of elements with classnames that look suspect
+        class_list = node['class'].to_s.downcase.split(" ")
+        return true if class_list.any?{ %w(comments-show comments actionbar related-stories navigation nodisplay sidebar admz hidden header footer social share).include?(_1) }
+        id_list = node['id'].to_s.downcase.split(" ")
+        return true if id_list.any?{ %w(header navigation ad admz sidebar related-stories hidden).include?(_1) }
+        # Get rid of elements with inline styles that look suspect
+        style_list = node['style'].to_s.downcase.split(";")
+        return true if style_list.any?{ ["display: none", "display:none"].include?(_1) }
+        # Don't remove images, etc in this step, which never have text content:
+        return false if %w(img picture figure).include?(node.name.downcase)
+        # Don't remove if any descendants are imgs, etc:
+        return false if node.xpath(".//img | .//picture | .//figure").any?
+        # Using XPath to find any descendant text node that contains non-whitespace characters.
+        node.xpath(".//text()[normalize-space()]").empty?
+      end
+    end
+  end
+end

data/lib/detergent/node_scorer.rb ADDED Viewed

@@ -0,0 +1,74 @@
+# LOL yes, I'm doing this.
+class BasicObject
+  def readify_score=(new_score)
+    @readify_score = new_score
+  end
+  def readify_score
+    @readify_score
+  end
+end
+module Detergent
+  class NodeScorer
+    def score(node)
+      return node.readify_score unless node.readify_score.nil?
+      return 0 unless node.element?
+      score = 0
+      tag = node.name.downcase
+      # Positive indicators for main content
+      score += 100 if tag == 'article'
+      score += 50 if tag == 'main'
+      score += 25 if node['role'].to_s.downcase == 'main'
+      # Count paragraphs - strong indicator of article content
+      paragraph_count = node.xpath('.//p').count
+      score += paragraph_count * 5
+      # Calculate text length
+      begin
+        text_content = node.xpath('.//text()').map(&:text).join
+        text_length = text_content.strip.length
+        score += (text_length / 100).to_i  # 1 point per 100 characters
+      rescue StandardError
+        text_length = 0
+      end
+      # Calculate link density (high link density suggests navigation)
+      link_count = node.xpath('.//a').count
+      if text_length > 0
+        link_density = (link_count.to_f / (text_length / 100.0))
+        score -= (link_density * 10).to_i  # Penalize high link density
+      end
+      # Bonus for images/figures (suggests article content)
+      media_count = node.xpath('.//img | .//picture | .//figure').count
+      score += media_count * 10
+      # Bonus for long paragraphs
+      begin
+        long_paragraphs = node.xpath('.//p').select { |p| p.text.strip.length > 100 }.count
+        score += long_paragraphs * 15
+      rescue StandardError
+        # Do nothing
+      end
+      # Bonus for blockquotes and lists (often in articles)
+      score += node.xpath('.//blockquote').count * 10
+      score += node.xpath('.//ul | .//ol').count * 5
+      # Penalty for suspicious classes/ids
+      classes = node['class'].to_s.downcase
+      ids = node['id'].to_s.downcase
+      score -= 50 if classes.include?('sidebar') || ids.include?('sidebar')
+      score -= 50 if classes.include?('comment') || ids.include?('comment')
+      score -= 70 if classes.include?('ad') || ids.include?('ad')
+      score -= 70 if classes.include?('social') || ids.include?('social')
+      node.readify_score = [score, 0].max  # Don't return negative scores
+    end
+  end
+end

data/lib/detergent/version.rb ADDED Viewed

@@ -0,0 +1,3 @@
+module Detergent
+  VERSION = "1.0.0"
+end

data/lib/detergent.rb ADDED Viewed

@@ -0,0 +1,10 @@
+require "nokogiri"
+require_relative "detergent/version"
+require_relative "detergent/node_scorer"
+require_relative "detergent/matchers/obvious_junk_matcher"
+require_relative "detergent/matchers/removable_node_matcher"
+require_relative "detergent/cleaner"
+module Detergent
+end

metadata ADDED Viewed

@@ -0,0 +1,93 @@
+--- !ruby/object:Gem::Specification
+name: detergent
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Jeff McFadden
+bindir: bin
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.15'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.15'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+description: 'Detergent scrubs junk out of web pages: it removes ads, navigation,
+  scripts, and other cruft, then extracts the main article content from an HTML document.'
+email:
+- 55709+jeffmcfadden@users.noreply.github.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE
+- README.md
+- lib/detergent.rb
+- lib/detergent/cleaner.rb
+- lib/detergent/matchers/obvious_junk_matcher.rb
+- lib/detergent/matchers/removable_node_matcher.rb
+- lib/detergent/node_scorer.rb
+- lib/detergent/version.rb
+homepage: https://github.com/jeffmcfadden/detergent
+licenses:
+- WTFPL
+metadata:
+  homepage_uri: https://github.com/jeffmcfadden/detergent
+  source_code_uri: https://github.com/jeffmcfadden/detergent
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.1.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.6.9
+specification_version: 4
+summary: Cleans up website HTML, extracting the main content.
+test_files: []