detergent 2.2.0 → 2.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1e00871e3ab04f35424e5c563328b1e036bf14a10ec9738da79c81462135e963
-  data.tar.gz: b600be23e126d45cdfcc8e058398a1096274693be3d985f3051511e718f7c248
+  metadata.gz: 27c1381c7afa9541cb1285a2f31c418d9c1d392fd5990abb7c1ee4a5ef630f24
+  data.tar.gz: 4935ce56bfef6771ab9cdbd2a431d9931ab374df7e4dffa876c75f95ee3c671c
 SHA512:
-  metadata.gz: 5263b607e0e15a25b15ab1df4c846f5228a7c188daf6a78811a67e91bf06fdc46b2b9e188fc4158657223e8665df5de894e3b1a6611be5886ab2489c7d63210c
-  data.tar.gz: '0884b42ce893b34ff696234b0d094d9a0a8613c5d9eee3b979308c9214eb4a0b67b5247a9b7cb33e0af9e9320922995dd234c5844e0e3169e899da4e6ab687fe'
+  metadata.gz: 5b5f9a0ce17b8ac37ccfa764780709f4bff83a3c527c7b2d5bd68ed633369aeea93a662d81e680d3b4e90679b518478a72cebb8a24b97990df9a857aebf55dcd
+  data.tar.gz: 7030d8857c008565da627a56b4071de605bf3b5ecc6b5bdebd91a1bc20b5db991c14953b81da910567b9810d257089449dccfa0e33ed2fc7bfb82e390dda3111

data/README.md

@@ -31,6 +31,43 @@ clean_html = cleaner.clean(dirty_html)
 title = cleaner.title(dirty_html)
 ```
 
+## Command line
+
+```
+detergent article.html                    # extracted content as Markdown
+detergent --format text https://example.com/post
+detergent --format html article.html     # cleaned standalone document
+detergent --format title article.html
+```
+
+## Debugging
+
+When extraction picks the wrong node (or nothing at all), the Inspector
+reports every decision: what each pass removed, the top-scoring nodes with
+point-by-point score breakdowns, and whether the best candidate cleared the
+minimum content score.
+
+```
+$ detergent --format debug page.html
+
+Detergent 2.3.0 debug report
+Title: "Hacker News"
+Verdict: NO MAIN CONTENT (best score 10 < threshold 25)
+
+First pass (obvious junk): removed 1 nodes (script x1)
+
+Top scoring nodes after first pass:
+     10  td
+           +10  1 media elements
+      0  tr#48825749.athing.submission          20.Tenda firmware (multiple versions) co...
+           -34  link density 3.45 (3 links)
+...
+```
+
+Or from Ruby: `puts Detergent::Inspector.new.analyze(html)` — the returned
+report object also exposes `located?`, `best_score`, `top_nodes`, and
+`removals` for programmatic use.
+
 ## How it works
 
 1. A first pass prunes obvious junk (scripts, styles, iframes, nav, headers, footers, hidden elements).

data/exe/detergent

@@ -0,0 +1,58 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "detergent"
+require "optparse"
+
+format = "markdown"
+
+parser = OptionParser.new do |opts|
+  opts.banner = <<~BANNER
+    Usage: detergent [options] <file or URL>
+
+    Extracts the main content from a web page.
+  BANNER
+
+  opts.on("-f", "--format FORMAT", %w[html markdown text title debug],
+          "Output format: html, markdown, text, title, or debug (default: markdown)") do |value|
+    format = value
+  end
+
+  opts.on("-v", "--version", "Print the version") do
+    puts Detergent::VERSION
+    exit
+  end
+end
+parser.parse!
+
+source = ARGV.shift
+abort parser.to_s if source.nil?
+
+def fetch(url, limit = 5)
+  abort "Error: too many redirects" if limit.zero?
+
+  response = Net::HTTP.get_response(URI(url))
+  case response
+  when Net::HTTPRedirection
+    fetch(URI.join(url, response["location"]).to_s, limit - 1)
+  when Net::HTTPSuccess
+    response.body
+  else
+    abort "Error: HTTP #{response.code} fetching #{url}"
+  end
+end
+
+html = if source.match?(%r{\Ahttps?://})
+  require "net/http"
+  fetch(source)
+else
+  File.read(source)
+end
+
+case format
+when "html" then puts Detergent.clean(html)
+when "markdown" then puts Detergent.markdown(html)
+when "text" then puts Detergent.text(html)
+when "title" then puts Detergent::Cleaner.new.title(html)
+when "debug" then puts Detergent::Inspector.new.analyze(html)
+end

data/lib/detergent/cleaner.rb

@@ -4,7 +4,11 @@ module Detergent
   # Orchestrates cleaning: prunes obvious junk, locates the main content,
   # scrubs it, and renders the result back out as a standalone document.
   class Cleaner
-    def initialize
+    # observer (optional) receives instrumentation callbacks during
+    # extraction: node_removed(node, pass:) and content_pruned(body, scorer).
+    # Used by Inspector to build debug reports.
+    def initialize(observer: nil)
+      @observer = observer
       @obvious_junk_matcher = Matchers::ObviousJunkMatcher.new
       @removable_node_matcher = Matchers::RemovableNodeMatcher.new
     end
@@ -50,7 +54,9 @@ module Detergent
 
         # Score caches are only valid for a single parse of a single
         # document, so the scorer and locator are built per extract.
-        content = ContentLocator.new(NodeScorer.new).locate(body)
+        scorer = NodeScorer.new
+        @observer&.content_pruned(body, scorer)
+        content = ContentLocator.new(scorer).locate(body)
 
         # Apply second-pass cleaning to the content
         if content
@@ -91,6 +97,7 @@ module Detergent
 
         # Match first so we never bother walking a subtree we're removing
         if matcher.match?(child)
+          @observer&.node_removed(child, pass: :first_pass)
           child.remove
         else
           prune(node: child, matcher: matcher)
@@ -111,8 +118,10 @@ module Detergent
           # Remove the child if it qualifies as "empty"
 
           if child.name.downcase == "aside" && !within_article
+            @observer&.node_removed(child, pass: :second_pass)
             child.remove
           elsif removable?(child)
+            @observer&.node_removed(child, pass: :second_pass)
             child.remove
           else
             strip_junk_attributes(child)

data/lib/detergent/inspector.rb

@@ -0,0 +1,139 @@
+# frozen_string_literal: true
+
+module Detergent
+  # Instruments a full extraction run and reports every decision made
+  # along the way: what the first pass removed, how the tree scored (with
+  # per-node score breakdowns), whether the best candidate cleared the
+  # minimum-score threshold, which content root was chosen, and what the
+  # second pass removed.
+  #
+  #   puts Detergent::Inspector.new.analyze(html)
+  #
+  # Also available from the command line:
+  #
+  #   detergent --format debug page.html
+  class Inspector
+    TOP_NODE_COUNT = 10
+    BREAKDOWN_COUNT = 3
+
+    ScoredNode = Struct.new(:score, :descriptor, :preview, :breakdown)
+
+    class Report
+      attr_accessor :title, :best_score, :content_root, :content_root_size, :top_nodes
+      attr_reader :removals
+
+      def initialize
+        @removals = { first_pass: [], second_pass: [] }
+        @top_nodes = []
+        @best_score = 0
+      end
+
+      def located?
+        !content_root.nil?
+      end
+
+      def to_s
+        lines = ["Detergent #{Detergent::VERSION} debug report"]
+        lines << "Title: #{title.inspect}"
+        lines << verdict
+        lines << ""
+        lines << removal_summary(:first_pass, "First pass (obvious junk)")
+        lines << ""
+        lines << "Top scoring nodes after first pass:"
+
+        top_nodes.each do |node|
+          lines << format("  %5d  %-38s %s", node.score, node.descriptor, node.preview)
+          node.breakdown&.each do |label, points|
+            lines << format("         %+5d  %s", points, label)
+          end
+        end
+
+        lines << ""
+        lines << if located?
+          "Content root: #{content_root} (#{content_root_size} chars of HTML)"
+        else
+          "Content root: (none)"
+        end
+        lines << removal_summary(:second_pass, "Second pass (within content)")
+        lines.join("\n")
+      end
+
+      private
+
+      def verdict
+        threshold = ContentLocator::MINIMUM_CONTENT_SCORE
+
+        if located?
+          "Verdict: content located (best score #{best_score}, threshold #{threshold})"
+        else
+          "Verdict: NO MAIN CONTENT (best score #{best_score} < threshold #{threshold})"
+        end
+      end
+
+      def removal_summary(pass, label)
+        removed = removals[pass]
+        return "#{label}: removed 0 nodes" if removed.empty?
+
+        tally = removed.tally.sort_by { |_tag, count| -count }
+                       .map { |tag, count| "#{tag} x#{count}" }.join(", ")
+        "#{label}: removed #{removed.length} nodes (#{tally})"
+      end
+    end
+
+    def analyze(html)
+      @report = Report.new
+
+      title, content = Cleaner.new(observer: self).extract(html)
+      @report.title = title
+
+      if content
+        @report.content_root = descriptor(content)
+        @report.content_root_size = content.to_s.length
+      end
+
+      @report
+    end
+
+    # -- Cleaner observer callbacks --
+
+    def node_removed(node, pass:)
+      @report.removals[pass] << node.name.downcase
+    end
+
+    # Called after the first pass, before content location, with the same
+    # scorer the locator will use — so the report matches its decisions.
+    def content_pruned(body, scorer)
+      scored = ([body] + body.xpath(".//*").to_a)
+               .map { |node| [scorer.score(node), node] }
+               .sort_by { |score, _node| -score }
+               .first(TOP_NODE_COUNT)
+
+      # Materialize descriptors/previews now; the tree mutates after this.
+      @report.top_nodes = scored.each_with_index.map do |(score, node), index|
+        breakdown = index < BREAKDOWN_COUNT ? scorer.explain(node) : nil
+        ScoredNode.new(score, descriptor(node), preview(node), breakdown)
+      end
+
+      @report.best_score = scored.first&.first || 0
+    end
+
+    private
+
+    def descriptor(node)
+      desc = +node.name.downcase
+
+      id = node['id'].to_s
+      desc << "##{id}" unless id.empty?
+
+      classes = node['class'].to_s.split(" ")
+      desc << ".#{classes.first(3).join('.')}" unless classes.empty?
+
+      desc
+    end
+
+    def preview(node)
+      text = node.text.gsub(/\s+/, " ").strip
+      text.length > 60 ? "#{text[0, 60]}..." : text
+    end
+  end
+end

data/lib/detergent/node_scorer.rb

@@ -35,47 +35,63 @@ module Detergent
       return cached unless cached.nil?
       return 0 unless node.element?
 
-      @scores[node] = compute_score(node)
+      total = components_for(node).sum { |_label, points| points }
+      @scores[node] = [total, 0].max # Don't return negative scores
+    end
+
+    # Returns the score as [label, points] pairs explaining where every
+    # point came from. Debugging aid used by Inspector.
+    def explain(node)
+      return [] unless node.element?
+
+      components_for(node)
     end
 
     private
 
-    def compute_score(node)
+    def components_for(node)
       stats = stats_for(node)
-      score = 0
       tag = node.name.downcase
+      parts = []
 
       # Positive indicators for main content
-      score += ARTICLE_TAG_BONUS if tag == 'article'
-      score += MAIN_TAG_BONUS if tag == 'main'
-      score += MAIN_ROLE_BONUS if node['role'].to_s.downcase == 'main'
+      parts << ["<article> tag", ARTICLE_TAG_BONUS] if tag == 'article'
+      parts << ["<main> tag", MAIN_TAG_BONUS] if tag == 'main'
+      parts << ["role=main", MAIN_ROLE_BONUS] if node['role'].to_s.downcase == 'main'
 
       # Paragraphs and raw text are strong indicators of article content
-      score += stats[:paragraphs] * POINTS_PER_PARAGRAPH
-      score += stats[:text_length] / CHARS_PER_TEXT_POINT
-      score += stats[:long_paragraphs] * LONG_PARAGRAPH_BONUS
+      if stats[:paragraphs] > 0
+        parts << ["#{stats[:paragraphs]} paragraphs", stats[:paragraphs] * POINTS_PER_PARAGRAPH]
+      end
+      if (text_points = stats[:text_length] / CHARS_PER_TEXT_POINT) > 0
+        parts << ["#{stats[:text_length]} chars of text", text_points]
+      end
+      if stats[:long_paragraphs] > 0
+        parts << ["#{stats[:long_paragraphs]} long paragraphs", stats[:long_paragraphs] * LONG_PARAGRAPH_BONUS]
+      end
 
       # High link density suggests navigation
-      if stats[:text_length] > 0
+      if stats[:text_length] > 0 && stats[:links] > 0
         link_density = stats[:links].to_f / (stats[:text_length] / 100.0)
-        score -= (link_density * LINK_DENSITY_PENALTY).to_i
+        parts << ["link density #{link_density.round(2)} (#{stats[:links]} links)",
+                  -(link_density * LINK_DENSITY_PENALTY).to_i]
       end
 
       # Media, blockquotes, and lists all suggest article content
-      score += stats[:media] * POINTS_PER_MEDIA
-      score += stats[:blockquotes] * POINTS_PER_BLOCKQUOTE
-      score += stats[:lists] * POINTS_PER_LIST
+      parts << ["#{stats[:media]} media elements", stats[:media] * POINTS_PER_MEDIA] if stats[:media] > 0
+      parts << ["#{stats[:blockquotes]} blockquotes", stats[:blockquotes] * POINTS_PER_BLOCKQUOTE] if stats[:blockquotes] > 0
+      parts << ["#{stats[:lists]} lists", stats[:lists] * POINTS_PER_LIST] if stats[:lists] > 0
 
       # Penalty for suspicious classes/ids
       classes = node['class'].to_s.downcase
       ids = node['id'].to_s.downcase
 
-      score -= SIDEBAR_PENALTY if classes.include?('sidebar') || ids.include?('sidebar')
-      score -= COMMENT_PENALTY if classes.include?('comment') || ids.include?('comment')
-      score -= AD_PENALTY if classes.include?('ad') || ids.include?('ad')
-      score -= SOCIAL_PENALTY if classes.include?('social') || ids.include?('social')
+      parts << ["suspect 'sidebar' class/id", -SIDEBAR_PENALTY] if classes.include?('sidebar') || ids.include?('sidebar')
+      parts << ["suspect 'comment' class/id", -COMMENT_PENALTY] if classes.include?('comment') || ids.include?('comment')
+      parts << ["suspect 'ad' class/id", -AD_PENALTY] if classes.include?('ad') || ids.include?('ad')
+      parts << ["suspect 'social' class/id", -SOCIAL_PENALTY] if classes.include?('social') || ids.include?('social')
 
-      [score, 0].max # Don't return negative scores
+      parts
     end
 
     # Statistics about a node's descendants (not the node itself),

data/lib/detergent/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Detergent
-  VERSION = "2.2.0"
+  VERSION = "2.3.0"
 end

data/lib/detergent.rb

@@ -51,3 +51,4 @@ require_relative "detergent/text_renderer"
 require_relative "detergent/matchers/obvious_junk_matcher"
 require_relative "detergent/matchers/removable_node_matcher"
 require_relative "detergent/cleaner"
+require_relative "detergent/inspector"

metadata

@@ -1,11 +1,11 @@
 --- !ruby/object:Gem::Specification
 name: detergent
 version: !ruby/object:Gem::Version
-  version: 2.2.0
+  version: 2.3.0
 platform: ruby
 authors:
 - Jeff McFadden
-bindir: bin
+bindir: exe
 cert_chain: []
 date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
@@ -55,15 +55,18 @@ description: 'Detergent scrubs junk out of web pages: it removes ads, navigation
   scripts, and other cruft, then extracts the main article content from an HTML document.'
 email:
 - 55709+jeffmcfadden@users.noreply.github.com
-executables: []
+executables:
+- detergent
 extensions: []
 extra_rdoc_files: []
 files:
 - LICENSE
 - README.md
+- exe/detergent
 - lib/detergent.rb
 - lib/detergent/cleaner.rb
 - lib/detergent/content_locator.rb
+- lib/detergent/inspector.rb
 - lib/detergent/markdown_renderer.rb
 - lib/detergent/matchers/obvious_junk_matcher.rb
 - lib/detergent/matchers/removable_node_matcher.rb