detergent 2.2.0 → 2.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1e00871e3ab04f35424e5c563328b1e036bf14a10ec9738da79c81462135e963
-  data.tar.gz: b600be23e126d45cdfcc8e058398a1096274693be3d985f3051511e718f7c248
+  metadata.gz: fe5c0dc45668a4c14b991433606aec83c6fd2672e93533632d608e66b858f03a
+  data.tar.gz: eada52c178f26669036609d5431f85245cd06d5de6b3920b010ba91a587ba18c
 SHA512:
-  metadata.gz: 5263b607e0e15a25b15ab1df4c846f5228a7c188daf6a78811a67e91bf06fdc46b2b9e188fc4158657223e8665df5de894e3b1a6611be5886ab2489c7d63210c
-  data.tar.gz: '0884b42ce893b34ff696234b0d094d9a0a8613c5d9eee3b979308c9214eb4a0b67b5247a9b7cb33e0af9e9320922995dd234c5844e0e3169e899da4e6ab687fe'
+  metadata.gz: 9ed056835cc1475822f443caa51610abc1dab4000dccf97f1058971f15cf8d9ceb6dbf44c7a91b079b4d6b886a8f239f1ec1408a53311e8d335d17cb34174886
+  data.tar.gz: e0ff4553f215510c0aebfcbbe35b2a5cca86ebfc8c755f597157a4c565b9950ad6b188b32610a92c7661635307ac388968a5e6a2fcd157e87463e2582c51810e

data/README.md

@@ -31,6 +31,43 @@ clean_html = cleaner.clean(dirty_html)
 title = cleaner.title(dirty_html)
 ```
 
+## Command line
+
+```
+detergent article.html                    # extracted content as Markdown
+detergent --format text https://example.com/post
+detergent --format html article.html     # cleaned standalone document
+detergent --format title article.html
+```
+
+## Debugging
+
+When extraction picks the wrong node (or nothing at all), the Inspector
+reports every decision: what each pass removed, the top-scoring nodes with
+point-by-point score breakdowns, and whether the best candidate cleared the
+minimum content score.
+
+```
+$ detergent --format debug page.html
+
+Detergent 2.3.0 debug report
+Title: "Hacker News"
+Verdict: NO MAIN CONTENT (best score 10 < threshold 25)
+
+First pass (obvious junk): removed 1 nodes (script x1)
+
+Top scoring nodes after first pass:
+     10  td
+           +10  1 media elements
+      0  tr#48825749.athing.submission          20.Tenda firmware (multiple versions) co...
+           -34  link density 3.45 (3 links)
+...
+```
+
+Or from Ruby: `puts Detergent::Inspector.new.analyze(html)` — the returned
+report object also exposes `located?`, `best_score`, `top_nodes`, and
+`removals` for programmatic use.
+
 ## How it works
 
 1. A first pass prunes obvious junk (scripts, styles, iframes, nav, headers, footers, hidden elements).
@@ -38,6 +75,11 @@ title = cleaner.title(dirty_html)
 3. The highest-scoring node is located and walked up to an appropriate semantic container.
 4. A second pass cleans that content of empty and suspect elements.
 
+Pages with no article content but a substantial set of title-like links —
+index pages and link aggregators like the Hacker News front page — fall
+back to link-list extraction: the result is a clean list of the page's
+links instead of an article.
+
 ## License
 
 [WTFPL](LICENSE)

data/exe/detergent

@@ -0,0 +1,58 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "detergent"
+require "optparse"
+
+format = "markdown"
+
+parser = OptionParser.new do |opts|
+  opts.banner = <<~BANNER
+    Usage: detergent [options] <file or URL>
+
+    Extracts the main content from a web page.
+  BANNER
+
+  opts.on("-f", "--format FORMAT", %w[html markdown text title debug],
+          "Output format: html, markdown, text, title, or debug (default: markdown)") do |value|
+    format = value
+  end
+
+  opts.on("-v", "--version", "Print the version") do
+    puts Detergent::VERSION
+    exit
+  end
+end
+parser.parse!
+
+source = ARGV.shift
+abort parser.to_s if source.nil?
+
+def fetch(url, limit = 5)
+  abort "Error: too many redirects" if limit.zero?
+
+  response = Net::HTTP.get_response(URI(url))
+  case response
+  when Net::HTTPRedirection
+    fetch(URI.join(url, response["location"]).to_s, limit - 1)
+  when Net::HTTPSuccess
+    response.body
+  else
+    abort "Error: HTTP #{response.code} fetching #{url}"
+  end
+end
+
+html = if source.match?(%r{\Ahttps?://})
+  require "net/http"
+  fetch(source)
+else
+  File.read(source)
+end
+
+case format
+when "html" then puts Detergent.clean(html)
+when "markdown" then puts Detergent.markdown(html)
+when "text" then puts Detergent.text(html)
+when "title" then puts Detergent::Cleaner.new.title(html)
+when "debug" then puts Detergent::Inspector.new.analyze(html)
+end

data/lib/detergent/cleaner.rb

@@ -4,7 +4,12 @@ module Detergent
   # Orchestrates cleaning: prunes obvious junk, locates the main content,
   # scrubs it, and renders the result back out as a standalone document.
   class Cleaner
-    def initialize
+    # observer (optional) receives instrumentation callbacks during
+    # extraction: node_removed(node, pass:), content_pruned(body, scorer),
+    # and extraction_strategy(strategy). Used by Inspector to build debug
+    # reports.
+    def initialize(observer: nil)
+      @observer = observer
       @obvious_junk_matcher = Matchers::ObviousJunkMatcher.new
       @removable_node_matcher = Matchers::RemovableNodeMatcher.new
     end
@@ -50,7 +55,19 @@ module Detergent
 
         # Score caches are only valid for a single parse of a single
         # document, so the scorer and locator are built per extract.
-        content = ContentLocator.new(NodeScorer.new).locate(body)
+        scorer = NodeScorer.new
+        @observer&.content_pruned(body, scorer)
+        content = ContentLocator.new(scorer).locate(body)
+        strategy = content ? :article : nil
+
+        # No article-shaped content: the page may be a link index (front
+        # page, aggregator), whose content is its list of links.
+        if content.nil?
+          content = LinkListExtractor.new.extract(body)
+          strategy = :link_list if content
+        end
+
+        @observer&.extraction_strategy(strategy)
 
         # Apply second-pass cleaning to the content
         if content
@@ -91,6 +108,7 @@ module Detergent
 
         # Match first so we never bother walking a subtree we're removing
         if matcher.match?(child)
+          @observer&.node_removed(child, pass: :first_pass)
           child.remove
         else
           prune(node: child, matcher: matcher)
@@ -111,8 +129,10 @@ module Detergent
           # Remove the child if it qualifies as "empty"
 
           if child.name.downcase == "aside" && !within_article
+            @observer&.node_removed(child, pass: :second_pass)
             child.remove
           elsif removable?(child)
+            @observer&.node_removed(child, pass: :second_pass)
             child.remove
           else
             strip_junk_attributes(child)

data/lib/detergent/inspector.rb

@@ -0,0 +1,146 @@
+# frozen_string_literal: true
+
+module Detergent
+  # Instruments a full extraction run and reports every decision made
+  # along the way: what the first pass removed, how the tree scored (with
+  # per-node score breakdowns), whether the best candidate cleared the
+  # minimum-score threshold, which content root was chosen, and what the
+  # second pass removed.
+  #
+  #   puts Detergent::Inspector.new.analyze(html)
+  #
+  # Also available from the command line:
+  #
+  #   detergent --format debug page.html
+  class Inspector
+    TOP_NODE_COUNT = 10
+    BREAKDOWN_COUNT = 3
+
+    ScoredNode = Struct.new(:score, :descriptor, :preview, :breakdown)
+
+    class Report
+      attr_accessor :title, :best_score, :content_root, :content_root_size, :top_nodes, :strategy
+      attr_reader :removals
+
+      def initialize
+        @removals = { first_pass: [], second_pass: [] }
+        @top_nodes = []
+        @best_score = 0
+      end
+
+      def located?
+        !content_root.nil?
+      end
+
+      def to_s
+        lines = ["Detergent #{Detergent::VERSION} debug report"]
+        lines << "Title: #{title.inspect}"
+        lines << verdict
+        lines << ""
+        lines << removal_summary(:first_pass, "First pass (obvious junk)")
+        lines << ""
+        lines << "Top scoring nodes after first pass:"
+
+        top_nodes.each do |node|
+          lines << format("  %5d  %-38s %s", node.score, node.descriptor, node.preview)
+          node.breakdown&.each do |label, points|
+            lines << format("         %+5d  %s", points, label)
+          end
+        end
+
+        lines << ""
+        lines << if located?
+          "Content root: #{content_root} (#{content_root_size} chars of HTML)"
+        else
+          "Content root: (none)"
+        end
+        lines << removal_summary(:second_pass, "Second pass (within content)")
+        lines.join("\n")
+      end
+
+      private
+
+      def verdict
+        threshold = ContentLocator::MINIMUM_CONTENT_SCORE
+
+        case strategy
+        when :article
+          "Verdict: article content located (best score #{best_score}, threshold #{threshold})"
+        when :link_list
+          "Verdict: no article content (best score #{best_score} < threshold #{threshold}); extracted link list instead"
+        else
+          "Verdict: NO MAIN CONTENT (best score #{best_score} < threshold #{threshold})"
+        end
+      end
+
+      def removal_summary(pass, label)
+        removed = removals[pass]
+        return "#{label}: removed 0 nodes" if removed.empty?
+
+        tally = removed.tally.sort_by { |_tag, count| -count }
+                       .map { |tag, count| "#{tag} x#{count}" }.join(", ")
+        "#{label}: removed #{removed.length} nodes (#{tally})"
+      end
+    end
+
+    def analyze(html)
+      @report = Report.new
+
+      title, content = Cleaner.new(observer: self).extract(html)
+      @report.title = title
+
+      if content
+        @report.content_root = descriptor(content)
+        @report.content_root_size = content.to_s.length
+      end
+
+      @report
+    end
+
+    # -- Cleaner observer callbacks --
+
+    def node_removed(node, pass:)
+      @report.removals[pass] << node.name.downcase
+    end
+
+    # Called after the first pass, before content location, with the same
+    # scorer the locator will use — so the report matches its decisions.
+    def content_pruned(body, scorer)
+      scored = ([body] + body.xpath(".//*").to_a)
+               .map { |node| [scorer.score(node), node] }
+               .sort_by { |score, _node| -score }
+               .first(TOP_NODE_COUNT)
+
+      # Materialize descriptors/previews now; the tree mutates after this.
+      @report.top_nodes = scored.each_with_index.map do |(score, node), index|
+        breakdown = index < BREAKDOWN_COUNT ? scorer.explain(node) : nil
+        ScoredNode.new(score, descriptor(node), preview(node), breakdown)
+      end
+
+      @report.best_score = scored.first&.first || 0
+    end
+
+    def extraction_strategy(strategy)
+      @report.strategy = strategy
+    end
+
+    private
+
+    def descriptor(node)
+      desc = +node.name.downcase
+
+      id = node['id'].to_s
+      desc << "##{id}" unless id.empty?
+
+      classes = node['class'].to_s.split(" ")
+      desc << ".#{classes.first(3).join('.')}" unless classes.empty?
+
+      desc
+    end
+
+    def preview(node)
+      text = node.text.gsub(/\s+/, " ").strip
+      text.length > 60 ? "#{text[0, 60]}..." : text
+    end
+  end
+end

data/lib/detergent/link_list_extractor.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module Detergent
+  # Fallback extractor for index pages: link aggregators, front pages, and
+  # directories (e.g. the Hacker News front page) where the "content" is a
+  # list of links to other pages. The article locator correctly rejects
+  # these — link density is its strongest navigation signal — so when it
+  # finds nothing, this extractor looks for a substantial collection of
+  # title-like links and returns them as a synthetic list node that flows
+  # through the normal cleaning and rendering pipeline.
+  class LinkListExtractor
+    # Links with less text than this are treated as chrome ("login",
+    # "hide", "42 comments"), not titles.
+    MIN_LINK_TEXT_LENGTH = 15
+
+    # Fewer title-like links than this isn't an index page.
+    MIN_LINKS = 10
+
+    # Returns a synthetic <ul> of the page's title-like links, or nil if
+    # the page doesn't look like a link index.
+    def extract(body)
+      links = title_like_links(body)
+      return nil if links.length < MIN_LINKS
+
+      build_list(body.document, links)
+    end
+
+    private
+
+    def title_like_links(body)
+      seen_hrefs = {}
+
+      body.css("a").filter_map do |link|
+        href = link["href"].to_s
+        next if href.empty? || href.start_with?("#", "javascript:")
+
+        text = link.text.gsub(/\s+/, " ").strip
+        next if text.length < MIN_LINK_TEXT_LENGTH
+        next if seen_hrefs[href]
+
+        seen_hrefs[href] = true
+        [text, href]
+      end
+    end
+
+    def build_list(doc, links)
+      list = Nokogiri::XML::Node.new("ul", doc)
+
+      links.each do |text, href|
+        item = Nokogiri::XML::Node.new("li", doc)
+        anchor = Nokogiri::XML::Node.new("a", doc)
+        anchor["href"] = href
+        anchor.content = text
+        item.add_child(anchor)
+        list.add_child(item)
+      end
+
+      list
+    end
+  end
+end

data/lib/detergent/markdown_renderer.rb

@@ -9,7 +9,11 @@ module Detergent
   # image, ...) so TextRenderer can subclass this and strip the syntax.
   class MarkdownRenderer
     def render(node)
-      render_blocks(node).gsub(/\n{3,}/, "\n\n").strip
+      # Route through render_block so a root that is itself a block
+      # element (a ul, a blockquote) gets its block treatment; generic
+      # containers and fragments fall through to render_blocks.
+      output = node.element? ? render_block(node) : render_blocks(node)
+      output.gsub(/\n{3,}/, "\n\n").strip
     end
 
     private

data/lib/detergent/node_scorer.rb

@@ -35,47 +35,63 @@ module Detergent
       return cached unless cached.nil?
       return 0 unless node.element?
 
-      @scores[node] = compute_score(node)
+      total = components_for(node).sum { |_label, points| points }
+      @scores[node] = [total, 0].max # Don't return negative scores
+    end
+
+    # Returns the score as [label, points] pairs explaining where every
+    # point came from. Debugging aid used by Inspector.
+    def explain(node)
+      return [] unless node.element?
+
+      components_for(node)
     end
 
     private
 
-    def compute_score(node)
+    def components_for(node)
       stats = stats_for(node)
-      score = 0
       tag = node.name.downcase
+      parts = []
 
       # Positive indicators for main content
-      score += ARTICLE_TAG_BONUS if tag == 'article'
-      score += MAIN_TAG_BONUS if tag == 'main'
-      score += MAIN_ROLE_BONUS if node['role'].to_s.downcase == 'main'
+      parts << ["<article> tag", ARTICLE_TAG_BONUS] if tag == 'article'
+      parts << ["<main> tag", MAIN_TAG_BONUS] if tag == 'main'
+      parts << ["role=main", MAIN_ROLE_BONUS] if node['role'].to_s.downcase == 'main'
 
       # Paragraphs and raw text are strong indicators of article content
-      score += stats[:paragraphs] * POINTS_PER_PARAGRAPH
-      score += stats[:text_length] / CHARS_PER_TEXT_POINT
-      score += stats[:long_paragraphs] * LONG_PARAGRAPH_BONUS
+      if stats[:paragraphs] > 0
+        parts << ["#{stats[:paragraphs]} paragraphs", stats[:paragraphs] * POINTS_PER_PARAGRAPH]
+      end
+      if (text_points = stats[:text_length] / CHARS_PER_TEXT_POINT) > 0
+        parts << ["#{stats[:text_length]} chars of text", text_points]
+      end
+      if stats[:long_paragraphs] > 0
+        parts << ["#{stats[:long_paragraphs]} long paragraphs", stats[:long_paragraphs] * LONG_PARAGRAPH_BONUS]
+      end
 
       # High link density suggests navigation
-      if stats[:text_length] > 0
+      if stats[:text_length] > 0 && stats[:links] > 0
         link_density = stats[:links].to_f / (stats[:text_length] / 100.0)
-        score -= (link_density * LINK_DENSITY_PENALTY).to_i
+        parts << ["link density #{link_density.round(2)} (#{stats[:links]} links)",
+                  -(link_density * LINK_DENSITY_PENALTY).to_i]
       end
 
       # Media, blockquotes, and lists all suggest article content
-      score += stats[:media] * POINTS_PER_MEDIA
-      score += stats[:blockquotes] * POINTS_PER_BLOCKQUOTE
-      score += stats[:lists] * POINTS_PER_LIST
+      parts << ["#{stats[:media]} media elements", stats[:media] * POINTS_PER_MEDIA] if stats[:media] > 0
+      parts << ["#{stats[:blockquotes]} blockquotes", stats[:blockquotes] * POINTS_PER_BLOCKQUOTE] if stats[:blockquotes] > 0
+      parts << ["#{stats[:lists]} lists", stats[:lists] * POINTS_PER_LIST] if stats[:lists] > 0
 
       # Penalty for suspicious classes/ids
       classes = node['class'].to_s.downcase
       ids = node['id'].to_s.downcase
 
-      score -= SIDEBAR_PENALTY if classes.include?('sidebar') || ids.include?('sidebar')
-      score -= COMMENT_PENALTY if classes.include?('comment') || ids.include?('comment')
-      score -= AD_PENALTY if classes.include?('ad') || ids.include?('ad')
-      score -= SOCIAL_PENALTY if classes.include?('social') || ids.include?('social')
+      parts << ["suspect 'sidebar' class/id", -SIDEBAR_PENALTY] if classes.include?('sidebar') || ids.include?('sidebar')
+      parts << ["suspect 'comment' class/id", -COMMENT_PENALTY] if classes.include?('comment') || ids.include?('comment')
+      parts << ["suspect 'ad' class/id", -AD_PENALTY] if classes.include?('ad') || ids.include?('ad')
+      parts << ["suspect 'social' class/id", -SOCIAL_PENALTY] if classes.include?('social') || ids.include?('social')
 
-      [score, 0].max # Don't return negative scores
+      parts
     end
 
     # Statistics about a node's descendants (not the node itself),

data/lib/detergent/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Detergent
-  VERSION = "2.2.0"
+  VERSION = "2.4.0"
 end

data/lib/detergent.rb

@@ -46,8 +46,10 @@ end
 require_relative "detergent/version"
 require_relative "detergent/node_scorer"
 require_relative "detergent/content_locator"
+require_relative "detergent/link_list_extractor"
 require_relative "detergent/markdown_renderer"
 require_relative "detergent/text_renderer"
 require_relative "detergent/matchers/obvious_junk_matcher"
 require_relative "detergent/matchers/removable_node_matcher"
 require_relative "detergent/cleaner"
+require_relative "detergent/inspector"

metadata

@@ -1,11 +1,11 @@
 --- !ruby/object:Gem::Specification
 name: detergent
 version: !ruby/object:Gem::Version
-  version: 2.2.0
+  version: 2.4.0
 platform: ruby
 authors:
 - Jeff McFadden
-bindir: bin
+bindir: exe
 cert_chain: []
 date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
@@ -55,15 +55,19 @@ description: 'Detergent scrubs junk out of web pages: it removes ads, navigation
   scripts, and other cruft, then extracts the main article content from an HTML document.'
 email:
 - 55709+jeffmcfadden@users.noreply.github.com
-executables: []
+executables:
+- detergent
 extensions: []
 extra_rdoc_files: []
 files:
 - LICENSE
 - README.md
+- exe/detergent
 - lib/detergent.rb
 - lib/detergent/cleaner.rb
 - lib/detergent/content_locator.rb
+- lib/detergent/inspector.rb
+- lib/detergent/link_list_extractor.rb
 - lib/detergent/markdown_renderer.rb
 - lib/detergent/matchers/obvious_junk_matcher.rb
 - lib/detergent/matchers/removable_node_matcher.rb