detergent 2.3.0 → 2.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 27c1381c7afa9541cb1285a2f31c418d9c1d392fd5990abb7c1ee4a5ef630f24
-  data.tar.gz: 4935ce56bfef6771ab9cdbd2a431d9931ab374df7e4dffa876c75f95ee3c671c
+  metadata.gz: fe5c0dc45668a4c14b991433606aec83c6fd2672e93533632d608e66b858f03a
+  data.tar.gz: eada52c178f26669036609d5431f85245cd06d5de6b3920b010ba91a587ba18c
 SHA512:
-  metadata.gz: 5b5f9a0ce17b8ac37ccfa764780709f4bff83a3c527c7b2d5bd68ed633369aeea93a662d81e680d3b4e90679b518478a72cebb8a24b97990df9a857aebf55dcd
-  data.tar.gz: 7030d8857c008565da627a56b4071de605bf3b5ecc6b5bdebd91a1bc20b5db991c14953b81da910567b9810d257089449dccfa0e33ed2fc7bfb82e390dda3111
+  metadata.gz: 9ed056835cc1475822f443caa51610abc1dab4000dccf97f1058971f15cf8d9ceb6dbf44c7a91b079b4d6b886a8f239f1ec1408a53311e8d335d17cb34174886
+  data.tar.gz: e0ff4553f215510c0aebfcbbe35b2a5cca86ebfc8c755f597157a4c565b9950ad6b188b32610a92c7661635307ac388968a5e6a2fcd157e87463e2582c51810e

data/README.md

@@ -75,6 +75,11 @@ report object also exposes `located?`, `best_score`, `top_nodes`, and
 3. The highest-scoring node is located and walked up to an appropriate semantic container.
 4. A second pass cleans that content of empty and suspect elements.
 
+Pages with no article content but a substantial set of title-like links —
+index pages and link aggregators like the Hacker News front page — fall
+back to link-list extraction: the result is a clean list of the page's
+links instead of an article.
+
 ## License
 
 [WTFPL](LICENSE)

data/lib/detergent/cleaner.rb

@@ -5,8 +5,9 @@ module Detergent
   # scrubs it, and renders the result back out as a standalone document.
   class Cleaner
     # observer (optional) receives instrumentation callbacks during
-    # extraction: node_removed(node, pass:) and content_pruned(body, scorer).
-    # Used by Inspector to build debug reports.
+    # extraction: node_removed(node, pass:), content_pruned(body, scorer),
+    # and extraction_strategy(strategy). Used by Inspector to build debug
+    # reports.
     def initialize(observer: nil)
       @observer = observer
       @obvious_junk_matcher = Matchers::ObviousJunkMatcher.new
@@ -57,6 +58,16 @@ module Detergent
         scorer = NodeScorer.new
         @observer&.content_pruned(body, scorer)
         content = ContentLocator.new(scorer).locate(body)
+        strategy = content ? :article : nil
+
+        # No article-shaped content: the page may be a link index (front
+        # page, aggregator), whose content is its list of links.
+        if content.nil?
+          content = LinkListExtractor.new.extract(body)
+          strategy = :link_list if content
+        end
+
+        @observer&.extraction_strategy(strategy)
 
         # Apply second-pass cleaning to the content
         if content

data/lib/detergent/inspector.rb

@@ -19,7 +19,7 @@ module Detergent
     ScoredNode = Struct.new(:score, :descriptor, :preview, :breakdown)
 
     class Report
-      attr_accessor :title, :best_score, :content_root, :content_root_size, :top_nodes
+      attr_accessor :title, :best_score, :content_root, :content_root_size, :top_nodes, :strategy
       attr_reader :removals
 
       def initialize
@@ -63,8 +63,11 @@ module Detergent
       def verdict
         threshold = ContentLocator::MINIMUM_CONTENT_SCORE
 
-        if located?
-          "Verdict: content located (best score #{best_score}, threshold #{threshold})"
+        case strategy
+        when :article
+          "Verdict: article content located (best score #{best_score}, threshold #{threshold})"
+        when :link_list
+          "Verdict: no article content (best score #{best_score} < threshold #{threshold}); extracted link list instead"
         else
           "Verdict: NO MAIN CONTENT (best score #{best_score} < threshold #{threshold})"
         end
@@ -117,6 +120,10 @@ module Detergent
       @report.best_score = scored.first&.first || 0
     end
 
+    def extraction_strategy(strategy)
+      @report.strategy = strategy
+    end
+
     private
 
     def descriptor(node)

data/lib/detergent/link_list_extractor.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module Detergent
+  # Fallback extractor for index pages: link aggregators, front pages, and
+  # directories (e.g. the Hacker News front page) where the "content" is a
+  # list of links to other pages. The article locator correctly rejects
+  # these — link density is its strongest navigation signal — so when it
+  # finds nothing, this extractor looks for a substantial collection of
+  # title-like links and returns them as a synthetic list node that flows
+  # through the normal cleaning and rendering pipeline.
+  class LinkListExtractor
+    # Links with less text than this are treated as chrome ("login",
+    # "hide", "42 comments"), not titles.
+    MIN_LINK_TEXT_LENGTH = 15
+
+    # Fewer title-like links than this isn't an index page.
+    MIN_LINKS = 10
+
+    # Returns a synthetic <ul> of the page's title-like links, or nil if
+    # the page doesn't look like a link index.
+    def extract(body)
+      links = title_like_links(body)
+      return nil if links.length < MIN_LINKS
+
+      build_list(body.document, links)
+    end
+
+    private
+
+    def title_like_links(body)
+      seen_hrefs = {}
+
+      body.css("a").filter_map do |link|
+        href = link["href"].to_s
+        next if href.empty? || href.start_with?("#", "javascript:")
+
+        text = link.text.gsub(/\s+/, " ").strip
+        next if text.length < MIN_LINK_TEXT_LENGTH
+        next if seen_hrefs[href]
+
+        seen_hrefs[href] = true
+        [text, href]
+      end
+    end
+
+    def build_list(doc, links)
+      list = Nokogiri::XML::Node.new("ul", doc)
+
+      links.each do |text, href|
+        item = Nokogiri::XML::Node.new("li", doc)
+        anchor = Nokogiri::XML::Node.new("a", doc)
+        anchor["href"] = href
+        anchor.content = text
+        item.add_child(anchor)
+        list.add_child(item)
+      end
+
+      list
+    end
+  end
+end

data/lib/detergent/markdown_renderer.rb

@@ -9,7 +9,11 @@ module Detergent
   # image, ...) so TextRenderer can subclass this and strip the syntax.
   class MarkdownRenderer
     def render(node)
-      render_blocks(node).gsub(/\n{3,}/, "\n\n").strip
+      # Route through render_block so a root that is itself a block
+      # element (a ul, a blockquote) gets its block treatment; generic
+      # containers and fragments fall through to render_blocks.
+      output = node.element? ? render_block(node) : render_blocks(node)
+      output.gsub(/\n{3,}/, "\n\n").strip
     end
 
     private

data/lib/detergent/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Detergent
-  VERSION = "2.3.0"
+  VERSION = "2.4.0"
 end

data/lib/detergent.rb

@@ -46,6 +46,7 @@ end
 require_relative "detergent/version"
 require_relative "detergent/node_scorer"
 require_relative "detergent/content_locator"
+require_relative "detergent/link_list_extractor"
 require_relative "detergent/markdown_renderer"
 require_relative "detergent/text_renderer"
 require_relative "detergent/matchers/obvious_junk_matcher"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: detergent
 version: !ruby/object:Gem::Version
-  version: 2.3.0
+  version: 2.4.0
 platform: ruby
 authors:
 - Jeff McFadden
@@ -67,6 +67,7 @@ files:
 - lib/detergent/cleaner.rb
 - lib/detergent/content_locator.rb
 - lib/detergent/inspector.rb
+- lib/detergent/link_list_extractor.rb
 - lib/detergent/markdown_renderer.rb
 - lib/detergent/matchers/obvious_junk_matcher.rb
 - lib/detergent/matchers/removable_node_matcher.rb