detergent 2.3.0 → 2.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 27c1381c7afa9541cb1285a2f31c418d9c1d392fd5990abb7c1ee4a5ef630f24
4
- data.tar.gz: 4935ce56bfef6771ab9cdbd2a431d9931ab374df7e4dffa876c75f95ee3c671c
3
+ metadata.gz: c3bb169a806bef1d0892b1ce40252c85ac66699615e2a38f1bb62d76f3c37fab
4
+ data.tar.gz: 0adf4788cd10809e6fce2bed2ecfb7b62ce585ab9b0aa713fb23fa42d016efcf
5
5
  SHA512:
6
- metadata.gz: 5b5f9a0ce17b8ac37ccfa764780709f4bff83a3c527c7b2d5bd68ed633369aeea93a662d81e680d3b4e90679b518478a72cebb8a24b97990df9a857aebf55dcd
7
- data.tar.gz: 7030d8857c008565da627a56b4071de605bf3b5ecc6b5bdebd91a1bc20b5db991c14953b81da910567b9810d257089449dccfa0e33ed2fc7bfb82e390dda3111
6
+ metadata.gz: 96e7c0ee1917a0eebd8e3bc31bf26c9700119739693869ba36593333bb13c4bfba364cb254555cfa14a5b6ce7165e0c12f59c7e19d75f676fd65117d5d42daab
7
+ data.tar.gz: 40d3f376a47dbf5748786d030b49fda3e3842ca0ca921afa8c90c89972564e9555690b74c6f87e37fb69777bd1e9865e9b8c994de840f93d323b4f83b0974ccc
data/README.md CHANGED
@@ -31,6 +31,18 @@ clean_html = cleaner.clean(dirty_html)
31
31
  title = cleaner.title(dirty_html)
32
32
  ```
33
33
 
34
+ ## Extraction behavior
35
+
36
+ Detergent prefers common article-body markers (including schema.org
37
+ `articleBody`, `entry-content`, and `story-body`) when they contain substantive
38
+ prose rather than mostly links. Otherwise it uses content scoring and the
39
+ existing link-list fallback. Navigation, ads, sharing widgets, newsletters, and
40
+ recommendations are pruned before scoring; ordinary asides inside an article
41
+ remain available. Extraction is heuristic and does not bypass paywalls.
42
+
43
+ Detergent is a content extractor, not an HTML security sanitizer. Sanitize its
44
+ output before rendering untrusted HTML in an application.
45
+
34
46
  ## Command line
35
47
 
36
48
  ```
@@ -75,6 +87,11 @@ report object also exposes `located?`, `best_score`, `top_nodes`, and
75
87
  3. The highest-scoring node is located and walked up to an appropriate semantic container.
76
88
  4. A second pass cleans that content of empty and suspect elements.
77
89
 
90
+ Pages with no article content but a substantial set of title-like links —
91
+ index pages and link aggregators like the Hacker News front page — fall
92
+ back to link-list extraction: the result is a clean list of the page's
93
+ links instead of an article.
94
+
78
95
  ## License
79
96
 
80
97
  [WTFPL](LICENSE)
@@ -5,8 +5,9 @@ module Detergent
5
5
  # scrubs it, and renders the result back out as a standalone document.
6
6
  class Cleaner
7
7
  # observer (optional) receives instrumentation callbacks during
8
- # extraction: node_removed(node, pass:) and content_pruned(body, scorer).
9
- # Used by Inspector to build debug reports.
8
+ # extraction: node_removed(node, pass:), content_pruned(body, scorer),
9
+ # and extraction_strategy(strategy). Used by Inspector to build debug
10
+ # reports.
10
11
  def initialize(observer: nil)
11
12
  @observer = observer
12
13
  @obvious_junk_matcher = Matchers::ObviousJunkMatcher.new
@@ -57,6 +58,16 @@ module Detergent
57
58
  scorer = NodeScorer.new
58
59
  @observer&.content_pruned(body, scorer)
59
60
  content = ContentLocator.new(scorer).locate(body)
61
+ strategy = content ? :article : nil
62
+
63
+ # No article-shaped content: the page may be a link index (front
64
+ # page, aggregator), whose content is its list of links.
65
+ if content.nil?
66
+ content = LinkListExtractor.new.extract(body)
67
+ strategy = :link_list if content
68
+ end
69
+
70
+ @observer&.extraction_strategy(strategy)
60
71
 
61
72
  # Apply second-pass cleaning to the content
62
73
  if content
@@ -6,6 +6,7 @@ module Detergent
6
6
  # the best root container for the content.
7
7
  class ContentLocator
8
8
  SEMANTIC_CONTAINERS = %w[article main section].freeze
9
+ BODY_SELECTORS = '[itemprop~="articleBody"], .entry-content, .post-content, .article-content, .article-body, .story-body'.freeze
9
10
 
10
11
  # A parent must beat the best root found so far by this ratio to be
11
12
  # promoted — not just score marginally higher from including the child.
@@ -25,6 +26,17 @@ module Detergent
25
26
  # Returns the main content root node, or nil if the page doesn't
26
27
  # appear to have main content at all.
27
28
  def locate(body)
29
+ # Prefer a publisher's body marker only when it contains substantive prose
30
+ # and little linked text. Empty wrappers and link indexes still use scoring.
31
+ candidates = body.css(BODY_SELECTORS).select do |node|
32
+ text_length = node.text.strip.length
33
+ prose_length = node.css("p, pre, blockquote").sum { |child| child.text.strip.length }
34
+ linked_length = node.css("a").sum { |link| link.text.strip.length }
35
+ prose_length >= 80 && score(node).positive? &&
36
+ linked_length < text_length * 0.5
37
+ end
38
+ return candidates.max_by { |node| score(node) } unless candidates.empty?
39
+
28
40
  highest_scoring = find_highest_scoring_node(body)
29
41
  return nil unless highest_scoring
30
42
  return nil if score(highest_scoring) < MINIMUM_CONTENT_SCORE
@@ -19,7 +19,7 @@ module Detergent
19
19
  ScoredNode = Struct.new(:score, :descriptor, :preview, :breakdown)
20
20
 
21
21
  class Report
22
- attr_accessor :title, :best_score, :content_root, :content_root_size, :top_nodes
22
+ attr_accessor :title, :best_score, :content_root, :content_root_size, :top_nodes, :strategy
23
23
  attr_reader :removals
24
24
 
25
25
  def initialize
@@ -63,8 +63,11 @@ module Detergent
63
63
  def verdict
64
64
  threshold = ContentLocator::MINIMUM_CONTENT_SCORE
65
65
 
66
- if located?
67
- "Verdict: content located (best score #{best_score}, threshold #{threshold})"
66
+ case strategy
67
+ when :article
68
+ "Verdict: article content located (best score #{best_score}, threshold #{threshold})"
69
+ when :link_list
70
+ "Verdict: no article content (best score #{best_score} < threshold #{threshold}); extracted link list instead"
68
71
  else
69
72
  "Verdict: NO MAIN CONTENT (best score #{best_score} < threshold #{threshold})"
70
73
  end
@@ -117,6 +120,10 @@ module Detergent
117
120
  @report.best_score = scored.first&.first || 0
118
121
  end
119
122
 
123
+ def extraction_strategy(strategy)
124
+ @report.strategy = strategy
125
+ end
126
+
120
127
  private
121
128
 
122
129
  def descriptor(node)
@@ -0,0 +1,61 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Detergent
4
+ # Fallback extractor for index pages: link aggregators, front pages, and
5
+ # directories (e.g. the Hacker News front page) where the "content" is a
6
+ # list of links to other pages. The article locator correctly rejects
7
+ # these — link density is its strongest navigation signal — so when it
8
+ # finds nothing, this extractor looks for a substantial collection of
9
+ # title-like links and returns them as a synthetic list node that flows
10
+ # through the normal cleaning and rendering pipeline.
11
+ class LinkListExtractor
12
+ # Links with less text than this are treated as chrome ("login",
13
+ # "hide", "42 comments"), not titles.
14
+ MIN_LINK_TEXT_LENGTH = 15
15
+
16
+ # Fewer title-like links than this isn't an index page.
17
+ MIN_LINKS = 10
18
+
19
+ # Returns a synthetic <ul> of the page's title-like links, or nil if
20
+ # the page doesn't look like a link index.
21
+ def extract(body)
22
+ links = title_like_links(body)
23
+ return nil if links.length < MIN_LINKS
24
+
25
+ build_list(body.document, links)
26
+ end
27
+
28
+ private
29
+
30
+ def title_like_links(body)
31
+ seen_hrefs = {}
32
+
33
+ body.css("a").filter_map do |link|
34
+ href = link["href"].to_s
35
+ next if href.empty? || href.start_with?("#", "javascript:")
36
+
37
+ text = link.text.gsub(/\s+/, " ").strip
38
+ next if text.length < MIN_LINK_TEXT_LENGTH
39
+ next if seen_hrefs[href]
40
+
41
+ seen_hrefs[href] = true
42
+ [text, href]
43
+ end
44
+ end
45
+
46
+ def build_list(doc, links)
47
+ list = Nokogiri::XML::Node.new("ul", doc)
48
+
49
+ links.each do |text, href|
50
+ item = Nokogiri::XML::Node.new("li", doc)
51
+ anchor = Nokogiri::XML::Node.new("a", doc)
52
+ anchor["href"] = href
53
+ anchor.content = text
54
+ item.add_child(anchor)
55
+ list.add_child(item)
56
+ end
57
+
58
+ list
59
+ end
60
+ end
61
+ end
@@ -9,7 +9,11 @@ module Detergent
9
9
  # image, ...) so TextRenderer can subclass this and strip the syntax.
10
10
  class MarkdownRenderer
11
11
  def render(node)
12
- render_blocks(node).gsub(/\n{3,}/, "\n\n").strip
12
+ # Route through render_block so a root that is itself a block
13
+ # element (a ul, a blockquote) gets its block treatment; generic
14
+ # containers and fragments fall through to render_blocks.
15
+ output = node.element? ? render_block(node) : render_blocks(node)
16
+ output.gsub(/\n{3,}/, "\n\n").strip
13
17
  end
14
18
 
15
19
  private
@@ -5,6 +5,9 @@ module Detergent
5
5
  # First-pass matcher: junk that can be removed on sight, before any
6
6
  # content scoring happens.
7
7
  class ObviousJunkMatcher
8
+ STRUCTURAL_TAGS = %w[div section ul ol li aside].freeze
9
+ CHROME_NAME = /(?:\A|[-_\s])(?:menu|navigation|sidebar|related|recommendations|newsletter|subscribe|social|share|sharing|comments?|disqus|advertisement|advertising|advert|ad|ads|promo|cookie)(?:\z|[-_\s])/i
10
+
8
11
  def match?(node)
9
12
  tag = node.name.downcase
10
13
 
@@ -12,12 +15,19 @@ module Detergent
12
15
  return true if JUNK_TAGS.include?(tag)
13
16
 
14
17
  # Remove hidden elements
18
+ return true if node.key?('hidden')
15
19
  return true if node['aria-hidden'] == 'true'
16
20
  return true if Detergent.display_none?(node)
17
21
 
18
22
  # Remove structural navigation
19
23
  return true if CHROME_TAGS.include?(tag)
20
- return true if node['role'].to_s.downcase == 'navigation'
24
+ return true if %w[navigation banner contentinfo].include?(node['role'].to_s.downcase)
25
+
26
+ # Match whole class/id components, not substrings such as "ad" in
27
+ # "reading". Remove these before scoring so widgets cannot inflate a page.
28
+ if STRUCTURAL_TAGS.include?(tag)
29
+ return true if CHROME_NAME.match?([node['class'], node['id']].compact.join(' '))
30
+ end
21
31
 
22
32
  false
23
33
  end
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Detergent
4
- VERSION = "2.3.0"
4
+ VERSION = "2.5.0"
5
5
  end
data/lib/detergent.rb CHANGED
@@ -18,7 +18,7 @@ module Detergent
18
18
 
19
19
  def self.display_none?(node)
20
20
  style = node["style"].to_s.downcase
21
- style.include?("display:none") || style.include?("display: none")
21
+ /(?:display\s*:\s*none|visibility\s*:\s*hidden)/.match?(style)
22
22
  end
23
23
 
24
24
  # Returns a cleaned, standalone HTML document.
@@ -46,6 +46,7 @@ end
46
46
  require_relative "detergent/version"
47
47
  require_relative "detergent/node_scorer"
48
48
  require_relative "detergent/content_locator"
49
+ require_relative "detergent/link_list_extractor"
49
50
  require_relative "detergent/markdown_renderer"
50
51
  require_relative "detergent/text_renderer"
51
52
  require_relative "detergent/matchers/obvious_junk_matcher"
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: detergent
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.3.0
4
+ version: 2.5.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Jeff McFadden
@@ -67,6 +67,7 @@ files:
67
67
  - lib/detergent/cleaner.rb
68
68
  - lib/detergent/content_locator.rb
69
69
  - lib/detergent/inspector.rb
70
+ - lib/detergent/link_list_extractor.rb
70
71
  - lib/detergent/markdown_renderer.rb
71
72
  - lib/detergent/matchers/obvious_junk_matcher.rb
72
73
  - lib/detergent/matchers/removable_node_matcher.rb