detergent 2.4.0 → 2.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: fe5c0dc45668a4c14b991433606aec83c6fd2672e93533632d608e66b858f03a
4
- data.tar.gz: eada52c178f26669036609d5431f85245cd06d5de6b3920b010ba91a587ba18c
3
+ metadata.gz: c3bb169a806bef1d0892b1ce40252c85ac66699615e2a38f1bb62d76f3c37fab
4
+ data.tar.gz: 0adf4788cd10809e6fce2bed2ecfb7b62ce585ab9b0aa713fb23fa42d016efcf
5
5
  SHA512:
6
- metadata.gz: 9ed056835cc1475822f443caa51610abc1dab4000dccf97f1058971f15cf8d9ceb6dbf44c7a91b079b4d6b886a8f239f1ec1408a53311e8d335d17cb34174886
7
- data.tar.gz: e0ff4553f215510c0aebfcbbe35b2a5cca86ebfc8c755f597157a4c565b9950ad6b188b32610a92c7661635307ac388968a5e6a2fcd157e87463e2582c51810e
6
+ metadata.gz: 96e7c0ee1917a0eebd8e3bc31bf26c9700119739693869ba36593333bb13c4bfba364cb254555cfa14a5b6ce7165e0c12f59c7e19d75f676fd65117d5d42daab
7
+ data.tar.gz: 40d3f376a47dbf5748786d030b49fda3e3842ca0ca921afa8c90c89972564e9555690b74c6f87e37fb69777bd1e9865e9b8c994de840f93d323b4f83b0974ccc
data/README.md CHANGED
@@ -31,6 +31,18 @@ clean_html = cleaner.clean(dirty_html)
31
31
  title = cleaner.title(dirty_html)
32
32
  ```
33
33
 
34
+ ## Extraction behavior
35
+
36
+ Detergent prefers common article-body markers (including schema.org
37
+ `articleBody`, `entry-content`, and `story-body`) when they contain substantive
38
+ prose rather than mostly links. Otherwise it uses content scoring and the
39
+ existing link-list fallback. Navigation, ads, sharing widgets, newsletters, and
40
+ recommendations are pruned before scoring; ordinary asides inside an article
41
+ remain available. Extraction is heuristic and does not bypass paywalls.
42
+
43
+ Detergent is a content extractor, not an HTML security sanitizer. Sanitize its
44
+ output before rendering untrusted HTML in an application.
45
+
34
46
  ## Command line
35
47
 
36
48
  ```
@@ -6,6 +6,7 @@ module Detergent
6
6
  # the best root container for the content.
7
7
  class ContentLocator
8
8
  SEMANTIC_CONTAINERS = %w[article main section].freeze
9
+ BODY_SELECTORS = '[itemprop~="articleBody"], .entry-content, .post-content, .article-content, .article-body, .story-body'.freeze
9
10
 
10
11
  # A parent must beat the best root found so far by this ratio to be
11
12
  # promoted — not just score marginally higher from including the child.
@@ -25,6 +26,17 @@ module Detergent
25
26
  # Returns the main content root node, or nil if the page doesn't
26
27
  # appear to have main content at all.
27
28
  def locate(body)
29
+ # Prefer a publisher's body marker only when it contains substantive prose
30
+ # and little linked text. Empty wrappers and link indexes still use scoring.
31
+ candidates = body.css(BODY_SELECTORS).select do |node|
32
+ text_length = node.text.strip.length
33
+ prose_length = node.css("p, pre, blockquote").sum { |child| child.text.strip.length }
34
+ linked_length = node.css("a").sum { |link| link.text.strip.length }
35
+ prose_length >= 80 && score(node).positive? &&
36
+ linked_length < text_length * 0.5
37
+ end
38
+ return candidates.max_by { |node| score(node) } unless candidates.empty?
39
+
28
40
  highest_scoring = find_highest_scoring_node(body)
29
41
  return nil unless highest_scoring
30
42
  return nil if score(highest_scoring) < MINIMUM_CONTENT_SCORE
@@ -5,6 +5,9 @@ module Detergent
5
5
  # First-pass matcher: junk that can be removed on sight, before any
6
6
  # content scoring happens.
7
7
  class ObviousJunkMatcher
8
+ STRUCTURAL_TAGS = %w[div section ul ol li aside].freeze
9
+ CHROME_NAME = /(?:\A|[-_\s])(?:menu|navigation|sidebar|related|recommendations|newsletter|subscribe|social|share|sharing|comments?|disqus|advertisement|advertising|advert|ad|ads|promo|cookie)(?:\z|[-_\s])/i
10
+
8
11
  def match?(node)
9
12
  tag = node.name.downcase
10
13
 
@@ -12,12 +15,19 @@ module Detergent
12
15
  return true if JUNK_TAGS.include?(tag)
13
16
 
14
17
  # Remove hidden elements
18
+ return true if node.key?('hidden')
15
19
  return true if node['aria-hidden'] == 'true'
16
20
  return true if Detergent.display_none?(node)
17
21
 
18
22
  # Remove structural navigation
19
23
  return true if CHROME_TAGS.include?(tag)
20
- return true if node['role'].to_s.downcase == 'navigation'
24
+ return true if %w[navigation banner contentinfo].include?(node['role'].to_s.downcase)
25
+
26
+ # Match whole class/id components, not substrings such as "ad" in
27
+ # "reading". Remove these before scoring so widgets cannot inflate a page.
28
+ if STRUCTURAL_TAGS.include?(tag)
29
+ return true if CHROME_NAME.match?([node['class'], node['id']].compact.join(' '))
30
+ end
21
31
 
22
32
  false
23
33
  end
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Detergent
4
- VERSION = "2.4.0"
4
+ VERSION = "2.5.0"
5
5
  end
data/lib/detergent.rb CHANGED
@@ -18,7 +18,7 @@ module Detergent
18
18
 
19
19
  def self.display_none?(node)
20
20
  style = node["style"].to_s.downcase
21
- style.include?("display:none") || style.include?("display: none")
21
+ /(?:display\s*:\s*none|visibility\s*:\s*hidden)/.match?(style)
22
22
  end
23
23
 
24
24
  # Returns a cleaned, standalone HTML document.
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: detergent
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.4.0
4
+ version: 2.5.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Jeff McFadden