detergent 2.4.0 → 2.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +12 -0
- data/lib/detergent/content_locator.rb +12 -0
- data/lib/detergent/matchers/obvious_junk_matcher.rb +11 -1
- data/lib/detergent/version.rb +1 -1
- data/lib/detergent.rb +1 -1
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: c3bb169a806bef1d0892b1ce40252c85ac66699615e2a38f1bb62d76f3c37fab
|
|
4
|
+
data.tar.gz: 0adf4788cd10809e6fce2bed2ecfb7b62ce585ab9b0aa713fb23fa42d016efcf
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 96e7c0ee1917a0eebd8e3bc31bf26c9700119739693869ba36593333bb13c4bfba364cb254555cfa14a5b6ce7165e0c12f59c7e19d75f676fd65117d5d42daab
|
|
7
|
+
data.tar.gz: 40d3f376a47dbf5748786d030b49fda3e3842ca0ca921afa8c90c89972564e9555690b74c6f87e37fb69777bd1e9865e9b8c994de840f93d323b4f83b0974ccc
|
data/README.md
CHANGED
|
@@ -31,6 +31,18 @@ clean_html = cleaner.clean(dirty_html)
|
|
|
31
31
|
title = cleaner.title(dirty_html)
|
|
32
32
|
```
|
|
33
33
|
|
|
34
|
+
## Extraction behavior
|
|
35
|
+
|
|
36
|
+
Detergent prefers common article-body markers (including schema.org
|
|
37
|
+
`articleBody`, `entry-content`, and `story-body`) when they contain substantive
|
|
38
|
+
prose rather than mostly links. Otherwise it uses content scoring and the
|
|
39
|
+
existing link-list fallback. Navigation, ads, sharing widgets, newsletters, and
|
|
40
|
+
recommendations are pruned before scoring; ordinary asides inside an article
|
|
41
|
+
remain available. Extraction is heuristic and does not bypass paywalls.
|
|
42
|
+
|
|
43
|
+
Detergent is a content extractor, not an HTML security sanitizer. Sanitize its
|
|
44
|
+
output before rendering untrusted HTML in an application.
|
|
45
|
+
|
|
34
46
|
## Command line
|
|
35
47
|
|
|
36
48
|
```
|
|
@@ -6,6 +6,7 @@ module Detergent
|
|
|
6
6
|
# the best root container for the content.
|
|
7
7
|
class ContentLocator
|
|
8
8
|
SEMANTIC_CONTAINERS = %w[article main section].freeze
|
|
9
|
+
BODY_SELECTORS = '[itemprop~="articleBody"], .entry-content, .post-content, .article-content, .article-body, .story-body'.freeze
|
|
9
10
|
|
|
10
11
|
# A parent must beat the best root found so far by this ratio to be
|
|
11
12
|
# promoted — not just score marginally higher from including the child.
|
|
@@ -25,6 +26,17 @@ module Detergent
|
|
|
25
26
|
# Returns the main content root node, or nil if the page doesn't
|
|
26
27
|
# appear to have main content at all.
|
|
27
28
|
def locate(body)
|
|
29
|
+
# Prefer a publisher's body marker only when it contains substantive prose
|
|
30
|
+
# and little linked text. Empty wrappers and link indexes still use scoring.
|
|
31
|
+
candidates = body.css(BODY_SELECTORS).select do |node|
|
|
32
|
+
text_length = node.text.strip.length
|
|
33
|
+
prose_length = node.css("p, pre, blockquote").sum { |child| child.text.strip.length }
|
|
34
|
+
linked_length = node.css("a").sum { |link| link.text.strip.length }
|
|
35
|
+
prose_length >= 80 && score(node).positive? &&
|
|
36
|
+
linked_length < text_length * 0.5
|
|
37
|
+
end
|
|
38
|
+
return candidates.max_by { |node| score(node) } unless candidates.empty?
|
|
39
|
+
|
|
28
40
|
highest_scoring = find_highest_scoring_node(body)
|
|
29
41
|
return nil unless highest_scoring
|
|
30
42
|
return nil if score(highest_scoring) < MINIMUM_CONTENT_SCORE
|
|
@@ -5,6 +5,9 @@ module Detergent
|
|
|
5
5
|
# First-pass matcher: junk that can be removed on sight, before any
|
|
6
6
|
# content scoring happens.
|
|
7
7
|
class ObviousJunkMatcher
|
|
8
|
+
STRUCTURAL_TAGS = %w[div section ul ol li aside].freeze
|
|
9
|
+
CHROME_NAME = /(?:\A|[-_\s])(?:menu|navigation|sidebar|related|recommendations|newsletter|subscribe|social|share|sharing|comments?|disqus|advertisement|advertising|advert|ad|ads|promo|cookie)(?:\z|[-_\s])/i
|
|
10
|
+
|
|
8
11
|
def match?(node)
|
|
9
12
|
tag = node.name.downcase
|
|
10
13
|
|
|
@@ -12,12 +15,19 @@ module Detergent
|
|
|
12
15
|
return true if JUNK_TAGS.include?(tag)
|
|
13
16
|
|
|
14
17
|
# Remove hidden elements
|
|
18
|
+
return true if node.key?('hidden')
|
|
15
19
|
return true if node['aria-hidden'] == 'true'
|
|
16
20
|
return true if Detergent.display_none?(node)
|
|
17
21
|
|
|
18
22
|
# Remove structural navigation
|
|
19
23
|
return true if CHROME_TAGS.include?(tag)
|
|
20
|
-
return true if node['role'].to_s.downcase
|
|
24
|
+
return true if %w[navigation banner contentinfo].include?(node['role'].to_s.downcase)
|
|
25
|
+
|
|
26
|
+
# Match whole class/id components, not substrings such as "ad" in
|
|
27
|
+
# "reading". Remove these before scoring so widgets cannot inflate a page.
|
|
28
|
+
if STRUCTURAL_TAGS.include?(tag)
|
|
29
|
+
return true if CHROME_NAME.match?([node['class'], node['id']].compact.join(' '))
|
|
30
|
+
end
|
|
21
31
|
|
|
22
32
|
false
|
|
23
33
|
end
|
data/lib/detergent/version.rb
CHANGED
data/lib/detergent.rb
CHANGED
|
@@ -18,7 +18,7 @@ module Detergent
|
|
|
18
18
|
|
|
19
19
|
def self.display_none?(node)
|
|
20
20
|
style = node["style"].to_s.downcase
|
|
21
|
-
|
|
21
|
+
/(?:display\s*:\s*none|visibility\s*:\s*hidden)/.match?(style)
|
|
22
22
|
end
|
|
23
23
|
|
|
24
24
|
# Returns a cleaned, standalone HTML document.
|