detergent 2.3.0 → 2.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +17 -0
- data/lib/detergent/cleaner.rb +13 -2
- data/lib/detergent/content_locator.rb +12 -0
- data/lib/detergent/inspector.rb +10 -3
- data/lib/detergent/link_list_extractor.rb +61 -0
- data/lib/detergent/markdown_renderer.rb +5 -1
- data/lib/detergent/matchers/obvious_junk_matcher.rb +11 -1
- data/lib/detergent/version.rb +1 -1
- data/lib/detergent.rb +2 -1
- metadata +2 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: c3bb169a806bef1d0892b1ce40252c85ac66699615e2a38f1bb62d76f3c37fab
|
|
4
|
+
data.tar.gz: 0adf4788cd10809e6fce2bed2ecfb7b62ce585ab9b0aa713fb23fa42d016efcf
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 96e7c0ee1917a0eebd8e3bc31bf26c9700119739693869ba36593333bb13c4bfba364cb254555cfa14a5b6ce7165e0c12f59c7e19d75f676fd65117d5d42daab
|
|
7
|
+
data.tar.gz: 40d3f376a47dbf5748786d030b49fda3e3842ca0ca921afa8c90c89972564e9555690b74c6f87e37fb69777bd1e9865e9b8c994de840f93d323b4f83b0974ccc
|
data/README.md
CHANGED
|
@@ -31,6 +31,18 @@ clean_html = cleaner.clean(dirty_html)
|
|
|
31
31
|
title = cleaner.title(dirty_html)
|
|
32
32
|
```
|
|
33
33
|
|
|
34
|
+
## Extraction behavior
|
|
35
|
+
|
|
36
|
+
Detergent prefers common article-body markers (including schema.org
|
|
37
|
+
`articleBody`, `entry-content`, and `story-body`) when they contain substantive
|
|
38
|
+
prose rather than mostly links. Otherwise it uses content scoring and the
|
|
39
|
+
existing link-list fallback. Navigation, ads, sharing widgets, newsletters, and
|
|
40
|
+
recommendations are pruned before scoring; ordinary asides inside an article
|
|
41
|
+
remain available. Extraction is heuristic and does not bypass paywalls.
|
|
42
|
+
|
|
43
|
+
Detergent is a content extractor, not an HTML security sanitizer. Sanitize its
|
|
44
|
+
output before rendering untrusted HTML in an application.
|
|
45
|
+
|
|
34
46
|
## Command line
|
|
35
47
|
|
|
36
48
|
```
|
|
@@ -75,6 +87,11 @@ report object also exposes `located?`, `best_score`, `top_nodes`, and
|
|
|
75
87
|
3. The highest-scoring node is located and walked up to an appropriate semantic container.
|
|
76
88
|
4. A second pass cleans that content of empty and suspect elements.
|
|
77
89
|
|
|
90
|
+
Pages with no article content but a substantial set of title-like links —
|
|
91
|
+
index pages and link aggregators like the Hacker News front page — fall
|
|
92
|
+
back to link-list extraction: the result is a clean list of the page's
|
|
93
|
+
links instead of an article.
|
|
94
|
+
|
|
78
95
|
## License
|
|
79
96
|
|
|
80
97
|
[WTFPL](LICENSE)
|
data/lib/detergent/cleaner.rb
CHANGED
|
@@ -5,8 +5,9 @@ module Detergent
|
|
|
5
5
|
# scrubs it, and renders the result back out as a standalone document.
|
|
6
6
|
class Cleaner
|
|
7
7
|
# observer (optional) receives instrumentation callbacks during
|
|
8
|
-
# extraction: node_removed(node, pass:)
|
|
9
|
-
# Used by Inspector to build debug
|
|
8
|
+
# extraction: node_removed(node, pass:), content_pruned(body, scorer),
|
|
9
|
+
# and extraction_strategy(strategy). Used by Inspector to build debug
|
|
10
|
+
# reports.
|
|
10
11
|
def initialize(observer: nil)
|
|
11
12
|
@observer = observer
|
|
12
13
|
@obvious_junk_matcher = Matchers::ObviousJunkMatcher.new
|
|
@@ -57,6 +58,16 @@ module Detergent
|
|
|
57
58
|
scorer = NodeScorer.new
|
|
58
59
|
@observer&.content_pruned(body, scorer)
|
|
59
60
|
content = ContentLocator.new(scorer).locate(body)
|
|
61
|
+
strategy = content ? :article : nil
|
|
62
|
+
|
|
63
|
+
# No article-shaped content: the page may be a link index (front
|
|
64
|
+
# page, aggregator), whose content is its list of links.
|
|
65
|
+
if content.nil?
|
|
66
|
+
content = LinkListExtractor.new.extract(body)
|
|
67
|
+
strategy = :link_list if content
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
@observer&.extraction_strategy(strategy)
|
|
60
71
|
|
|
61
72
|
# Apply second-pass cleaning to the content
|
|
62
73
|
if content
|
|
@@ -6,6 +6,7 @@ module Detergent
|
|
|
6
6
|
# the best root container for the content.
|
|
7
7
|
class ContentLocator
|
|
8
8
|
SEMANTIC_CONTAINERS = %w[article main section].freeze
|
|
9
|
+
BODY_SELECTORS = '[itemprop~="articleBody"], .entry-content, .post-content, .article-content, .article-body, .story-body'.freeze
|
|
9
10
|
|
|
10
11
|
# A parent must beat the best root found so far by this ratio to be
|
|
11
12
|
# promoted — not just score marginally higher from including the child.
|
|
@@ -25,6 +26,17 @@ module Detergent
|
|
|
25
26
|
# Returns the main content root node, or nil if the page doesn't
|
|
26
27
|
# appear to have main content at all.
|
|
27
28
|
def locate(body)
|
|
29
|
+
# Prefer a publisher's body marker only when it contains substantive prose
|
|
30
|
+
# and little linked text. Empty wrappers and link indexes still use scoring.
|
|
31
|
+
candidates = body.css(BODY_SELECTORS).select do |node|
|
|
32
|
+
text_length = node.text.strip.length
|
|
33
|
+
prose_length = node.css("p, pre, blockquote").sum { |child| child.text.strip.length }
|
|
34
|
+
linked_length = node.css("a").sum { |link| link.text.strip.length }
|
|
35
|
+
prose_length >= 80 && score(node).positive? &&
|
|
36
|
+
linked_length < text_length * 0.5
|
|
37
|
+
end
|
|
38
|
+
return candidates.max_by { |node| score(node) } unless candidates.empty?
|
|
39
|
+
|
|
28
40
|
highest_scoring = find_highest_scoring_node(body)
|
|
29
41
|
return nil unless highest_scoring
|
|
30
42
|
return nil if score(highest_scoring) < MINIMUM_CONTENT_SCORE
|
data/lib/detergent/inspector.rb
CHANGED
|
@@ -19,7 +19,7 @@ module Detergent
|
|
|
19
19
|
ScoredNode = Struct.new(:score, :descriptor, :preview, :breakdown)
|
|
20
20
|
|
|
21
21
|
class Report
|
|
22
|
-
attr_accessor :title, :best_score, :content_root, :content_root_size, :top_nodes
|
|
22
|
+
attr_accessor :title, :best_score, :content_root, :content_root_size, :top_nodes, :strategy
|
|
23
23
|
attr_reader :removals
|
|
24
24
|
|
|
25
25
|
def initialize
|
|
@@ -63,8 +63,11 @@ module Detergent
|
|
|
63
63
|
def verdict
|
|
64
64
|
threshold = ContentLocator::MINIMUM_CONTENT_SCORE
|
|
65
65
|
|
|
66
|
-
|
|
67
|
-
|
|
66
|
+
case strategy
|
|
67
|
+
when :article
|
|
68
|
+
"Verdict: article content located (best score #{best_score}, threshold #{threshold})"
|
|
69
|
+
when :link_list
|
|
70
|
+
"Verdict: no article content (best score #{best_score} < threshold #{threshold}); extracted link list instead"
|
|
68
71
|
else
|
|
69
72
|
"Verdict: NO MAIN CONTENT (best score #{best_score} < threshold #{threshold})"
|
|
70
73
|
end
|
|
@@ -117,6 +120,10 @@ module Detergent
|
|
|
117
120
|
@report.best_score = scored.first&.first || 0
|
|
118
121
|
end
|
|
119
122
|
|
|
123
|
+
def extraction_strategy(strategy)
|
|
124
|
+
@report.strategy = strategy
|
|
125
|
+
end
|
|
126
|
+
|
|
120
127
|
private
|
|
121
128
|
|
|
122
129
|
def descriptor(node)
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Detergent
|
|
4
|
+
# Fallback extractor for index pages: link aggregators, front pages, and
|
|
5
|
+
# directories (e.g. the Hacker News front page) where the "content" is a
|
|
6
|
+
# list of links to other pages. The article locator correctly rejects
|
|
7
|
+
# these — link density is its strongest navigation signal — so when it
|
|
8
|
+
# finds nothing, this extractor looks for a substantial collection of
|
|
9
|
+
# title-like links and returns them as a synthetic list node that flows
|
|
10
|
+
# through the normal cleaning and rendering pipeline.
|
|
11
|
+
class LinkListExtractor
|
|
12
|
+
# Links with less text than this are treated as chrome ("login",
|
|
13
|
+
# "hide", "42 comments"), not titles.
|
|
14
|
+
MIN_LINK_TEXT_LENGTH = 15
|
|
15
|
+
|
|
16
|
+
# Fewer title-like links than this isn't an index page.
|
|
17
|
+
MIN_LINKS = 10
|
|
18
|
+
|
|
19
|
+
# Returns a synthetic <ul> of the page's title-like links, or nil if
|
|
20
|
+
# the page doesn't look like a link index.
|
|
21
|
+
def extract(body)
|
|
22
|
+
links = title_like_links(body)
|
|
23
|
+
return nil if links.length < MIN_LINKS
|
|
24
|
+
|
|
25
|
+
build_list(body.document, links)
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
private
|
|
29
|
+
|
|
30
|
+
def title_like_links(body)
|
|
31
|
+
seen_hrefs = {}
|
|
32
|
+
|
|
33
|
+
body.css("a").filter_map do |link|
|
|
34
|
+
href = link["href"].to_s
|
|
35
|
+
next if href.empty? || href.start_with?("#", "javascript:")
|
|
36
|
+
|
|
37
|
+
text = link.text.gsub(/\s+/, " ").strip
|
|
38
|
+
next if text.length < MIN_LINK_TEXT_LENGTH
|
|
39
|
+
next if seen_hrefs[href]
|
|
40
|
+
|
|
41
|
+
seen_hrefs[href] = true
|
|
42
|
+
[text, href]
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
def build_list(doc, links)
|
|
47
|
+
list = Nokogiri::XML::Node.new("ul", doc)
|
|
48
|
+
|
|
49
|
+
links.each do |text, href|
|
|
50
|
+
item = Nokogiri::XML::Node.new("li", doc)
|
|
51
|
+
anchor = Nokogiri::XML::Node.new("a", doc)
|
|
52
|
+
anchor["href"] = href
|
|
53
|
+
anchor.content = text
|
|
54
|
+
item.add_child(anchor)
|
|
55
|
+
list.add_child(item)
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
list
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
end
|
|
@@ -9,7 +9,11 @@ module Detergent
|
|
|
9
9
|
# image, ...) so TextRenderer can subclass this and strip the syntax.
|
|
10
10
|
class MarkdownRenderer
|
|
11
11
|
def render(node)
|
|
12
|
-
|
|
12
|
+
# Route through render_block so a root that is itself a block
|
|
13
|
+
# element (a ul, a blockquote) gets its block treatment; generic
|
|
14
|
+
# containers and fragments fall through to render_blocks.
|
|
15
|
+
output = node.element? ? render_block(node) : render_blocks(node)
|
|
16
|
+
output.gsub(/\n{3,}/, "\n\n").strip
|
|
13
17
|
end
|
|
14
18
|
|
|
15
19
|
private
|
|
@@ -5,6 +5,9 @@ module Detergent
|
|
|
5
5
|
# First-pass matcher: junk that can be removed on sight, before any
|
|
6
6
|
# content scoring happens.
|
|
7
7
|
class ObviousJunkMatcher
|
|
8
|
+
STRUCTURAL_TAGS = %w[div section ul ol li aside].freeze
|
|
9
|
+
CHROME_NAME = /(?:\A|[-_\s])(?:menu|navigation|sidebar|related|recommendations|newsletter|subscribe|social|share|sharing|comments?|disqus|advertisement|advertising|advert|ad|ads|promo|cookie)(?:\z|[-_\s])/i
|
|
10
|
+
|
|
8
11
|
def match?(node)
|
|
9
12
|
tag = node.name.downcase
|
|
10
13
|
|
|
@@ -12,12 +15,19 @@ module Detergent
|
|
|
12
15
|
return true if JUNK_TAGS.include?(tag)
|
|
13
16
|
|
|
14
17
|
# Remove hidden elements
|
|
18
|
+
return true if node.key?('hidden')
|
|
15
19
|
return true if node['aria-hidden'] == 'true'
|
|
16
20
|
return true if Detergent.display_none?(node)
|
|
17
21
|
|
|
18
22
|
# Remove structural navigation
|
|
19
23
|
return true if CHROME_TAGS.include?(tag)
|
|
20
|
-
return true if node['role'].to_s.downcase
|
|
24
|
+
return true if %w[navigation banner contentinfo].include?(node['role'].to_s.downcase)
|
|
25
|
+
|
|
26
|
+
# Match whole class/id components, not substrings such as "ad" in
|
|
27
|
+
# "reading". Remove these before scoring so widgets cannot inflate a page.
|
|
28
|
+
if STRUCTURAL_TAGS.include?(tag)
|
|
29
|
+
return true if CHROME_NAME.match?([node['class'], node['id']].compact.join(' '))
|
|
30
|
+
end
|
|
21
31
|
|
|
22
32
|
false
|
|
23
33
|
end
|
data/lib/detergent/version.rb
CHANGED
data/lib/detergent.rb
CHANGED
|
@@ -18,7 +18,7 @@ module Detergent
|
|
|
18
18
|
|
|
19
19
|
def self.display_none?(node)
|
|
20
20
|
style = node["style"].to_s.downcase
|
|
21
|
-
|
|
21
|
+
/(?:display\s*:\s*none|visibility\s*:\s*hidden)/.match?(style)
|
|
22
22
|
end
|
|
23
23
|
|
|
24
24
|
# Returns a cleaned, standalone HTML document.
|
|
@@ -46,6 +46,7 @@ end
|
|
|
46
46
|
require_relative "detergent/version"
|
|
47
47
|
require_relative "detergent/node_scorer"
|
|
48
48
|
require_relative "detergent/content_locator"
|
|
49
|
+
require_relative "detergent/link_list_extractor"
|
|
49
50
|
require_relative "detergent/markdown_renderer"
|
|
50
51
|
require_relative "detergent/text_renderer"
|
|
51
52
|
require_relative "detergent/matchers/obvious_junk_matcher"
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: detergent
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 2.
|
|
4
|
+
version: 2.5.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Jeff McFadden
|
|
@@ -67,6 +67,7 @@ files:
|
|
|
67
67
|
- lib/detergent/cleaner.rb
|
|
68
68
|
- lib/detergent/content_locator.rb
|
|
69
69
|
- lib/detergent/inspector.rb
|
|
70
|
+
- lib/detergent/link_list_extractor.rb
|
|
70
71
|
- lib/detergent/markdown_renderer.rb
|
|
71
72
|
- lib/detergent/matchers/obvious_junk_matcher.rb
|
|
72
73
|
- lib/detergent/matchers/removable_node_matcher.rb
|