readability-rb 0.5.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/readability/cleaner.rb +1 -1
- data/lib/readability/document.rb +2 -2
- data/lib/readability/metadata.rb +11 -9
- data/lib/readability/utils.rb +4 -4
- data/lib/readability/version.rb +1 -1
- metadata +3 -3
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 9face7faf604c0ce94f2ad0db7bd6537bb0b46ec34703e8d8711f67e11249071
|
|
4
|
+
data.tar.gz: 8ce547233f221524924bf3678eecee1b9da566d16ac025586050a480693f6861
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: f90592e96640dee76abced9686165b9bb3d20254df78c157c70108929234c3bfd5bc2049472263c5e7ce32fac07627402beb64a1f1bca46510a15c0b146f07c5
|
|
7
|
+
data.tar.gz: 6b79072fb4874ec9881cce4c3b44dd90880fb3fb31a7b9c20b0130db658ac7a806f2c29d1113e4484d25ef7776eede73809edf37f3467d18e58361b0d95c67b4
|
data/lib/readability/cleaner.rb
CHANGED
|
@@ -449,7 +449,7 @@ module Readability
|
|
|
449
449
|
|
|
450
450
|
inner_text = get_inner_text(node)
|
|
451
451
|
|
|
452
|
-
if inner_text.
|
|
452
|
+
if inner_text.count(",") < 10
|
|
453
453
|
tag_counts = Hash.new(0)
|
|
454
454
|
node.css("p, img, li, input").each { |n| tag_counts[n.name] += 1 }
|
|
455
455
|
p_count = tag_counts["p"]
|
data/lib/readability/document.rb
CHANGED
|
@@ -170,7 +170,7 @@ module Readability
|
|
|
170
170
|
nxt = get_next_node(nxt)
|
|
171
171
|
end
|
|
172
172
|
end
|
|
173
|
-
@article_byline = (item_prop_name_node || node).text
|
|
173
|
+
@article_byline = js_trim((item_prop_name_node || node).text)
|
|
174
174
|
node = remove_and_get_next(node)
|
|
175
175
|
next
|
|
176
176
|
end
|
|
@@ -290,7 +290,7 @@ module Readability
|
|
|
290
290
|
content_score += 1
|
|
291
291
|
|
|
292
292
|
# Add points for any commas within this paragraph.
|
|
293
|
-
content_score += inner_text.split(COMMAS).length
|
|
293
|
+
content_score += inner_text.split(COMMAS, -1).length
|
|
294
294
|
|
|
295
295
|
# For every 100 characters in this paragraph, add another point. Up to 3 points.
|
|
296
296
|
content_score += [inner_text.length / 100, 3].min
|
data/lib/readability/metadata.rb
CHANGED
|
@@ -41,7 +41,7 @@ module Readability
|
|
|
41
41
|
|
|
42
42
|
if parsed.is_a?(Array)
|
|
43
43
|
parsed = parsed.find do |it|
|
|
44
|
-
Array(it["@type"]).any? { |t| t.match?(JSON_LD_ARTICLE_TYPES) }
|
|
44
|
+
Array(it["@type"]).any? { |t| t.is_a?(String) && t.match?(JSON_LD_ARTICLE_TYPES) }
|
|
45
45
|
end
|
|
46
46
|
next unless parsed
|
|
47
47
|
end
|
|
@@ -64,6 +64,8 @@ module Readability
|
|
|
64
64
|
end
|
|
65
65
|
end
|
|
66
66
|
|
|
67
|
+
# Unlike JS, accept an array @type, which schema.org allows. JS calls
|
|
68
|
+
# @type.match, which throws for an array, so it skips the script.
|
|
67
69
|
next if !parsed || !parsed["@type"] || !Array(parsed["@type"]).any? { |t| t.is_a?(String) && t.match?(JSON_LD_ARTICLE_TYPES) }
|
|
68
70
|
|
|
69
71
|
metadata = {}
|
|
@@ -88,13 +90,13 @@ module Readability
|
|
|
88
90
|
|
|
89
91
|
if parsed["author"]
|
|
90
92
|
if parsed["author"].is_a?(Hash) && parsed["author"]["name"].is_a?(String)
|
|
91
|
-
metadata["byline"] = parsed["author"]["name"]
|
|
93
|
+
metadata["byline"] = js_trim(parsed["author"]["name"])
|
|
92
94
|
elsif parsed["author"].is_a?(Array) &&
|
|
93
|
-
parsed["author"][0] &&
|
|
95
|
+
parsed["author"][0].is_a?(Hash) &&
|
|
94
96
|
parsed["author"][0]["name"].is_a?(String)
|
|
95
97
|
metadata["byline"] = parsed["author"]
|
|
96
|
-
.select { |author| author && author["name"].is_a?(String) }
|
|
97
|
-
.map { |author| author["name"]
|
|
98
|
+
.select { |author| author.is_a?(Hash) && author["name"].is_a?(String) }
|
|
99
|
+
.map { |author| js_trim(author["name"]) }
|
|
98
100
|
.join(", ")
|
|
99
101
|
end
|
|
100
102
|
end
|
|
@@ -110,8 +112,8 @@ module Readability
|
|
|
110
112
|
if parsed["datePublished"].is_a?(String)
|
|
111
113
|
metadata["date_published"] = parsed["datePublished"].strip
|
|
112
114
|
end
|
|
113
|
-
rescue
|
|
114
|
-
#
|
|
115
|
+
rescue => e
|
|
116
|
+
# Skip malformed JSON or JSON with an unexpected shape, as JS does
|
|
115
117
|
log(e.message) if respond_to?(:log, true)
|
|
116
118
|
end
|
|
117
119
|
end
|
|
@@ -225,7 +227,7 @@ module Readability
|
|
|
225
227
|
if matches
|
|
226
228
|
# Convert to lowercase, and remove any whitespace
|
|
227
229
|
name = matches[0].downcase.gsub(/\s/, "")
|
|
228
|
-
values[name] = content
|
|
230
|
+
values[name] = js_trim(content)
|
|
229
231
|
end
|
|
230
232
|
end
|
|
231
233
|
|
|
@@ -234,7 +236,7 @@ module Readability
|
|
|
234
236
|
if content
|
|
235
237
|
# Convert to lowercase, remove whitespace, convert dots to colons
|
|
236
238
|
name = name.downcase.gsub(/\s/, "").gsub(".", ":")
|
|
237
|
-
values[name] = content
|
|
239
|
+
values[name] = js_trim(content)
|
|
238
240
|
end
|
|
239
241
|
end
|
|
240
242
|
end
|
data/lib/readability/utils.rb
CHANGED
|
@@ -33,10 +33,10 @@ module Readability
|
|
|
33
33
|
end
|
|
34
34
|
|
|
35
35
|
# JS-compatible trim that also strips Unicode whitespace like \u00A0 (NBSP)
|
|
36
|
-
# JS's String.prototype.trim() strips all Unicode whitespace
|
|
36
|
+
# JS's String.prototype.trim() strips all Unicode whitespace and \uFEFF (BOM).
|
|
37
37
|
# Ruby's String#strip only removes ASCII whitespace.
|
|
38
38
|
def js_trim(str)
|
|
39
|
-
str.gsub(/\A[\
|
|
39
|
+
str.gsub(/\A[[:space:]\uFEFF]+|[[:space:]\uFEFF]+\z/, "")
|
|
40
40
|
end
|
|
41
41
|
|
|
42
42
|
# Port of _getInnerText (JS line 2084)
|
|
@@ -211,9 +211,9 @@ module Readability
|
|
|
211
211
|
end
|
|
212
212
|
|
|
213
213
|
# Port of _isUrl (JS line 442)
|
|
214
|
+
# JS's URL constructor accepts any absolute URL, so any scheme counts.
|
|
214
215
|
def is_url?(str)
|
|
215
|
-
|
|
216
|
-
uri.is_a?(URI::HTTP) || uri.is_a?(URI::HTTPS)
|
|
216
|
+
!URI.parse(str).scheme.nil?
|
|
217
217
|
rescue URI::InvalidURIError, URI::InvalidComponentError
|
|
218
218
|
false
|
|
219
219
|
end
|
data/lib/readability/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: readability-rb
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.7.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Andy Croll
|
|
@@ -58,14 +58,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
|
58
58
|
requirements:
|
|
59
59
|
- - ">="
|
|
60
60
|
- !ruby/object:Gem::Version
|
|
61
|
-
version: '3.
|
|
61
|
+
version: '3.3'
|
|
62
62
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
63
63
|
requirements:
|
|
64
64
|
- - ">="
|
|
65
65
|
- !ruby/object:Gem::Version
|
|
66
66
|
version: '0'
|
|
67
67
|
requirements: []
|
|
68
|
-
rubygems_version:
|
|
68
|
+
rubygems_version: 4.0.20
|
|
69
69
|
specification_version: 4
|
|
70
70
|
summary: Extract readable article content from HTML pages
|
|
71
71
|
test_files: []
|