readability-rb 0.6.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/readability/cleaner.rb +1 -1
- data/lib/readability/document.rb +1 -1
- data/lib/readability/metadata.rb +10 -9
- data/lib/readability/regexps.rb +4 -3
- data/lib/readability/version.rb +1 -1
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 1a1b61ac01f8639dc5ebd9219be2c129f739913aa31b80755985cce993586947
|
|
4
|
+
data.tar.gz: '0093eca156a35705d9bed99d33e31f0cfbdb23f4421c844a29b22b8ee17c7a52'
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 952f8342972eba74060d8302febc3ff5a6f6f1f87de0d5b98ec50a2a0ed9476592a1d32ea195f61c0dd03473611799586e6a59150b2145979d9c76db5b1c92b9
|
|
7
|
+
data.tar.gz: 5e277f3a164204f8def9ba19d7e0fc3f85dfc39bf9aaaa25b8fd8110336fa9bb20c74ad796ddb07b35ef48d272a63f48212d1ea0a9767bf9cc1c012a3e54cd42
|
data/lib/readability/cleaner.rb
CHANGED
|
@@ -449,7 +449,7 @@ module Readability
|
|
|
449
449
|
|
|
450
450
|
inner_text = get_inner_text(node)
|
|
451
451
|
|
|
452
|
-
if inner_text.
|
|
452
|
+
if inner_text.count(",") < 10
|
|
453
453
|
tag_counts = Hash.new(0)
|
|
454
454
|
node.css("p, img, li, input").each { |n| tag_counts[n.name] += 1 }
|
|
455
455
|
p_count = tag_counts["p"]
|
data/lib/readability/document.rb
CHANGED
|
@@ -290,7 +290,7 @@ module Readability
|
|
|
290
290
|
content_score += 1
|
|
291
291
|
|
|
292
292
|
# Add points for any commas within this paragraph.
|
|
293
|
-
content_score += inner_text.split(COMMAS).length
|
|
293
|
+
content_score += inner_text.split(COMMAS, -1).length
|
|
294
294
|
|
|
295
295
|
# For every 100 characters in this paragraph, add another point. Up to 3 points.
|
|
296
296
|
content_score += [inner_text.length / 100, 3].min
|
data/lib/readability/metadata.rb
CHANGED
|
@@ -128,7 +128,7 @@ module Readability
|
|
|
128
128
|
orig_title = ""
|
|
129
129
|
|
|
130
130
|
begin
|
|
131
|
-
cur_title = orig_title = (@doc.at_css("title")&.text
|
|
131
|
+
cur_title = orig_title = js_trim(@doc.at_css("title")&.text || "")
|
|
132
132
|
|
|
133
133
|
# If title came back as something other than a string (shouldn't happen
|
|
134
134
|
# with Nokogiri, but match JS logic)
|
|
@@ -140,16 +140,17 @@ module Readability
|
|
|
140
140
|
end
|
|
141
141
|
|
|
142
142
|
title_had_hierarchical_separators = false
|
|
143
|
-
|
|
143
|
+
space = '[[:space:]\uFEFF]'
|
|
144
|
+
word_count = ->(str) { str.split(/#{space}+/, -1).length }
|
|
144
145
|
|
|
145
146
|
# Title separator characters — exact JS source string from line 597
|
|
146
147
|
title_separators = '\|\-\u2013\u2014\\\\\/>»'
|
|
147
148
|
|
|
148
|
-
if cur_title.match?(
|
|
149
|
-
title_had_hierarchical_separators = cur_title.match?(
|
|
149
|
+
if cur_title.match?(/#{space}[#{title_separators}]#{space}/)
|
|
150
|
+
title_had_hierarchical_separators = cur_title.match?(/#{space}[\\\/>\u00BB]#{space}/)
|
|
150
151
|
|
|
151
152
|
# Find all separator positions and remove everything after the last one
|
|
152
|
-
all_separators = orig_title.to_enum(:scan,
|
|
153
|
+
all_separators = orig_title.to_enum(:scan, /#{space}[#{title_separators}]#{space}/i).map { Regexp.last_match }
|
|
153
154
|
cur_title = orig_title[0, all_separators.last.begin(0)]
|
|
154
155
|
|
|
155
156
|
# If the resulting title is too short, remove the first part instead
|
|
@@ -159,8 +160,8 @@ module Readability
|
|
|
159
160
|
elsif cur_title.include?(": ")
|
|
160
161
|
# Check if we have a heading containing this exact string
|
|
161
162
|
headings = get_all_nodes_with_tag(@doc, ["h1", "h2"])
|
|
162
|
-
trimmed_title = cur_title
|
|
163
|
-
match = headings.any? { |heading| heading.text
|
|
163
|
+
trimmed_title = js_trim(cur_title)
|
|
164
|
+
match = headings.any? { |heading| js_trim(heading.text) == trimmed_title }
|
|
164
165
|
|
|
165
166
|
# If we don't, extract the title out of the original title string
|
|
166
167
|
unless match
|
|
@@ -182,7 +183,7 @@ module Readability
|
|
|
182
183
|
end
|
|
183
184
|
end
|
|
184
185
|
|
|
185
|
-
cur_title = cur_title.
|
|
186
|
+
cur_title = js_trim(cur_title).gsub(NORMALIZE, " ")
|
|
186
187
|
|
|
187
188
|
# If we now have 4 words or fewer as our title, and either no
|
|
188
189
|
# 'hierarchical' separators (\, /, > or ») were found in the original
|
|
@@ -192,7 +193,7 @@ module Readability
|
|
|
192
193
|
if cur_title_word_count <= 4 &&
|
|
193
194
|
(!title_had_hierarchical_separators ||
|
|
194
195
|
cur_title_word_count !=
|
|
195
|
-
word_count.call(orig_title.gsub(
|
|
196
|
+
word_count.call(orig_title.gsub(/#{space}[#{title_separators}]#{space}/, "")) - 1)
|
|
196
197
|
cur_title = orig_title
|
|
197
198
|
end
|
|
198
199
|
|
data/lib/readability/regexps.rb
CHANGED
|
@@ -33,7 +33,8 @@ module Readability
|
|
|
33
33
|
|
|
34
34
|
REPLACE_FONTS = /<(\/?)font[^>]*>/i
|
|
35
35
|
|
|
36
|
-
|
|
36
|
+
# JS \s matches Unicode spaces such as NBSP; Ruby's \s is ASCII-only.
|
|
37
|
+
NORMALIZE = /[[:space:]\uFEFF]{2,}/
|
|
37
38
|
|
|
38
39
|
VIDEOS = /\/\/(www\.)?((dailymotion|youtube|youtube-nocookie|player\.vimeo|v\.qq|bilibili|live.bilibili)\.com|(archive|upload\.wikimedia)\.org|player\.twitch\.tv)/i
|
|
39
40
|
|
|
@@ -45,9 +46,9 @@ module Readability
|
|
|
45
46
|
|
|
46
47
|
TOKENIZE = /\W+/
|
|
47
48
|
|
|
48
|
-
WHITESPACE = /\A\
|
|
49
|
+
WHITESPACE = /\A[[:space:]\uFEFF]*\z/
|
|
49
50
|
|
|
50
|
-
HAS_CONTENT =
|
|
51
|
+
HAS_CONTENT = /[^[:space:]\uFEFF]\z/
|
|
51
52
|
|
|
52
53
|
HASH_URL = /\A#.+/
|
|
53
54
|
|
data/lib/readability/version.rb
CHANGED