simple-rss 2.1.0 → 2.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +174 -0
- data/README.md +492 -8
- data/Rakefile +1 -1
- data/examples/digest.rb +19 -0
- data/examples/discover.rb +14 -0
- data/examples/feedbag.rb +10 -0
- data/lib/simple-rss/discovery.rb +107 -0
- data/lib/simple-rss/entry_normalizer.rb +356 -0
- data/lib/simple-rss/http_client.rb +216 -0
- data/lib/simple-rss/json_entry_normalizer.rb +147 -0
- data/lib/simple-rss/json_feed.rb +176 -0
- data/lib/simple-rss/normalized_entry.rb +79 -0
- data/lib/simple-rss/request_errors.rb +21 -0
- data/lib/simple-rss/request_policy.rb +72 -0
- data/lib/simple-rss/xml_element.rb +136 -0
- data/lib/simple-rss.rb +437 -71
- data/simple-rss.gemspec +5 -5
- data/test/base/category_parsing_test.rb +217 -0
- data/test/base/date_ordering_test.rb +95 -0
- data/test/base/discovery_dependency_test.rb +29 -0
- data/test/base/discovery_test.rb +151 -0
- data/test/base/discovery_transport_test.rb +413 -0
- data/test/base/enumerable_test.rb +16 -0
- data/test/base/feed_merging_and_diffing_test.rb +140 -0
- data/test/base/feedbag_integration_test.rb +21 -0
- data/test/base/fetch_integration_test.rb +25 -0
- data/test/base/fetch_test.rb +0 -27
- data/test/base/filtering_and_validation_test.rb +187 -0
- data/test/base/json_feed_test.rb +372 -0
- data/test/base/media_and_enclosure_helpers_test.rb +84 -0
- data/test/base/normalized_entries_test.rb +410 -0
- data/test/base/normalized_fetch_test.rb +132 -0
- data/test/base/relation_links_test.rb +162 -0
- data/test/data/atom_categories.xml +23 -0
- data/test/data/atom_nested_link.xml +13 -0
- data/test/data/discovery.html +28 -0
- data/test/data/json_feed_1.json +75 -0
- data/test/data/json_feed_1_1.json +78 -0
- data/test/data/mixed_dates.xml +55 -0
- data/test/data/normalized_atom.xml +22 -0
- data/test/data/normalized_rss.xml +21 -0
- data/test/data/rss_categories.xml +15 -0
- data/test/support/http_server.rb +44 -0
- data/test/support/replace_method.rb +14 -0
- metadata +46 -10
data/lib/simple-rss.rb
CHANGED
|
@@ -3,25 +3,32 @@
|
|
|
3
3
|
require "cgi"
|
|
4
4
|
require "time"
|
|
5
5
|
|
|
6
|
-
class SimpleRSS
|
|
6
|
+
class SimpleRSS # rubocop:disable Metrics/ClassLength
|
|
7
7
|
# @rbs skip
|
|
8
8
|
include Enumerable
|
|
9
9
|
|
|
10
10
|
# @rbs!
|
|
11
11
|
# include Enumerable[Hash[Symbol, untyped]]
|
|
12
12
|
|
|
13
|
-
VERSION = "2.
|
|
13
|
+
VERSION = "2.3.0".freeze
|
|
14
14
|
|
|
15
15
|
# @rbs @items: Array[Hash[Symbol, untyped]]
|
|
16
16
|
# @rbs @source: String
|
|
17
17
|
# @rbs @options: Hash[Symbol, untyped]
|
|
18
|
+
# @rbs @json_feed: JsonFeed?
|
|
18
19
|
# @rbs @etag: String?
|
|
19
20
|
# @rbs @last_modified: String?
|
|
21
|
+
# @rbs @entry_contexts: Hash[Hash[Symbol, untyped], Hash[Symbol, untyped]]
|
|
20
22
|
|
|
21
23
|
attr_reader :items #: Array[Hash[Symbol, untyped]]
|
|
22
24
|
attr_reader :source #: String
|
|
23
25
|
attr_reader :etag #: String?
|
|
24
26
|
attr_reader :last_modified #: String?
|
|
27
|
+
attr_reader :source_url #: String?
|
|
28
|
+
attr_reader :raw_json #: Hash[String, untyped]?
|
|
29
|
+
attr_reader :home_page_url, :feed_url, :favicon, :next_url, :user_comment #: String?
|
|
30
|
+
attr_reader :authors, :hubs #: untyped
|
|
31
|
+
attr_reader :expired #: bool?
|
|
25
32
|
alias entries items #: Array[Hash[Symbol, untyped]]
|
|
26
33
|
|
|
27
34
|
@@feed_tags = %i[
|
|
@@ -52,6 +59,9 @@ class SimpleRSS
|
|
|
52
59
|
media:title media:thumbnail#url media:thumbnail#height media:thumbnail#width
|
|
53
60
|
media:credit media:credit#role
|
|
54
61
|
media:category media:category#scheme
|
|
62
|
+
media:description
|
|
63
|
+
enclosure#url enclosure#type enclosure#length
|
|
64
|
+
itunes:duration itunes:image#href
|
|
55
65
|
]
|
|
56
66
|
|
|
57
67
|
# @rbs (untyped, ?Hash[Symbol, untyped]) -> void
|
|
@@ -60,6 +70,11 @@ class SimpleRSS
|
|
|
60
70
|
@items = [] #: Array[Hash[Symbol, untyped]]
|
|
61
71
|
@options = {} #: Hash[Symbol, untyped]
|
|
62
72
|
@options.update(options)
|
|
73
|
+
@source_url = options[:source_url]
|
|
74
|
+
@json_feed = nil
|
|
75
|
+
@raw_json = nil
|
|
76
|
+
@entry_contexts = {} #: Hash[Hash[Symbol, untyped], Hash[Symbol, untyped]]
|
|
77
|
+
@entry_contexts.compare_by_identity
|
|
63
78
|
|
|
64
79
|
parse
|
|
65
80
|
end
|
|
@@ -70,6 +85,22 @@ class SimpleRSS
|
|
|
70
85
|
end
|
|
71
86
|
alias feed channel
|
|
72
87
|
|
|
88
|
+
# @rbs (?source_url: String?, ?mappings: Hash[Symbol, untyped]) -> Array[NormalizedEntry]
|
|
89
|
+
def normalized_entries(source_url: nil, mappings: {})
|
|
90
|
+
json_feed = @json_feed
|
|
91
|
+
raise ArgumentError, "XML mappings are not supported for JSON Feed" if json_feed && !mappings.empty?
|
|
92
|
+
return items.map { |item| json_feed.normalized_entry(item, source_url: source_url || @source_url) } if json_feed
|
|
93
|
+
|
|
94
|
+
EntryNormalizer.validate_mappings(mappings)
|
|
95
|
+
feed_authors = normalized_feed_authors
|
|
96
|
+
items.map do |item|
|
|
97
|
+
context = @entry_contexts[item] || raise(SimpleRSSError, "Cannot normalize an item without its original XML source")
|
|
98
|
+
element = XmlElement.new(context[:name], context[:attributes], context[:content], context[:parent])
|
|
99
|
+
EntryNormalizer.new(element, item, source_url: source_url || @source_url, mappings: mappings,
|
|
100
|
+
raw_xml: context[:xml], feed_authors: feed_authors).entry
|
|
101
|
+
end
|
|
102
|
+
end
|
|
103
|
+
|
|
73
104
|
# Iterate over all items in the feed
|
|
74
105
|
#
|
|
75
106
|
# @rbs () { (Hash[Symbol, untyped]) -> void } -> self
|
|
@@ -92,12 +123,112 @@ class SimpleRSS
|
|
|
92
123
|
#
|
|
93
124
|
# @rbs (?Integer) -> Array[Hash[Symbol, untyped]]
|
|
94
125
|
def latest(count = 10)
|
|
95
|
-
items
|
|
126
|
+
sorted_items_by_date(items).first(count)
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
# @rbs () -> Symbol
|
|
130
|
+
def feed_type
|
|
131
|
+
return :json_feed if @json_feed
|
|
132
|
+
|
|
133
|
+
atom_namespaced_feed = source.match?(/<(atom:)?feed\b[^>]*xmlns(:\w+)?=['"][^'"]*atom/i)
|
|
134
|
+
return :atom if atom_namespaced_feed
|
|
135
|
+
return :rss2 if source.match?(/<rss[^>]*version=['"]2/i)
|
|
136
|
+
return :rss1 if source.match?(/<rdf:RDF/i)
|
|
137
|
+
return :rss09 if source.match?(/<rss[^>]*version=['"]0\.9/i)
|
|
138
|
+
|
|
139
|
+
:unknown
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
# @rbs () -> bool
|
|
143
|
+
def valid?
|
|
144
|
+
return true if @json_feed
|
|
145
|
+
|
|
146
|
+
return false if items.empty?
|
|
147
|
+
|
|
148
|
+
title_value = instance_variable_get(:@title)
|
|
149
|
+
link_value = instance_variable_get(:@link)
|
|
150
|
+
return true if title_value || link_value
|
|
151
|
+
|
|
152
|
+
false
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
# @rbs (Time) -> Array[Hash[Symbol, untyped]]
|
|
156
|
+
def items_since(time)
|
|
157
|
+
items.select do |item|
|
|
158
|
+
date = item_date(item)
|
|
159
|
+
date && date > time
|
|
160
|
+
end
|
|
161
|
+
end
|
|
162
|
+
|
|
163
|
+
# @rbs (String) -> Array[Hash[Symbol, untyped]]
|
|
164
|
+
def items_by_category(name)
|
|
165
|
+
query = name.to_s.downcase
|
|
166
|
+
|
|
167
|
+
items.select do |item|
|
|
168
|
+
category = item[:category]
|
|
169
|
+
next false if category.nil?
|
|
170
|
+
|
|
171
|
+
category_matches_query?(category, query)
|
|
172
|
+
end
|
|
173
|
+
end
|
|
174
|
+
|
|
175
|
+
# @rbs (String) -> Array[Hash[Symbol, untyped]]
|
|
176
|
+
def search(query)
|
|
177
|
+
pattern = Regexp.new(Regexp.escape(query.to_s), Regexp::IGNORECASE)
|
|
178
|
+
|
|
179
|
+
items.select do |item|
|
|
180
|
+
searchable_fields(item).any? { |field| field.to_s.match?(pattern) }
|
|
181
|
+
end
|
|
182
|
+
end
|
|
183
|
+
|
|
184
|
+
# @rbs (*SimpleRSS) -> Array[Hash[Symbol, untyped]]
|
|
185
|
+
def merge(*feeds)
|
|
186
|
+
all_items = [items, *feeds.map(&:items)].flatten
|
|
187
|
+
keyed_items, unkeyed_items = all_items.partition { |item| !item_key(item).nil? }
|
|
188
|
+
dedupe_items(sorted_items_by_date(keyed_items) + unkeyed_items)
|
|
189
|
+
end
|
|
190
|
+
|
|
191
|
+
# @rbs (SimpleRSS) -> Hash[Symbol, Array[Hash[Symbol, untyped]]]
|
|
192
|
+
def diff(other)
|
|
193
|
+
other_keys = keyed_item_set(other.items)
|
|
194
|
+
current_keys = keyed_item_set(items)
|
|
195
|
+
|
|
196
|
+
{
|
|
197
|
+
added: select_new_keyed_items(other.items, current_keys),
|
|
198
|
+
removed: select_new_keyed_items(items, other_keys)
|
|
199
|
+
}
|
|
200
|
+
end
|
|
201
|
+
|
|
202
|
+
# @rbs () -> self
|
|
203
|
+
def dedupe
|
|
204
|
+
@items = dedupe_items(items)
|
|
205
|
+
self
|
|
206
|
+
end
|
|
207
|
+
|
|
208
|
+
# @rbs () -> Array[Hash[Symbol, untyped]]
|
|
209
|
+
def enclosures
|
|
210
|
+
items.filter_map do |item|
|
|
211
|
+
enclosure_url = item[:enclosure_url]
|
|
212
|
+
next if blank_value?(enclosure_url)
|
|
213
|
+
|
|
214
|
+
{
|
|
215
|
+
url: enclosure_url,
|
|
216
|
+
type: item[:enclosure_type],
|
|
217
|
+
length: item[:enclosure_length],
|
|
218
|
+
item: item
|
|
219
|
+
}
|
|
220
|
+
end
|
|
221
|
+
end
|
|
222
|
+
|
|
223
|
+
# @rbs () -> Array[String]
|
|
224
|
+
def images
|
|
225
|
+
items.flat_map { |item| item_image_urls(item) }.uniq
|
|
96
226
|
end
|
|
97
227
|
|
|
98
228
|
# @rbs (?Hash[Symbol, untyped]) -> Hash[Symbol, untyped]
|
|
99
229
|
def as_json(_options = {})
|
|
100
|
-
|
|
230
|
+
raw_json = @raw_json
|
|
231
|
+
hash = raw_json ? raw_json.transform_keys(&:to_sym) : {} #: Hash[Symbol, untyped]
|
|
101
232
|
|
|
102
233
|
@@feed_tags.each do |tag|
|
|
103
234
|
tag_cleaned = clean_tag(tag)
|
|
@@ -122,6 +253,8 @@ class SimpleRSS
|
|
|
122
253
|
|
|
123
254
|
# @rbs (?format: Symbol) -> String
|
|
124
255
|
def to_xml(format: :rss2)
|
|
256
|
+
raise SimpleRSSError, "JSON Feed to XML conversion is not supported" if @json_feed
|
|
257
|
+
|
|
125
258
|
case format
|
|
126
259
|
when :rss2 then to_rss2_xml
|
|
127
260
|
when :atom then to_atom_xml
|
|
@@ -157,6 +290,22 @@ class SimpleRSS
|
|
|
157
290
|
new source, options
|
|
158
291
|
end
|
|
159
292
|
|
|
293
|
+
# @rbs (untyped, ?Hash[Symbol, untyped]) -> bool
|
|
294
|
+
def valid?(source, options = {})
|
|
295
|
+
parse(source, options)
|
|
296
|
+
true
|
|
297
|
+
rescue StandardError
|
|
298
|
+
false
|
|
299
|
+
end
|
|
300
|
+
|
|
301
|
+
# @rbs (*SimpleRSS) -> Array[Hash[Symbol, untyped]]
|
|
302
|
+
def merge(*feeds)
|
|
303
|
+
first_feed = feeds.first
|
|
304
|
+
return [] if first_feed.nil?
|
|
305
|
+
|
|
306
|
+
first_feed.merge(*feeds.drop(1))
|
|
307
|
+
end
|
|
308
|
+
|
|
160
309
|
# Fetch and parse a feed from a URL
|
|
161
310
|
# Returns nil if conditional GET returns 304 Not Modified
|
|
162
311
|
#
|
|
@@ -165,85 +314,68 @@ class SimpleRSS
|
|
|
165
314
|
require "net/http"
|
|
166
315
|
require "uri"
|
|
167
316
|
|
|
168
|
-
|
|
169
|
-
response =
|
|
317
|
+
require_relative "simple-rss/http_client"
|
|
318
|
+
response, final_uri = HTTPClient.new(options).get(url)
|
|
170
319
|
|
|
171
320
|
return nil if response.is_a?(Net::HTTPNotModified)
|
|
172
321
|
|
|
173
322
|
raise SimpleRSSError, "HTTP #{response.code}: #{response.message}" unless response.is_a?(Net::HTTPSuccess)
|
|
174
323
|
|
|
175
|
-
body = response.body.force_encoding(Encoding::UTF_8)
|
|
176
|
-
feed = parse(body, options)
|
|
324
|
+
body = (response.body || "").force_encoding(Encoding::UTF_8)
|
|
325
|
+
feed = parse(body, options.merge(source_url: final_uri.to_s))
|
|
177
326
|
feed.instance_variable_set(:@etag, response["ETag"])
|
|
178
327
|
feed.instance_variable_set(:@last_modified, response["Last-Modified"])
|
|
179
328
|
feed
|
|
180
329
|
end
|
|
181
330
|
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
http = build_http(uri, options)
|
|
187
|
-
request = build_request(uri, options)
|
|
188
|
-
|
|
189
|
-
response = http.request(request)
|
|
190
|
-
handle_redirect(response, options) || response
|
|
331
|
+
# @rbs (String, ?Hash[Symbol, untyped]) -> Array[Hash[Symbol, untyped]]
|
|
332
|
+
def discover(url, options = {})
|
|
333
|
+
require_relative "simple-rss/discovery"
|
|
334
|
+
Discovery.new(options).discover(url)
|
|
191
335
|
end
|
|
336
|
+
end
|
|
192
337
|
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
timeout = options[:timeout]
|
|
200
|
-
if timeout
|
|
201
|
-
http.open_timeout = timeout
|
|
202
|
-
http.read_timeout = timeout
|
|
203
|
-
end
|
|
204
|
-
|
|
205
|
-
http
|
|
206
|
-
end
|
|
338
|
+
DATE_TAGS = %i[pubDate lastBuildDate published updated expirationDate modified dc:date].freeze
|
|
339
|
+
STRIP_HTML_TAGS = %i[author contributor skipHours skipDays].freeze
|
|
340
|
+
ATOM_NAMESPACE = "http://www.w3.org/2005/Atom".freeze
|
|
341
|
+
RSS_NAMESPACES = [nil, "", "http://purl.org/rss/1.0/", "http://my.netscape.com/rdf/simple/0.9/"].freeze
|
|
342
|
+
XML_TAG_PATTERN = %r{<!--.*?-->|<!\[CDATA\[.*?\]\]>|<\?.*?\?>|<(/?)([\w:.-]+)((?:[^<>"']|"[^"]*"|'[^']*')*)>}m
|
|
207
343
|
|
|
208
|
-
|
|
209
|
-
def build_request(uri, options)
|
|
210
|
-
request = Net::HTTP::Get.new(uri)
|
|
211
|
-
request["User-Agent"] = "SimpleRSS/#{VERSION}"
|
|
344
|
+
private
|
|
212
345
|
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
346
|
+
# @rbs () -> Array[XmlElement]
|
|
347
|
+
def normalized_feed_authors
|
|
348
|
+
document = XmlElement.new("", "", @source, { namespaces: {}, base_urls: [] })
|
|
349
|
+
feed = document.children.find { |element| element.matches?("feed", ATOM_NAMESPACE) }
|
|
350
|
+
return [] unless feed
|
|
216
351
|
|
|
217
|
-
|
|
218
|
-
|
|
352
|
+
feed.children.select { |element| element.matches?("author", ATOM_NAMESPACE) }
|
|
353
|
+
end
|
|
219
354
|
|
|
220
|
-
|
|
355
|
+
# @rbs () -> void
|
|
356
|
+
def parse
|
|
357
|
+
prefix = @source.b.sub(/\A\xEF\xBB\xBF/n, "").lstrip
|
|
358
|
+
return parse_xml unless prefix.match?(/\A(?:[\{\["0-9-]|true\b|false\b|null\b)/n)
|
|
359
|
+
|
|
360
|
+
json_feed = JsonFeed.new(@source)
|
|
361
|
+
@json_feed = json_feed
|
|
362
|
+
@raw_json = json_feed.document
|
|
363
|
+
JsonFeed::FEED_FIELDS.each do |field|
|
|
364
|
+
instance_variable_set("@#{field}", json_feed.document[field])
|
|
365
|
+
self.class.attr_reader(field)
|
|
221
366
|
end
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
location = response["Location"]
|
|
229
|
-
return nil unless location
|
|
230
|
-
|
|
231
|
-
redirects = (options[:_redirects] || 0) + 1
|
|
232
|
-
raise SimpleRSSError, "Too many redirects" if redirects > 5
|
|
233
|
-
|
|
234
|
-
new_options = options.merge(_redirects: redirects)
|
|
235
|
-
perform_fetch(URI.parse(location), new_options)
|
|
367
|
+
@link = json_feed.document["home_page_url"]
|
|
368
|
+
self.class.attr_reader(:link)
|
|
369
|
+
@items = json_feed.items
|
|
370
|
+
@items.each do |item|
|
|
371
|
+
item.define_singleton_method(:method_missing) { |name, *_args| self[name] }
|
|
372
|
+
add_item_media_helpers(item)
|
|
236
373
|
end
|
|
237
374
|
end
|
|
238
375
|
|
|
239
|
-
DATE_TAGS = %i[pubDate lastBuildDate published updated expirationDate modified dc:date].freeze
|
|
240
|
-
STRIP_HTML_TAGS = %i[author contributor skipHours skipDays].freeze
|
|
241
|
-
|
|
242
|
-
private
|
|
243
|
-
|
|
244
376
|
# @rbs () -> void
|
|
245
|
-
def
|
|
246
|
-
raise SimpleRSSError, "Poorly formatted feed" unless @source =~ %r{<(channel|feed).*?>.*?</(channel|feed)
|
|
377
|
+
def parse_xml
|
|
378
|
+
raise SimpleRSSError, "Poorly formatted feed" unless @source =~ %r{<(channel|feed).*?>.*?</(channel|feed)>|<(channel|feed)\b[^>]*?/\s*>}mi
|
|
247
379
|
|
|
248
380
|
# Feed's title and link
|
|
249
381
|
feed_content = Regexp.last_match(1) if @source =~ %r{(.*?)<(rss:|atom:)?(item|entry).*?>.*?</(rss:|atom:)?(item|entry)>}mi
|
|
@@ -283,18 +415,57 @@ class SimpleRSS
|
|
|
283
415
|
end
|
|
284
416
|
|
|
285
417
|
# RSS items' title, link, and description
|
|
286
|
-
|
|
418
|
+
namespace_contexts = entry_contexts
|
|
419
|
+
entry_pattern = %r{<(rss:|atom:)?(item|entry)([\s][^>]*)?>(.*?)</(rss:|atom:)?(item|entry)>}mi
|
|
420
|
+
position = 0
|
|
421
|
+
while (match = entry_pattern.match(@source, position))
|
|
422
|
+
position = match.end(0)
|
|
287
423
|
item = {} #: Hash[Symbol, untyped]
|
|
424
|
+
parent_context = namespace_contexts[match.begin(0)]
|
|
425
|
+
namespaces = parent_context && parent_context[:namespaces].merge(namespace_attributes(match[3].to_s))
|
|
426
|
+
@entry_contexts[item] = {
|
|
427
|
+
name: "#{match[1]}#{match[2]}", attributes: match[3].to_s, content: match[4].to_s,
|
|
428
|
+
parent: parent_context || { namespaces: {}, base_urls: [] }, xml: match[0]
|
|
429
|
+
}
|
|
288
430
|
@@item_tags.each do |tag|
|
|
289
431
|
next if tag.to_s.strip.empty?
|
|
290
432
|
|
|
291
|
-
|
|
433
|
+
if tag == :category
|
|
434
|
+
next unless namespaces
|
|
435
|
+
|
|
436
|
+
parse_category_tag(item, match[4].to_s, namespaces, element_namespace("#{match[1]}#{match[2]}", namespaces))
|
|
437
|
+
next
|
|
438
|
+
end
|
|
439
|
+
|
|
440
|
+
parse_item_tag(item, tag, match[4], match[3])
|
|
292
441
|
end
|
|
293
|
-
item.define_singleton_method(:method_missing) { |name,
|
|
442
|
+
item.define_singleton_method(:method_missing) { |name, *_args| self[name] }
|
|
443
|
+
add_item_media_helpers(item)
|
|
294
444
|
@items << item
|
|
295
445
|
end
|
|
296
446
|
end
|
|
297
447
|
|
|
448
|
+
# @rbs (Hash[Symbol, untyped]) -> void
|
|
449
|
+
def add_item_media_helpers(item)
|
|
450
|
+
item.define_singleton_method(:has_media?) do
|
|
451
|
+
[
|
|
452
|
+
self[:media_content_url],
|
|
453
|
+
self[:media_thumbnail_url],
|
|
454
|
+
self[:enclosure_url],
|
|
455
|
+
self[:itunes_image_href]
|
|
456
|
+
].any? { |value| !value.nil? && !value.to_s.strip.empty? }
|
|
457
|
+
end
|
|
458
|
+
|
|
459
|
+
item.define_singleton_method(:media_url) do
|
|
460
|
+
[
|
|
461
|
+
self[:media_content_url],
|
|
462
|
+
self[:media_thumbnail_url],
|
|
463
|
+
self[:enclosure_url],
|
|
464
|
+
self[:itunes_image_href]
|
|
465
|
+
].find { |value| !value.nil? && !value.to_s.strip.empty? }
|
|
466
|
+
end
|
|
467
|
+
end
|
|
468
|
+
|
|
298
469
|
# @rbs (Hash[Symbol, untyped], Symbol, String?, String?) -> void
|
|
299
470
|
def parse_item_tag(item, tag, content, item_attrs = nil)
|
|
300
471
|
return if content.nil?
|
|
@@ -309,15 +480,120 @@ class SimpleRSS
|
|
|
309
480
|
|
|
310
481
|
# @rbs (Hash[Symbol, untyped], String, String) -> void
|
|
311
482
|
def parse_rel_tag(item, tag_str, content)
|
|
312
|
-
tag, rel = tag_str.split("+")
|
|
483
|
+
tag, rel = tag_str.split("+", 2)
|
|
313
484
|
return unless tag && rel
|
|
314
485
|
|
|
315
|
-
|
|
316
|
-
|
|
486
|
+
value = if tag == "link"
|
|
487
|
+
link_relation_href(content, rel)
|
|
488
|
+
else
|
|
489
|
+
content =~ %r{<(rss:|atom:)?#{tag}(.*?)rel=['"]#{rel}['"](.*?)>(.*?)</(rss:|atom:)?#{tag}>}mi ||
|
|
490
|
+
content =~ %r{<(rss:|atom:)?#{tag}(.*?)rel=['"]#{rel}['"](.*?)/\s*>}mi
|
|
491
|
+
|
|
492
|
+
return unless Regexp.last_match(3) || Regexp.last_match(4)
|
|
493
|
+
|
|
494
|
+
clean_content(tag.to_sym, Regexp.last_match(3), Regexp.last_match(4))
|
|
495
|
+
end
|
|
496
|
+
return if value.nil?
|
|
497
|
+
|
|
498
|
+
item[clean_tag("#{tag}+#{rel}")] = value
|
|
499
|
+
item[clean_tag("#{tag}_#{rel}")] = value
|
|
500
|
+
end
|
|
501
|
+
|
|
502
|
+
# @rbs (String, String) -> String?
|
|
503
|
+
def link_relation_href(content, relation)
|
|
504
|
+
attributes = entry_link_attributes(content).find { |link| link["rel"]&.casecmp?(relation) }
|
|
505
|
+
return unless attributes
|
|
506
|
+
|
|
507
|
+
attributes["href"]
|
|
508
|
+
end
|
|
509
|
+
|
|
510
|
+
# @rbs (String) -> Array[Hash[String, String]]
|
|
511
|
+
def entry_link_attributes(content)
|
|
512
|
+
child_elements(content).filter_map do |tag, attributes, _body|
|
|
513
|
+
next unless %w[link atom:link rss:link].include?(tag.downcase)
|
|
317
514
|
|
|
318
|
-
|
|
515
|
+
xml_attributes(attributes).transform_keys(&:downcase)
|
|
516
|
+
end
|
|
517
|
+
end
|
|
518
|
+
|
|
519
|
+
# @rbs (String) -> Array[[String, String, String?]]
|
|
520
|
+
def child_elements(content)
|
|
521
|
+
XmlElement.child_elements(content)
|
|
522
|
+
end
|
|
523
|
+
|
|
524
|
+
# @rbs (Hash[Symbol, untyped], String, Hash[String, String], String?) -> void
|
|
525
|
+
def parse_category_tag(item, content, namespaces, entry_namespace)
|
|
526
|
+
values = child_elements(content).filter_map do |tag, raw_attributes, body|
|
|
527
|
+
next unless tag.split(":").last&.casecmp?("category")
|
|
528
|
+
|
|
529
|
+
attributes = xml_attributes(raw_attributes)
|
|
530
|
+
namespace = element_namespace(tag, namespaces.merge(namespace_attributes(raw_attributes)))
|
|
531
|
+
if namespace == ATOM_NAMESPACE
|
|
532
|
+
term = CGI.unescapeHTML(attributes["term"].to_s).strip
|
|
533
|
+
next term unless term.empty?
|
|
534
|
+
|
|
535
|
+
next
|
|
536
|
+
end
|
|
537
|
+
|
|
538
|
+
next if entry_namespace == ATOM_NAMESPACE || !RSS_NAMESPACES.include?(namespace)
|
|
539
|
+
next if tag.include?(":") && namespace.nil?
|
|
540
|
+
next if body.nil? && (array_tag?(:category) || !raw_attributes.rstrip.end_with?("/"))
|
|
541
|
+
|
|
542
|
+
unescape(body.to_s)
|
|
543
|
+
end
|
|
544
|
+
return if values.empty?
|
|
545
|
+
|
|
546
|
+
item[:category] = array_tag?(:category) ? values : values.first
|
|
547
|
+
end
|
|
548
|
+
|
|
549
|
+
# @rbs () -> Hash[Integer, Hash[Symbol, untyped]]
|
|
550
|
+
def entry_contexts
|
|
551
|
+
contexts = {} #: Hash[Integer, Hash[Symbol, untyped]]
|
|
552
|
+
scopes = [{ namespaces: {}, base_urls: [] }] #: Array[Hash[Symbol, untyped]]
|
|
553
|
+
position = 0
|
|
554
|
+
|
|
555
|
+
while (token = XML_TAG_PATTERN.match(@source, position))
|
|
556
|
+
position = token.end(0)
|
|
557
|
+
attributes = token[3]
|
|
558
|
+
next unless attributes
|
|
559
|
+
|
|
560
|
+
if token[1] == "/"
|
|
561
|
+
scopes.pop if scopes.size > 1
|
|
562
|
+
next
|
|
563
|
+
end
|
|
564
|
+
|
|
565
|
+
parent = scopes.fetch(-1)
|
|
566
|
+
tag = token[2].to_s.split(":").last
|
|
567
|
+
contexts[token.begin(0).to_i] = parent if %w[item entry].include?(tag&.downcase)
|
|
568
|
+
base_url = xml_attributes(attributes)["xml:base"]
|
|
569
|
+
base_urls = parent[:base_urls].dup
|
|
570
|
+
base_urls << CGI.unescapeHTML(base_url) if base_url
|
|
571
|
+
context = {
|
|
572
|
+
namespaces: parent[:namespaces].merge(namespace_attributes(attributes)),
|
|
573
|
+
base_urls: base_urls
|
|
574
|
+
}
|
|
575
|
+
scopes << context unless attributes.rstrip.end_with?("/")
|
|
576
|
+
end
|
|
577
|
+
|
|
578
|
+
contexts
|
|
579
|
+
end
|
|
580
|
+
|
|
581
|
+
# @rbs (String) -> Hash[String, String]
|
|
582
|
+
def namespace_attributes(attributes)
|
|
583
|
+
xml_attributes(attributes)
|
|
584
|
+
.select { |name, _value| name == "xmlns" || name.start_with?("xmlns:") }
|
|
585
|
+
.transform_values { |value| CGI.unescapeHTML(value) }
|
|
586
|
+
end
|
|
587
|
+
|
|
588
|
+
# @rbs (String, Hash[String, String]) -> String?
|
|
589
|
+
def element_namespace(tag, namespaces)
|
|
590
|
+
key = tag.include?(":") ? "xmlns:#{tag.split(":").first}" : "xmlns"
|
|
591
|
+
namespaces[key]
|
|
592
|
+
end
|
|
319
593
|
|
|
320
|
-
|
|
594
|
+
# @rbs (String) -> Hash[String, String]
|
|
595
|
+
def xml_attributes(attributes)
|
|
596
|
+
XmlElement.attributes(attributes)
|
|
321
597
|
end
|
|
322
598
|
|
|
323
599
|
# @rbs (String, String?) -> void
|
|
@@ -412,6 +688,88 @@ class SimpleRSS
|
|
|
412
688
|
tag.to_s.tr(":", "_").intern
|
|
413
689
|
end
|
|
414
690
|
|
|
691
|
+
# @rbs (untyped, String) -> bool
|
|
692
|
+
def category_matches_query?(category, query)
|
|
693
|
+
return category.any? { |value| value.to_s.downcase.include?(query) } if category.is_a?(Array)
|
|
694
|
+
|
|
695
|
+
category.to_s.downcase.include?(query)
|
|
696
|
+
end
|
|
697
|
+
|
|
698
|
+
# @rbs (Hash[Symbol, untyped]) -> Array[untyped]
|
|
699
|
+
def searchable_fields(item)
|
|
700
|
+
[item[:title], item[:description], item[:summary], item[:content]]
|
|
701
|
+
end
|
|
702
|
+
|
|
703
|
+
# @rbs (Array[Hash[Symbol, untyped]]) -> Set[String]
|
|
704
|
+
def keyed_item_set(item_list)
|
|
705
|
+
item_list.each_with_object(Set.new) do |item, keys|
|
|
706
|
+
key = item_key(item)
|
|
707
|
+
next if key.nil?
|
|
708
|
+
|
|
709
|
+
keys.add(key)
|
|
710
|
+
end
|
|
711
|
+
end
|
|
712
|
+
|
|
713
|
+
# @rbs (Array[Hash[Symbol, untyped]], Set[String]) -> Array[Hash[Symbol, untyped]]
|
|
714
|
+
def select_new_keyed_items(item_list, known_keys)
|
|
715
|
+
item_list.select do |item|
|
|
716
|
+
key = item_key(item)
|
|
717
|
+
!key.nil? && !known_keys.include?(key)
|
|
718
|
+
end
|
|
719
|
+
end
|
|
720
|
+
|
|
721
|
+
# @rbs (Array[Hash[Symbol, untyped]]) -> Array[Hash[Symbol, untyped]]
|
|
722
|
+
def sorted_items_by_date(item_list)
|
|
723
|
+
item_list.sort_by.with_index do |item, index|
|
|
724
|
+
date = item_date(item)
|
|
725
|
+
[date ? -date.to_r : Float::INFINITY, index]
|
|
726
|
+
end
|
|
727
|
+
end
|
|
728
|
+
|
|
729
|
+
# @rbs (Array[Hash[Symbol, untyped]]) -> Array[Hash[Symbol, untyped]]
|
|
730
|
+
def dedupe_items(item_list)
|
|
731
|
+
seen_keys = Set.new
|
|
732
|
+
|
|
733
|
+
item_list.each_with_object([]) do |item, unique_items|
|
|
734
|
+
key = item_key(item)
|
|
735
|
+
|
|
736
|
+
if key.nil?
|
|
737
|
+
unique_items << item
|
|
738
|
+
next
|
|
739
|
+
end
|
|
740
|
+
|
|
741
|
+
next if seen_keys.include?(key)
|
|
742
|
+
|
|
743
|
+
seen_keys.add(key)
|
|
744
|
+
unique_items << item
|
|
745
|
+
end
|
|
746
|
+
end
|
|
747
|
+
|
|
748
|
+
# @rbs (Hash[Symbol, untyped]) -> String?
|
|
749
|
+
def item_key(item)
|
|
750
|
+
key = item[:guid] || item[:id] || item[:link]
|
|
751
|
+
return nil if key.nil?
|
|
752
|
+
|
|
753
|
+
key.to_s
|
|
754
|
+
end
|
|
755
|
+
|
|
756
|
+
# @rbs (Hash[Symbol, untyped]) -> Time?
|
|
757
|
+
def item_date(item)
|
|
758
|
+
[item[:pubDate], item[:updated], item[:published]].find { |date| date.is_a?(Time) }
|
|
759
|
+
end
|
|
760
|
+
|
|
761
|
+
# @rbs (Hash[Symbol, untyped]) -> Array[String]
|
|
762
|
+
def item_image_urls(item)
|
|
763
|
+
[item[:media_thumbnail_url], item[:media_content_url], item[:itunes_image_href]]
|
|
764
|
+
.compact
|
|
765
|
+
.reject { |url| blank_value?(url) }
|
|
766
|
+
end
|
|
767
|
+
|
|
768
|
+
# @rbs (untyped) -> bool
|
|
769
|
+
def blank_value?(value)
|
|
770
|
+
value.to_s.strip.empty?
|
|
771
|
+
end
|
|
772
|
+
|
|
415
773
|
# @rbs (untyped) -> untyped
|
|
416
774
|
def serialize_value(value)
|
|
417
775
|
case value
|
|
@@ -545,5 +903,13 @@ class SimpleRSS
|
|
|
545
903
|
end
|
|
546
904
|
end
|
|
547
905
|
|
|
548
|
-
|
|
906
|
+
require_relative "simple-rss/xml_element"
|
|
907
|
+
require_relative "simple-rss/normalized_entry"
|
|
908
|
+
require_relative "simple-rss/json_feed"
|
|
909
|
+
require_relative "simple-rss/json_entry_normalizer"
|
|
910
|
+
require_relative "simple-rss/entry_normalizer"
|
|
911
|
+
|
|
912
|
+
class SimpleRSSError < StandardError # rubocop:disable Style/OneClassPerFile
|
|
549
913
|
end
|
|
914
|
+
|
|
915
|
+
require_relative "simple-rss/request_errors"
|
data/simple-rss.gemspec
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
Gem::Specification.new do |s|
|
|
2
2
|
s.name = "simple-rss"
|
|
3
|
-
s.version = "2.
|
|
4
|
-
s.
|
|
5
|
-
s.summary = "A simple, flexible, extensible, and liberal RSS and Atom reader for Ruby. It is designed to be backwards compatible with the standard RSS parser, but will never do RSS generation."
|
|
3
|
+
s.version = "2.3.0"
|
|
4
|
+
s.summary = "A flexible RSS, Atom, and JSON Feed reader for Ruby."
|
|
6
5
|
s.email = "lucas@rufy.com"
|
|
7
6
|
s.homepage = "https://github.com/cardmagic/simple-rss"
|
|
8
|
-
s.
|
|
7
|
+
s.metadata["changelog_uri"] = "https://github.com/cardmagic/simple-rss/blob/master/CHANGELOG.md"
|
|
8
|
+
s.description = "Parse RSS, Atom, and JSON Feed with normalized entries, HTTP fetching, website feed discovery, and JSON/XML serialization."
|
|
9
9
|
s.authors = ["Lucas Carlson"]
|
|
10
|
-
s.files = Dir["lib/**/*", "test/**/*", "LICENSE", "README.md", "Rakefile", "simple-rss.gemspec"]
|
|
10
|
+
s.files = Dir["lib/**/*", "examples/**/*", "test/**/*", "LICENSE", "README.md", "CHANGELOG.md", "Rakefile", "simple-rss.gemspec"]
|
|
11
11
|
s.required_ruby_version = ">= 3.1"
|
|
12
12
|
s.add_development_dependency "rake"
|
|
13
13
|
s.add_development_dependency "rdoc"
|