simple-rss 2.1.0 → 2.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +174 -0
  3. data/README.md +492 -8
  4. data/Rakefile +1 -1
  5. data/examples/digest.rb +19 -0
  6. data/examples/discover.rb +14 -0
  7. data/examples/feedbag.rb +10 -0
  8. data/lib/simple-rss/discovery.rb +107 -0
  9. data/lib/simple-rss/entry_normalizer.rb +356 -0
  10. data/lib/simple-rss/http_client.rb +216 -0
  11. data/lib/simple-rss/json_entry_normalizer.rb +147 -0
  12. data/lib/simple-rss/json_feed.rb +176 -0
  13. data/lib/simple-rss/normalized_entry.rb +79 -0
  14. data/lib/simple-rss/request_errors.rb +21 -0
  15. data/lib/simple-rss/request_policy.rb +72 -0
  16. data/lib/simple-rss/xml_element.rb +136 -0
  17. data/lib/simple-rss.rb +437 -71
  18. data/simple-rss.gemspec +5 -5
  19. data/test/base/category_parsing_test.rb +217 -0
  20. data/test/base/date_ordering_test.rb +95 -0
  21. data/test/base/discovery_dependency_test.rb +29 -0
  22. data/test/base/discovery_test.rb +151 -0
  23. data/test/base/discovery_transport_test.rb +413 -0
  24. data/test/base/enumerable_test.rb +16 -0
  25. data/test/base/feed_merging_and_diffing_test.rb +140 -0
  26. data/test/base/feedbag_integration_test.rb +21 -0
  27. data/test/base/fetch_integration_test.rb +25 -0
  28. data/test/base/fetch_test.rb +0 -27
  29. data/test/base/filtering_and_validation_test.rb +187 -0
  30. data/test/base/json_feed_test.rb +372 -0
  31. data/test/base/media_and_enclosure_helpers_test.rb +84 -0
  32. data/test/base/normalized_entries_test.rb +410 -0
  33. data/test/base/normalized_fetch_test.rb +132 -0
  34. data/test/base/relation_links_test.rb +162 -0
  35. data/test/data/atom_categories.xml +23 -0
  36. data/test/data/atom_nested_link.xml +13 -0
  37. data/test/data/discovery.html +28 -0
  38. data/test/data/json_feed_1.json +75 -0
  39. data/test/data/json_feed_1_1.json +78 -0
  40. data/test/data/mixed_dates.xml +55 -0
  41. data/test/data/normalized_atom.xml +22 -0
  42. data/test/data/normalized_rss.xml +21 -0
  43. data/test/data/rss_categories.xml +15 -0
  44. data/test/support/http_server.rb +44 -0
  45. data/test/support/replace_method.rb +14 -0
  46. metadata +46 -10
data/lib/simple-rss.rb CHANGED
@@ -3,25 +3,32 @@
3
3
  require "cgi"
4
4
  require "time"
5
5
 
6
- class SimpleRSS
6
+ class SimpleRSS # rubocop:disable Metrics/ClassLength
7
7
  # @rbs skip
8
8
  include Enumerable
9
9
 
10
10
  # @rbs!
11
11
  # include Enumerable[Hash[Symbol, untyped]]
12
12
 
13
- VERSION = "2.1.0".freeze
13
+ VERSION = "2.3.0".freeze
14
14
 
15
15
  # @rbs @items: Array[Hash[Symbol, untyped]]
16
16
  # @rbs @source: String
17
17
  # @rbs @options: Hash[Symbol, untyped]
18
+ # @rbs @json_feed: JsonFeed?
18
19
  # @rbs @etag: String?
19
20
  # @rbs @last_modified: String?
21
+ # @rbs @entry_contexts: Hash[Hash[Symbol, untyped], Hash[Symbol, untyped]]
20
22
 
21
23
  attr_reader :items #: Array[Hash[Symbol, untyped]]
22
24
  attr_reader :source #: String
23
25
  attr_reader :etag #: String?
24
26
  attr_reader :last_modified #: String?
27
+ attr_reader :source_url #: String?
28
+ attr_reader :raw_json #: Hash[String, untyped]?
29
+ attr_reader :home_page_url, :feed_url, :favicon, :next_url, :user_comment #: String?
30
+ attr_reader :authors, :hubs #: untyped
31
+ attr_reader :expired #: bool?
25
32
  alias entries items #: Array[Hash[Symbol, untyped]]
26
33
 
27
34
  @@feed_tags = %i[
@@ -52,6 +59,9 @@ class SimpleRSS
52
59
  media:title media:thumbnail#url media:thumbnail#height media:thumbnail#width
53
60
  media:credit media:credit#role
54
61
  media:category media:category#scheme
62
+ media:description
63
+ enclosure#url enclosure#type enclosure#length
64
+ itunes:duration itunes:image#href
55
65
  ]
56
66
 
57
67
  # @rbs (untyped, ?Hash[Symbol, untyped]) -> void
@@ -60,6 +70,11 @@ class SimpleRSS
60
70
  @items = [] #: Array[Hash[Symbol, untyped]]
61
71
  @options = {} #: Hash[Symbol, untyped]
62
72
  @options.update(options)
73
+ @source_url = options[:source_url]
74
+ @json_feed = nil
75
+ @raw_json = nil
76
+ @entry_contexts = {} #: Hash[Hash[Symbol, untyped], Hash[Symbol, untyped]]
77
+ @entry_contexts.compare_by_identity
63
78
 
64
79
  parse
65
80
  end
@@ -70,6 +85,22 @@ class SimpleRSS
70
85
  end
71
86
  alias feed channel
72
87
 
88
+ # @rbs (?source_url: String?, ?mappings: Hash[Symbol, untyped]) -> Array[NormalizedEntry]
89
+ def normalized_entries(source_url: nil, mappings: {})
90
+ json_feed = @json_feed
91
+ raise ArgumentError, "XML mappings are not supported for JSON Feed" if json_feed && !mappings.empty?
92
+ return items.map { |item| json_feed.normalized_entry(item, source_url: source_url || @source_url) } if json_feed
93
+
94
+ EntryNormalizer.validate_mappings(mappings)
95
+ feed_authors = normalized_feed_authors
96
+ items.map do |item|
97
+ context = @entry_contexts[item] || raise(SimpleRSSError, "Cannot normalize an item without its original XML source")
98
+ element = XmlElement.new(context[:name], context[:attributes], context[:content], context[:parent])
99
+ EntryNormalizer.new(element, item, source_url: source_url || @source_url, mappings: mappings,
100
+ raw_xml: context[:xml], feed_authors: feed_authors).entry
101
+ end
102
+ end
103
+
73
104
  # Iterate over all items in the feed
74
105
  #
75
106
  # @rbs () { (Hash[Symbol, untyped]) -> void } -> self
@@ -92,12 +123,112 @@ class SimpleRSS
92
123
  #
93
124
  # @rbs (?Integer) -> Array[Hash[Symbol, untyped]]
94
125
  def latest(count = 10)
95
- items.sort_by { |item| item[:pubDate] || item[:updated] || Time.at(0) }.reverse.first(count)
126
+ sorted_items_by_date(items).first(count)
127
+ end
128
+
129
+ # @rbs () -> Symbol
130
+ def feed_type
131
+ return :json_feed if @json_feed
132
+
133
+ atom_namespaced_feed = source.match?(/<(atom:)?feed\b[^>]*xmlns(:\w+)?=['"][^'"]*atom/i)
134
+ return :atom if atom_namespaced_feed
135
+ return :rss2 if source.match?(/<rss[^>]*version=['"]2/i)
136
+ return :rss1 if source.match?(/<rdf:RDF/i)
137
+ return :rss09 if source.match?(/<rss[^>]*version=['"]0\.9/i)
138
+
139
+ :unknown
140
+ end
141
+
142
+ # @rbs () -> bool
143
+ def valid?
144
+ return true if @json_feed
145
+
146
+ return false if items.empty?
147
+
148
+ title_value = instance_variable_get(:@title)
149
+ link_value = instance_variable_get(:@link)
150
+ return true if title_value || link_value
151
+
152
+ false
153
+ end
154
+
155
+ # @rbs (Time) -> Array[Hash[Symbol, untyped]]
156
+ def items_since(time)
157
+ items.select do |item|
158
+ date = item_date(item)
159
+ date && date > time
160
+ end
161
+ end
162
+
163
+ # @rbs (String) -> Array[Hash[Symbol, untyped]]
164
+ def items_by_category(name)
165
+ query = name.to_s.downcase
166
+
167
+ items.select do |item|
168
+ category = item[:category]
169
+ next false if category.nil?
170
+
171
+ category_matches_query?(category, query)
172
+ end
173
+ end
174
+
175
+ # @rbs (String) -> Array[Hash[Symbol, untyped]]
176
+ def search(query)
177
+ pattern = Regexp.new(Regexp.escape(query.to_s), Regexp::IGNORECASE)
178
+
179
+ items.select do |item|
180
+ searchable_fields(item).any? { |field| field.to_s.match?(pattern) }
181
+ end
182
+ end
183
+
184
+ # @rbs (*SimpleRSS) -> Array[Hash[Symbol, untyped]]
185
+ def merge(*feeds)
186
+ all_items = [items, *feeds.map(&:items)].flatten
187
+ keyed_items, unkeyed_items = all_items.partition { |item| !item_key(item).nil? }
188
+ dedupe_items(sorted_items_by_date(keyed_items) + unkeyed_items)
189
+ end
190
+
191
+ # @rbs (SimpleRSS) -> Hash[Symbol, Array[Hash[Symbol, untyped]]]
192
+ def diff(other)
193
+ other_keys = keyed_item_set(other.items)
194
+ current_keys = keyed_item_set(items)
195
+
196
+ {
197
+ added: select_new_keyed_items(other.items, current_keys),
198
+ removed: select_new_keyed_items(items, other_keys)
199
+ }
200
+ end
201
+
202
+ # @rbs () -> self
203
+ def dedupe
204
+ @items = dedupe_items(items)
205
+ self
206
+ end
207
+
208
+ # @rbs () -> Array[Hash[Symbol, untyped]]
209
+ def enclosures
210
+ items.filter_map do |item|
211
+ enclosure_url = item[:enclosure_url]
212
+ next if blank_value?(enclosure_url)
213
+
214
+ {
215
+ url: enclosure_url,
216
+ type: item[:enclosure_type],
217
+ length: item[:enclosure_length],
218
+ item: item
219
+ }
220
+ end
221
+ end
222
+
223
+ # @rbs () -> Array[String]
224
+ def images
225
+ items.flat_map { |item| item_image_urls(item) }.uniq
96
226
  end
97
227
 
98
228
  # @rbs (?Hash[Symbol, untyped]) -> Hash[Symbol, untyped]
99
229
  def as_json(_options = {})
100
- hash = {} #: Hash[Symbol, untyped]
230
+ raw_json = @raw_json
231
+ hash = raw_json ? raw_json.transform_keys(&:to_sym) : {} #: Hash[Symbol, untyped]
101
232
 
102
233
  @@feed_tags.each do |tag|
103
234
  tag_cleaned = clean_tag(tag)
@@ -122,6 +253,8 @@ class SimpleRSS
122
253
 
123
254
  # @rbs (?format: Symbol) -> String
124
255
  def to_xml(format: :rss2)
256
+ raise SimpleRSSError, "JSON Feed to XML conversion is not supported" if @json_feed
257
+
125
258
  case format
126
259
  when :rss2 then to_rss2_xml
127
260
  when :atom then to_atom_xml
@@ -157,6 +290,22 @@ class SimpleRSS
157
290
  new source, options
158
291
  end
159
292
 
293
+ # @rbs (untyped, ?Hash[Symbol, untyped]) -> bool
294
+ def valid?(source, options = {})
295
+ parse(source, options)
296
+ true
297
+ rescue StandardError
298
+ false
299
+ end
300
+
301
+ # @rbs (*SimpleRSS) -> Array[Hash[Symbol, untyped]]
302
+ def merge(*feeds)
303
+ first_feed = feeds.first
304
+ return [] if first_feed.nil?
305
+
306
+ first_feed.merge(*feeds.drop(1))
307
+ end
308
+
160
309
  # Fetch and parse a feed from a URL
161
310
  # Returns nil if conditional GET returns 304 Not Modified
162
311
  #
@@ -165,85 +314,68 @@ class SimpleRSS
165
314
  require "net/http"
166
315
  require "uri"
167
316
 
168
- uri = URI.parse(url)
169
- response = perform_fetch(uri, options)
317
+ require_relative "simple-rss/http_client"
318
+ response, final_uri = HTTPClient.new(options).get(url)
170
319
 
171
320
  return nil if response.is_a?(Net::HTTPNotModified)
172
321
 
173
322
  raise SimpleRSSError, "HTTP #{response.code}: #{response.message}" unless response.is_a?(Net::HTTPSuccess)
174
323
 
175
- body = response.body.force_encoding(Encoding::UTF_8)
176
- feed = parse(body, options)
324
+ body = (response.body || "").force_encoding(Encoding::UTF_8)
325
+ feed = parse(body, options.merge(source_url: final_uri.to_s))
177
326
  feed.instance_variable_set(:@etag, response["ETag"])
178
327
  feed.instance_variable_set(:@last_modified, response["Last-Modified"])
179
328
  feed
180
329
  end
181
330
 
182
- private
183
-
184
- # @rbs (untyped, Hash[Symbol, untyped]) -> untyped
185
- def perform_fetch(uri, options)
186
- http = build_http(uri, options)
187
- request = build_request(uri, options)
188
-
189
- response = http.request(request)
190
- handle_redirect(response, options) || response
331
+ # @rbs (String, ?Hash[Symbol, untyped]) -> Array[Hash[Symbol, untyped]]
332
+ def discover(url, options = {})
333
+ require_relative "simple-rss/discovery"
334
+ Discovery.new(options).discover(url)
191
335
  end
336
+ end
192
337
 
193
- # @rbs (untyped, Hash[Symbol, untyped]) -> untyped
194
- def build_http(uri, options)
195
- host = uri.host || raise(SimpleRSSError, "Invalid URL: missing host")
196
- http = Net::HTTP.new(host, uri.port)
197
- http.use_ssl = uri.scheme == "https"
198
-
199
- timeout = options[:timeout]
200
- if timeout
201
- http.open_timeout = timeout
202
- http.read_timeout = timeout
203
- end
204
-
205
- http
206
- end
338
+ DATE_TAGS = %i[pubDate lastBuildDate published updated expirationDate modified dc:date].freeze
339
+ STRIP_HTML_TAGS = %i[author contributor skipHours skipDays].freeze
340
+ ATOM_NAMESPACE = "http://www.w3.org/2005/Atom".freeze
341
+ RSS_NAMESPACES = [nil, "", "http://purl.org/rss/1.0/", "http://my.netscape.com/rdf/simple/0.9/"].freeze
342
+ XML_TAG_PATTERN = %r{<!--.*?-->|<!\[CDATA\[.*?\]\]>|<\?.*?\?>|<(/?)([\w:.-]+)((?:[^<>"']|"[^"]*"|'[^']*')*)>}m
207
343
 
208
- # @rbs (untyped, Hash[Symbol, untyped]) -> untyped
209
- def build_request(uri, options)
210
- request = Net::HTTP::Get.new(uri)
211
- request["User-Agent"] = "SimpleRSS/#{VERSION}"
344
+ private
212
345
 
213
- # Conditional GET headers
214
- request["If-None-Match"] = options[:etag] if options[:etag]
215
- request["If-Modified-Since"] = options[:last_modified] if options[:last_modified]
346
+ # @rbs () -> Array[XmlElement]
347
+ def normalized_feed_authors
348
+ document = XmlElement.new("", "", @source, { namespaces: {}, base_urls: [] })
349
+ feed = document.children.find { |element| element.matches?("feed", ATOM_NAMESPACE) }
350
+ return [] unless feed
216
351
 
217
- # Custom headers
218
- options[:headers]&.each { |key, value| request[key] = value }
352
+ feed.children.select { |element| element.matches?("author", ATOM_NAMESPACE) }
353
+ end
219
354
 
220
- request
355
+ # @rbs () -> void
356
+ def parse
357
+ prefix = @source.b.sub(/\A\xEF\xBB\xBF/n, "").lstrip
358
+ return parse_xml unless prefix.match?(/\A(?:[\{\["0-9-]|true\b|false\b|null\b)/n)
359
+
360
+ json_feed = JsonFeed.new(@source)
361
+ @json_feed = json_feed
362
+ @raw_json = json_feed.document
363
+ JsonFeed::FEED_FIELDS.each do |field|
364
+ instance_variable_set("@#{field}", json_feed.document[field])
365
+ self.class.attr_reader(field)
221
366
  end
222
-
223
- # @rbs (untyped, Hash[Symbol, untyped]) -> untyped
224
- def handle_redirect(response, options)
225
- return nil unless response.is_a?(Net::HTTPRedirection)
226
- return nil if options[:follow_redirects] == false
227
-
228
- location = response["Location"]
229
- return nil unless location
230
-
231
- redirects = (options[:_redirects] || 0) + 1
232
- raise SimpleRSSError, "Too many redirects" if redirects > 5
233
-
234
- new_options = options.merge(_redirects: redirects)
235
- perform_fetch(URI.parse(location), new_options)
367
+ @link = json_feed.document["home_page_url"]
368
+ self.class.attr_reader(:link)
369
+ @items = json_feed.items
370
+ @items.each do |item|
371
+ item.define_singleton_method(:method_missing) { |name, *_args| self[name] }
372
+ add_item_media_helpers(item)
236
373
  end
237
374
  end
238
375
 
239
- DATE_TAGS = %i[pubDate lastBuildDate published updated expirationDate modified dc:date].freeze
240
- STRIP_HTML_TAGS = %i[author contributor skipHours skipDays].freeze
241
-
242
- private
243
-
244
376
  # @rbs () -> void
245
- def parse
246
- raise SimpleRSSError, "Poorly formatted feed" unless @source =~ %r{<(channel|feed).*?>.*?</(channel|feed)>}mi
377
+ def parse_xml
378
+ raise SimpleRSSError, "Poorly formatted feed" unless @source =~ %r{<(channel|feed).*?>.*?</(channel|feed)>|<(channel|feed)\b[^>]*?/\s*>}mi
247
379
 
248
380
  # Feed's title and link
249
381
  feed_content = Regexp.last_match(1) if @source =~ %r{(.*?)<(rss:|atom:)?(item|entry).*?>.*?</(rss:|atom:)?(item|entry)>}mi
@@ -283,18 +415,57 @@ class SimpleRSS
283
415
  end
284
416
 
285
417
  # RSS items' title, link, and description
286
- @source.scan(%r{<(rss:|atom:)?(item|entry)([\s][^>]*)?>(.*?)</(rss:|atom:)?(item|entry)>}mi) do |match|
418
+ namespace_contexts = entry_contexts
419
+ entry_pattern = %r{<(rss:|atom:)?(item|entry)([\s][^>]*)?>(.*?)</(rss:|atom:)?(item|entry)>}mi
420
+ position = 0
421
+ while (match = entry_pattern.match(@source, position))
422
+ position = match.end(0)
287
423
  item = {} #: Hash[Symbol, untyped]
424
+ parent_context = namespace_contexts[match.begin(0)]
425
+ namespaces = parent_context && parent_context[:namespaces].merge(namespace_attributes(match[3].to_s))
426
+ @entry_contexts[item] = {
427
+ name: "#{match[1]}#{match[2]}", attributes: match[3].to_s, content: match[4].to_s,
428
+ parent: parent_context || { namespaces: {}, base_urls: [] }, xml: match[0]
429
+ }
288
430
  @@item_tags.each do |tag|
289
431
  next if tag.to_s.strip.empty?
290
432
 
291
- parse_item_tag(item, tag, match[3], match[2])
433
+ if tag == :category
434
+ next unless namespaces
435
+
436
+ parse_category_tag(item, match[4].to_s, namespaces, element_namespace("#{match[1]}#{match[2]}", namespaces))
437
+ next
438
+ end
439
+
440
+ parse_item_tag(item, tag, match[4], match[3])
292
441
  end
293
- item.define_singleton_method(:method_missing) { |name, *| self[name] }
442
+ item.define_singleton_method(:method_missing) { |name, *_args| self[name] }
443
+ add_item_media_helpers(item)
294
444
  @items << item
295
445
  end
296
446
  end
297
447
 
448
+ # @rbs (Hash[Symbol, untyped]) -> void
449
+ def add_item_media_helpers(item)
450
+ item.define_singleton_method(:has_media?) do
451
+ [
452
+ self[:media_content_url],
453
+ self[:media_thumbnail_url],
454
+ self[:enclosure_url],
455
+ self[:itunes_image_href]
456
+ ].any? { |value| !value.nil? && !value.to_s.strip.empty? }
457
+ end
458
+
459
+ item.define_singleton_method(:media_url) do
460
+ [
461
+ self[:media_content_url],
462
+ self[:media_thumbnail_url],
463
+ self[:enclosure_url],
464
+ self[:itunes_image_href]
465
+ ].find { |value| !value.nil? && !value.to_s.strip.empty? }
466
+ end
467
+ end
468
+
298
469
  # @rbs (Hash[Symbol, untyped], Symbol, String?, String?) -> void
299
470
  def parse_item_tag(item, tag, content, item_attrs = nil)
300
471
  return if content.nil?
@@ -309,15 +480,120 @@ class SimpleRSS
309
480
 
310
481
  # @rbs (Hash[Symbol, untyped], String, String) -> void
311
482
  def parse_rel_tag(item, tag_str, content)
312
- tag, rel = tag_str.split("+")
483
+ tag, rel = tag_str.split("+", 2)
313
484
  return unless tag && rel
314
485
 
315
- content =~ %r{<(rss:|atom:)?#{tag}(.*?)rel=['"]#{rel}['"](.*?)>(.*?)</(rss:|atom:)?#{tag}>}mi ||
316
- content =~ %r{<(rss:|atom:)?#{tag}(.*?)rel=['"]#{rel}['"](.*?)/\s*>}mi
486
+ value = if tag == "link"
487
+ link_relation_href(content, rel)
488
+ else
489
+ content =~ %r{<(rss:|atom:)?#{tag}(.*?)rel=['"]#{rel}['"](.*?)>(.*?)</(rss:|atom:)?#{tag}>}mi ||
490
+ content =~ %r{<(rss:|atom:)?#{tag}(.*?)rel=['"]#{rel}['"](.*?)/\s*>}mi
491
+
492
+ return unless Regexp.last_match(3) || Regexp.last_match(4)
493
+
494
+ clean_content(tag.to_sym, Regexp.last_match(3), Regexp.last_match(4))
495
+ end
496
+ return if value.nil?
497
+
498
+ item[clean_tag("#{tag}+#{rel}")] = value
499
+ item[clean_tag("#{tag}_#{rel}")] = value
500
+ end
501
+
502
+ # @rbs (String, String) -> String?
503
+ def link_relation_href(content, relation)
504
+ attributes = entry_link_attributes(content).find { |link| link["rel"]&.casecmp?(relation) }
505
+ return unless attributes
506
+
507
+ attributes["href"]
508
+ end
509
+
510
+ # @rbs (String) -> Array[Hash[String, String]]
511
+ def entry_link_attributes(content)
512
+ child_elements(content).filter_map do |tag, attributes, _body|
513
+ next unless %w[link atom:link rss:link].include?(tag.downcase)
317
514
 
318
- return unless Regexp.last_match(3) || Regexp.last_match(4)
515
+ xml_attributes(attributes).transform_keys(&:downcase)
516
+ end
517
+ end
518
+
519
+ # @rbs (String) -> Array[[String, String, String?]]
520
+ def child_elements(content)
521
+ XmlElement.child_elements(content)
522
+ end
523
+
524
+ # @rbs (Hash[Symbol, untyped], String, Hash[String, String], String?) -> void
525
+ def parse_category_tag(item, content, namespaces, entry_namespace)
526
+ values = child_elements(content).filter_map do |tag, raw_attributes, body|
527
+ next unless tag.split(":").last&.casecmp?("category")
528
+
529
+ attributes = xml_attributes(raw_attributes)
530
+ namespace = element_namespace(tag, namespaces.merge(namespace_attributes(raw_attributes)))
531
+ if namespace == ATOM_NAMESPACE
532
+ term = CGI.unescapeHTML(attributes["term"].to_s).strip
533
+ next term unless term.empty?
534
+
535
+ next
536
+ end
537
+
538
+ next if entry_namespace == ATOM_NAMESPACE || !RSS_NAMESPACES.include?(namespace)
539
+ next if tag.include?(":") && namespace.nil?
540
+ next if body.nil? && (array_tag?(:category) || !raw_attributes.rstrip.end_with?("/"))
541
+
542
+ unescape(body.to_s)
543
+ end
544
+ return if values.empty?
545
+
546
+ item[:category] = array_tag?(:category) ? values : values.first
547
+ end
548
+
549
+ # @rbs () -> Hash[Integer, Hash[Symbol, untyped]]
550
+ def entry_contexts
551
+ contexts = {} #: Hash[Integer, Hash[Symbol, untyped]]
552
+ scopes = [{ namespaces: {}, base_urls: [] }] #: Array[Hash[Symbol, untyped]]
553
+ position = 0
554
+
555
+ while (token = XML_TAG_PATTERN.match(@source, position))
556
+ position = token.end(0)
557
+ attributes = token[3]
558
+ next unless attributes
559
+
560
+ if token[1] == "/"
561
+ scopes.pop if scopes.size > 1
562
+ next
563
+ end
564
+
565
+ parent = scopes.fetch(-1)
566
+ tag = token[2].to_s.split(":").last
567
+ contexts[token.begin(0).to_i] = parent if %w[item entry].include?(tag&.downcase)
568
+ base_url = xml_attributes(attributes)["xml:base"]
569
+ base_urls = parent[:base_urls].dup
570
+ base_urls << CGI.unescapeHTML(base_url) if base_url
571
+ context = {
572
+ namespaces: parent[:namespaces].merge(namespace_attributes(attributes)),
573
+ base_urls: base_urls
574
+ }
575
+ scopes << context unless attributes.rstrip.end_with?("/")
576
+ end
577
+
578
+ contexts
579
+ end
580
+
581
+ # @rbs (String) -> Hash[String, String]
582
+ def namespace_attributes(attributes)
583
+ xml_attributes(attributes)
584
+ .select { |name, _value| name == "xmlns" || name.start_with?("xmlns:") }
585
+ .transform_values { |value| CGI.unescapeHTML(value) }
586
+ end
587
+
588
+ # @rbs (String, Hash[String, String]) -> String?
589
+ def element_namespace(tag, namespaces)
590
+ key = tag.include?(":") ? "xmlns:#{tag.split(":").first}" : "xmlns"
591
+ namespaces[key]
592
+ end
319
593
 
320
- item[clean_tag("#{tag}+#{rel}")] = clean_content(tag.to_sym, Regexp.last_match(3), Regexp.last_match(4))
594
+ # @rbs (String) -> Hash[String, String]
595
+ def xml_attributes(attributes)
596
+ XmlElement.attributes(attributes)
321
597
  end
322
598
 
323
599
  # @rbs (String, String?) -> void
@@ -412,6 +688,88 @@ class SimpleRSS
412
688
  tag.to_s.tr(":", "_").intern
413
689
  end
414
690
 
691
+ # @rbs (untyped, String) -> bool
692
+ def category_matches_query?(category, query)
693
+ return category.any? { |value| value.to_s.downcase.include?(query) } if category.is_a?(Array)
694
+
695
+ category.to_s.downcase.include?(query)
696
+ end
697
+
698
+ # @rbs (Hash[Symbol, untyped]) -> Array[untyped]
699
+ def searchable_fields(item)
700
+ [item[:title], item[:description], item[:summary], item[:content]]
701
+ end
702
+
703
+ # @rbs (Array[Hash[Symbol, untyped]]) -> Set[String]
704
+ def keyed_item_set(item_list)
705
+ item_list.each_with_object(Set.new) do |item, keys|
706
+ key = item_key(item)
707
+ next if key.nil?
708
+
709
+ keys.add(key)
710
+ end
711
+ end
712
+
713
+ # @rbs (Array[Hash[Symbol, untyped]], Set[String]) -> Array[Hash[Symbol, untyped]]
714
+ def select_new_keyed_items(item_list, known_keys)
715
+ item_list.select do |item|
716
+ key = item_key(item)
717
+ !key.nil? && !known_keys.include?(key)
718
+ end
719
+ end
720
+
721
+ # @rbs (Array[Hash[Symbol, untyped]]) -> Array[Hash[Symbol, untyped]]
722
+ def sorted_items_by_date(item_list)
723
+ item_list.sort_by.with_index do |item, index|
724
+ date = item_date(item)
725
+ [date ? -date.to_r : Float::INFINITY, index]
726
+ end
727
+ end
728
+
729
+ # @rbs (Array[Hash[Symbol, untyped]]) -> Array[Hash[Symbol, untyped]]
730
+ def dedupe_items(item_list)
731
+ seen_keys = Set.new
732
+
733
+ item_list.each_with_object([]) do |item, unique_items|
734
+ key = item_key(item)
735
+
736
+ if key.nil?
737
+ unique_items << item
738
+ next
739
+ end
740
+
741
+ next if seen_keys.include?(key)
742
+
743
+ seen_keys.add(key)
744
+ unique_items << item
745
+ end
746
+ end
747
+
748
+ # @rbs (Hash[Symbol, untyped]) -> String?
749
+ def item_key(item)
750
+ key = item[:guid] || item[:id] || item[:link]
751
+ return nil if key.nil?
752
+
753
+ key.to_s
754
+ end
755
+
756
+ # @rbs (Hash[Symbol, untyped]) -> Time?
757
+ def item_date(item)
758
+ [item[:pubDate], item[:updated], item[:published]].find { |date| date.is_a?(Time) }
759
+ end
760
+
761
+ # @rbs (Hash[Symbol, untyped]) -> Array[String]
762
+ def item_image_urls(item)
763
+ [item[:media_thumbnail_url], item[:media_content_url], item[:itunes_image_href]]
764
+ .compact
765
+ .reject { |url| blank_value?(url) }
766
+ end
767
+
768
+ # @rbs (untyped) -> bool
769
+ def blank_value?(value)
770
+ value.to_s.strip.empty?
771
+ end
772
+
415
773
  # @rbs (untyped) -> untyped
416
774
  def serialize_value(value)
417
775
  case value
@@ -545,5 +903,13 @@ class SimpleRSS
545
903
  end
546
904
  end
547
905
 
548
- class SimpleRSSError < StandardError
906
+ require_relative "simple-rss/xml_element"
907
+ require_relative "simple-rss/normalized_entry"
908
+ require_relative "simple-rss/json_feed"
909
+ require_relative "simple-rss/json_entry_normalizer"
910
+ require_relative "simple-rss/entry_normalizer"
911
+
912
+ class SimpleRSSError < StandardError # rubocop:disable Style/OneClassPerFile
549
913
  end
914
+
915
+ require_relative "simple-rss/request_errors"
data/simple-rss.gemspec CHANGED
@@ -1,13 +1,13 @@
1
1
  Gem::Specification.new do |s|
2
2
  s.name = "simple-rss"
3
- s.version = "2.1.0"
4
- s.date = "2025-12-29"
5
- s.summary = "A simple, flexible, extensible, and liberal RSS and Atom reader for Ruby. It is designed to be backwards compatible with the standard RSS parser, but will never do RSS generation."
3
+ s.version = "2.3.0"
4
+ s.summary = "A flexible RSS, Atom, and JSON Feed reader for Ruby."
6
5
  s.email = "lucas@rufy.com"
7
6
  s.homepage = "https://github.com/cardmagic/simple-rss"
8
- s.description = "A simple, flexible, extensible, and liberal RSS and Atom reader for Ruby. It is designed to be backwards compatible with the standard RSS parser, but will never do RSS generation."
7
+ s.metadata["changelog_uri"] = "https://github.com/cardmagic/simple-rss/blob/master/CHANGELOG.md"
8
+ s.description = "Parse RSS, Atom, and JSON Feed with normalized entries, HTTP fetching, website feed discovery, and JSON/XML serialization."
9
9
  s.authors = ["Lucas Carlson"]
10
- s.files = Dir["lib/**/*", "test/**/*", "LICENSE", "README.md", "Rakefile", "simple-rss.gemspec"]
10
+ s.files = Dir["lib/**/*", "examples/**/*", "test/**/*", "LICENSE", "README.md", "CHANGELOG.md", "Rakefile", "simple-rss.gemspec"]
11
11
  s.required_ruby_version = ">= 3.1"
12
12
  s.add_development_dependency "rake"
13
13
  s.add_development_dependency "rdoc"