simple-rss 2.2.0 → 2.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +174 -0
- data/README.md +492 -8
- data/Rakefile +1 -1
- data/examples/digest.rb +19 -0
- data/examples/discover.rb +14 -0
- data/examples/feedbag.rb +10 -0
- data/lib/simple-rss/discovery.rb +107 -0
- data/lib/simple-rss/entry_normalizer.rb +356 -0
- data/lib/simple-rss/http_client.rb +216 -0
- data/lib/simple-rss/json_entry_normalizer.rb +147 -0
- data/lib/simple-rss/json_feed.rb +176 -0
- data/lib/simple-rss/normalized_entry.rb +79 -0
- data/lib/simple-rss/request_errors.rb +21 -0
- data/lib/simple-rss/request_policy.rb +72 -0
- data/lib/simple-rss/xml_element.rb +136 -0
- data/lib/simple-rss.rb +225 -77
- data/simple-rss.gemspec +5 -5
- data/test/base/category_parsing_test.rb +217 -0
- data/test/base/date_ordering_test.rb +95 -0
- data/test/base/discovery_dependency_test.rb +29 -0
- data/test/base/discovery_test.rb +151 -0
- data/test/base/discovery_transport_test.rb +413 -0
- data/test/base/enumerable_test.rb +16 -0
- data/test/base/feedbag_integration_test.rb +21 -0
- data/test/base/json_feed_test.rb +372 -0
- data/test/base/normalized_entries_test.rb +410 -0
- data/test/base/normalized_fetch_test.rb +132 -0
- data/test/base/relation_links_test.rb +162 -0
- data/test/data/atom_categories.xml +23 -0
- data/test/data/atom_nested_link.xml +13 -0
- data/test/data/discovery.html +28 -0
- data/test/data/json_feed_1.json +75 -0
- data/test/data/json_feed_1_1.json +78 -0
- data/test/data/mixed_dates.xml +55 -0
- data/test/data/normalized_atom.xml +22 -0
- data/test/data/normalized_rss.xml +21 -0
- data/test/data/rss_categories.xml +15 -0
- data/test/support/http_server.rb +44 -0
- data/test/support/replace_method.rb +14 -0
- metadata +42 -10
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
# rbs_inline: enabled
|
|
2
|
+
|
|
3
|
+
require "uri"
|
|
4
|
+
require "date"
|
|
5
|
+
|
|
6
|
+
class SimpleRSS::JsonEntryNormalizer
|
|
7
|
+
RFC3339_TIMESTAMP = /\A\d{4}-\d{2}-\d{2}[tT]
|
|
8
|
+
(?:[01]\d|2[0-3]):[0-5]\d:(?:[0-5]\d|60)(?:\.\d+)?
|
|
9
|
+
(?:[zZ]|[+-](?:[01]\d|2[0-3]):[0-5]\d)\z/x
|
|
10
|
+
FIELDS = {
|
|
11
|
+
title: "title", content_html: "content_html", content_text: "content_text", summary: "summary"
|
|
12
|
+
}.freeze
|
|
13
|
+
URL_FIELDS = %i[url external_url image banner_image].freeze
|
|
14
|
+
|
|
15
|
+
# @rbs @item: Hash[String, untyped]
|
|
16
|
+
# @rbs @feed: Hash[String, untyped]
|
|
17
|
+
# @rbs @source_url: String?
|
|
18
|
+
# @rbs @values: Hash[Symbol, untyped]
|
|
19
|
+
# @rbs @issues: Array[Hash[Symbol, untyped]]
|
|
20
|
+
# @rbs @sources: Hash[Symbol, untyped]
|
|
21
|
+
|
|
22
|
+
# @rbs (Hash[String, untyped], Hash[String, untyped], ?source_url: String?) -> void
|
|
23
|
+
def initialize(item, feed, source_url: nil)
|
|
24
|
+
@item = item
|
|
25
|
+
@feed = feed
|
|
26
|
+
@source_url = source_url || feed["feed_url"]
|
|
27
|
+
@issues = [] #: Array[Hash[Symbol, untyped]]
|
|
28
|
+
@sources = {} #: Hash[Symbol, untyped]
|
|
29
|
+
@values = { raw: item, issues: @issues, field_sources: @sources } #: Hash[Symbol, untyped]
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
# @rbs () -> SimpleRSS::NormalizedEntry
|
|
33
|
+
def entry
|
|
34
|
+
assign(:identifier, @item["id"].to_s, "id")
|
|
35
|
+
FIELDS.each { |field, source| assign(field, @item[source], source) }
|
|
36
|
+
URL_FIELDS.each { |field| assign(field, resolve_url(@item[field.to_s], field, field.to_s), field.to_s) }
|
|
37
|
+
assign(:published_at, read_date("date_published", :published_at), "date_published")
|
|
38
|
+
assign(:updated_at, read_date("date_modified", :updated_at), "date_modified")
|
|
39
|
+
@values[:summary_type] = :text if @item["summary"]
|
|
40
|
+
@values[:content_base_url] = @values[:url] || @source_url
|
|
41
|
+
read_language
|
|
42
|
+
read_categories
|
|
43
|
+
read_authors
|
|
44
|
+
@values[:attachments] = (@item["attachments"] || []).each_with_index.map { |attachment, index| read_attachment(attachment, index) }
|
|
45
|
+
@values[:links] = %i[url external_url].filter_map do |field|
|
|
46
|
+
next unless @values[field]
|
|
47
|
+
|
|
48
|
+
{ url: @values[field], rel: field == :url ? "alternate" : "related", media_type: nil, source: field.to_s, raw: @item[field.to_s] }
|
|
49
|
+
end
|
|
50
|
+
SimpleRSS::NormalizedEntry.new(@values)
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
private
|
|
54
|
+
|
|
55
|
+
# @rbs (Symbol, untyped, String) -> void
|
|
56
|
+
def assign(field, value, source)
|
|
57
|
+
return if value.nil?
|
|
58
|
+
|
|
59
|
+
@values[field] = value
|
|
60
|
+
@sources[field] = source
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
# @rbs (Symbol, Symbol, untyped, String) -> void
|
|
64
|
+
def issue(field, code, value, source)
|
|
65
|
+
@issues << { field: field, code: code, value: value, source: source }
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
# @rbs (String, Symbol) -> Time?
|
|
69
|
+
def read_date(source, field)
|
|
70
|
+
value = @item[source]
|
|
71
|
+
return if value.nil?
|
|
72
|
+
return DateTime.rfc3339(value, Date::GREGORIAN).to_time if value.is_a?(String) && value.match?(RFC3339_TIMESTAMP)
|
|
73
|
+
|
|
74
|
+
issue(field, :invalid_date, value, source)
|
|
75
|
+
nil
|
|
76
|
+
rescue ArgumentError, RangeError
|
|
77
|
+
issue(field, :invalid_date, value, source)
|
|
78
|
+
nil
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
# @rbs (String?, Symbol, String) -> String?
|
|
82
|
+
def resolve_url(value, field, source)
|
|
83
|
+
return if value.nil? || value.strip.empty?
|
|
84
|
+
|
|
85
|
+
value = value.strip
|
|
86
|
+
return value if URI.parse(value).absolute?
|
|
87
|
+
|
|
88
|
+
source_url = @source_url
|
|
89
|
+
return URI.join(source_url, value).to_s if source_url && URI.parse(source_url).absolute?
|
|
90
|
+
|
|
91
|
+
issue(field, :relative_url_without_base, value, source)
|
|
92
|
+
value
|
|
93
|
+
rescue URI::Error
|
|
94
|
+
issue(field, :invalid_url, value, source)
|
|
95
|
+
value
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
# @rbs () -> void
|
|
99
|
+
def read_language
|
|
100
|
+
return unless @feed["version"] == SimpleRSS::JsonFeed::VERSIONS.last
|
|
101
|
+
|
|
102
|
+
object = @item.key?("language") ? @item : @feed
|
|
103
|
+
source = object.equal?(@item) ? "language" : "feed.language"
|
|
104
|
+
assign(:language, object["language"], source)
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
# @rbs () -> void
|
|
108
|
+
def read_categories
|
|
109
|
+
tags = @item["tags"] || []
|
|
110
|
+
categories = tags.map(&:strip).reject(&:empty?).uniq
|
|
111
|
+
assign(:categories, categories, "tags") if @item.key?("tags")
|
|
112
|
+
@values[:category_details] = tags.reject { |tag| tag.strip.empty? }.map { |tag| { term: tag.strip, label: nil, scheme: nil, source: "tags", raw: tag } }
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
# @rbs () -> void
|
|
116
|
+
def read_authors
|
|
117
|
+
object = @item
|
|
118
|
+
plural = @feed["version"] == SimpleRSS::JsonFeed::VERSIONS.last
|
|
119
|
+
object = @feed unless (plural && object.key?("authors")) || object.key?("author")
|
|
120
|
+
source = plural && object.key?("authors") ? "authors" : "author"
|
|
121
|
+
authors = source == "authors" ? object[source] : [object[source]].compact
|
|
122
|
+
source = "feed.#{source}" if object.equal?(@feed)
|
|
123
|
+
records = (authors || []).map do |author|
|
|
124
|
+
{ name: author["name"], email: nil, url: resolve_url(author["url"], :authors, source),
|
|
125
|
+
avatar: resolve_url(author["avatar"], :authors, source), raw: author }
|
|
126
|
+
end
|
|
127
|
+
assign(:authors, records, source) unless records.empty?
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
# @rbs (Hash[String, untyped], Integer) -> Hash[Symbol, untyped]
|
|
131
|
+
def read_attachment(attachment, index)
|
|
132
|
+
source = "attachments[#{index}]"
|
|
133
|
+
{ url: resolve_url(attachment["url"], :attachments, "#{source}.url"), media_type: attachment["mime_type"],
|
|
134
|
+
title: attachment["title"], size_in_bytes: read_number(attachment, "size_in_bytes", source),
|
|
135
|
+
duration_in_seconds: read_number(attachment, "duration_in_seconds", source), source: source, raw: attachment }
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
# @rbs (Hash[String, untyped], String, String) -> Numeric?
|
|
139
|
+
def read_number(attachment, field, source)
|
|
140
|
+
value = attachment[field]
|
|
141
|
+
return if value.nil?
|
|
142
|
+
return value if value.is_a?(Numeric) && value >= 0 && value.finite?
|
|
143
|
+
|
|
144
|
+
issue(:attachments, :invalid_number, value, "#{source}.#{field}")
|
|
145
|
+
nil
|
|
146
|
+
end
|
|
147
|
+
end
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
# rbs_inline: enabled
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
|
|
5
|
+
class SimpleRSS::JsonFeed
|
|
6
|
+
VERSIONS = %w[https://jsonfeed.org/version/1 https://jsonfeed.org/version/1.1].freeze
|
|
7
|
+
FEED_FIELDS = %w[title description home_page_url feed_url icon favicon language author authors expired next_url hubs user_comment].freeze
|
|
8
|
+
FEED_STRINGS = %w[description home_page_url feed_url icon favicon next_url user_comment].freeze
|
|
9
|
+
ITEM_STRINGS = %w[url external_url title summary content_html content_text image banner_image].freeze
|
|
10
|
+
|
|
11
|
+
attr_reader :document #: Hash[String, untyped]
|
|
12
|
+
attr_reader :items #: Array[Hash[Symbol, untyped]]
|
|
13
|
+
|
|
14
|
+
# @rbs @originals: Hash[Hash[Symbol, untyped], Hash[String, untyped]]
|
|
15
|
+
|
|
16
|
+
# @rbs (String) -> void
|
|
17
|
+
def initialize(source)
|
|
18
|
+
source = source.b.sub(/\A\xEF\xBB\xBF/n, "").force_encoding(Encoding::UTF_8)
|
|
19
|
+
raise SimpleRSSError, "Malformed JSON Feed: invalid UTF-8" unless source.valid_encoding?
|
|
20
|
+
|
|
21
|
+
@document = JSON.parse(source)
|
|
22
|
+
validate
|
|
23
|
+
freeze_data(@document)
|
|
24
|
+
@originals = {} #: Hash[Hash[Symbol, untyped], Hash[String, untyped]]
|
|
25
|
+
@originals.compare_by_identity
|
|
26
|
+
@items = @document.fetch("items").map do |original|
|
|
27
|
+
item = legacy_item(original)
|
|
28
|
+
@originals[item] = original
|
|
29
|
+
item
|
|
30
|
+
end
|
|
31
|
+
rescue JSON::ParserError, EncodingError => e
|
|
32
|
+
raise SimpleRSSError, "Malformed JSON Feed: #{e.message}"
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
# @rbs (Hash[Symbol, untyped], ?source_url: String?) -> SimpleRSS::NormalizedEntry
|
|
36
|
+
def normalized_entry(item, source_url: nil)
|
|
37
|
+
original = @originals[item] || raise(SimpleRSSError, "Cannot normalize an item without its original JSON source")
|
|
38
|
+
SimpleRSS::JsonEntryNormalizer.new(original, document, source_url: source_url).entry
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
private
|
|
42
|
+
|
|
43
|
+
# @rbs () -> void
|
|
44
|
+
def validate
|
|
45
|
+
check_type(document, Hash, "feed")
|
|
46
|
+
validate_numbers(document)
|
|
47
|
+
required_string(document, "version", "feed")
|
|
48
|
+
raise SimpleRSSError, "Unsupported JSON Feed version: #{document["version"].inspect}" unless VERSIONS.include?(document["version"])
|
|
49
|
+
|
|
50
|
+
required_string(document, "title", "feed")
|
|
51
|
+
check_type(document["items"], Array, "items")
|
|
52
|
+
optional_strings(document, FEED_STRINGS, "feed")
|
|
53
|
+
optional_strings(document, ["language"], "feed") if version_1_1?
|
|
54
|
+
validate_expiration
|
|
55
|
+
validate_authors(document, "feed")
|
|
56
|
+
optional_array(document, "hubs", "feed").each_with_index do |hub, index|
|
|
57
|
+
check_type(hub, Hash, "hubs[#{index}]")
|
|
58
|
+
%w[type url].each { |field| required_string(hub, field, "hubs[#{index}]") }
|
|
59
|
+
end
|
|
60
|
+
document["items"].each_with_index { |item, index| validate_item(item, "items[#{index}]") }
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
# @rbs (untyped) -> void
|
|
64
|
+
def validate_numbers(value)
|
|
65
|
+
case value
|
|
66
|
+
when Hash then value.each_value { |child| validate_numbers(child) }
|
|
67
|
+
when Array then value.each { |child| validate_numbers(child) }
|
|
68
|
+
when Float
|
|
69
|
+
raise SimpleRSSError, "JSON Feed number exceeds the supported range" unless value.finite?
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
# @rbs () -> void
|
|
74
|
+
def validate_expiration
|
|
75
|
+
return unless document.key?("expired")
|
|
76
|
+
return if [true, false].include?(document["expired"])
|
|
77
|
+
|
|
78
|
+
raise SimpleRSSError, "JSON Feed feed.expired must be a boolean"
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
# @rbs (untyped, String) -> void
|
|
82
|
+
def validate_item(item, path)
|
|
83
|
+
check_type(item, Hash, path)
|
|
84
|
+
identifier = item["id"]
|
|
85
|
+
unless (identifier.is_a?(String) || identifier.is_a?(Numeric)) && !identifier.to_s.strip.empty?
|
|
86
|
+
raise SimpleRSSError, "JSON Feed #{path}.id must be a nonblank string or number"
|
|
87
|
+
end
|
|
88
|
+
unless %w[content_html content_text].any? { |field| item[field].is_a?(String) }
|
|
89
|
+
raise SimpleRSSError, "JSON Feed #{path} requires content_html or content_text"
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
optional_strings(item, ITEM_STRINGS, path)
|
|
93
|
+
optional_strings(item, ["language"], path) if version_1_1?
|
|
94
|
+
validate_authors(item, path)
|
|
95
|
+
optional_array(item, "tags", path).each { |tag| check_type(tag, String, "#{path}.tags[]") }
|
|
96
|
+
optional_array(item, "attachments", path).each_with_index do |attachment, index|
|
|
97
|
+
attachment_path = "#{path}.attachments[#{index}]"
|
|
98
|
+
check_type(attachment, Hash, attachment_path)
|
|
99
|
+
%w[url mime_type].each { |field| required_string(attachment, field, attachment_path) }
|
|
100
|
+
optional_strings(attachment, ["title"], attachment_path)
|
|
101
|
+
end
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
# @rbs (Hash[String, untyped], String) -> void
|
|
105
|
+
def validate_authors(object, path)
|
|
106
|
+
if version_1_1? && object.key?("authors")
|
|
107
|
+
optional_array(object, "authors", path).each_with_index { |author, index| validate_author(author, "#{path}.authors[#{index}]") }
|
|
108
|
+
return
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
validate_author(object["author"], "#{path}.author") if object.key?("author")
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
# @rbs (untyped, String) -> void
|
|
115
|
+
def validate_author(author, path)
|
|
116
|
+
check_type(author, Hash, path)
|
|
117
|
+
fields = %w[name url avatar]
|
|
118
|
+
optional_strings(author, fields, path)
|
|
119
|
+
return if fields.any? { |field| author[field].is_a?(String) }
|
|
120
|
+
|
|
121
|
+
raise SimpleRSSError, "JSON Feed #{path} requires name, url, or avatar"
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
# @rbs (Hash[String, untyped], Array[String], String) -> void
|
|
125
|
+
def optional_strings(object, fields, path)
|
|
126
|
+
fields.each { |field| check_type(object[field], String, "#{path}.#{field}") if object.key?(field) }
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
# @rbs (Hash[String, untyped], String, String) -> Array[untyped]
|
|
130
|
+
def optional_array(object, field, path)
|
|
131
|
+
return [] unless object.key?(field)
|
|
132
|
+
|
|
133
|
+
check_type(object[field], Array, "#{path}.#{field}")
|
|
134
|
+
object[field]
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
# @rbs (Hash[String, untyped], String, String) -> void
|
|
138
|
+
def required_string(object, field, path)
|
|
139
|
+
check_type(object[field], String, "#{path}.#{field}")
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
# @rbs (untyped, untyped, String) -> void
|
|
143
|
+
def check_type(value, type, path)
|
|
144
|
+
return if value.is_a?(type)
|
|
145
|
+
|
|
146
|
+
raise SimpleRSSError, "JSON Feed #{path} must be a #{type}"
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
# @rbs () -> bool
|
|
150
|
+
def version_1_1?
|
|
151
|
+
document["version"] == VERSIONS.last
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
# @rbs (Hash[String, untyped]) -> Hash[Symbol, untyped]
|
|
155
|
+
def legacy_item(original)
|
|
156
|
+
entry = SimpleRSS::JsonEntryNormalizer.new(original, document).entry
|
|
157
|
+
attachment = entry.attachments.first || {}
|
|
158
|
+
original.transform_keys(&:to_sym).merge(
|
|
159
|
+
id: entry.identifier, guid: entry.identifier, link: entry.url,
|
|
160
|
+
description: entry.summary || entry.content_text || entry.content_html,
|
|
161
|
+
content: entry.content_html || entry.content_text, category: entry.categories.dup,
|
|
162
|
+
pubDate: entry.published_at || original["date_published"], updated: entry.updated_at || original["date_modified"],
|
|
163
|
+
enclosure_url: attachment[:url], enclosure_type: attachment[:media_type], enclosure_length: attachment[:size_in_bytes],
|
|
164
|
+
media_thumbnail_url: entry.image, media_content_url: entry.banner_image
|
|
165
|
+
)
|
|
166
|
+
end
|
|
167
|
+
|
|
168
|
+
# @rbs (untyped) -> void
|
|
169
|
+
def freeze_data(value)
|
|
170
|
+
case value
|
|
171
|
+
when Hash then value.each_value { |child| freeze_data(child) }
|
|
172
|
+
when Array then value.each { |child| freeze_data(child) }
|
|
173
|
+
end
|
|
174
|
+
value.freeze
|
|
175
|
+
end
|
|
176
|
+
end
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
# rbs_inline: enabled
|
|
2
|
+
|
|
3
|
+
class SimpleRSS::NormalizedEntry
|
|
4
|
+
attr_reader :external_url, :image, :banner_image, :language #: String?
|
|
5
|
+
attr_reader :identifier, :url, :title #: String?
|
|
6
|
+
attr_reader :published_at, :updated_at #: Time?
|
|
7
|
+
attr_reader :content_html, :content_text, :content_url, :content_base_url, :summary #: String?
|
|
8
|
+
attr_reader :summary_type #: Symbol?
|
|
9
|
+
attr_reader :categories #: Array[String]
|
|
10
|
+
attr_reader :category_details, :attachments, :links, :authors, :issues #: Array[Hash[Symbol, untyped]]
|
|
11
|
+
attr_reader :raw #: Hash[Symbol | String, untyped]
|
|
12
|
+
attr_reader :field_sources #: Hash[Symbol, untyped]
|
|
13
|
+
attr_reader :raw_xml #: String?
|
|
14
|
+
|
|
15
|
+
# @rbs (Hash[Symbol, untyped]) -> void
|
|
16
|
+
def initialize(attributes)
|
|
17
|
+
values = immutable_copy(attributes)
|
|
18
|
+
empty_categories = [] #: Array[String]
|
|
19
|
+
empty_records = [] #: Array[Hash[Symbol, untyped]]
|
|
20
|
+
empty_data = {} #: Hash[Symbol, untyped]
|
|
21
|
+
empty_categories.freeze
|
|
22
|
+
empty_records.freeze
|
|
23
|
+
empty_data.freeze
|
|
24
|
+
@external_url = values[:external_url]
|
|
25
|
+
@image = values[:image]
|
|
26
|
+
@banner_image = values[:banner_image]
|
|
27
|
+
@language = values[:language]
|
|
28
|
+
@identifier = values[:identifier]
|
|
29
|
+
@url = values[:url]
|
|
30
|
+
@title = values[:title]
|
|
31
|
+
@published_at = values[:published_at]
|
|
32
|
+
@updated_at = values[:updated_at]
|
|
33
|
+
@content_html = values[:content_html]
|
|
34
|
+
@content_text = values[:content_text]
|
|
35
|
+
@content_url = values[:content_url]
|
|
36
|
+
@content_base_url = values[:content_base_url]
|
|
37
|
+
@summary = values[:summary]
|
|
38
|
+
@summary_type = values[:summary_type]
|
|
39
|
+
@categories = values.fetch(:categories, empty_categories)
|
|
40
|
+
@category_details = values.fetch(:category_details, empty_records)
|
|
41
|
+
@attachments = values.fetch(:attachments, empty_records)
|
|
42
|
+
@links = values.fetch(:links, empty_records)
|
|
43
|
+
@authors = values.fetch(:authors, empty_records)
|
|
44
|
+
@issues = values.fetch(:issues, empty_records)
|
|
45
|
+
@raw = values.fetch(:raw, empty_data)
|
|
46
|
+
@raw_xml = values[:raw_xml]
|
|
47
|
+
@field_sources = values.fetch(:field_sources, empty_data)
|
|
48
|
+
freeze
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
# @rbs () -> Time?
|
|
52
|
+
def effective_at
|
|
53
|
+
published_at || updated_at
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
# @rbs () -> Hash[Symbol, untyped]
|
|
57
|
+
def to_h
|
|
58
|
+
{
|
|
59
|
+
external_url: external_url, image: image, banner_image: banner_image, language: language,
|
|
60
|
+
identifier: identifier, url: url, title: title, published_at: published_at, updated_at: updated_at,
|
|
61
|
+
content_html: content_html, content_text: content_text, content_url: content_url, content_base_url: content_base_url,
|
|
62
|
+
summary: summary, summary_type: summary_type, categories: categories, category_details: category_details,
|
|
63
|
+
attachments: attachments, links: links, authors: authors, issues: issues, field_sources: field_sources,
|
|
64
|
+
raw: raw, raw_xml: raw_xml
|
|
65
|
+
}
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
private
|
|
69
|
+
|
|
70
|
+
# @rbs (untyped) -> untyped
|
|
71
|
+
def immutable_copy(value)
|
|
72
|
+
case value
|
|
73
|
+
when Hash then value.to_h { |key, child| [immutable_copy(key), immutable_copy(child)] }.freeze
|
|
74
|
+
when Array then value.map { |child| immutable_copy(child) }.freeze
|
|
75
|
+
when String, Time then value.dup.freeze
|
|
76
|
+
else value
|
|
77
|
+
end
|
|
78
|
+
end
|
|
79
|
+
end
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
# rbs_inline: enabled
|
|
2
|
+
|
|
3
|
+
class SimpleRSS
|
|
4
|
+
class RequestError < SimpleRSSError; end
|
|
5
|
+
class PolicyError < RequestError; end
|
|
6
|
+
class RequestTimeout < RequestError; end
|
|
7
|
+
class ResponseTooLarge < RequestError; end
|
|
8
|
+
class RedirectError < RequestError; end
|
|
9
|
+
class DiscoveryError < SimpleRSSError; end
|
|
10
|
+
class DiscoveryDependencyError < DiscoveryError; end
|
|
11
|
+
|
|
12
|
+
class HTTPError < RequestError
|
|
13
|
+
attr_reader :status_code #: Integer
|
|
14
|
+
|
|
15
|
+
# @rbs (Integer) -> void
|
|
16
|
+
def initialize(status_code)
|
|
17
|
+
@status_code = status_code
|
|
18
|
+
super("HTTP #{status_code} during feed discovery")
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
end
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
# rbs_inline: enabled
|
|
2
|
+
|
|
3
|
+
require "ipaddr"
|
|
4
|
+
require "resolv"
|
|
5
|
+
require "uri"
|
|
6
|
+
|
|
7
|
+
class SimpleRSS::RequestPolicy
|
|
8
|
+
BLOCKED_IPV4 = %w[
|
|
9
|
+
0.0.0.0/8 10.0.0.0/8 100.64.0.0/10 127.0.0.0/8 169.254.0.0/16 172.16.0.0/12
|
|
10
|
+
192.0.0.0/24 192.0.2.0/24 192.88.99.0/24 192.168.0.0/16 198.18.0.0/15
|
|
11
|
+
198.51.100.0/24 203.0.113.0/24 224.0.0.0/4 240.0.0.0/4
|
|
12
|
+
].map { |range| IPAddr.new(range).freeze }.freeze
|
|
13
|
+
GLOBAL_IPV6 = IPAddr.new("2000::/3").freeze
|
|
14
|
+
BLOCKED_IPV6 = %w[2001::/23 2001:db8::/32 2002::/16 3fff::/20].map { |range| IPAddr.new(range).freeze }.freeze
|
|
15
|
+
|
|
16
|
+
# @rbs @policy: untyped
|
|
17
|
+
|
|
18
|
+
# @rbs (untyped) -> void
|
|
19
|
+
def initialize(policy)
|
|
20
|
+
unless %i[public unrestricted].include?(policy) || policy.respond_to?(:call)
|
|
21
|
+
raise ArgumentError, "network_policy must be :public, :unrestricted, or a callable"
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
@policy = policy
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
# @rbs (String) -> untyped
|
|
28
|
+
def self.parse_url(value)
|
|
29
|
+
uri = URI.parse(value)
|
|
30
|
+
raise SimpleRSS::PolicyError, "Expected an absolute HTTP or HTTPS URL" unless uri.is_a?(URI::HTTP)
|
|
31
|
+
|
|
32
|
+
unless uri.hostname && !uri.hostname.to_s.empty? && uri.port&.between?(1, 65_535)
|
|
33
|
+
raise SimpleRSS::PolicyError, "Expected a valid host and port"
|
|
34
|
+
end
|
|
35
|
+
raise SimpleRSS::PolicyError, "Credentials in URLs are not supported" if uri.userinfo
|
|
36
|
+
|
|
37
|
+
uri.fragment = nil
|
|
38
|
+
uri.path = "/" if uri.path.to_s.empty?
|
|
39
|
+
uri.normalize
|
|
40
|
+
rescue URI::Error, TypeError
|
|
41
|
+
raise SimpleRSS::PolicyError, "Malformed HTTP or HTTPS URL"
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
# @rbs (untyped) -> String
|
|
45
|
+
def address(uri)
|
|
46
|
+
addresses = resolve(uri.hostname.to_s)
|
|
47
|
+
raise SimpleRSS::RequestError, "No destination addresses were resolved" if addresses.empty?
|
|
48
|
+
unless addresses.all? { |address| allowed?(uri, address) }
|
|
49
|
+
raise SimpleRSS::PolicyError, "Destination address is prohibited by network_policy"
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
(addresses.find(&:ipv4?) || addresses.fetch(0)).to_s
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
private
|
|
56
|
+
|
|
57
|
+
# @rbs (String) -> Array[untyped]
|
|
58
|
+
def resolve(hostname)
|
|
59
|
+
[IPAddr.new(hostname)]
|
|
60
|
+
rescue IPAddr::InvalidAddressError
|
|
61
|
+
Resolv.getaddresses(hostname).uniq.map { |address| IPAddr.new(address) }
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
# @rbs (untyped, untyped) -> bool
|
|
65
|
+
def allowed?(uri, address)
|
|
66
|
+
return true if @policy == :unrestricted
|
|
67
|
+
return @policy.call(uri.dup.freeze, address.dup.freeze) == true if @policy.respond_to?(:call)
|
|
68
|
+
return BLOCKED_IPV4.none? { |range| range.include?(address) } if address.ipv4?
|
|
69
|
+
|
|
70
|
+
GLOBAL_IPV6.include?(address) && BLOCKED_IPV6.none? { |range| range.include?(address) }
|
|
71
|
+
end
|
|
72
|
+
end
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
# rbs_inline: enabled
|
|
2
|
+
|
|
3
|
+
class SimpleRSS::XmlElement
|
|
4
|
+
attr_reader :name #: String
|
|
5
|
+
attr_reader :attributes #: Hash[String, String]
|
|
6
|
+
attr_reader :content #: String
|
|
7
|
+
attr_reader :namespaces #: Hash[String, String]
|
|
8
|
+
attr_reader :base_urls #: Array[String]
|
|
9
|
+
|
|
10
|
+
# @rbs (String, String, String?, Hash[Symbol, untyped]) -> void
|
|
11
|
+
def initialize(name, attributes, content, parent)
|
|
12
|
+
@name = name
|
|
13
|
+
@attributes = self.class.attributes(attributes).transform_values { |value| CGI.unescapeHTML(value) }
|
|
14
|
+
@content = content.to_s
|
|
15
|
+
@namespaces = parent.fetch(:namespaces).merge(@attributes.select { |key, _value| key == "xmlns" || key.start_with?("xmlns:") })
|
|
16
|
+
base_url = @attributes["xml:base"]
|
|
17
|
+
@base_urls = parent.fetch(:base_urls).dup
|
|
18
|
+
@base_urls << base_url if base_url
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
# @rbs () -> Array[SimpleRSS::XmlElement]
|
|
22
|
+
def children
|
|
23
|
+
self.class.child_elements(content).map do |name, attributes, body|
|
|
24
|
+
self.class.new(name, attributes, body, { namespaces: namespaces, base_urls: base_urls })
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
# @rbs () -> String?
|
|
29
|
+
def namespace
|
|
30
|
+
namespaces[name.include?(":") ? "xmlns:#{name.split(":").first}" : "xmlns"]
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
# @rbs (String, String?) -> bool
|
|
34
|
+
def matches?(local_name, namespace)
|
|
35
|
+
name.split(":").last == local_name && self.namespace == namespace
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# @rbs (String) -> bool
|
|
39
|
+
def rss?(local_name)
|
|
40
|
+
return false if name.include?(":") && namespace.nil?
|
|
41
|
+
|
|
42
|
+
name.split(":").last == local_name && SimpleRSS::RSS_NAMESPACES.include?(namespace)
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
# @rbs (String) -> bool
|
|
46
|
+
def selected?(selector)
|
|
47
|
+
if selector.start_with?("{")
|
|
48
|
+
namespace, local_name = selector.delete_prefix("{").split("}", 2)
|
|
49
|
+
return local_name ? matches?(local_name, namespace) : false
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
name == selector
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# @rbs () -> String
|
|
56
|
+
def text
|
|
57
|
+
content.split(/(<!\[CDATA\[.*?\]\]>)/m).map do |part|
|
|
58
|
+
part.start_with?("<![CDATA[") ? part[9...-3].to_s : CGI.unescapeHTML(part.gsub(/<!--.*?-->|<\?.*?\?>/m, ""))
|
|
59
|
+
end.join.strip
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
# @rbs () -> Hash[Symbol, untyped]
|
|
63
|
+
def raw
|
|
64
|
+
{ name: name, namespace: namespace, attributes: attributes, content: content }
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
# @rbs () -> String
|
|
68
|
+
def xhtml_content
|
|
69
|
+
scopes = [namespaces]
|
|
70
|
+
content.gsub(SimpleRSS::XML_TAG_PATTERN) do |token|
|
|
71
|
+
name = Regexp.last_match(2).to_s
|
|
72
|
+
attributes = Regexp.last_match(3)
|
|
73
|
+
closing = Regexp.last_match(1) == "/"
|
|
74
|
+
next token unless attributes
|
|
75
|
+
|
|
76
|
+
if closing
|
|
77
|
+
context = scopes.last || namespaces
|
|
78
|
+
scopes.pop if scopes.size > 1
|
|
79
|
+
else
|
|
80
|
+
declarations = self.class.attributes(attributes).select { |key, _value| key == "xmlns" || key.start_with?("xmlns:") }
|
|
81
|
+
context = (scopes.last || namespaces).merge(declarations.transform_values { |value| CGI.unescapeHTML(value) })
|
|
82
|
+
scopes << context unless attributes.rstrip.end_with?("/")
|
|
83
|
+
end
|
|
84
|
+
namespace = context[name.include?(":") ? "xmlns:#{name.split(":").first}" : "xmlns"]
|
|
85
|
+
next token unless namespace == "http://www.w3.org/1999/xhtml"
|
|
86
|
+
|
|
87
|
+
token.sub(name, name.split(":").last.to_s)
|
|
88
|
+
end.strip
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
# @rbs (String) -> Hash[String, String]
|
|
92
|
+
def self.attributes(attributes)
|
|
93
|
+
values = {} #: Hash[String, String]
|
|
94
|
+
attributes.scan(/([\w:.-]+)\s*=\s*(?:"([^"]*)"|'([^']*)')/m) do
|
|
95
|
+
name = Regexp.last_match(1)
|
|
96
|
+
value = Regexp.last_match(2) || Regexp.last_match(3)
|
|
97
|
+
values[name] = value if name && value
|
|
98
|
+
end
|
|
99
|
+
values
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
# @rbs (String) -> Array[[String, String, String?]]
|
|
103
|
+
def self.child_elements(content)
|
|
104
|
+
elements = [] #: Array[[String, String, String?]]
|
|
105
|
+
current_element = nil #: [String, String, String?]?
|
|
106
|
+
depth = 0
|
|
107
|
+
body_start = 0
|
|
108
|
+
position = 0
|
|
109
|
+
|
|
110
|
+
while (token = SimpleRSS::XML_TAG_PATTERN.match(content, position))
|
|
111
|
+
position = token.end(0)
|
|
112
|
+
attributes = token[3]
|
|
113
|
+
next unless attributes
|
|
114
|
+
|
|
115
|
+
if token[1] == "/"
|
|
116
|
+
depth = [depth - 1, 0].max
|
|
117
|
+
if depth.zero? && current_element
|
|
118
|
+
current_element[2] = content[body_start...token.begin(0)]
|
|
119
|
+
current_element = nil
|
|
120
|
+
end
|
|
121
|
+
next
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
self_closing = attributes.rstrip.end_with?("/")
|
|
125
|
+
if depth.zero?
|
|
126
|
+
element = [token[2].to_s, attributes, nil] #: [String, String, String?]
|
|
127
|
+
elements << element
|
|
128
|
+
current_element = element unless self_closing
|
|
129
|
+
body_start = token.end(0)
|
|
130
|
+
end
|
|
131
|
+
depth += 1 unless self_closing
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
elements
|
|
135
|
+
end
|
|
136
|
+
end
|