Icarus-Mod-Tools 2.6.0 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,5 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require "digest"
3
4
  require "google/cloud/firestore"
4
5
  require "tools/modinfo"
5
6
  require "tools/toolinfo"
@@ -40,19 +41,71 @@ module Icarus
40
41
  @tools ||= list(:tools)
41
42
  end
42
43
 
44
+ # Internal map of mod/tool doc ID => the modinfo/toolinfo URL it was last synced from, kept in
45
+ # meta/mod_sources and meta/tool_sources ({ sources: { "<doc id>" => "<url>" } }). Sync uses it to protect
46
+ # only the docs of a source that failed transiently (#16). Consumers never read it; mods/tools docs are unchanged.
47
+ def sources(type)
48
+ key = pluralize(type)
49
+ @sources ||= {}
50
+ @sources[key] ||= begin
51
+ data = @client.doc(sources_path(key)).get&.data
52
+ (data&.dig(:sources) || {}).transform_keys(&:to_s)
53
+ end
54
+ end
55
+
56
+ # Records (in memory) that doc +id+ came from +url+; #save_sources writes it
57
+ def record_source(type, id, url)
58
+ return if id.nil? || url.nil?
59
+
60
+ map = sources(type)
61
+ return if map[id.to_s] == url
62
+
63
+ map[id.to_s] = url
64
+ sources_changed(type) << id.to_s
65
+ end
66
+
67
+ # Writes the entries changed since the map was read or last saved. Only those keys are written (merged),
68
+ # so entries another sync host recorded meanwhile are kept.
69
+ def save_sources(type)
70
+ changed = sources_changed(type)
71
+ return true if changed.empty?
72
+
73
+ map = sources(type)
74
+ entries = changed.to_h { |id| [id, map.fetch(id) { Google::Cloud::Firestore::FieldValue.delete }] }
75
+ response = @client.doc(sources_path(pluralize(type))).set({ sources: entries }, merge: true)
76
+ success = response.is_a?(Google::Cloud::Firestore::CommitResponse::WriteResult)
77
+ changed.clear if success
78
+
79
+ success
80
+ end
81
+
82
+ # Gives doc +to+ the source of doc +from+ (cleanup copies +from+'s data onto +to+), and saves
83
+ def move_source(type, from:, to:)
84
+ url = sources(type)[from.to_s]
85
+ return true unless url
86
+
87
+ record_source(type, to, url)
88
+ save_sources(type)
89
+ end
90
+
43
91
  def find_by_type(type:, name:, author:)
44
92
  instance_variable_get(:"@#{type}").find { |obj| obj.name == name && obj.author == author }
45
93
  end
46
94
 
95
+ # A deterministic document id for a new :mods or :tools document, so a mod that is deleted and later
96
+ # re-added gets the same id back (#16). Author and name are case- and whitespace-normalised, and the
97
+ # collection is part of the key so a mod and a tool with the same author and name do not collide.
98
+ def stable_id(type, author:, name:)
99
+ key = [pluralize(type), author.to_s.strip.downcase, name.to_s.strip.downcase].join("\0")
100
+ Digest::SHA256.hexdigest(key)[0, 20]
101
+ end
102
+
47
103
  def update(type, payload, merge: false)
48
104
  raise "You must specify a payload to update" if payload&.empty? || payload.nil?
49
105
 
50
106
  response = case type.to_sym
51
- when :modinfo, :toolinfo
52
- update_array = (send(type) + [payload]).flatten.uniq
53
- @client.doc(collections.meta.send(type)).set({ list: update_array }, merge:) if update_array.any?
54
- when :repositories
55
- @client.doc(collections.meta.repositories).set({ list: payload }, merge:)
107
+ when :modinfo, :toolinfo, :repositories
108
+ update_list(type, [payload].flatten.uniq)
56
109
  when :mod, :tool
57
110
  create_or_update(pluralize(type), payload, merge:)
58
111
  else
@@ -62,6 +115,19 @@ module Icarus
62
115
  response.is_a?(Google::Cloud::Firestore::DocumentReference) || response.is_a?(Google::Cloud::Firestore::CommitResponse::WriteResult)
63
116
  end
64
117
 
118
+ # Overwrites the :mod or :tool document at +id+ with +data+ (merge: false), keeping the document's id.
119
+ # Used by `imt sync cleanup` to copy the newest duplicate's data onto the oldest one (#16).
120
+ def replace_data(type, id:, data:)
121
+ raise "Invalid type: #{type}" unless %i[mod tool].include?(type.to_sym)
122
+
123
+ collection = pluralize(type)
124
+ response = @client.doc("#{collections.send(collection)}/#{id}").set(data, merge: false)
125
+ success = response.is_a?(Google::Cloud::Firestore::CommitResponse::WriteResult)
126
+ replace_cached(collection, id, data) if success
127
+
128
+ success
129
+ end
130
+
65
131
  def delete(type, payload)
66
132
  case type.to_sym
67
133
  when :mod, :tool
@@ -71,13 +137,16 @@ module Icarus
71
137
  cache_var = type == :mod ? :@mods : :@tools
72
138
  cached_collection = instance_variable_get(cache_var)
73
139
  cached_collection&.delete_if { |item| item.id == payload.id }
140
+ forget_source(type, payload.id)
74
141
  end
75
142
  when :modinfo, :toolinfo, :repositories
76
- update_array = (send(type) - [payload]).flatten.uniq
77
-
78
- response = @client.doc(collections.meta.send(type)).set({ list: update_array })
143
+ # Removed on the server, so entries another host added meanwhile are kept
144
+ values = [payload].flatten
145
+ response = @client.doc(collections.meta.send(type)).set(
146
+ { list: Google::Cloud::Firestore::FieldValue.array_delete(*values) }, merge: true
147
+ )
79
148
  # Invalidate cache if Firestore update was successful
80
- instance_variable_set(:"@#{type}", update_array) if response.is_a?(Google::Cloud::Firestore::CommitResponse::WriteResult)
149
+ instance_variable_set(:"@#{type}", send(type) - values) if response.is_a?(Google::Cloud::Firestore::CommitResponse::WriteResult)
81
150
  else
82
151
  raise "Invalid type: #{type}"
83
152
  end
@@ -90,7 +159,8 @@ module Icarus
90
159
  def list(type)
91
160
  case type.to_sym
92
161
  when :modinfo, :toolinfo, :repositories
93
- @client.doc(collections.meta.send(type)).get[:list]
162
+ snapshot = @client.doc(collections.meta.send(type)).get
163
+ snapshot.missing? ? [] : snapshot[:list] || []
94
164
  when :mods, :tools
95
165
  @client.col(collections.send(type)).get.map do |doc|
96
166
  klass = type == :mods ? Icarus::Mod::Tools::Modinfo : Icarus::Mod::Tools::Toolinfo
@@ -101,20 +171,67 @@ module Icarus
101
171
  end
102
172
  end
103
173
 
174
+ # Adds +values+ to a meta list. The union happens on the server, so two sync hosts (or a sync and `imt add`)
175
+ # writing at once keep each other's additions. The cache is refreshed so a later stage of `sync all` sees them.
176
+ def update_list(type, values)
177
+ response = @client.doc(collections.meta.send(type)).set(
178
+ { list: Google::Cloud::Firestore::FieldValue.array_union(*values) }, merge: true
179
+ )
180
+ if response.is_a?(Google::Cloud::Firestore::CommitResponse::WriteResult)
181
+ instance_variable_set(:"@#{type}", (send(type) + values).uniq)
182
+ end
183
+
184
+ response
185
+ end
186
+
104
187
  def create_or_update(type, payload, merge:)
105
188
  doc_id = payload.id || find_by_type(type:, name: payload.name, author: payload.author)&.id
106
189
 
107
190
  return @client.doc("#{collections.send(type)}/#{doc_id}").set(payload.to_h, merge:) if doc_id
108
191
 
109
- doc_ref = @client.col(collections.send(type)).add(payload.to_h)
192
+ # New document: write it at its stable id (set creates it, or updates a doc a previous re-add left there).
193
+ # Only if that id is already held by a different mod (same author/name in another case) fall back to add.
194
+ cached_collection = instance_variable_get(:"@#{type}")
195
+ new_id = stable_id(type, author: payload.author, name: payload.name)
196
+ if cached_collection&.any? { |item| item.id == new_id }
197
+ response = @client.col(collections.send(type)).add(payload.to_h)
198
+ new_id = response.document_id
199
+ else
200
+ response = @client.doc("#{collections.send(type)}/#{new_id}").set(payload.to_h, merge:)
201
+ end
110
202
 
111
203
  # Update cache to prevent duplicate creation within the same sync run
112
- payload.id = doc_ref.document_id
113
- cache_var = :"@#{type}"
114
- cached_collection = instance_variable_get(cache_var)
204
+ payload.id = new_id
115
205
  cached_collection&.push(payload)
116
206
 
117
- doc_ref
207
+ response
208
+ end
209
+
210
+ def forget_source(type, id)
211
+ return unless sources(type).delete(id.to_s)
212
+
213
+ sources_changed(type) << id.to_s
214
+ save_sources(type)
215
+ end
216
+
217
+ # Doc IDs whose source entry changed (or was removed) since the map was read or last saved
218
+ def sources_changed(type)
219
+ @sources_changed ||= Hash.new { |hash, key| hash[key] = Set.new }
220
+ @sources_changed[pluralize(type)]
221
+ end
222
+
223
+ def sources_path(collection)
224
+ name = :"#{collection.chomp("s")}_sources"
225
+ collections.meta&.send(name) || "meta/#{name}"
226
+ end
227
+
228
+ def replace_cached(collection, id, data)
229
+ cached_collection = instance_variable_get(:"@#{collection}")
230
+ index = cached_collection&.index { |item| item.id == id }
231
+ return unless index
232
+
233
+ old = cached_collection[index]
234
+ cached_collection[index] = old.class.new(data, id:, created: old.created_at, updated: Time.now)
118
235
  end
119
236
 
120
237
  def pluralize(type)
@@ -12,6 +12,7 @@ module Icarus
12
12
  self.repository = repo if repo
13
13
  @client = Octokit::Client.new(access_token: Config.github.token)
14
14
  @resources = []
15
+ @cache = {}
15
16
  end
16
17
 
17
18
  def repository
@@ -22,6 +23,7 @@ module Icarus
22
23
 
23
24
  def repository=(repo)
24
25
  @resources = [] # reset the resources cache
26
+ @cache = {}
25
27
  @repository = repo.gsub(%r{https?://.*github\.com/}, "")
26
28
  end
27
29
 
@@ -29,33 +31,48 @@ module Icarus
29
31
  # path: the path to search in
30
32
  # cache: whether to use the cached resources
31
33
  # recursive: whether to recursively search subdirectories
34
+ # Results are cached per [path, recursive]; the top-level listing also lives in #resources
32
35
  def all_files(path: nil, cache: true, recursive: false, &block)
33
- # If we've already been called for this repository, use the cached resources
34
- use_cache = @resources.any? && cache
35
-
36
- if use_cache
37
- @resources.each { |file| block.call(file) } if block
38
- else
39
- begin
40
- @client.contents(repository, path:).each do |entry|
41
- if entry[:type] == "dir"
42
- all_files(path: entry[:path], cache: false, recursive: true, &block) if recursive
43
- next # we don't need directories in our output
44
- end
45
-
46
- block&.call(entry)
47
- @resources << entry # cache the file
48
- end
49
- rescue Octokit::NotFound
50
- warn "WARNING: Could not access #{repository}: 404 - not found"
51
- end
36
+ key = [path, recursive]
37
+ cached = key == [nil, false] ? @resources : @cache[key]
38
+
39
+ # If we've already been called with these arguments for this repository, use the cached resources
40
+ files = (cached if cache && cached&.any?) || fetch_files(path, recursive).tap do |found|
41
+ key == [nil, false] ? @resources = found : @cache[key] = found
42
+ end
43
+
44
+ if block
45
+ files.each { |file| block.call(file) }
46
+ return nil
52
47
  end
53
48
 
54
- @resources unless block
49
+ files
50
+ end
51
+
52
+ # Returns the first file whose name equals the given name (case-insensitive)
53
+ def find(name)
54
+ all_files { |file| return file if file[:name].to_s.casecmp?(name.to_s) }
55
55
  end
56
56
 
57
- def find(pattern)
58
- all_files { |file| return file if /#{pattern}/i.match?(file[:name]) }
57
+ private
58
+
59
+ # Fetches the files (not directories) under path from the API
60
+ def fetch_files(path, recursive)
61
+ found = []
62
+
63
+ @client.contents(repository, path:).each do |entry|
64
+ if entry[:type] == "dir"
65
+ found.concat(fetch_files(entry[:path], true)) if recursive
66
+ next # we don't need directories in our output
67
+ end
68
+
69
+ found << entry
70
+ end
71
+
72
+ found
73
+ rescue Octokit::NotFound
74
+ warn "WARNING: Could not access #{repository}: 404 - not found"
75
+ found
59
76
  end
60
77
  end
61
78
  end
@@ -1,5 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require "uri"
4
+
3
5
  module Icarus
4
6
  module Mod
5
7
  module Tools
@@ -7,9 +9,17 @@ module Icarus
7
9
  class Baseinfo
8
10
  attr_reader :data
9
11
  attr_accessor :id, :created_at, :updated_at
12
+ # The modinfo/toolinfo URL this entry was read from during sync; never written to the entry's doc
13
+ attr_accessor :source_url
10
14
 
11
15
  HASHKEYS = %i[name author version compatibility description files imageURL readmeURL].freeze
12
16
 
17
+ # Appended to warnings for input the stricter rules reject but the old ones accepted
18
+ FUTURE_ERROR = "(this will become an error in a future release)"
19
+
20
+ # Keys that may be read through method_missing even when absent from the data
21
+ EXTRA_KEYS = %i[files fileType fileURL].freeze
22
+
13
23
  # Match github.com URLs with /blob/ or /raw/ and convert to raw.githubusercontent.com
14
24
  GITHUB_BLOB_RAW_URL_PATTERN = %r{
15
25
  (https?)://(?:www\.)?github\.com/ # Protocol and GitHub domain (capture group 1)
@@ -31,7 +41,7 @@ module Icarus
31
41
  end
32
42
 
33
43
  def author_id
34
- author.downcase.gsub(/\s+/, "_")
44
+ author.to_s.downcase.gsub(/\s+/, "_")
35
45
  end
36
46
 
37
47
  def read(data)
@@ -68,11 +78,11 @@ module Icarus
68
78
  end
69
79
 
70
80
  def uniq_name
71
- "#{author.strip}/#{name.strip}"
81
+ "#{author.to_s.strip}/#{name.to_s.strip}"
72
82
  end
73
83
 
74
- def to_json(*args)
75
- JSON.generate(@data, *args)
84
+ def to_json(*)
85
+ JSON.generate(@data, *)
76
86
  end
77
87
 
78
88
  def to_h
@@ -90,11 +100,11 @@ module Icarus
90
100
  validate_version
91
101
 
92
102
  %w[name author description].each do |key|
93
- @errors << "#{key.capitalize} cannot be blank" unless validate_string(@data[key.to_sym])
103
+ validate_required_string(key, @data[key.to_sym])
94
104
  end
95
105
 
96
106
  %w[imageURL readmeURL].each do |key|
97
- @errors << "Invalid URL #{key.capitalize}: #{@data[key.to_sym] || "blank"}" unless validate_url(@data[key.to_sym])
107
+ check_url(@data[key.to_sym], "Invalid URL #{key.capitalize}: #{@data[key.to_sym] || "blank"}")
98
108
  end
99
109
 
100
110
  @validated = true
@@ -107,62 +117,116 @@ module Icarus
107
117
  end
108
118
 
109
119
  def file_types
110
- files&.keys || []
120
+ files.is_a?(Hash) ? files.keys : []
111
121
  end
112
122
 
113
123
  def file_urls
114
- files&.values || []
124
+ files.is_a?(Hash) ? files.values : []
115
125
  end
116
126
 
117
- def method_missing(method_name, *_args, &_block)
118
- @data[method_name.to_sym] if @data.key?(method_name.to_sym)
127
+ # Firestore#update guards its payload with #empty?
128
+ def empty?
129
+ @data.nil? || @data.empty?
130
+ end
131
+
132
+ # Data keys and known fields read as nil when absent; any other name (a typo) raises
133
+ def method_missing(method_name, *args, &)
134
+ return super unless known_key?(method_name)
135
+
136
+ @data[method_name.to_sym]
119
137
  end
120
138
 
121
139
  def respond_to_missing?(method_name, include_private = false)
122
- @data.key?(method_name.to_sym) || super
140
+ known_key?(method_name) || super
123
141
  end
124
142
 
125
143
  private
126
144
 
145
+ def known_key?(method_name)
146
+ return false if method_name.to_s.end_with?("=", "?", "!")
147
+
148
+ @data.key?(method_name.to_sym) || self.class::HASHKEYS.include?(method_name.to_sym) || EXTRA_KEYS.include?(method_name.to_sym)
149
+ end
150
+
151
+ # The file type must be exactly one of these, with an optional leading dot
127
152
  def filetype_pattern
153
+ /\A\.?(zip|pak|exmodz?)\z/i
154
+ end
155
+
156
+ # The previous, unanchored rule; a match here but not above is only a warning for now
157
+ def legacy_filetype_pattern
128
158
  /(zip|pak|exmodz?)/i
129
159
  end
130
160
 
131
- def validate_string(string)
132
- !(string.nil? || string.empty?)
161
+ def validate_required_string(key, value)
162
+ if value.nil? || value == ""
163
+ @errors << "#{key.capitalize} cannot be blank"
164
+ elsif !value.is_a?(String)
165
+ @errors << "#{key.capitalize} must be a string"
166
+ end
167
+ end
168
+
169
+ # :ok, :legacy (passed the old check only, so warn) or :invalid
170
+ def url_status(url)
171
+ return :ok if url.nil? || url == ""
172
+ return :invalid unless url.is_a?(String)
173
+ return :ok if strict_url?(url)
174
+
175
+ url.match?(URI::DEFAULT_PARSER.make_regexp) ? :legacy : :invalid
133
176
  end
134
177
 
135
- def validate_url(url)
136
- return true if url.nil? || url.empty?
178
+ def strict_url?(url)
179
+ uri = URI.parse(url)
180
+ %w[http https].include?(uri.scheme&.downcase) && !uri.host.to_s.empty?
181
+ rescue URI::Error
182
+ false
183
+ end
137
184
 
138
- url =~ URI::DEFAULT_PARSER.make_regexp
185
+ def check_url(url, message)
186
+ case url_status(url)
187
+ when :invalid then @errors << message
188
+ when :legacy then @warnings << "#{message} #{FUTURE_ERROR}"
189
+ end
139
190
  end
140
191
 
141
192
  def validate_files
142
193
  @warnings << "This mod uses deprecated fields (fileType and fileURL)" if @data.key?(:fileType) || @data.key?(:fileURL)
194
+
195
+ if !files.nil? && !files.is_a?(Hash)
196
+ @errors << "Files must be an object"
197
+ return
198
+ end
199
+
143
200
  @warnings << "files should not be empty" if file_types.empty?
144
201
 
145
202
  file_types.each { |file_type| validate_filetype(file_type) }
146
203
 
147
- file_urls.each do |file_url|
148
- @errors << "Invalid URL: #{file_url}" unless validate_url(file_url)
149
- end
204
+ file_urls.each { |file_url| check_url(file_url, "Invalid URL: #{file_url}") }
150
205
  end
151
206
 
152
207
  def validate_filetype(file_type)
153
- @errors << "Invalid fileType: #{file_type.upcase}" unless file_type&.match?(filetype_pattern)
208
+ name = file_type.to_s
209
+ return if name.match?(filetype_pattern)
210
+
211
+ if name.match?(legacy_filetype_pattern)
212
+ @warnings << "Invalid fileType: #{name.upcase} #{FUTURE_ERROR}"
213
+ else
214
+ @errors << "Invalid fileType: #{name.upcase}"
215
+ end
154
216
  end
155
217
 
156
218
  def validate_version
157
219
  if version.nil?
158
220
  @warnings << "Version was nil, it has been defaulted to 1.0"
221
+ elsif !version.is_a?(String)
222
+ @errors << "Version must be a string"
159
223
  else
160
- @warnings << "Version should be a version string" unless /^\d+[.\d+]*/.match?(version)
224
+ @warnings << "Version should be a version string" unless /\A\d+(\.\d+)*\z/.match?(version)
161
225
  end
162
226
  end
163
227
 
164
228
  def normalize_github_url(url)
165
- return url if url.nil? || url.empty?
229
+ return url unless url.is_a?(String) && !url.empty?
166
230
 
167
231
  if url.match?(GITHUB_BLOB_RAW_URL_PATTERN)
168
232
  normalized = url.gsub(GITHUB_BLOB_RAW_URL_PATTERN, '\1://raw.githubusercontent.com/\2/\3/\4')
@@ -9,7 +9,7 @@ module Icarus
9
9
  class Modinfo < Baseinfo
10
10
  def to_h
11
11
  db_hash = super
12
- db_hash[:meta] = {status:} # Add metadata
12
+ db_hash[:meta] = { status: } # Add metadata
13
13
 
14
14
  db_hash
15
15
  end
@@ -0,0 +1,102 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "paint"
4
+ require "firestore"
5
+ require "tools/sync/helpers"
6
+
7
+ module Icarus
8
+ module Mod
9
+ module Tools
10
+ module Sync
11
+ # Shared meta/modinfo -> mods and meta/toolinfo -> tools sync. Subclasses set TYPE (:mod or :tool),
12
+ # INFO (:modinfo or :toolinfo) and ENTITY (the Modinfo or Toolinfo class).
13
+ class Entities
14
+ include Helpers
15
+
16
+ def initialize(client: nil)
17
+ @firestore = client || Firestore.new
18
+ end
19
+
20
+ # Every entry we could read from the info URLs this run. URLs that failed transiently
21
+ # (see #transient_failures) contribute nothing here, so their entries must not be deleted.
22
+ def info_array
23
+ return @info_array if @info_array
24
+
25
+ @transient_failures = []
26
+ @info_array = @firestore.send(self.class::INFO).map do |url|
27
+ next unless url
28
+
29
+ entries_from(url)
30
+ rescue Icarus::Mod::Tools::Sync::RequestFailed => e
31
+ @transient_failures << url if e.transient?
32
+ warn "Skipped; Failed to retrieve #{url}#{" (transient; its existing entries are kept)" if e.transient?}"
33
+ next
34
+ rescue JSON::ParserError => e
35
+ @transient_failures << url
36
+ warn Paint["WARNING: Skipped; Invalid JSON in #{url}: #{e.message}", :red, :bright]
37
+ warn Paint["WARNING: Existing entries from #{url} are kept until it parses again; " \
38
+ "remove or rename the file to remove them", :red, :bright]
39
+ next
40
+ end.flatten.compact.uniq { |entry| [entry.name, entry.author] }
41
+ end
42
+
43
+ # The info URLs that could not be read this run for a reason that may clear up
44
+ # (network error, 5xx, rate limit, invalid JSON). Gone (404/410) URLs are not included.
45
+ def transient_failures
46
+ info_array
47
+ @transient_failures || []
48
+ end
49
+
50
+ def find(info)
51
+ @firestore.find_by_type(type: plural.to_s, name: info.name, author: info.author)&.id
52
+ end
53
+
54
+ def find_info(info)
55
+ @info_array.find { |entry| entry.name == info.name && entry.author == info.author }
56
+ end
57
+
58
+ def update(info)
59
+ success = @firestore.update(self.class::TYPE, info, merge: false)
60
+ @firestore.record_source(plural, info.id, info.source_url) if success && info.source_url
61
+
62
+ success
63
+ end
64
+
65
+ # doc ID => info URL it was last synced from (see Firestore#sources)
66
+ def sources
67
+ @firestore.sources(plural)
68
+ end
69
+
70
+ def save_sources
71
+ @firestore.save_sources(plural)
72
+ end
73
+
74
+ def delete(info)
75
+ @firestore.delete(self.class::TYPE, info)
76
+ end
77
+
78
+ private
79
+
80
+ def plural
81
+ :"#{self.class::TYPE}s"
82
+ end
83
+
84
+ # A body that is not a JSON object, or an entry that is not one, is treated as invalid JSON
85
+ # (transient), so one malformed file cannot abort the whole sync
86
+ def entries_from(url)
87
+ body = retrieve_from_url(url)
88
+ entries = body[plural] if body.is_a?(Hash)
89
+ raise JSON::ParserError, "no '#{plural}' array" unless entries.is_a?(Array)
90
+ raise JSON::ParserError, "'#{plural}' entries must be objects" unless entries.all?(Hash)
91
+
92
+ entries.filter_map do |data|
93
+ next unless data[:name].is_a?(String) && /[a-z0-9]+/i.match?(data[:name])
94
+
95
+ self.class::ENTITY.new(data).tap { |entry| entry.source_url = url }
96
+ end
97
+ end
98
+ end
99
+ end
100
+ end
101
+ end
102
+ end
@@ -9,14 +9,41 @@ module Icarus
9
9
  module Mod
10
10
  module Tools
11
11
  module Sync
12
- class RequestFailed < StandardError; end
12
+ # Raised when a source URL cannot be read. Subclasses say whether the failure is permanent
13
+ # (RequestGone: the author removed or renamed the file) or may clear up on a later run
14
+ # (RequestTransient). Sync only deletes catalog entries on a permanent failure (#16).
15
+ class RequestFailed < StandardError
16
+ def transient?
17
+ false
18
+ end
19
+ end
20
+
21
+ # The source is gone (HTTP 404 or 410): its entries may be removed from the catalog
22
+ class RequestGone < RequestFailed; end
23
+
24
+ # The source could not be read this run (timeout, network/DNS/TLS error, 5xx, 403/429 rate limits,
25
+ # or any other unexpected status): its entries must be kept until a run can read it again
26
+ class RequestTransient < RequestFailed
27
+ def transient?
28
+ true
29
+ end
30
+ end
13
31
 
14
32
  # Sync helper methods
15
33
  module Helpers
34
+ GONE_STATUS_CODES = %w[404 410].freeze
35
+
36
+ # Errors raised by Net::HTTP for a request that never got a usable response
37
+ NETWORK_ERRORS = [Timeout::Error, SocketError, SystemCallError, IOError, OpenSSL::SSL::SSLError, Net::ProtocolError].freeze
38
+
16
39
  def retrieve_from_url(url)
17
40
  raise RequestFailed, "Invalid URI: '#{url}'" unless url && url =~ URI::DEFAULT_PARSER.make_regexp
18
41
 
19
- uri = URI(url)
42
+ uri = begin
43
+ URI(url)
44
+ rescue URI::InvalidURIError
45
+ raise RequestFailed, "Invalid URI: '#{url}'"
46
+ end
20
47
  http = Net::HTTP.new(uri.host, uri.port)
21
48
 
22
49
  if uri.scheme == "https"
@@ -32,9 +59,16 @@ module Icarus
32
59
  }
33
60
  end
34
61
 
35
- res = http.get(uri.path.empty? ? "/" : uri.path)
62
+ res = begin
63
+ http.get(uri.request_uri)
64
+ rescue *NETWORK_ERRORS => e
65
+ raise RequestTransient, "HTTP Request failed for #{url} (#{e.class}): #{e.message}"
66
+ end
36
67
 
37
- raise RequestFailed, "HTTP Request failed for #{url} (#{res.code}): #{res.message}" unless res&.code == "200"
68
+ unless res&.code == "200"
69
+ error_class = GONE_STATUS_CODES.include?(res&.code) ? RequestGone : RequestTransient
70
+ raise error_class, "HTTP Request failed for #{url} (#{res&.code}): #{res&.message}"
71
+ end
38
72
 
39
73
  JSON.parse(res.body, symbolize_names: true)
40
74
  end