Icarus-Mod-Tools 2.7.0 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +55 -0
- data/.rubocop_todo.yml +142 -0
- data/.ruby-version +1 -1
- data/CHANGELOG.md +41 -0
- data/CLAUDE.md +4 -4
- data/Gemfile +4 -6
- data/Gemfile.lock +90 -71
- data/Guardfile +2 -0
- data/README.md +8 -2
- data/icarus-mod-tools.gemspec +4 -3
- data/lib/icarus/mod/cli/add.rb +53 -19
- data/lib/icarus/mod/cli/base.rb +4 -4
- data/lib/icarus/mod/cli/command.rb +6 -1
- data/lib/icarus/mod/cli/list.rb +19 -7
- data/lib/icarus/mod/cli/remove.rb +64 -62
- data/lib/icarus/mod/cli/subcommand_base.rb +2 -2
- data/lib/icarus/mod/cli/sync.rb +68 -23
- data/lib/icarus/mod/cli/validate.rb +1 -4
- data/lib/icarus/mod/config.rb +2 -2
- data/lib/icarus/mod/firestore.rb +38 -21
- data/lib/icarus/mod/github.rb +39 -22
- data/lib/icarus/mod/tools/baseinfo.rb +84 -22
- data/lib/icarus/mod/tools/modinfo.rb +1 -1
- data/lib/icarus/mod/tools/sync/entities.rb +102 -0
- data/lib/icarus/mod/tools/sync/helpers.rb +6 -2
- data/lib/icarus/mod/tools/sync/mods.rb +6 -79
- data/lib/icarus/mod/tools/sync/tools.rb +6 -81
- data/lib/icarus/mod/tools/toolinfo.rb +12 -8
- data/lib/icarus/mod/tools/validator.rb +7 -7
- data/lib/icarus/mod/version.rb +1 -1
- metadata +24 -8
- data/sig/database/sync/sync.rbs +0 -15
data/lib/icarus/mod/firestore.rb
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require "digest"
|
|
4
|
-
require "set"
|
|
5
4
|
require "google/cloud/firestore"
|
|
6
5
|
require "tools/modinfo"
|
|
7
6
|
require "tools/toolinfo"
|
|
@@ -62,17 +61,20 @@ module Icarus
|
|
|
62
61
|
return if map[id.to_s] == url
|
|
63
62
|
|
|
64
63
|
map[id.to_s] = url
|
|
65
|
-
sources_changed <<
|
|
64
|
+
sources_changed(type) << id.to_s
|
|
66
65
|
end
|
|
67
66
|
|
|
68
|
-
# Writes the
|
|
67
|
+
# Writes the entries changed since the map was read or last saved. Only those keys are written (merged),
|
|
68
|
+
# so entries another sync host recorded meanwhile are kept.
|
|
69
69
|
def save_sources(type)
|
|
70
|
-
|
|
71
|
-
return true
|
|
70
|
+
changed = sources_changed(type)
|
|
71
|
+
return true if changed.empty?
|
|
72
72
|
|
|
73
|
-
|
|
73
|
+
map = sources(type)
|
|
74
|
+
entries = changed.to_h { |id| [id, map.fetch(id) { Google::Cloud::Firestore::FieldValue.delete }] }
|
|
75
|
+
response = @client.doc(sources_path(pluralize(type))).set({ sources: entries }, merge: true)
|
|
74
76
|
success = response.is_a?(Google::Cloud::Firestore::CommitResponse::WriteResult)
|
|
75
|
-
|
|
77
|
+
changed.clear if success
|
|
76
78
|
|
|
77
79
|
success
|
|
78
80
|
end
|
|
@@ -102,11 +104,8 @@ module Icarus
|
|
|
102
104
|
raise "You must specify a payload to update" if payload&.empty? || payload.nil?
|
|
103
105
|
|
|
104
106
|
response = case type.to_sym
|
|
105
|
-
when :modinfo, :toolinfo
|
|
106
|
-
|
|
107
|
-
@client.doc(collections.meta.send(type)).set({ list: update_array }, merge:) if update_array.any?
|
|
108
|
-
when :repositories
|
|
109
|
-
@client.doc(collections.meta.repositories).set({ list: payload }, merge:)
|
|
107
|
+
when :modinfo, :toolinfo, :repositories
|
|
108
|
+
update_list(type, [payload].flatten.uniq)
|
|
110
109
|
when :mod, :tool
|
|
111
110
|
create_or_update(pluralize(type), payload, merge:)
|
|
112
111
|
else
|
|
@@ -141,11 +140,13 @@ module Icarus
|
|
|
141
140
|
forget_source(type, payload.id)
|
|
142
141
|
end
|
|
143
142
|
when :modinfo, :toolinfo, :repositories
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
response = @client.doc(collections.meta.send(type)).set(
|
|
143
|
+
# Removed on the server, so entries another host added meanwhile are kept
|
|
144
|
+
values = [payload].flatten
|
|
145
|
+
response = @client.doc(collections.meta.send(type)).set(
|
|
146
|
+
{ list: Google::Cloud::Firestore::FieldValue.array_delete(*values) }, merge: true
|
|
147
|
+
)
|
|
147
148
|
# Invalidate cache if Firestore update was successful
|
|
148
|
-
instance_variable_set(:"@#{type}",
|
|
149
|
+
instance_variable_set(:"@#{type}", send(type) - values) if response.is_a?(Google::Cloud::Firestore::CommitResponse::WriteResult)
|
|
149
150
|
else
|
|
150
151
|
raise "Invalid type: #{type}"
|
|
151
152
|
end
|
|
@@ -158,7 +159,8 @@ module Icarus
|
|
|
158
159
|
def list(type)
|
|
159
160
|
case type.to_sym
|
|
160
161
|
when :modinfo, :toolinfo, :repositories
|
|
161
|
-
@client.doc(collections.meta.send(type)).get
|
|
162
|
+
snapshot = @client.doc(collections.meta.send(type)).get
|
|
163
|
+
snapshot.missing? ? [] : snapshot[:list] || []
|
|
162
164
|
when :mods, :tools
|
|
163
165
|
@client.col(collections.send(type)).get.map do |doc|
|
|
164
166
|
klass = type == :mods ? Icarus::Mod::Tools::Modinfo : Icarus::Mod::Tools::Toolinfo
|
|
@@ -169,6 +171,19 @@ module Icarus
|
|
|
169
171
|
end
|
|
170
172
|
end
|
|
171
173
|
|
|
174
|
+
# Adds +values+ to a meta list. The union happens on the server, so two sync hosts (or a sync and `imt add`)
|
|
175
|
+
# writing at once keep each other's additions. The cache is refreshed so a later stage of `sync all` sees them.
|
|
176
|
+
def update_list(type, values)
|
|
177
|
+
response = @client.doc(collections.meta.send(type)).set(
|
|
178
|
+
{ list: Google::Cloud::Firestore::FieldValue.array_union(*values) }, merge: true
|
|
179
|
+
)
|
|
180
|
+
if response.is_a?(Google::Cloud::Firestore::CommitResponse::WriteResult)
|
|
181
|
+
instance_variable_set(:"@#{type}", (send(type) + values).uniq)
|
|
182
|
+
end
|
|
183
|
+
|
|
184
|
+
response
|
|
185
|
+
end
|
|
186
|
+
|
|
172
187
|
def create_or_update(type, payload, merge:)
|
|
173
188
|
doc_id = payload.id || find_by_type(type:, name: payload.name, author: payload.author)&.id
|
|
174
189
|
|
|
@@ -195,16 +210,18 @@ module Icarus
|
|
|
195
210
|
def forget_source(type, id)
|
|
196
211
|
return unless sources(type).delete(id.to_s)
|
|
197
212
|
|
|
198
|
-
sources_changed <<
|
|
213
|
+
sources_changed(type) << id.to_s
|
|
199
214
|
save_sources(type)
|
|
200
215
|
end
|
|
201
216
|
|
|
202
|
-
|
|
203
|
-
|
|
217
|
+
# Doc IDs whose source entry changed (or was removed) since the map was read or last saved
|
|
218
|
+
def sources_changed(type)
|
|
219
|
+
@sources_changed ||= Hash.new { |hash, key| hash[key] = Set.new }
|
|
220
|
+
@sources_changed[pluralize(type)]
|
|
204
221
|
end
|
|
205
222
|
|
|
206
223
|
def sources_path(collection)
|
|
207
|
-
name = :"#{collection.chomp(
|
|
224
|
+
name = :"#{collection.chomp("s")}_sources"
|
|
208
225
|
collections.meta&.send(name) || "meta/#{name}"
|
|
209
226
|
end
|
|
210
227
|
|
data/lib/icarus/mod/github.rb
CHANGED
|
@@ -12,6 +12,7 @@ module Icarus
|
|
|
12
12
|
self.repository = repo if repo
|
|
13
13
|
@client = Octokit::Client.new(access_token: Config.github.token)
|
|
14
14
|
@resources = []
|
|
15
|
+
@cache = {}
|
|
15
16
|
end
|
|
16
17
|
|
|
17
18
|
def repository
|
|
@@ -22,6 +23,7 @@ module Icarus
|
|
|
22
23
|
|
|
23
24
|
def repository=(repo)
|
|
24
25
|
@resources = [] # reset the resources cache
|
|
26
|
+
@cache = {}
|
|
25
27
|
@repository = repo.gsub(%r{https?://.*github\.com/}, "")
|
|
26
28
|
end
|
|
27
29
|
|
|
@@ -29,33 +31,48 @@ module Icarus
|
|
|
29
31
|
# path: the path to search in
|
|
30
32
|
# cache: whether to use the cached resources
|
|
31
33
|
# recursive: whether to recursively search subdirectories
|
|
34
|
+
# Results are cached per [path, recursive]; the top-level listing also lives in #resources
|
|
32
35
|
def all_files(path: nil, cache: true, recursive: false, &block)
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
end
|
|
45
|
-
|
|
46
|
-
block&.call(entry)
|
|
47
|
-
@resources << entry # cache the file
|
|
48
|
-
end
|
|
49
|
-
rescue Octokit::NotFound
|
|
50
|
-
warn "WARNING: Could not access #{repository}: 404 - not found"
|
|
51
|
-
end
|
|
36
|
+
key = [path, recursive]
|
|
37
|
+
cached = key == [nil, false] ? @resources : @cache[key]
|
|
38
|
+
|
|
39
|
+
# If we've already been called with these arguments for this repository, use the cached resources
|
|
40
|
+
files = (cached if cache && cached&.any?) || fetch_files(path, recursive).tap do |found|
|
|
41
|
+
key == [nil, false] ? @resources = found : @cache[key] = found
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
if block
|
|
45
|
+
files.each { |file| block.call(file) }
|
|
46
|
+
return nil
|
|
52
47
|
end
|
|
53
48
|
|
|
54
|
-
|
|
49
|
+
files
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
# Returns the first file whose name equals the given name (case-insensitive)
|
|
53
|
+
def find(name)
|
|
54
|
+
all_files { |file| return file if file[:name].to_s.casecmp?(name.to_s) }
|
|
55
55
|
end
|
|
56
56
|
|
|
57
|
-
|
|
58
|
-
|
|
57
|
+
private
|
|
58
|
+
|
|
59
|
+
# Fetches the files (not directories) under path from the API
|
|
60
|
+
def fetch_files(path, recursive)
|
|
61
|
+
found = []
|
|
62
|
+
|
|
63
|
+
@client.contents(repository, path:).each do |entry|
|
|
64
|
+
if entry[:type] == "dir"
|
|
65
|
+
found.concat(fetch_files(entry[:path], true)) if recursive
|
|
66
|
+
next # we don't need directories in our output
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
found << entry
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
found
|
|
73
|
+
rescue Octokit::NotFound
|
|
74
|
+
warn "WARNING: Could not access #{repository}: 404 - not found"
|
|
75
|
+
found
|
|
59
76
|
end
|
|
60
77
|
end
|
|
61
78
|
end
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require "uri"
|
|
4
|
+
|
|
3
5
|
module Icarus
|
|
4
6
|
module Mod
|
|
5
7
|
module Tools
|
|
@@ -12,6 +14,12 @@ module Icarus
|
|
|
12
14
|
|
|
13
15
|
HASHKEYS = %i[name author version compatibility description files imageURL readmeURL].freeze
|
|
14
16
|
|
|
17
|
+
# Appended to warnings for input the stricter rules reject but the old ones accepted
|
|
18
|
+
FUTURE_ERROR = "(this will become an error in a future release)"
|
|
19
|
+
|
|
20
|
+
# Keys that may be read through method_missing even when absent from the data
|
|
21
|
+
EXTRA_KEYS = %i[files fileType fileURL].freeze
|
|
22
|
+
|
|
15
23
|
# Match github.com URLs with /blob/ or /raw/ and convert to raw.githubusercontent.com
|
|
16
24
|
GITHUB_BLOB_RAW_URL_PATTERN = %r{
|
|
17
25
|
(https?)://(?:www\.)?github\.com/ # Protocol and GitHub domain (capture group 1)
|
|
@@ -33,7 +41,7 @@ module Icarus
|
|
|
33
41
|
end
|
|
34
42
|
|
|
35
43
|
def author_id
|
|
36
|
-
author.downcase.gsub(/\s+/, "_")
|
|
44
|
+
author.to_s.downcase.gsub(/\s+/, "_")
|
|
37
45
|
end
|
|
38
46
|
|
|
39
47
|
def read(data)
|
|
@@ -70,11 +78,11 @@ module Icarus
|
|
|
70
78
|
end
|
|
71
79
|
|
|
72
80
|
def uniq_name
|
|
73
|
-
"#{author.strip}/#{name.strip}"
|
|
81
|
+
"#{author.to_s.strip}/#{name.to_s.strip}"
|
|
74
82
|
end
|
|
75
83
|
|
|
76
|
-
def to_json(*
|
|
77
|
-
JSON.generate(@data, *
|
|
84
|
+
def to_json(*)
|
|
85
|
+
JSON.generate(@data, *)
|
|
78
86
|
end
|
|
79
87
|
|
|
80
88
|
def to_h
|
|
@@ -92,11 +100,11 @@ module Icarus
|
|
|
92
100
|
validate_version
|
|
93
101
|
|
|
94
102
|
%w[name author description].each do |key|
|
|
95
|
-
|
|
103
|
+
validate_required_string(key, @data[key.to_sym])
|
|
96
104
|
end
|
|
97
105
|
|
|
98
106
|
%w[imageURL readmeURL].each do |key|
|
|
99
|
-
@
|
|
107
|
+
check_url(@data[key.to_sym], "Invalid URL #{key.capitalize}: #{@data[key.to_sym] || "blank"}")
|
|
100
108
|
end
|
|
101
109
|
|
|
102
110
|
@validated = true
|
|
@@ -109,62 +117,116 @@ module Icarus
|
|
|
109
117
|
end
|
|
110
118
|
|
|
111
119
|
def file_types
|
|
112
|
-
files
|
|
120
|
+
files.is_a?(Hash) ? files.keys : []
|
|
113
121
|
end
|
|
114
122
|
|
|
115
123
|
def file_urls
|
|
116
|
-
files
|
|
124
|
+
files.is_a?(Hash) ? files.values : []
|
|
117
125
|
end
|
|
118
126
|
|
|
119
|
-
|
|
120
|
-
|
|
127
|
+
# Firestore#update guards its payload with #empty?
|
|
128
|
+
def empty?
|
|
129
|
+
@data.nil? || @data.empty?
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
# Data keys and known fields read as nil when absent; any other name (a typo) raises
|
|
133
|
+
def method_missing(method_name, *args, &)
|
|
134
|
+
return super unless known_key?(method_name)
|
|
135
|
+
|
|
136
|
+
@data[method_name.to_sym]
|
|
121
137
|
end
|
|
122
138
|
|
|
123
139
|
def respond_to_missing?(method_name, include_private = false)
|
|
124
|
-
|
|
140
|
+
known_key?(method_name) || super
|
|
125
141
|
end
|
|
126
142
|
|
|
127
143
|
private
|
|
128
144
|
|
|
145
|
+
def known_key?(method_name)
|
|
146
|
+
return false if method_name.to_s.end_with?("=", "?", "!")
|
|
147
|
+
|
|
148
|
+
@data.key?(method_name.to_sym) || self.class::HASHKEYS.include?(method_name.to_sym) || EXTRA_KEYS.include?(method_name.to_sym)
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
# The file type must be exactly one of these, with an optional leading dot
|
|
129
152
|
def filetype_pattern
|
|
153
|
+
/\A\.?(zip|pak|exmodz?)\z/i
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
# The previous, unanchored rule; a match here but not above is only a warning for now
|
|
157
|
+
def legacy_filetype_pattern
|
|
130
158
|
/(zip|pak|exmodz?)/i
|
|
131
159
|
end
|
|
132
160
|
|
|
133
|
-
def
|
|
134
|
-
|
|
161
|
+
def validate_required_string(key, value)
|
|
162
|
+
if value.nil? || value == ""
|
|
163
|
+
@errors << "#{key.capitalize} cannot be blank"
|
|
164
|
+
elsif !value.is_a?(String)
|
|
165
|
+
@errors << "#{key.capitalize} must be a string"
|
|
166
|
+
end
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
# :ok, :legacy (passed the old check only, so warn) or :invalid
|
|
170
|
+
def url_status(url)
|
|
171
|
+
return :ok if url.nil? || url == ""
|
|
172
|
+
return :invalid unless url.is_a?(String)
|
|
173
|
+
return :ok if strict_url?(url)
|
|
174
|
+
|
|
175
|
+
url.match?(URI::DEFAULT_PARSER.make_regexp) ? :legacy : :invalid
|
|
135
176
|
end
|
|
136
177
|
|
|
137
|
-
def
|
|
138
|
-
|
|
178
|
+
def strict_url?(url)
|
|
179
|
+
uri = URI.parse(url)
|
|
180
|
+
%w[http https].include?(uri.scheme&.downcase) && !uri.host.to_s.empty?
|
|
181
|
+
rescue URI::Error
|
|
182
|
+
false
|
|
183
|
+
end
|
|
139
184
|
|
|
140
|
-
|
|
185
|
+
def check_url(url, message)
|
|
186
|
+
case url_status(url)
|
|
187
|
+
when :invalid then @errors << message
|
|
188
|
+
when :legacy then @warnings << "#{message} #{FUTURE_ERROR}"
|
|
189
|
+
end
|
|
141
190
|
end
|
|
142
191
|
|
|
143
192
|
def validate_files
|
|
144
193
|
@warnings << "This mod uses deprecated fields (fileType and fileURL)" if @data.key?(:fileType) || @data.key?(:fileURL)
|
|
194
|
+
|
|
195
|
+
if !files.nil? && !files.is_a?(Hash)
|
|
196
|
+
@errors << "Files must be an object"
|
|
197
|
+
return
|
|
198
|
+
end
|
|
199
|
+
|
|
145
200
|
@warnings << "files should not be empty" if file_types.empty?
|
|
146
201
|
|
|
147
202
|
file_types.each { |file_type| validate_filetype(file_type) }
|
|
148
203
|
|
|
149
|
-
file_urls.each
|
|
150
|
-
@errors << "Invalid URL: #{file_url}" unless validate_url(file_url)
|
|
151
|
-
end
|
|
204
|
+
file_urls.each { |file_url| check_url(file_url, "Invalid URL: #{file_url}") }
|
|
152
205
|
end
|
|
153
206
|
|
|
154
207
|
def validate_filetype(file_type)
|
|
155
|
-
|
|
208
|
+
name = file_type.to_s
|
|
209
|
+
return if name.match?(filetype_pattern)
|
|
210
|
+
|
|
211
|
+
if name.match?(legacy_filetype_pattern)
|
|
212
|
+
@warnings << "Invalid fileType: #{name.upcase} #{FUTURE_ERROR}"
|
|
213
|
+
else
|
|
214
|
+
@errors << "Invalid fileType: #{name.upcase}"
|
|
215
|
+
end
|
|
156
216
|
end
|
|
157
217
|
|
|
158
218
|
def validate_version
|
|
159
219
|
if version.nil?
|
|
160
220
|
@warnings << "Version was nil, it has been defaulted to 1.0"
|
|
221
|
+
elsif !version.is_a?(String)
|
|
222
|
+
@errors << "Version must be a string"
|
|
161
223
|
else
|
|
162
|
-
@warnings << "Version should be a version string" unless
|
|
224
|
+
@warnings << "Version should be a version string" unless /\A\d+(\.\d+)*\z/.match?(version)
|
|
163
225
|
end
|
|
164
226
|
end
|
|
165
227
|
|
|
166
228
|
def normalize_github_url(url)
|
|
167
|
-
return url
|
|
229
|
+
return url unless url.is_a?(String) && !url.empty?
|
|
168
230
|
|
|
169
231
|
if url.match?(GITHUB_BLOB_RAW_URL_PATTERN)
|
|
170
232
|
normalized = url.gsub(GITHUB_BLOB_RAW_URL_PATTERN, '\1://raw.githubusercontent.com/\2/\3/\4')
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "paint"
|
|
4
|
+
require "firestore"
|
|
5
|
+
require "tools/sync/helpers"
|
|
6
|
+
|
|
7
|
+
module Icarus
|
|
8
|
+
module Mod
|
|
9
|
+
module Tools
|
|
10
|
+
module Sync
|
|
11
|
+
# Shared meta/modinfo -> mods and meta/toolinfo -> tools sync. Subclasses set TYPE (:mod or :tool),
|
|
12
|
+
# INFO (:modinfo or :toolinfo) and ENTITY (the Modinfo or Toolinfo class).
|
|
13
|
+
class Entities
|
|
14
|
+
include Helpers
|
|
15
|
+
|
|
16
|
+
def initialize(client: nil)
|
|
17
|
+
@firestore = client || Firestore.new
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
# Every entry we could read from the info URLs this run. URLs that failed transiently
|
|
21
|
+
# (see #transient_failures) contribute nothing here, so their entries must not be deleted.
|
|
22
|
+
def info_array
|
|
23
|
+
return @info_array if @info_array
|
|
24
|
+
|
|
25
|
+
@transient_failures = []
|
|
26
|
+
@info_array = @firestore.send(self.class::INFO).map do |url|
|
|
27
|
+
next unless url
|
|
28
|
+
|
|
29
|
+
entries_from(url)
|
|
30
|
+
rescue Icarus::Mod::Tools::Sync::RequestFailed => e
|
|
31
|
+
@transient_failures << url if e.transient?
|
|
32
|
+
warn "Skipped; Failed to retrieve #{url}#{" (transient; its existing entries are kept)" if e.transient?}"
|
|
33
|
+
next
|
|
34
|
+
rescue JSON::ParserError => e
|
|
35
|
+
@transient_failures << url
|
|
36
|
+
warn Paint["WARNING: Skipped; Invalid JSON in #{url}: #{e.message}", :red, :bright]
|
|
37
|
+
warn Paint["WARNING: Existing entries from #{url} are kept until it parses again; " \
|
|
38
|
+
"remove or rename the file to remove them", :red, :bright]
|
|
39
|
+
next
|
|
40
|
+
end.flatten.compact.uniq { |entry| [entry.name, entry.author] }
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# The info URLs that could not be read this run for a reason that may clear up
|
|
44
|
+
# (network error, 5xx, rate limit, invalid JSON). Gone (404/410) URLs are not included.
|
|
45
|
+
def transient_failures
|
|
46
|
+
info_array
|
|
47
|
+
@transient_failures || []
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def find(info)
|
|
51
|
+
@firestore.find_by_type(type: plural.to_s, name: info.name, author: info.author)&.id
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def find_info(info)
|
|
55
|
+
@info_array.find { |entry| entry.name == info.name && entry.author == info.author }
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
def update(info)
|
|
59
|
+
success = @firestore.update(self.class::TYPE, info, merge: false)
|
|
60
|
+
@firestore.record_source(plural, info.id, info.source_url) if success && info.source_url
|
|
61
|
+
|
|
62
|
+
success
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
# doc ID => info URL it was last synced from (see Firestore#sources)
|
|
66
|
+
def sources
|
|
67
|
+
@firestore.sources(plural)
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def save_sources
|
|
71
|
+
@firestore.save_sources(plural)
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
def delete(info)
|
|
75
|
+
@firestore.delete(self.class::TYPE, info)
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
private
|
|
79
|
+
|
|
80
|
+
def plural
|
|
81
|
+
:"#{self.class::TYPE}s"
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
# A body that is not a JSON object, or an entry that is not one, is treated as invalid JSON
|
|
85
|
+
# (transient), so one malformed file cannot abort the whole sync
|
|
86
|
+
def entries_from(url)
|
|
87
|
+
body = retrieve_from_url(url)
|
|
88
|
+
entries = body[plural] if body.is_a?(Hash)
|
|
89
|
+
raise JSON::ParserError, "no '#{plural}' array" unless entries.is_a?(Array)
|
|
90
|
+
raise JSON::ParserError, "'#{plural}' entries must be objects" unless entries.all?(Hash)
|
|
91
|
+
|
|
92
|
+
entries.filter_map do |data|
|
|
93
|
+
next unless data[:name].is_a?(String) && /[a-z0-9]+/i.match?(data[:name])
|
|
94
|
+
|
|
95
|
+
self.class::ENTITY.new(data).tap { |entry| entry.source_url = url }
|
|
96
|
+
end
|
|
97
|
+
end
|
|
98
|
+
end
|
|
99
|
+
end
|
|
100
|
+
end
|
|
101
|
+
end
|
|
102
|
+
end
|
|
@@ -39,7 +39,11 @@ module Icarus
|
|
|
39
39
|
def retrieve_from_url(url)
|
|
40
40
|
raise RequestFailed, "Invalid URI: '#{url}'" unless url && url =~ URI::DEFAULT_PARSER.make_regexp
|
|
41
41
|
|
|
42
|
-
uri =
|
|
42
|
+
uri = begin
|
|
43
|
+
URI(url)
|
|
44
|
+
rescue URI::InvalidURIError
|
|
45
|
+
raise RequestFailed, "Invalid URI: '#{url}'"
|
|
46
|
+
end
|
|
43
47
|
http = Net::HTTP.new(uri.host, uri.port)
|
|
44
48
|
|
|
45
49
|
if uri.scheme == "https"
|
|
@@ -56,7 +60,7 @@ module Icarus
|
|
|
56
60
|
end
|
|
57
61
|
|
|
58
62
|
res = begin
|
|
59
|
-
http.get(uri.
|
|
63
|
+
http.get(uri.request_uri)
|
|
60
64
|
rescue *NETWORK_ERRORS => e
|
|
61
65
|
raise RequestTransient, "HTTP Request failed for #{url} (#{e.class}): #{e.message}"
|
|
62
66
|
end
|
|
@@ -1,93 +1,20 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require "
|
|
4
|
-
require "firestore"
|
|
5
|
-
require "tools/sync/helpers"
|
|
3
|
+
require "tools/sync/entities"
|
|
6
4
|
|
|
7
5
|
module Icarus
|
|
8
6
|
module Mod
|
|
9
7
|
module Tools
|
|
10
8
|
module Sync
|
|
11
|
-
#
|
|
12
|
-
class Mods
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
@firestore = client || Firestore.new
|
|
17
|
-
end
|
|
9
|
+
# Syncs meta/modinfo to the mods collection (see Entities)
|
|
10
|
+
class Mods < Entities
|
|
11
|
+
TYPE = :mod
|
|
12
|
+
INFO = :modinfo
|
|
13
|
+
ENTITY = Icarus::Mod::Tools::Modinfo
|
|
18
14
|
|
|
19
15
|
def mods
|
|
20
16
|
@firestore.mods
|
|
21
17
|
end
|
|
22
|
-
|
|
23
|
-
# Every mod we could read from the modinfo URLs this run. URLs that failed transiently
|
|
24
|
-
# (see #transient_failures) contribute nothing here, so their mods must not be deleted.
|
|
25
|
-
def info_array
|
|
26
|
-
return @info_array if @info_array
|
|
27
|
-
|
|
28
|
-
@transient_failures = []
|
|
29
|
-
@info_array = @firestore.modinfo.map do |url|
|
|
30
|
-
mods_from(url)
|
|
31
|
-
rescue Icarus::Mod::Tools::Sync::RequestFailed => e
|
|
32
|
-
@transient_failures << url if e.transient?
|
|
33
|
-
warn "Skipped; Failed to retrieve #{url}#{' (transient; its existing entries are kept)' if e.transient?}"
|
|
34
|
-
next
|
|
35
|
-
rescue JSON::ParserError => e
|
|
36
|
-
@transient_failures << url
|
|
37
|
-
warn Paint["WARNING: Skipped; Invalid JSON in #{url}: #{e.message}", :red, :bright]
|
|
38
|
-
warn Paint["WARNING: Existing entries from #{url} are kept until it parses again; " \
|
|
39
|
-
"remove or rename the file to remove them", :red, :bright]
|
|
40
|
-
next
|
|
41
|
-
end.flatten.compact.uniq { |mod| [mod.name, mod.author] }
|
|
42
|
-
end
|
|
43
|
-
|
|
44
|
-
# The modinfo URLs that could not be read this run for a reason that may clear up
|
|
45
|
-
# (network error, 5xx, rate limit, invalid JSON). Gone (404/410) URLs are not included.
|
|
46
|
-
def transient_failures
|
|
47
|
-
info_array
|
|
48
|
-
@transient_failures || []
|
|
49
|
-
end
|
|
50
|
-
|
|
51
|
-
def find(modinfo)
|
|
52
|
-
@firestore.find_by_type(type: "mods", name: modinfo.name, author: modinfo.author)&.id
|
|
53
|
-
end
|
|
54
|
-
|
|
55
|
-
def find_info(modinfo)
|
|
56
|
-
@info_array.find { |mod| mod.name == modinfo.name && mod.author == modinfo.author }
|
|
57
|
-
end
|
|
58
|
-
|
|
59
|
-
def update(modinfo)
|
|
60
|
-
success = @firestore.update(:mod, modinfo, merge: false)
|
|
61
|
-
@firestore.record_source(:mods, modinfo.id, modinfo.source_url) if success && modinfo.source_url
|
|
62
|
-
|
|
63
|
-
success
|
|
64
|
-
end
|
|
65
|
-
|
|
66
|
-
# doc ID => modinfo URL it was last synced from (see Firestore#sources)
|
|
67
|
-
def sources
|
|
68
|
-
@firestore.sources(:mods)
|
|
69
|
-
end
|
|
70
|
-
|
|
71
|
-
def save_sources
|
|
72
|
-
@firestore.save_sources(:mods)
|
|
73
|
-
end
|
|
74
|
-
|
|
75
|
-
def delete(modinfo)
|
|
76
|
-
@firestore.delete(:mod, modinfo)
|
|
77
|
-
end
|
|
78
|
-
|
|
79
|
-
private
|
|
80
|
-
|
|
81
|
-
def mods_from(url)
|
|
82
|
-
mods = retrieve_from_url(url)[:mods]
|
|
83
|
-
raise JSON::ParserError, "no 'mods' array" unless mods.is_a?(Array)
|
|
84
|
-
|
|
85
|
-
mods.filter_map do |data|
|
|
86
|
-
next unless /[a-z0-9]+/i.match?(data[:name])
|
|
87
|
-
|
|
88
|
-
Icarus::Mod::Tools::Modinfo.new(data).tap { |mod| mod.source_url = url }
|
|
89
|
-
end
|
|
90
|
-
end
|
|
91
18
|
end
|
|
92
19
|
end
|
|
93
20
|
end
|