Icarus-Mod-Tools 2.7.0 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +55 -0
- data/.rubocop_todo.yml +142 -0
- data/.ruby-version +1 -1
- data/CHANGELOG.md +34 -0
- data/CLAUDE.md +4 -4
- data/Gemfile +3 -5
- data/Gemfile.lock +18 -27
- data/Guardfile +2 -0
- data/README.md +5 -2
- data/icarus-mod-tools.gemspec +2 -1
- data/lib/icarus/mod/cli/add.rb +53 -19
- data/lib/icarus/mod/cli/base.rb +4 -4
- data/lib/icarus/mod/cli/command.rb +6 -1
- data/lib/icarus/mod/cli/list.rb +19 -7
- data/lib/icarus/mod/cli/remove.rb +64 -62
- data/lib/icarus/mod/cli/subcommand_base.rb +2 -2
- data/lib/icarus/mod/cli/sync.rb +47 -18
- data/lib/icarus/mod/cli/validate.rb +1 -4
- data/lib/icarus/mod/config.rb +2 -2
- data/lib/icarus/mod/firestore.rb +38 -21
- data/lib/icarus/mod/github.rb +39 -22
- data/lib/icarus/mod/tools/baseinfo.rb +84 -22
- data/lib/icarus/mod/tools/modinfo.rb +1 -1
- data/lib/icarus/mod/tools/sync/entities.rb +102 -0
- data/lib/icarus/mod/tools/sync/helpers.rb +6 -2
- data/lib/icarus/mod/tools/sync/mods.rb +6 -79
- data/lib/icarus/mod/tools/sync/tools.rb +6 -81
- data/lib/icarus/mod/tools/toolinfo.rb +12 -8
- data/lib/icarus/mod/tools/validator.rb +7 -7
- data/lib/icarus/mod/version.rb +1 -1
- metadata +20 -4
- data/sig/database/sync/sync.rbs +0 -15
data/lib/icarus/mod/cli/list.rb
CHANGED
|
@@ -45,7 +45,7 @@ module Icarus
|
|
|
45
45
|
|
|
46
46
|
no_commands do
|
|
47
47
|
def list_for_type(type)
|
|
48
|
-
klass =
|
|
48
|
+
klass = type == :mods ? Icarus::Mod::Tools::Modinfo : Icarus::Mod::Tools::Toolinfo
|
|
49
49
|
valid_keys = klass::HASHKEYS + [:updated_at]
|
|
50
50
|
sort_field = options[:sort]&.to_sym
|
|
51
51
|
filter = !options[:filter].empty?
|
|
@@ -54,9 +54,15 @@ module Icarus
|
|
|
54
54
|
filter_field = options[:filter].first&.to_sym
|
|
55
55
|
filter_value = options[:filter].last&.to_s
|
|
56
56
|
|
|
57
|
-
|
|
57
|
+
unless options[:filter].empty? || options[:filter]&.count == 2
|
|
58
|
+
raise Icarus::Mod::Tools::Error,
|
|
59
|
+
"Invalid filter option #{options[:filter]}"
|
|
60
|
+
end
|
|
58
61
|
|
|
59
|
-
|
|
62
|
+
unless filter_field && valid_keys.include?(filter_field)
|
|
63
|
+
raise Icarus::Mod::Tools::Error,
|
|
64
|
+
"Invalid filter field '#{filter_field}'"
|
|
65
|
+
end
|
|
60
66
|
end
|
|
61
67
|
|
|
62
68
|
raise Icarus::Mod::Tools::Error, "Invalid sort field '#{sort_field}'" unless valid_keys.include?(sort_field)
|
|
@@ -67,7 +73,8 @@ module Icarus
|
|
|
67
73
|
items = Firestore.new.send(type)
|
|
68
74
|
|
|
69
75
|
# Filter by field
|
|
70
|
-
|
|
76
|
+
# Case-insensitive substring match; the value is literal text, not a regex
|
|
77
|
+
items.select! { |item| item.send(filter_field).to_s.downcase.include?(filter_value.to_s.downcase) } if filter_field
|
|
71
78
|
|
|
72
79
|
if items.empty?
|
|
73
80
|
puts "no entries found" if verbose?
|
|
@@ -90,16 +97,21 @@ module Icarus
|
|
|
90
97
|
end
|
|
91
98
|
|
|
92
99
|
# Sort by field, optionally subsorting by name
|
|
93
|
-
|
|
100
|
+
sorted = if sort_field == :name
|
|
101
|
+
items.sort_by(&:name)
|
|
102
|
+
else
|
|
103
|
+
items.sort_by { |item| [item.send(sort_field).to_s, item.name.to_s] }
|
|
104
|
+
end
|
|
105
|
+
sorted.each do |item|
|
|
94
106
|
data_format = "%-<name>50s %-<author>20s v%-<version>10s%-<updated_at>20s"
|
|
95
107
|
data_format += " %-<id>20s %<description>s" if verbose > 1
|
|
96
108
|
|
|
97
|
-
puts format(data_format, item.to_h.merge(id: item.id, updated_at: item.updated_at
|
|
109
|
+
puts format(data_format, item.to_h.merge(id: item.id, updated_at: item.updated_at&.strftime("%Y-%m-%d %H:%M:%S")))
|
|
98
110
|
end
|
|
99
111
|
|
|
100
112
|
puts "Total: #{items.count}" if verbose?
|
|
101
113
|
rescue Icarus::Mod::Tools::Error => e
|
|
102
|
-
|
|
114
|
+
warn e.message
|
|
103
115
|
exit 1
|
|
104
116
|
end
|
|
105
117
|
end
|
|
@@ -11,6 +11,7 @@ module Icarus
|
|
|
11
11
|
# rubocop:disable Style/GlobalVars
|
|
12
12
|
class Remove < SubcommandBase
|
|
13
13
|
include Tools::Sync::Helpers
|
|
14
|
+
|
|
14
15
|
class_option :dry_run, type: :boolean, default: false, desc: "Dry run (no changes will be made)"
|
|
15
16
|
|
|
16
17
|
desc "repos REPO", "Removes an entry from 'meta/repos/list' and cascades to associated mods/tools"
|
|
@@ -152,10 +153,14 @@ module Icarus
|
|
|
152
153
|
|
|
153
154
|
def cascade_delete_repo(repo_name)
|
|
154
155
|
@delete_failures = []
|
|
156
|
+
@failed_entity_fetches = []
|
|
157
|
+
|
|
158
|
+
# The stored entry may be a full URL; match on the "owner/repo" part
|
|
159
|
+
owner_repo = repo_name.sub(%r{\Ahttps?://(www\.)?github\.com/}i, "").sub(%r{/+\z}, "")
|
|
155
160
|
|
|
156
|
-
# Find all modinfo URLs belonging to this repository
|
|
157
|
-
#
|
|
158
|
-
repo_pattern = %r{/#{Regexp.escape(
|
|
161
|
+
# Find all modinfo/toolinfo URLs belonging to this repository. Anchored to the host and the full
|
|
162
|
+
# "owner/repo" path component so another repo's subdirectory or a similarly named repo can't match.
|
|
163
|
+
repo_pattern = %r{\Ahttps?://(raw\.githubusercontent\.com|github\.com)/#{Regexp.escape(owner_repo)}/}i
|
|
159
164
|
modinfo_urls = firestore.modinfo.select { |url| url.match?(repo_pattern) }
|
|
160
165
|
toolinfo_urls = firestore.toolinfo.select { |url| url.match?(repo_pattern) }
|
|
161
166
|
|
|
@@ -168,28 +173,20 @@ module Icarus
|
|
|
168
173
|
return
|
|
169
174
|
end
|
|
170
175
|
|
|
171
|
-
# Delete
|
|
172
|
-
modinfo_urls.each
|
|
173
|
-
|
|
174
|
-
track_delete(:modinfo, url) { firestore.delete(:modinfo, url) }
|
|
175
|
-
|
|
176
|
-
# Find and delete associated mods
|
|
177
|
-
delete_entities_from_url(url, :mod)
|
|
178
|
-
end
|
|
179
|
-
|
|
180
|
-
# Delete toolinfo URLs and their associated tools
|
|
181
|
-
toolinfo_urls.each do |url|
|
|
182
|
-
puts Paint[" Removing toolinfo: #{url}", :black] if verbose?
|
|
183
|
-
track_delete(:toolinfo, url) { firestore.delete(:toolinfo, url) }
|
|
184
|
-
|
|
185
|
-
# Find and delete associated tools
|
|
186
|
-
delete_entities_from_url(url, :tool)
|
|
187
|
-
end
|
|
176
|
+
# Delete each URL's entities, then the URL itself, but only if all of its entities went
|
|
177
|
+
modinfo_urls.each { |url| delete_info_url(url, :modinfo, :mod) }
|
|
178
|
+
toolinfo_urls.each { |url| delete_info_url(url, :toolinfo, :tool) }
|
|
188
179
|
|
|
189
180
|
# Report any failures
|
|
190
181
|
report_delete_failures
|
|
191
182
|
report_fetch_failures
|
|
192
183
|
|
|
184
|
+
if @delete_failures.any? || @failed_entity_fetches.any?
|
|
185
|
+
# Keep the repo entry so the removal can be retried; entries still listed would otherwise be orphaned
|
|
186
|
+
warn Paint["Repository not removed (kept so the removal can be retried): #{repo_name}", :red]
|
|
187
|
+
exit 1
|
|
188
|
+
end
|
|
189
|
+
|
|
193
190
|
# Finally, remove the repository
|
|
194
191
|
if firestore.delete(:repositories, repo_name)
|
|
195
192
|
puts Paint["Successfully removed repository and all associated entries: #{repo_name}", :green]
|
|
@@ -199,39 +196,57 @@ module Icarus
|
|
|
199
196
|
end
|
|
200
197
|
end
|
|
201
198
|
|
|
202
|
-
def
|
|
203
|
-
#
|
|
204
|
-
begin
|
|
205
|
-
data = retrieve_from_url(url)
|
|
206
|
-
entities = data[type == :mod ? :mods : :tools] || []
|
|
207
|
-
|
|
208
|
-
entities.each do |entity_data|
|
|
209
|
-
# Find ALL matching entities in Firestore by name and author
|
|
210
|
-
collection = type == :mod ? firestore.mods : firestore.tools
|
|
211
|
-
matching_entities = collection.select do |e|
|
|
212
|
-
e.name == entity_data[:name] && e.author == entity_data[:author]
|
|
213
|
-
end
|
|
199
|
+
def delete_info_url(url, info_type, type)
|
|
200
|
+
puts Paint[" Removing #{info_type}: #{url}", :black] if verbose?
|
|
214
201
|
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
matching_entities.each do |entity|
|
|
222
|
-
puts Paint[" Removing #{type}: #{entity.name} (ID: #{entity.id})", :black] if verbose?
|
|
223
|
-
track_delete(type, "#{entity.name} (#{entity.id})") { firestore.delete(type, entity) }
|
|
224
|
-
end
|
|
202
|
+
failures_before = @delete_failures.size + @failed_entity_fetches.size
|
|
203
|
+
begin
|
|
204
|
+
entities_for_url(url, type).each do |entity|
|
|
205
|
+
puts Paint[" Removing #{type}: #{entity.name} (ID: #{entity.id})", :black] if verbose?
|
|
206
|
+
track_delete(type, "#{entity.name} (#{entity.id})") { firestore.delete(type, entity) }
|
|
225
207
|
end
|
|
226
208
|
rescue Tools::Sync::RequestFailed, SocketError, IOError, SystemCallError, Timeout::Error,
|
|
227
209
|
JSON::ParserError => e
|
|
228
|
-
@failed_entity_fetches ||= []
|
|
229
210
|
@failed_entity_fetches << { url: url, error: e.class.name, message: e.message }
|
|
230
211
|
warn Paint["Warning: Could not fetch #{url} to remove entities: #{e.message}", :yellow]
|
|
231
212
|
end
|
|
213
|
+
|
|
214
|
+
# Keep the URL listed while any of its entities remain
|
|
215
|
+
return if (@delete_failures.size + @failed_entity_fetches.size) > failures_before
|
|
216
|
+
|
|
217
|
+
track_delete(info_type, url) { firestore.delete(info_type, url) }
|
|
218
|
+
end
|
|
219
|
+
|
|
220
|
+
# Entities (mods or tools) that came from +url+: the docs the source map attributes to it, plus, for docs
|
|
221
|
+
# with no recorded source, those matching a name+author in the file. Docs mapped to another URL
|
|
222
|
+
# (e.g. a fork publishing the same name/author) are never touched.
|
|
223
|
+
def entities_for_url(url, type)
|
|
224
|
+
collection = type == :mod ? firestore.mods : firestore.tools
|
|
225
|
+
sources = firestore.sources(type == :mod ? :mods : :tools)
|
|
226
|
+
|
|
227
|
+
entities = collection.select { |e| sources[e.id.to_s] == url }
|
|
228
|
+
unmapped = collection.reject { |e| sources.key?(e.id.to_s) }
|
|
229
|
+
return entities if unmapped.empty?
|
|
230
|
+
|
|
231
|
+
begin
|
|
232
|
+
data = retrieve_from_url(url)
|
|
233
|
+
rescue Tools::Sync::RequestGone
|
|
234
|
+
# File removed by its author: only the mapped docs can be attributed to it
|
|
235
|
+
return entities
|
|
236
|
+
end
|
|
237
|
+
|
|
238
|
+
(data[type == :mod ? :mods : :tools] || []).each do |entity_data|
|
|
239
|
+
matching = unmapped.select { |e| e.name == entity_data[:name] && e.author == entity_data[:author] }
|
|
240
|
+
if matching.size > 1 && verbose?
|
|
241
|
+
warn Paint[" Note: Found #{matching.size} entities matching '#{entity_data[:name]}' by #{entity_data[:author]}", :yellow]
|
|
242
|
+
end
|
|
243
|
+
entities.concat(matching)
|
|
244
|
+
end
|
|
245
|
+
|
|
246
|
+
entities.uniq(&:id)
|
|
232
247
|
end
|
|
233
248
|
|
|
234
|
-
def preview_cascade_deletions(
|
|
249
|
+
def preview_cascade_deletions(_repo_name, modinfo_urls, toolinfo_urls)
|
|
235
250
|
mods = []
|
|
236
251
|
tools = []
|
|
237
252
|
|
|
@@ -254,20 +269,7 @@ module Icarus
|
|
|
254
269
|
end
|
|
255
270
|
|
|
256
271
|
def fetch_entities_from_url(url, type)
|
|
257
|
-
|
|
258
|
-
entity_data_list = data[type == :mod ? :mods : :tools] || []
|
|
259
|
-
|
|
260
|
-
collection = type == :mod ? firestore.mods : firestore.tools
|
|
261
|
-
entities = []
|
|
262
|
-
|
|
263
|
-
entity_data_list.each do |entity_data|
|
|
264
|
-
matching_entities = collection.select do |e|
|
|
265
|
-
e.name == entity_data[:name] && e.author == entity_data[:author]
|
|
266
|
-
end
|
|
267
|
-
entities.concat(matching_entities)
|
|
268
|
-
end
|
|
269
|
-
|
|
270
|
-
entities
|
|
272
|
+
entities_for_url(url, type)
|
|
271
273
|
rescue Tools::Sync::RequestFailed, SocketError, IOError, SystemCallError, Timeout::Error, JSON::ParserError => e
|
|
272
274
|
warn Paint["Warning: Could not fetch #{url}: #{e.message}", :yellow] if verbose?
|
|
273
275
|
nil
|
|
@@ -292,10 +294,10 @@ module Icarus
|
|
|
292
294
|
preview[:toolinfo_urls].each { |url| puts " • #{url}" }
|
|
293
295
|
end
|
|
294
296
|
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
297
|
+
return unless preview[:tools].any?
|
|
298
|
+
|
|
299
|
+
puts " - Tools: #{preview[:tools].size}"
|
|
300
|
+
preview[:tools].each { |tool| puts " • #{tool.name} by #{tool.author} (ID: #{tool.id})" }
|
|
299
301
|
end
|
|
300
302
|
|
|
301
303
|
def track_delete(type, identifier)
|
|
@@ -8,8 +8,8 @@ module Icarus
|
|
|
8
8
|
# Base class for all subcommands
|
|
9
9
|
class SubcommandBase < Base
|
|
10
10
|
class_option :verbose,
|
|
11
|
-
|
|
12
|
-
|
|
11
|
+
aliases: "-v", type: :boolean, repeatable: true, default: [true],
|
|
12
|
+
desc: "Increase verbosity. May be repeated for even more verbosity."
|
|
13
13
|
|
|
14
14
|
no_commands do
|
|
15
15
|
def check_false
|
data/lib/icarus/mod/cli/sync.rb
CHANGED
|
@@ -13,21 +13,26 @@ module Icarus
|
|
|
13
13
|
class Sync < SubcommandBase
|
|
14
14
|
$firestore = nil
|
|
15
15
|
|
|
16
|
+
# Stage 2 refuses to delete more than this share of a collection (or to empty it) without --force
|
|
17
|
+
MASS_DELETE_RATIO = 0.25
|
|
18
|
+
MASS_DELETE_MIN = 5
|
|
19
|
+
|
|
16
20
|
class_option :dry_run, type: :boolean, default: false, desc: "Dry run (no changes will be made)"
|
|
21
|
+
class_option :force, type: :boolean, default: false,
|
|
22
|
+
desc: "Let mods/tools sync delete over #{(MASS_DELETE_RATIO * 100).to_i}% of a collection"
|
|
17
23
|
|
|
18
24
|
desc "all", "Run all sync jobs"
|
|
19
25
|
def all
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
26
|
+
# Each stage exits 1 on failure; run the rest anyway, then exit 1 if any failed
|
|
27
|
+
failed = %i[toolinfo tools modinfo mods].map do |stage|
|
|
28
|
+
puts "Running #{stage.to_s.capitalize} Sync..." if verbose?
|
|
29
|
+
invoke stage
|
|
30
|
+
false
|
|
31
|
+
rescue SystemExit => e
|
|
32
|
+
!e.success?
|
|
33
|
+
end
|
|
28
34
|
|
|
29
|
-
|
|
30
|
-
invoke :mods
|
|
35
|
+
exit 1 if failed.any?
|
|
31
36
|
end
|
|
32
37
|
|
|
33
38
|
desc "modinfo", "Reads from 'meta/repos/list' and Syncs any modinfo files we find (github only for now)"
|
|
@@ -89,8 +94,8 @@ module Icarus
|
|
|
89
94
|
|
|
90
95
|
if options[:dry_run]
|
|
91
96
|
puts Paint["Dry run; no changes made. Would have deleted #{deleted_count} duplicate #{type}.", :yellow] if verbose?
|
|
92
|
-
|
|
93
|
-
puts "Deleted #{deleted_count} duplicate #{type}."
|
|
97
|
+
elsif verbose?
|
|
98
|
+
puts "Deleted #{deleted_count} duplicate #{type}."
|
|
94
99
|
end
|
|
95
100
|
end
|
|
96
101
|
|
|
@@ -174,7 +179,7 @@ module Icarus
|
|
|
174
179
|
kept.each do |list|
|
|
175
180
|
url = sources[list.id.to_s]
|
|
176
181
|
reason = url ? "its source could not be read (#{url})" : "no recorded source yet, and a source failed"
|
|
177
|
-
warn Paint[" #{list.author ||
|
|
182
|
+
warn Paint[" #{list.author || "NoOne"}/#{list.name || "Unnamed"} (#{list.id}): #{reason}", :yellow]
|
|
178
183
|
end
|
|
179
184
|
end
|
|
180
185
|
|
|
@@ -212,6 +217,7 @@ module Icarus
|
|
|
212
217
|
end
|
|
213
218
|
rescue Icarus::Mod::Tools::Error => e
|
|
214
219
|
warn e.message
|
|
220
|
+
exit 1
|
|
215
221
|
end
|
|
216
222
|
|
|
217
223
|
def sync_list(type)
|
|
@@ -223,10 +229,12 @@ module Icarus
|
|
|
223
229
|
info_array = sync.info_array
|
|
224
230
|
|
|
225
231
|
puts "Retrieving List Data..." if verbose?
|
|
226
|
-
|
|
232
|
+
# A snapshot: creating a doc appends it to the cached collection, which would dilute the delete guard
|
|
233
|
+
list_array = sync.send(type).dup
|
|
227
234
|
|
|
228
235
|
return if options[:check]
|
|
229
236
|
|
|
237
|
+
updating = !options[:dry_run]
|
|
230
238
|
puts "Updating List Data..." if verbose?
|
|
231
239
|
info_array.each do |list|
|
|
232
240
|
verb = "Creating"
|
|
@@ -248,7 +256,7 @@ module Icarus
|
|
|
248
256
|
|
|
249
257
|
if verbose > 1
|
|
250
258
|
print format("#{verb} %-<name>60s",
|
|
251
|
-
name: "'#{list.author ||
|
|
259
|
+
name: "'#{list.author || "NoOne"}/#{list.name || "Unnamed"}'")
|
|
252
260
|
end
|
|
253
261
|
|
|
254
262
|
if options[:dry_run]
|
|
@@ -257,7 +265,7 @@ module Icarus
|
|
|
257
265
|
end
|
|
258
266
|
|
|
259
267
|
response = sync.update(list)
|
|
260
|
-
puts success_or_failure(
|
|
268
|
+
puts success_or_failure(response) if verbose > 1
|
|
261
269
|
end
|
|
262
270
|
|
|
263
271
|
if options[:dry_run]
|
|
@@ -272,16 +280,37 @@ module Icarus
|
|
|
272
280
|
|
|
273
281
|
return unless delete_array.any?
|
|
274
282
|
|
|
283
|
+
refuse_mass_delete(type, delete_array, list_array, info_array)
|
|
284
|
+
|
|
275
285
|
puts "Deleting outdated items..." if verbose?
|
|
276
286
|
delete_array.each do |list|
|
|
277
|
-
print "Deleting '#{list.author ||
|
|
287
|
+
print "Deleting '#{list.author || "NoOne"}/#{list.name || "Unnamed"}'#{" " * 20}" if verbose > 1
|
|
278
288
|
response = sync.delete(list)
|
|
279
|
-
puts success_or_failure(
|
|
289
|
+
puts success_or_failure(response) if verbose > 1
|
|
280
290
|
end
|
|
281
291
|
|
|
282
292
|
puts "Deleted #{delete_array.count} outdated items" if verbose?
|
|
283
293
|
rescue Icarus::Mod::Tools::Error => e
|
|
284
294
|
warn e.message
|
|
295
|
+
exit 1
|
|
296
|
+
rescue StandardError
|
|
297
|
+
# Keep the sources of the docs written before a failure, so the next run can protect them (#16)
|
|
298
|
+
sync&.save_sources if updating
|
|
299
|
+
raise
|
|
300
|
+
end
|
|
301
|
+
|
|
302
|
+
# A wave of 404s (a GitHub outage, a mass rename) looks like every author removing their file, so stage 2
|
|
303
|
+
# will not delete more than MASS_DELETE_RATIO of a collection, or empty it, unless run with --force
|
|
304
|
+
def refuse_mass_delete(type, delete_array, list_array, info_array)
|
|
305
|
+
return if options[:force]
|
|
306
|
+
|
|
307
|
+
emptying = info_array.empty? && delete_array.size >= list_array.size
|
|
308
|
+
too_many = delete_array.size >= MASS_DELETE_MIN && delete_array.size > list_array.size * MASS_DELETE_RATIO
|
|
309
|
+
return unless emptying || too_many
|
|
310
|
+
|
|
311
|
+
raise Icarus::Mod::Tools::Error,
|
|
312
|
+
Paint["Refusing to delete #{delete_array.size} of #{list_array.size} #{type}; " \
|
|
313
|
+
"check the sources, then rerun with --force if this is intended", :red, :bright]
|
|
285
314
|
end
|
|
286
315
|
end
|
|
287
316
|
end
|
|
@@ -23,9 +23,8 @@ module Icarus
|
|
|
23
23
|
validator = Icarus::Mod::Tools::Validator.new(type)
|
|
24
24
|
|
|
25
25
|
puts "Validating Entries..." if verbose?
|
|
26
|
-
max_length = validator.array.map { |info| info.uniq_name.length }.max + 1
|
|
26
|
+
max_length = (validator.array.map { |info| info.uniq_name.length }.max || 0) + 1
|
|
27
27
|
|
|
28
|
-
# rubocop:disable Lint/FormatParameterMismatch
|
|
29
28
|
validator.array.each do |info|
|
|
30
29
|
print Paint[format("%s %-#{max_length}s", "Running validation steps on", info.uniq_name), :cyan, :bright] if verbose > 1
|
|
31
30
|
|
|
@@ -47,8 +46,6 @@ module Icarus
|
|
|
47
46
|
puts info.warnings.map { |warning| Paint["#{warning} in #{info.uniq_name}", :yellow] }.join("\n")
|
|
48
47
|
puts "\n" if verbose > 1
|
|
49
48
|
end
|
|
50
|
-
# rubocop:enable Lint/FormatParameterMismatch
|
|
51
|
-
|
|
52
49
|
exit exit_code
|
|
53
50
|
end
|
|
54
51
|
end
|
data/lib/icarus/mod/config.rb
CHANGED
data/lib/icarus/mod/firestore.rb
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require "digest"
|
|
4
|
-
require "set"
|
|
5
4
|
require "google/cloud/firestore"
|
|
6
5
|
require "tools/modinfo"
|
|
7
6
|
require "tools/toolinfo"
|
|
@@ -62,17 +61,20 @@ module Icarus
|
|
|
62
61
|
return if map[id.to_s] == url
|
|
63
62
|
|
|
64
63
|
map[id.to_s] = url
|
|
65
|
-
sources_changed <<
|
|
64
|
+
sources_changed(type) << id.to_s
|
|
66
65
|
end
|
|
67
66
|
|
|
68
|
-
# Writes the
|
|
67
|
+
# Writes the entries changed since the map was read or last saved. Only those keys are written (merged),
|
|
68
|
+
# so entries another sync host recorded meanwhile are kept.
|
|
69
69
|
def save_sources(type)
|
|
70
|
-
|
|
71
|
-
return true
|
|
70
|
+
changed = sources_changed(type)
|
|
71
|
+
return true if changed.empty?
|
|
72
72
|
|
|
73
|
-
|
|
73
|
+
map = sources(type)
|
|
74
|
+
entries = changed.to_h { |id| [id, map.fetch(id) { Google::Cloud::Firestore::FieldValue.delete }] }
|
|
75
|
+
response = @client.doc(sources_path(pluralize(type))).set({ sources: entries }, merge: true)
|
|
74
76
|
success = response.is_a?(Google::Cloud::Firestore::CommitResponse::WriteResult)
|
|
75
|
-
|
|
77
|
+
changed.clear if success
|
|
76
78
|
|
|
77
79
|
success
|
|
78
80
|
end
|
|
@@ -102,11 +104,8 @@ module Icarus
|
|
|
102
104
|
raise "You must specify a payload to update" if payload&.empty? || payload.nil?
|
|
103
105
|
|
|
104
106
|
response = case type.to_sym
|
|
105
|
-
when :modinfo, :toolinfo
|
|
106
|
-
|
|
107
|
-
@client.doc(collections.meta.send(type)).set({ list: update_array }, merge:) if update_array.any?
|
|
108
|
-
when :repositories
|
|
109
|
-
@client.doc(collections.meta.repositories).set({ list: payload }, merge:)
|
|
107
|
+
when :modinfo, :toolinfo, :repositories
|
|
108
|
+
update_list(type, [payload].flatten.uniq)
|
|
110
109
|
when :mod, :tool
|
|
111
110
|
create_or_update(pluralize(type), payload, merge:)
|
|
112
111
|
else
|
|
@@ -141,11 +140,13 @@ module Icarus
|
|
|
141
140
|
forget_source(type, payload.id)
|
|
142
141
|
end
|
|
143
142
|
when :modinfo, :toolinfo, :repositories
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
response = @client.doc(collections.meta.send(type)).set(
|
|
143
|
+
# Removed on the server, so entries another host added meanwhile are kept
|
|
144
|
+
values = [payload].flatten
|
|
145
|
+
response = @client.doc(collections.meta.send(type)).set(
|
|
146
|
+
{ list: Google::Cloud::Firestore::FieldValue.array_delete(*values) }, merge: true
|
|
147
|
+
)
|
|
147
148
|
# Invalidate cache if Firestore update was successful
|
|
148
|
-
instance_variable_set(:"@#{type}",
|
|
149
|
+
instance_variable_set(:"@#{type}", send(type) - values) if response.is_a?(Google::Cloud::Firestore::CommitResponse::WriteResult)
|
|
149
150
|
else
|
|
150
151
|
raise "Invalid type: #{type}"
|
|
151
152
|
end
|
|
@@ -158,7 +159,8 @@ module Icarus
|
|
|
158
159
|
def list(type)
|
|
159
160
|
case type.to_sym
|
|
160
161
|
when :modinfo, :toolinfo, :repositories
|
|
161
|
-
@client.doc(collections.meta.send(type)).get
|
|
162
|
+
snapshot = @client.doc(collections.meta.send(type)).get
|
|
163
|
+
snapshot.missing? ? [] : snapshot[:list] || []
|
|
162
164
|
when :mods, :tools
|
|
163
165
|
@client.col(collections.send(type)).get.map do |doc|
|
|
164
166
|
klass = type == :mods ? Icarus::Mod::Tools::Modinfo : Icarus::Mod::Tools::Toolinfo
|
|
@@ -169,6 +171,19 @@ module Icarus
|
|
|
169
171
|
end
|
|
170
172
|
end
|
|
171
173
|
|
|
174
|
+
# Adds +values+ to a meta list. The union happens on the server, so two sync hosts (or a sync and `imt add`)
|
|
175
|
+
# writing at once keep each other's additions. The cache is refreshed so a later stage of `sync all` sees them.
|
|
176
|
+
def update_list(type, values)
|
|
177
|
+
response = @client.doc(collections.meta.send(type)).set(
|
|
178
|
+
{ list: Google::Cloud::Firestore::FieldValue.array_union(*values) }, merge: true
|
|
179
|
+
)
|
|
180
|
+
if response.is_a?(Google::Cloud::Firestore::CommitResponse::WriteResult)
|
|
181
|
+
instance_variable_set(:"@#{type}", (send(type) + values).uniq)
|
|
182
|
+
end
|
|
183
|
+
|
|
184
|
+
response
|
|
185
|
+
end
|
|
186
|
+
|
|
172
187
|
def create_or_update(type, payload, merge:)
|
|
173
188
|
doc_id = payload.id || find_by_type(type:, name: payload.name, author: payload.author)&.id
|
|
174
189
|
|
|
@@ -195,16 +210,18 @@ module Icarus
|
|
|
195
210
|
def forget_source(type, id)
|
|
196
211
|
return unless sources(type).delete(id.to_s)
|
|
197
212
|
|
|
198
|
-
sources_changed <<
|
|
213
|
+
sources_changed(type) << id.to_s
|
|
199
214
|
save_sources(type)
|
|
200
215
|
end
|
|
201
216
|
|
|
202
|
-
|
|
203
|
-
|
|
217
|
+
# Doc IDs whose source entry changed (or was removed) since the map was read or last saved
|
|
218
|
+
def sources_changed(type)
|
|
219
|
+
@sources_changed ||= Hash.new { |hash, key| hash[key] = Set.new }
|
|
220
|
+
@sources_changed[pluralize(type)]
|
|
204
221
|
end
|
|
205
222
|
|
|
206
223
|
def sources_path(collection)
|
|
207
|
-
name = :"#{collection.chomp(
|
|
224
|
+
name = :"#{collection.chomp("s")}_sources"
|
|
208
225
|
collections.meta&.send(name) || "meta/#{name}"
|
|
209
226
|
end
|
|
210
227
|
|
data/lib/icarus/mod/github.rb
CHANGED
|
@@ -12,6 +12,7 @@ module Icarus
|
|
|
12
12
|
self.repository = repo if repo
|
|
13
13
|
@client = Octokit::Client.new(access_token: Config.github.token)
|
|
14
14
|
@resources = []
|
|
15
|
+
@cache = {}
|
|
15
16
|
end
|
|
16
17
|
|
|
17
18
|
def repository
|
|
@@ -22,6 +23,7 @@ module Icarus
|
|
|
22
23
|
|
|
23
24
|
def repository=(repo)
|
|
24
25
|
@resources = [] # reset the resources cache
|
|
26
|
+
@cache = {}
|
|
25
27
|
@repository = repo.gsub(%r{https?://.*github\.com/}, "")
|
|
26
28
|
end
|
|
27
29
|
|
|
@@ -29,33 +31,48 @@ module Icarus
|
|
|
29
31
|
# path: the path to search in
|
|
30
32
|
# cache: whether to use the cached resources
|
|
31
33
|
# recursive: whether to recursively search subdirectories
|
|
34
|
+
# Results are cached per [path, recursive]; the top-level listing also lives in #resources
|
|
32
35
|
def all_files(path: nil, cache: true, recursive: false, &block)
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
end
|
|
45
|
-
|
|
46
|
-
block&.call(entry)
|
|
47
|
-
@resources << entry # cache the file
|
|
48
|
-
end
|
|
49
|
-
rescue Octokit::NotFound
|
|
50
|
-
warn "WARNING: Could not access #{repository}: 404 - not found"
|
|
51
|
-
end
|
|
36
|
+
key = [path, recursive]
|
|
37
|
+
cached = key == [nil, false] ? @resources : @cache[key]
|
|
38
|
+
|
|
39
|
+
# If we've already been called with these arguments for this repository, use the cached resources
|
|
40
|
+
files = (cached if cache && cached&.any?) || fetch_files(path, recursive).tap do |found|
|
|
41
|
+
key == [nil, false] ? @resources = found : @cache[key] = found
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
if block
|
|
45
|
+
files.each { |file| block.call(file) }
|
|
46
|
+
return nil
|
|
52
47
|
end
|
|
53
48
|
|
|
54
|
-
|
|
49
|
+
files
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
# Returns the first file whose name equals the given name (case-insensitive)
|
|
53
|
+
def find(name)
|
|
54
|
+
all_files { |file| return file if file[:name].to_s.casecmp?(name.to_s) }
|
|
55
55
|
end
|
|
56
56
|
|
|
57
|
-
|
|
58
|
-
|
|
57
|
+
private
|
|
58
|
+
|
|
59
|
+
# Fetches the files (not directories) under path from the API
|
|
60
|
+
def fetch_files(path, recursive)
|
|
61
|
+
found = []
|
|
62
|
+
|
|
63
|
+
@client.contents(repository, path:).each do |entry|
|
|
64
|
+
if entry[:type] == "dir"
|
|
65
|
+
found.concat(fetch_files(entry[:path], true)) if recursive
|
|
66
|
+
next # we don't need directories in our output
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
found << entry
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
found
|
|
73
|
+
rescue Octokit::NotFound
|
|
74
|
+
warn "WARNING: Could not access #{repository}: 404 - not found"
|
|
75
|
+
found
|
|
59
76
|
end
|
|
60
77
|
end
|
|
61
78
|
end
|