Icarus-Mod-Tools 2.6.0 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +55 -0
- data/.rubocop_todo.yml +142 -0
- data/.ruby-version +1 -1
- data/CHANGELOG.md +48 -0
- data/CLAUDE.md +6 -4
- data/Gemfile +3 -5
- data/Gemfile.lock +18 -27
- data/Guardfile +2 -0
- data/README.md +9 -1
- data/icarus-mod-tools.gemspec +2 -1
- data/imtconfig.sample.json +2 -0
- data/lib/icarus/mod/cli/add.rb +53 -19
- data/lib/icarus/mod/cli/base.rb +4 -4
- data/lib/icarus/mod/cli/command.rb +6 -1
- data/lib/icarus/mod/cli/list.rb +19 -7
- data/lib/icarus/mod/cli/remove.rb +65 -62
- data/lib/icarus/mod/cli/subcommand_base.rb +2 -2
- data/lib/icarus/mod/cli/sync.rb +139 -35
- data/lib/icarus/mod/cli/validate.rb +1 -4
- data/lib/icarus/mod/config.rb +2 -2
- data/lib/icarus/mod/firestore.rb +132 -15
- data/lib/icarus/mod/github.rb +39 -22
- data/lib/icarus/mod/tools/baseinfo.rb +86 -22
- data/lib/icarus/mod/tools/modinfo.rb +1 -1
- data/lib/icarus/mod/tools/sync/entities.rb +102 -0
- data/lib/icarus/mod/tools/sync/helpers.rb +38 -4
- data/lib/icarus/mod/tools/sync/modinfo_list.rb +30 -10
- data/lib/icarus/mod/tools/sync/mods.rb +6 -37
- data/lib/icarus/mod/tools/sync/toolinfo_list.rb +30 -10
- data/lib/icarus/mod/tools/sync/tools.rb +6 -39
- data/lib/icarus/mod/tools/toolinfo.rb +12 -8
- data/lib/icarus/mod/tools/validator.rb +7 -7
- data/lib/icarus/mod/version.rb +1 -1
- metadata +20 -4
- data/sig/database/sync/sync.rbs +0 -15
data/lib/icarus/mod/cli/list.rb
CHANGED
|
@@ -45,7 +45,7 @@ module Icarus
|
|
|
45
45
|
|
|
46
46
|
no_commands do
|
|
47
47
|
def list_for_type(type)
|
|
48
|
-
klass =
|
|
48
|
+
klass = type == :mods ? Icarus::Mod::Tools::Modinfo : Icarus::Mod::Tools::Toolinfo
|
|
49
49
|
valid_keys = klass::HASHKEYS + [:updated_at]
|
|
50
50
|
sort_field = options[:sort]&.to_sym
|
|
51
51
|
filter = !options[:filter].empty?
|
|
@@ -54,9 +54,15 @@ module Icarus
|
|
|
54
54
|
filter_field = options[:filter].first&.to_sym
|
|
55
55
|
filter_value = options[:filter].last&.to_s
|
|
56
56
|
|
|
57
|
-
|
|
57
|
+
unless options[:filter].empty? || options[:filter]&.count == 2
|
|
58
|
+
raise Icarus::Mod::Tools::Error,
|
|
59
|
+
"Invalid filter option #{options[:filter]}"
|
|
60
|
+
end
|
|
58
61
|
|
|
59
|
-
|
|
62
|
+
unless filter_field && valid_keys.include?(filter_field)
|
|
63
|
+
raise Icarus::Mod::Tools::Error,
|
|
64
|
+
"Invalid filter field '#{filter_field}'"
|
|
65
|
+
end
|
|
60
66
|
end
|
|
61
67
|
|
|
62
68
|
raise Icarus::Mod::Tools::Error, "Invalid sort field '#{sort_field}'" unless valid_keys.include?(sort_field)
|
|
@@ -67,7 +73,8 @@ module Icarus
|
|
|
67
73
|
items = Firestore.new.send(type)
|
|
68
74
|
|
|
69
75
|
# Filter by field
|
|
70
|
-
|
|
76
|
+
# Case-insensitive substring match; the value is literal text, not a regex
|
|
77
|
+
items.select! { |item| item.send(filter_field).to_s.downcase.include?(filter_value.to_s.downcase) } if filter_field
|
|
71
78
|
|
|
72
79
|
if items.empty?
|
|
73
80
|
puts "no entries found" if verbose?
|
|
@@ -90,16 +97,21 @@ module Icarus
|
|
|
90
97
|
end
|
|
91
98
|
|
|
92
99
|
# Sort by field, optionally subsorting by name
|
|
93
|
-
|
|
100
|
+
sorted = if sort_field == :name
|
|
101
|
+
items.sort_by(&:name)
|
|
102
|
+
else
|
|
103
|
+
items.sort_by { |item| [item.send(sort_field).to_s, item.name.to_s] }
|
|
104
|
+
end
|
|
105
|
+
sorted.each do |item|
|
|
94
106
|
data_format = "%-<name>50s %-<author>20s v%-<version>10s%-<updated_at>20s"
|
|
95
107
|
data_format += " %-<id>20s %<description>s" if verbose > 1
|
|
96
108
|
|
|
97
|
-
puts format(data_format, item.to_h.merge(id: item.id, updated_at: item.updated_at
|
|
109
|
+
puts format(data_format, item.to_h.merge(id: item.id, updated_at: item.updated_at&.strftime("%Y-%m-%d %H:%M:%S")))
|
|
98
110
|
end
|
|
99
111
|
|
|
100
112
|
puts "Total: #{items.count}" if verbose?
|
|
101
113
|
rescue Icarus::Mod::Tools::Error => e
|
|
102
|
-
|
|
114
|
+
warn e.message
|
|
103
115
|
exit 1
|
|
104
116
|
end
|
|
105
117
|
end
|
|
@@ -11,6 +11,7 @@ module Icarus
|
|
|
11
11
|
# rubocop:disable Style/GlobalVars
|
|
12
12
|
class Remove < SubcommandBase
|
|
13
13
|
include Tools::Sync::Helpers
|
|
14
|
+
|
|
14
15
|
class_option :dry_run, type: :boolean, default: false, desc: "Dry run (no changes will be made)"
|
|
15
16
|
|
|
16
17
|
desc "repos REPO", "Removes an entry from 'meta/repos/list' and cascades to associated mods/tools"
|
|
@@ -152,10 +153,14 @@ module Icarus
|
|
|
152
153
|
|
|
153
154
|
def cascade_delete_repo(repo_name)
|
|
154
155
|
@delete_failures = []
|
|
156
|
+
@failed_entity_fetches = []
|
|
157
|
+
|
|
158
|
+
# The stored entry may be a full URL; match on the "owner/repo" part
|
|
159
|
+
owner_repo = repo_name.sub(%r{\Ahttps?://(www\.)?github\.com/}i, "").sub(%r{/+\z}, "")
|
|
155
160
|
|
|
156
|
-
# Find all modinfo URLs belonging to this repository
|
|
157
|
-
#
|
|
158
|
-
repo_pattern = %r{/#{Regexp.escape(
|
|
161
|
+
# Find all modinfo/toolinfo URLs belonging to this repository. Anchored to the host and the full
|
|
162
|
+
# "owner/repo" path component so another repo's subdirectory or a similarly named repo can't match.
|
|
163
|
+
repo_pattern = %r{\Ahttps?://(raw\.githubusercontent\.com|github\.com)/#{Regexp.escape(owner_repo)}/}i
|
|
159
164
|
modinfo_urls = firestore.modinfo.select { |url| url.match?(repo_pattern) }
|
|
160
165
|
toolinfo_urls = firestore.toolinfo.select { |url| url.match?(repo_pattern) }
|
|
161
166
|
|
|
@@ -168,28 +173,20 @@ module Icarus
|
|
|
168
173
|
return
|
|
169
174
|
end
|
|
170
175
|
|
|
171
|
-
# Delete
|
|
172
|
-
modinfo_urls.each
|
|
173
|
-
|
|
174
|
-
track_delete(:modinfo, url) { firestore.delete(:modinfo, url) }
|
|
175
|
-
|
|
176
|
-
# Find and delete associated mods
|
|
177
|
-
delete_entities_from_url(url, :mod)
|
|
178
|
-
end
|
|
179
|
-
|
|
180
|
-
# Delete toolinfo URLs and their associated tools
|
|
181
|
-
toolinfo_urls.each do |url|
|
|
182
|
-
puts Paint[" Removing toolinfo: #{url}", :black] if verbose?
|
|
183
|
-
track_delete(:toolinfo, url) { firestore.delete(:toolinfo, url) }
|
|
184
|
-
|
|
185
|
-
# Find and delete associated tools
|
|
186
|
-
delete_entities_from_url(url, :tool)
|
|
187
|
-
end
|
|
176
|
+
# Delete each URL's entities, then the URL itself, but only if all of its entities went
|
|
177
|
+
modinfo_urls.each { |url| delete_info_url(url, :modinfo, :mod) }
|
|
178
|
+
toolinfo_urls.each { |url| delete_info_url(url, :toolinfo, :tool) }
|
|
188
179
|
|
|
189
180
|
# Report any failures
|
|
190
181
|
report_delete_failures
|
|
191
182
|
report_fetch_failures
|
|
192
183
|
|
|
184
|
+
if @delete_failures.any? || @failed_entity_fetches.any?
|
|
185
|
+
# Keep the repo entry so the removal can be retried; entries still listed would otherwise be orphaned
|
|
186
|
+
warn Paint["Repository not removed (kept so the removal can be retried): #{repo_name}", :red]
|
|
187
|
+
exit 1
|
|
188
|
+
end
|
|
189
|
+
|
|
193
190
|
# Finally, remove the repository
|
|
194
191
|
if firestore.delete(:repositories, repo_name)
|
|
195
192
|
puts Paint["Successfully removed repository and all associated entries: #{repo_name}", :green]
|
|
@@ -199,38 +196,57 @@ module Icarus
|
|
|
199
196
|
end
|
|
200
197
|
end
|
|
201
198
|
|
|
202
|
-
def
|
|
203
|
-
#
|
|
199
|
+
def delete_info_url(url, info_type, type)
|
|
200
|
+
puts Paint[" Removing #{info_type}: #{url}", :black] if verbose?
|
|
201
|
+
|
|
202
|
+
failures_before = @delete_failures.size + @failed_entity_fetches.size
|
|
204
203
|
begin
|
|
205
|
-
|
|
206
|
-
|
|
204
|
+
entities_for_url(url, type).each do |entity|
|
|
205
|
+
puts Paint[" Removing #{type}: #{entity.name} (ID: #{entity.id})", :black] if verbose?
|
|
206
|
+
track_delete(type, "#{entity.name} (#{entity.id})") { firestore.delete(type, entity) }
|
|
207
|
+
end
|
|
208
|
+
rescue Tools::Sync::RequestFailed, SocketError, IOError, SystemCallError, Timeout::Error,
|
|
209
|
+
JSON::ParserError => e
|
|
210
|
+
@failed_entity_fetches << { url: url, error: e.class.name, message: e.message }
|
|
211
|
+
warn Paint["Warning: Could not fetch #{url} to remove entities: #{e.message}", :yellow]
|
|
212
|
+
end
|
|
207
213
|
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
collection = type == :mod ? firestore.mods : firestore.tools
|
|
211
|
-
matching_entities = collection.select do |e|
|
|
212
|
-
e.name == entity_data[:name] && e.author == entity_data[:author]
|
|
213
|
-
end
|
|
214
|
+
# Keep the URL listed while any of its entities remain
|
|
215
|
+
return if (@delete_failures.size + @failed_entity_fetches.size) > failures_before
|
|
214
216
|
|
|
215
|
-
|
|
217
|
+
track_delete(info_type, url) { firestore.delete(info_type, url) }
|
|
218
|
+
end
|
|
219
|
+
|
|
220
|
+
# Entities (mods or tools) that came from +url+: the docs the source map attributes to it, plus, for docs
|
|
221
|
+
# with no recorded source, those matching a name+author in the file. Docs mapped to another URL
|
|
222
|
+
# (e.g. a fork publishing the same name/author) are never touched.
|
|
223
|
+
def entities_for_url(url, type)
|
|
224
|
+
collection = type == :mod ? firestore.mods : firestore.tools
|
|
225
|
+
sources = firestore.sources(type == :mod ? :mods : :tools)
|
|
216
226
|
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
227
|
+
entities = collection.select { |e| sources[e.id.to_s] == url }
|
|
228
|
+
unmapped = collection.reject { |e| sources.key?(e.id.to_s) }
|
|
229
|
+
return entities if unmapped.empty?
|
|
220
230
|
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
231
|
+
begin
|
|
232
|
+
data = retrieve_from_url(url)
|
|
233
|
+
rescue Tools::Sync::RequestGone
|
|
234
|
+
# File removed by its author: only the mapped docs can be attributed to it
|
|
235
|
+
return entities
|
|
236
|
+
end
|
|
237
|
+
|
|
238
|
+
(data[type == :mod ? :mods : :tools] || []).each do |entity_data|
|
|
239
|
+
matching = unmapped.select { |e| e.name == entity_data[:name] && e.author == entity_data[:author] }
|
|
240
|
+
if matching.size > 1 && verbose?
|
|
241
|
+
warn Paint[" Note: Found #{matching.size} entities matching '#{entity_data[:name]}' by #{entity_data[:author]}", :yellow]
|
|
225
242
|
end
|
|
226
|
-
|
|
227
|
-
@failed_entity_fetches ||= []
|
|
228
|
-
@failed_entity_fetches << { url: url, error: e.class.name, message: e.message }
|
|
229
|
-
warn Paint["Warning: Could not fetch #{url} to remove entities: #{e.message}", :yellow]
|
|
243
|
+
entities.concat(matching)
|
|
230
244
|
end
|
|
245
|
+
|
|
246
|
+
entities.uniq(&:id)
|
|
231
247
|
end
|
|
232
248
|
|
|
233
|
-
def preview_cascade_deletions(
|
|
249
|
+
def preview_cascade_deletions(_repo_name, modinfo_urls, toolinfo_urls)
|
|
234
250
|
mods = []
|
|
235
251
|
tools = []
|
|
236
252
|
|
|
@@ -253,21 +269,8 @@ module Icarus
|
|
|
253
269
|
end
|
|
254
270
|
|
|
255
271
|
def fetch_entities_from_url(url, type)
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
collection = type == :mod ? firestore.mods : firestore.tools
|
|
260
|
-
entities = []
|
|
261
|
-
|
|
262
|
-
entity_data_list.each do |entity_data|
|
|
263
|
-
matching_entities = collection.select do |e|
|
|
264
|
-
e.name == entity_data[:name] && e.author == entity_data[:author]
|
|
265
|
-
end
|
|
266
|
-
entities.concat(matching_entities)
|
|
267
|
-
end
|
|
268
|
-
|
|
269
|
-
entities
|
|
270
|
-
rescue SocketError, IOError, SystemCallError, Timeout::Error, JSON::ParserError => e
|
|
272
|
+
entities_for_url(url, type)
|
|
273
|
+
rescue Tools::Sync::RequestFailed, SocketError, IOError, SystemCallError, Timeout::Error, JSON::ParserError => e
|
|
271
274
|
warn Paint["Warning: Could not fetch #{url}: #{e.message}", :yellow] if verbose?
|
|
272
275
|
nil
|
|
273
276
|
end
|
|
@@ -291,10 +294,10 @@ module Icarus
|
|
|
291
294
|
preview[:toolinfo_urls].each { |url| puts " • #{url}" }
|
|
292
295
|
end
|
|
293
296
|
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
297
|
+
return unless preview[:tools].any?
|
|
298
|
+
|
|
299
|
+
puts " - Tools: #{preview[:tools].size}"
|
|
300
|
+
preview[:tools].each { |tool| puts " • #{tool.name} by #{tool.author} (ID: #{tool.id})" }
|
|
298
301
|
end
|
|
299
302
|
|
|
300
303
|
def track_delete(type, identifier)
|
|
@@ -8,8 +8,8 @@ module Icarus
|
|
|
8
8
|
# Base class for all subcommands
|
|
9
9
|
class SubcommandBase < Base
|
|
10
10
|
class_option :verbose,
|
|
11
|
-
|
|
12
|
-
|
|
11
|
+
aliases: "-v", type: :boolean, repeatable: true, default: [true],
|
|
12
|
+
desc: "Increase verbosity. May be repeated for even more verbosity."
|
|
13
13
|
|
|
14
14
|
no_commands do
|
|
15
15
|
def check_false
|
data/lib/icarus/mod/cli/sync.rb
CHANGED
|
@@ -13,21 +13,26 @@ module Icarus
|
|
|
13
13
|
class Sync < SubcommandBase
|
|
14
14
|
$firestore = nil
|
|
15
15
|
|
|
16
|
+
# Stage 2 refuses to delete more than this share of a collection (or to empty it) without --force
|
|
17
|
+
MASS_DELETE_RATIO = 0.25
|
|
18
|
+
MASS_DELETE_MIN = 5
|
|
19
|
+
|
|
16
20
|
class_option :dry_run, type: :boolean, default: false, desc: "Dry run (no changes will be made)"
|
|
21
|
+
class_option :force, type: :boolean, default: false,
|
|
22
|
+
desc: "Let mods/tools sync delete over #{(MASS_DELETE_RATIO * 100).to_i}% of a collection"
|
|
17
23
|
|
|
18
24
|
desc "all", "Run all sync jobs"
|
|
19
25
|
def all
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
26
|
+
# Each stage exits 1 on failure; run the rest anyway, then exit 1 if any failed
|
|
27
|
+
failed = %i[toolinfo tools modinfo mods].map do |stage|
|
|
28
|
+
puts "Running #{stage.to_s.capitalize} Sync..." if verbose?
|
|
29
|
+
invoke stage
|
|
30
|
+
false
|
|
31
|
+
rescue SystemExit => e
|
|
32
|
+
!e.success?
|
|
33
|
+
end
|
|
28
34
|
|
|
29
|
-
|
|
30
|
-
invoke :mods
|
|
35
|
+
exit 1 if failed.any?
|
|
31
36
|
end
|
|
32
37
|
|
|
33
38
|
desc "modinfo", "Reads from 'meta/repos/list' and Syncs any modinfo files we find (github only for now)"
|
|
@@ -80,32 +85,101 @@ module Icarus
|
|
|
80
85
|
|
|
81
86
|
puts "Found #{duplicates.length} duplicate #{singular_type}(s) to clean up:" if verbose?
|
|
82
87
|
|
|
88
|
+
deleted_count = 0
|
|
83
89
|
duplicates.each do |key, items|
|
|
84
90
|
name, author = key
|
|
85
|
-
# Sort by updated_at descending, keep the most recent
|
|
86
|
-
sorted = items.sort_by { |item| item.updated_at || Time.at(0) }.reverse
|
|
87
|
-
keeper = sorted.first
|
|
88
|
-
to_delete = sorted[1..]
|
|
89
|
-
|
|
90
91
|
puts " #{author}/#{name}: #{items.length} entries" if verbose?
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
to_delete.each do |item|
|
|
94
|
-
if options[:dry_run]
|
|
95
|
-
puts Paint[" Would delete: #{item.id} (updated: #{item.updated_at})", :yellow] if verbose?
|
|
96
|
-
else
|
|
97
|
-
puts " Deleting: #{item.id} (updated: #{item.updated_at})" if verbose?
|
|
98
|
-
response = firestore.delete(singular_type, item)
|
|
99
|
-
puts " #{success_or_failure(response)}" if verbose > 1
|
|
100
|
-
end
|
|
101
|
-
end
|
|
92
|
+
deleted_count += merge_duplicates(singular_type, items)
|
|
102
93
|
end
|
|
103
94
|
|
|
104
|
-
deleted_count = duplicates.values.sum { |items| items.length - 1 }
|
|
105
95
|
if options[:dry_run]
|
|
106
96
|
puts Paint["Dry run; no changes made. Would have deleted #{deleted_count} duplicate #{type}.", :yellow] if verbose?
|
|
107
|
-
|
|
108
|
-
puts "Deleted #{deleted_count} duplicate #{type}."
|
|
97
|
+
elsif verbose?
|
|
98
|
+
puts "Deleted #{deleted_count} duplicate #{type}."
|
|
99
|
+
end
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
# Keeps the oldest entry (the ID clients already hold), copies the most recently updated entry's
|
|
103
|
+
# data onto it, then deletes the rest. Returns how many entries were (or would be) deleted.
|
|
104
|
+
def merge_duplicates(singular_type, items)
|
|
105
|
+
keeper = items.min_by { |item| oldest_first_key(item) }
|
|
106
|
+
newest = items.min_by { |item| newest_first_key(item) }
|
|
107
|
+
to_delete = items.reject { |item| item.equal?(keeper) }
|
|
108
|
+
|
|
109
|
+
puts " Keeping: #{keeper.id} (created: #{keeper.created_at}, updated: #{keeper.updated_at})" if verbose?
|
|
110
|
+
return 0 unless copy_onto_keeper(singular_type, keeper, newest)
|
|
111
|
+
|
|
112
|
+
to_delete.each do |item|
|
|
113
|
+
if options[:dry_run]
|
|
114
|
+
puts Paint[" Would delete: #{item.id} (updated: #{item.updated_at})", :yellow] if verbose?
|
|
115
|
+
else
|
|
116
|
+
puts " Deleting: #{item.id} (updated: #{item.updated_at})" if verbose?
|
|
117
|
+
response = firestore.delete(singular_type, item)
|
|
118
|
+
puts " #{success_or_failure(response)}" if verbose > 1
|
|
119
|
+
end
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
to_delete.length
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
def copy_onto_keeper(singular_type, keeper, newest)
|
|
126
|
+
return true if newest.equal?(keeper)
|
|
127
|
+
|
|
128
|
+
copying = "data from #{newest.id} (updated: #{newest.updated_at}) onto #{keeper.id}"
|
|
129
|
+
if options[:dry_run]
|
|
130
|
+
puts Paint[" Would copy #{copying}", :yellow] if verbose?
|
|
131
|
+
return true
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
puts " Copying #{copying}" if verbose?
|
|
135
|
+
if firestore.replace_data(singular_type, id: keeper.id, data: newest.data)
|
|
136
|
+
firestore.move_source(singular_type, from: newest.id, to: keeper.id)
|
|
137
|
+
return true
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
warn Paint[" Failed to copy #{newest.id} onto #{keeper.id} for #{keeper.author}/#{keeper.name}; " \
|
|
141
|
+
"nothing deleted for this entry", :red]
|
|
142
|
+
false
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
# Earliest created_at, then updated_at, then id: deterministic even with missing timestamps
|
|
146
|
+
def oldest_first_key(item)
|
|
147
|
+
time = item.created_at || item.updated_at
|
|
148
|
+
[time ? time.to_r : Float::INFINITY, item.id.to_s]
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
# Latest updated_at, then created_at; ties go to the lowest id
|
|
152
|
+
def newest_first_key(item)
|
|
153
|
+
time = item.updated_at || item.created_at
|
|
154
|
+
[time ? -time.to_r : Float::INFINITY, item.id.to_s]
|
|
155
|
+
end
|
|
156
|
+
|
|
157
|
+
# Drops from +delete_array+ every doc that may only be missing because its source could not be read
|
|
158
|
+
# this run (#16): a doc mapped to a transiently failed URL, or, while any URL failed, a doc with no
|
|
159
|
+
# recorded source yet. A source that is gone (404/410) is not a transient failure, so authors still
|
|
160
|
+
# control removal, and docs from every other source are deleted as normal.
|
|
161
|
+
def protect_transient_sources(sync, delete_array)
|
|
162
|
+
failed_urls = sync.transient_failures
|
|
163
|
+
return delete_array if failed_urls.empty?
|
|
164
|
+
|
|
165
|
+
sources = sync.sources
|
|
166
|
+
kept, deletable = delete_array.partition do |list|
|
|
167
|
+
url = sources[list.id.to_s]
|
|
168
|
+
url.nil? || failed_urls.include?(url)
|
|
169
|
+
end
|
|
170
|
+
warn_kept(failed_urls, kept, sources) if kept.any?
|
|
171
|
+
|
|
172
|
+
deletable
|
|
173
|
+
end
|
|
174
|
+
|
|
175
|
+
def warn_kept(failed_urls, kept, sources)
|
|
176
|
+
warn Paint["WARNING: #{failed_urls.count} source(s) could not be read this run:", :yellow, :bright]
|
|
177
|
+
failed_urls.each { |url| warn Paint[" #{url}", :yellow] }
|
|
178
|
+
warn Paint["WARNING: Kept #{kept.count} outdated item(s) instead of deleting them:", :yellow, :bright]
|
|
179
|
+
kept.each do |list|
|
|
180
|
+
url = sources[list.id.to_s]
|
|
181
|
+
reason = url ? "its source could not be read (#{url})" : "no recorded source yet, and a source failed"
|
|
182
|
+
warn Paint[" #{list.author || "NoOne"}/#{list.name || "Unnamed"} (#{list.id}): #{reason}", :yellow]
|
|
109
183
|
end
|
|
110
184
|
end
|
|
111
185
|
|
|
@@ -124,6 +198,11 @@ module Icarus
|
|
|
124
198
|
puts "Retrieving Info Array..." if verbose?
|
|
125
199
|
info_array = sync.data(repositories, verbose: verbose > 1)&.map(&:download_url)&.compact
|
|
126
200
|
|
|
201
|
+
# The list update only ever adds URLs, so a repository we could not search keeps its entries (#16)
|
|
202
|
+
sync.transient_failures.each do |repo|
|
|
203
|
+
warn Paint["WARNING: Could not search #{repo} this run; its existing #{type} entries were kept", :yellow]
|
|
204
|
+
end
|
|
205
|
+
|
|
127
206
|
raise Icarus::Mod::Tools::Error, "no .json files found for #{type}" unless info_array&.any?
|
|
128
207
|
|
|
129
208
|
if options[:dry_run]
|
|
@@ -138,6 +217,7 @@ module Icarus
|
|
|
138
217
|
end
|
|
139
218
|
rescue Icarus::Mod::Tools::Error => e
|
|
140
219
|
warn e.message
|
|
220
|
+
exit 1
|
|
141
221
|
end
|
|
142
222
|
|
|
143
223
|
def sync_list(type)
|
|
@@ -149,10 +229,12 @@ module Icarus
|
|
|
149
229
|
info_array = sync.info_array
|
|
150
230
|
|
|
151
231
|
puts "Retrieving List Data..." if verbose?
|
|
152
|
-
|
|
232
|
+
# A snapshot: creating a doc appends it to the cached collection, which would dilute the delete guard
|
|
233
|
+
list_array = sync.send(type).dup
|
|
153
234
|
|
|
154
235
|
return if options[:check]
|
|
155
236
|
|
|
237
|
+
updating = !options[:dry_run]
|
|
156
238
|
puts "Updating List Data..." if verbose?
|
|
157
239
|
info_array.each do |list|
|
|
158
240
|
verb = "Creating"
|
|
@@ -174,7 +256,7 @@ module Icarus
|
|
|
174
256
|
|
|
175
257
|
if verbose > 1
|
|
176
258
|
print format("#{verb} %-<name>60s",
|
|
177
|
-
name: "'#{list.author ||
|
|
259
|
+
name: "'#{list.author || "NoOne"}/#{list.name || "Unnamed"}'")
|
|
178
260
|
end
|
|
179
261
|
|
|
180
262
|
if options[:dry_run]
|
|
@@ -183,7 +265,7 @@ module Icarus
|
|
|
183
265
|
end
|
|
184
266
|
|
|
185
267
|
response = sync.update(list)
|
|
186
|
-
puts success_or_failure(
|
|
268
|
+
puts success_or_failure(response) if verbose > 1
|
|
187
269
|
end
|
|
188
270
|
|
|
189
271
|
if options[:dry_run]
|
|
@@ -192,21 +274,43 @@ module Icarus
|
|
|
192
274
|
end
|
|
193
275
|
|
|
194
276
|
puts "Created/Updated #{info_array.count} Items" if verbose?
|
|
277
|
+
sync.save_sources
|
|
195
278
|
|
|
196
|
-
delete_array = list_array.filter { |list| sync.find_info(list).nil? }
|
|
279
|
+
delete_array = protect_transient_sources(sync, list_array.filter { |list| sync.find_info(list).nil? })
|
|
197
280
|
|
|
198
281
|
return unless delete_array.any?
|
|
199
282
|
|
|
283
|
+
refuse_mass_delete(type, delete_array, list_array, info_array)
|
|
284
|
+
|
|
200
285
|
puts "Deleting outdated items..." if verbose?
|
|
201
286
|
delete_array.each do |list|
|
|
202
|
-
print "Deleting '#{list.author ||
|
|
287
|
+
print "Deleting '#{list.author || "NoOne"}/#{list.name || "Unnamed"}'#{" " * 20}" if verbose > 1
|
|
203
288
|
response = sync.delete(list)
|
|
204
|
-
puts success_or_failure(
|
|
289
|
+
puts success_or_failure(response) if verbose > 1
|
|
205
290
|
end
|
|
206
291
|
|
|
207
292
|
puts "Deleted #{delete_array.count} outdated items" if verbose?
|
|
208
293
|
rescue Icarus::Mod::Tools::Error => e
|
|
209
294
|
warn e.message
|
|
295
|
+
exit 1
|
|
296
|
+
rescue StandardError
|
|
297
|
+
# Keep the sources of the docs written before a failure, so the next run can protect them (#16)
|
|
298
|
+
sync&.save_sources if updating
|
|
299
|
+
raise
|
|
300
|
+
end
|
|
301
|
+
|
|
302
|
+
# A wave of 404s (a GitHub outage, a mass rename) looks like every author removing their file, so stage 2
|
|
303
|
+
# will not delete more than MASS_DELETE_RATIO of a collection, or empty it, unless run with --force
|
|
304
|
+
def refuse_mass_delete(type, delete_array, list_array, info_array)
|
|
305
|
+
return if options[:force]
|
|
306
|
+
|
|
307
|
+
emptying = info_array.empty? && delete_array.size >= list_array.size
|
|
308
|
+
too_many = delete_array.size >= MASS_DELETE_MIN && delete_array.size > list_array.size * MASS_DELETE_RATIO
|
|
309
|
+
return unless emptying || too_many
|
|
310
|
+
|
|
311
|
+
raise Icarus::Mod::Tools::Error,
|
|
312
|
+
Paint["Refusing to delete #{delete_array.size} of #{list_array.size} #{type}; " \
|
|
313
|
+
"check the sources, then rerun with --force if this is intended", :red, :bright]
|
|
210
314
|
end
|
|
211
315
|
end
|
|
212
316
|
end
|
|
@@ -23,9 +23,8 @@ module Icarus
|
|
|
23
23
|
validator = Icarus::Mod::Tools::Validator.new(type)
|
|
24
24
|
|
|
25
25
|
puts "Validating Entries..." if verbose?
|
|
26
|
-
max_length = validator.array.map { |info| info.uniq_name.length }.max + 1
|
|
26
|
+
max_length = (validator.array.map { |info| info.uniq_name.length }.max || 0) + 1
|
|
27
27
|
|
|
28
|
-
# rubocop:disable Lint/FormatParameterMismatch
|
|
29
28
|
validator.array.each do |info|
|
|
30
29
|
print Paint[format("%s %-#{max_length}s", "Running validation steps on", info.uniq_name), :cyan, :bright] if verbose > 1
|
|
31
30
|
|
|
@@ -47,8 +46,6 @@ module Icarus
|
|
|
47
46
|
puts info.warnings.map { |warning| Paint["#{warning} in #{info.uniq_name}", :yellow] }.join("\n")
|
|
48
47
|
puts "\n" if verbose > 1
|
|
49
48
|
end
|
|
50
|
-
# rubocop:enable Lint/FormatParameterMismatch
|
|
51
|
-
|
|
52
49
|
exit exit_code
|
|
53
50
|
end
|
|
54
51
|
end
|