Icarus-Mod-Tools 2.7.0 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +55 -0
- data/.rubocop_todo.yml +142 -0
- data/.ruby-version +1 -1
- data/CHANGELOG.md +41 -0
- data/CLAUDE.md +4 -4
- data/Gemfile +4 -6
- data/Gemfile.lock +90 -71
- data/Guardfile +2 -0
- data/README.md +8 -2
- data/icarus-mod-tools.gemspec +4 -3
- data/lib/icarus/mod/cli/add.rb +53 -19
- data/lib/icarus/mod/cli/base.rb +4 -4
- data/lib/icarus/mod/cli/command.rb +6 -1
- data/lib/icarus/mod/cli/list.rb +19 -7
- data/lib/icarus/mod/cli/remove.rb +64 -62
- data/lib/icarus/mod/cli/subcommand_base.rb +2 -2
- data/lib/icarus/mod/cli/sync.rb +68 -23
- data/lib/icarus/mod/cli/validate.rb +1 -4
- data/lib/icarus/mod/config.rb +2 -2
- data/lib/icarus/mod/firestore.rb +38 -21
- data/lib/icarus/mod/github.rb +39 -22
- data/lib/icarus/mod/tools/baseinfo.rb +84 -22
- data/lib/icarus/mod/tools/modinfo.rb +1 -1
- data/lib/icarus/mod/tools/sync/entities.rb +102 -0
- data/lib/icarus/mod/tools/sync/helpers.rb +6 -2
- data/lib/icarus/mod/tools/sync/mods.rb +6 -79
- data/lib/icarus/mod/tools/sync/tools.rb +6 -81
- data/lib/icarus/mod/tools/toolinfo.rb +12 -8
- data/lib/icarus/mod/tools/validator.rb +7 -7
- data/lib/icarus/mod/version.rb +1 -1
- metadata +24 -8
- data/sig/database/sync/sync.rbs +0 -15
data/icarus-mod-tools.gemspec
CHANGED
|
@@ -11,7 +11,7 @@ Gem::Specification.new do |spec|
|
|
|
11
11
|
spec.summary = "Various tools for Icarus Modding"
|
|
12
12
|
spec.description = spec.summary
|
|
13
13
|
spec.homepage = "https://github.com/DonovanMods/icarus-mod-tools"
|
|
14
|
-
spec.required_ruby_version = ">=
|
|
14
|
+
spec.required_ruby_version = ">= 4.0"
|
|
15
15
|
|
|
16
16
|
spec.metadata["homepage_uri"] = spec.homepage
|
|
17
17
|
spec.metadata["source_code_uri"] = spec.homepage
|
|
@@ -31,8 +31,9 @@ Gem::Specification.new do |spec|
|
|
|
31
31
|
|
|
32
32
|
# Uncomment to register a new dependency of your gem
|
|
33
33
|
# spec.add_dependency "example-gem", "~> 1.0"
|
|
34
|
-
spec.add_dependency "google-cloud-firestore", "~>
|
|
35
|
-
spec.add_dependency "octokit", "~>
|
|
34
|
+
spec.add_dependency "google-cloud-firestore", "~> 3.3"
|
|
35
|
+
spec.add_dependency "octokit", "~> 10.0"
|
|
36
|
+
spec.add_dependency "ostruct", "~> 0.6"
|
|
36
37
|
spec.add_dependency "paint", "~> 2.3"
|
|
37
38
|
spec.add_dependency "thor", "~> 1.2"
|
|
38
39
|
# For more information and examples about making a new gem, check out our
|
data/lib/icarus/mod/cli/add.rb
CHANGED
|
@@ -8,34 +8,26 @@ module Icarus
|
|
|
8
8
|
module CLI
|
|
9
9
|
# Sync CLI command definitions
|
|
10
10
|
class Add < SubcommandBase
|
|
11
|
+
class_option :dry_run, type: :boolean, default: false, desc: "Dry run (validate and show what would be added, but change nothing)"
|
|
12
|
+
|
|
11
13
|
desc "modinfo", "Adds an entry to 'meta/modinfo/list'"
|
|
12
14
|
def modinfo(item)
|
|
13
|
-
|
|
14
|
-
payload = [firestore.modinfo, item].flatten.compact
|
|
15
|
-
|
|
16
|
-
puts firestore.update(:modinfo, payload, merge: true) ? "Success" : "Failure"
|
|
15
|
+
add_to_list(:modinfo, :modinfo, item)
|
|
17
16
|
end
|
|
18
17
|
|
|
19
18
|
desc "toolinfo", "Adds an entry to 'meta/toolinfo/list'"
|
|
20
19
|
def toolinfo(item)
|
|
21
|
-
|
|
22
|
-
payload = [firestore.toolinfo, item].flatten.compact
|
|
23
|
-
|
|
24
|
-
puts firestore.update(:toolinfo, payload, merge: true) ? "Success" : "Failure"
|
|
20
|
+
add_to_list(:toolinfo, :toolinfo, item)
|
|
25
21
|
end
|
|
26
22
|
|
|
27
23
|
desc "repos", "Adds an entry to 'meta/repos/list'"
|
|
28
24
|
def repos(item)
|
|
29
|
-
|
|
30
|
-
payload = [firestore.repositories, item].flatten.compact
|
|
31
|
-
|
|
32
|
-
puts firestore.update(:repositories, payload, merge: true) ? "Success" : "Failure"
|
|
25
|
+
add_to_list(:repositories, :repositories, item)
|
|
33
26
|
end
|
|
34
27
|
|
|
35
28
|
desc "mod", "Adds an entry to 'mods'"
|
|
36
29
|
method_option :modinfo, type: :string, required: true, default: "modinfo.json", desc: "Path to the modinfo.json file"
|
|
37
30
|
def mod
|
|
38
|
-
firestore = Firestore.new
|
|
39
31
|
data = options[:modinfo]
|
|
40
32
|
|
|
41
33
|
if data.nil? || !File.exist?(data)
|
|
@@ -43,15 +35,57 @@ module Icarus
|
|
|
43
35
|
exit 1
|
|
44
36
|
end
|
|
45
37
|
|
|
46
|
-
|
|
47
|
-
|
|
38
|
+
begin
|
|
39
|
+
parsed = JSON.parse(File.read(data), symbolize_names: true)
|
|
40
|
+
rescue JSON::ParserError => e
|
|
41
|
+
warn "Invalid JSON in #{data}: #{e.message.lines.first.strip}"
|
|
42
|
+
exit 1
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
mods = parsed[:mods] if parsed.is_a?(Hash)
|
|
46
|
+
|
|
47
|
+
unless mods.is_a?(Array)
|
|
48
|
+
warn "Invalid data file: no 'mods' list found in #{data}"
|
|
49
|
+
exit 1
|
|
50
|
+
end
|
|
48
51
|
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
+
# Validate everything up front so a bad entry never leaves a partial add behind
|
|
53
|
+
modinfos = mods.map { |mod| Icarus::Mod::Tools::Modinfo.new(mod.is_a?(Hash) ? mod : {}) }
|
|
54
|
+
invalid = modinfos.reject(&:valid?)
|
|
55
|
+
|
|
56
|
+
if invalid.any?
|
|
57
|
+
invalid.each { |modinfo| warn "Invalid modinfo: #{modinfo.errors}" }
|
|
58
|
+
exit 1
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
if options[:dry_run]
|
|
62
|
+
modinfos.each { |modinfo| puts Paint["Would add: #{modinfo.name}", :yellow] }
|
|
63
|
+
return
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
firestore = Firestore.new
|
|
67
|
+
results = modinfos.map { |modinfo| report(firestore.update(:mod, modinfo, merge: true)) }
|
|
68
|
+
|
|
69
|
+
exit 1 unless results.all?
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
no_commands do
|
|
73
|
+
# Appends item to one of the meta lists, exiting 1 if the update fails
|
|
74
|
+
def add_to_list(type, collection, item)
|
|
75
|
+
if options[:dry_run]
|
|
76
|
+
puts Paint["Would add to #{type}: #{item}", :yellow]
|
|
77
|
+
return
|
|
52
78
|
end
|
|
53
79
|
|
|
54
|
-
|
|
80
|
+
firestore = Firestore.new
|
|
81
|
+
payload = [firestore.send(collection), item].flatten.compact
|
|
82
|
+
|
|
83
|
+
exit 1 unless report(firestore.update(type, payload, merge: true))
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
def report(success)
|
|
87
|
+
puts success ? "Success" : "Failure"
|
|
88
|
+
success
|
|
55
89
|
end
|
|
56
90
|
end
|
|
57
91
|
end
|
data/lib/icarus/mod/cli/base.rb
CHANGED
|
@@ -11,12 +11,12 @@ module Icarus
|
|
|
11
11
|
# The Base CLI class for Icarus Mod Tools. This is inherited by all subcommands.
|
|
12
12
|
class Base < Thor
|
|
13
13
|
class_option :config,
|
|
14
|
-
|
|
15
|
-
|
|
14
|
+
aliases: "-C", type: :string, default: File.join(Dir.home, "/.imtconfig.json"),
|
|
15
|
+
desc: "Path to the config file"
|
|
16
16
|
|
|
17
17
|
class_option :version,
|
|
18
|
-
|
|
19
|
-
|
|
18
|
+
aliases: "-V", type: :boolean,
|
|
19
|
+
desc: "Print the version and exit"
|
|
20
20
|
|
|
21
21
|
def self.exit_on_failure?
|
|
22
22
|
true
|
|
@@ -26,7 +26,12 @@ module Icarus
|
|
|
26
26
|
exit 1
|
|
27
27
|
end
|
|
28
28
|
|
|
29
|
-
|
|
29
|
+
begin
|
|
30
|
+
Icarus::Mod::Config.read(options[:config])
|
|
31
|
+
rescue JSON::ParserError => e
|
|
32
|
+
warn "Could not find or read Config from '#{options[:config]}' - the file is malformed JSON (#{e.message.lines.first.strip})"
|
|
33
|
+
exit 1
|
|
34
|
+
end
|
|
30
35
|
end
|
|
31
36
|
|
|
32
37
|
desc "sync", "Syncs the databases"
|
data/lib/icarus/mod/cli/list.rb
CHANGED
|
@@ -45,7 +45,7 @@ module Icarus
|
|
|
45
45
|
|
|
46
46
|
no_commands do
|
|
47
47
|
def list_for_type(type)
|
|
48
|
-
klass =
|
|
48
|
+
klass = type == :mods ? Icarus::Mod::Tools::Modinfo : Icarus::Mod::Tools::Toolinfo
|
|
49
49
|
valid_keys = klass::HASHKEYS + [:updated_at]
|
|
50
50
|
sort_field = options[:sort]&.to_sym
|
|
51
51
|
filter = !options[:filter].empty?
|
|
@@ -54,9 +54,15 @@ module Icarus
|
|
|
54
54
|
filter_field = options[:filter].first&.to_sym
|
|
55
55
|
filter_value = options[:filter].last&.to_s
|
|
56
56
|
|
|
57
|
-
|
|
57
|
+
unless options[:filter].empty? || options[:filter]&.count == 2
|
|
58
|
+
raise Icarus::Mod::Tools::Error,
|
|
59
|
+
"Invalid filter option #{options[:filter]}"
|
|
60
|
+
end
|
|
58
61
|
|
|
59
|
-
|
|
62
|
+
unless filter_field && valid_keys.include?(filter_field)
|
|
63
|
+
raise Icarus::Mod::Tools::Error,
|
|
64
|
+
"Invalid filter field '#{filter_field}'"
|
|
65
|
+
end
|
|
60
66
|
end
|
|
61
67
|
|
|
62
68
|
raise Icarus::Mod::Tools::Error, "Invalid sort field '#{sort_field}'" unless valid_keys.include?(sort_field)
|
|
@@ -67,7 +73,8 @@ module Icarus
|
|
|
67
73
|
items = Firestore.new.send(type)
|
|
68
74
|
|
|
69
75
|
# Filter by field
|
|
70
|
-
|
|
76
|
+
# Case-insensitive substring match; the value is literal text, not a regex
|
|
77
|
+
items.select! { |item| item.send(filter_field).to_s.downcase.include?(filter_value.to_s.downcase) } if filter_field
|
|
71
78
|
|
|
72
79
|
if items.empty?
|
|
73
80
|
puts "no entries found" if verbose?
|
|
@@ -90,16 +97,21 @@ module Icarus
|
|
|
90
97
|
end
|
|
91
98
|
|
|
92
99
|
# Sort by field, optionally subsorting by name
|
|
93
|
-
|
|
100
|
+
sorted = if sort_field == :name
|
|
101
|
+
items.sort_by(&:name)
|
|
102
|
+
else
|
|
103
|
+
items.sort_by { |item| [item.send(sort_field).to_s, item.name.to_s] }
|
|
104
|
+
end
|
|
105
|
+
sorted.each do |item|
|
|
94
106
|
data_format = "%-<name>50s %-<author>20s v%-<version>10s%-<updated_at>20s"
|
|
95
107
|
data_format += " %-<id>20s %<description>s" if verbose > 1
|
|
96
108
|
|
|
97
|
-
puts format(data_format, item.to_h.merge(id: item.id, updated_at: item.updated_at
|
|
109
|
+
puts format(data_format, item.to_h.merge(id: item.id, updated_at: item.updated_at&.strftime("%Y-%m-%d %H:%M:%S")))
|
|
98
110
|
end
|
|
99
111
|
|
|
100
112
|
puts "Total: #{items.count}" if verbose?
|
|
101
113
|
rescue Icarus::Mod::Tools::Error => e
|
|
102
|
-
|
|
114
|
+
warn e.message
|
|
103
115
|
exit 1
|
|
104
116
|
end
|
|
105
117
|
end
|
|
@@ -11,6 +11,7 @@ module Icarus
|
|
|
11
11
|
# rubocop:disable Style/GlobalVars
|
|
12
12
|
class Remove < SubcommandBase
|
|
13
13
|
include Tools::Sync::Helpers
|
|
14
|
+
|
|
14
15
|
class_option :dry_run, type: :boolean, default: false, desc: "Dry run (no changes will be made)"
|
|
15
16
|
|
|
16
17
|
desc "repos REPO", "Removes an entry from 'meta/repos/list' and cascades to associated mods/tools"
|
|
@@ -152,10 +153,14 @@ module Icarus
|
|
|
152
153
|
|
|
153
154
|
def cascade_delete_repo(repo_name)
|
|
154
155
|
@delete_failures = []
|
|
156
|
+
@failed_entity_fetches = []
|
|
157
|
+
|
|
158
|
+
# The stored entry may be a full URL; match on the "owner/repo" part
|
|
159
|
+
owner_repo = repo_name.sub(%r{\Ahttps?://(www\.)?github\.com/}i, "").sub(%r{/+\z}, "")
|
|
155
160
|
|
|
156
|
-
# Find all modinfo URLs belonging to this repository
|
|
157
|
-
#
|
|
158
|
-
repo_pattern = %r{/#{Regexp.escape(
|
|
161
|
+
# Find all modinfo/toolinfo URLs belonging to this repository. Anchored to the host and the full
|
|
162
|
+
# "owner/repo" path component so another repo's subdirectory or a similarly named repo can't match.
|
|
163
|
+
repo_pattern = %r{\Ahttps?://(raw\.githubusercontent\.com|github\.com)/#{Regexp.escape(owner_repo)}/}i
|
|
159
164
|
modinfo_urls = firestore.modinfo.select { |url| url.match?(repo_pattern) }
|
|
160
165
|
toolinfo_urls = firestore.toolinfo.select { |url| url.match?(repo_pattern) }
|
|
161
166
|
|
|
@@ -168,28 +173,20 @@ module Icarus
|
|
|
168
173
|
return
|
|
169
174
|
end
|
|
170
175
|
|
|
171
|
-
# Delete
|
|
172
|
-
modinfo_urls.each
|
|
173
|
-
|
|
174
|
-
track_delete(:modinfo, url) { firestore.delete(:modinfo, url) }
|
|
175
|
-
|
|
176
|
-
# Find and delete associated mods
|
|
177
|
-
delete_entities_from_url(url, :mod)
|
|
178
|
-
end
|
|
179
|
-
|
|
180
|
-
# Delete toolinfo URLs and their associated tools
|
|
181
|
-
toolinfo_urls.each do |url|
|
|
182
|
-
puts Paint[" Removing toolinfo: #{url}", :black] if verbose?
|
|
183
|
-
track_delete(:toolinfo, url) { firestore.delete(:toolinfo, url) }
|
|
184
|
-
|
|
185
|
-
# Find and delete associated tools
|
|
186
|
-
delete_entities_from_url(url, :tool)
|
|
187
|
-
end
|
|
176
|
+
# Delete each URL's entities, then the URL itself, but only if all of its entities went
|
|
177
|
+
modinfo_urls.each { |url| delete_info_url(url, :modinfo, :mod) }
|
|
178
|
+
toolinfo_urls.each { |url| delete_info_url(url, :toolinfo, :tool) }
|
|
188
179
|
|
|
189
180
|
# Report any failures
|
|
190
181
|
report_delete_failures
|
|
191
182
|
report_fetch_failures
|
|
192
183
|
|
|
184
|
+
if @delete_failures.any? || @failed_entity_fetches.any?
|
|
185
|
+
# Keep the repo entry so the removal can be retried; entries still listed would otherwise be orphaned
|
|
186
|
+
warn Paint["Repository not removed (kept so the removal can be retried): #{repo_name}", :red]
|
|
187
|
+
exit 1
|
|
188
|
+
end
|
|
189
|
+
|
|
193
190
|
# Finally, remove the repository
|
|
194
191
|
if firestore.delete(:repositories, repo_name)
|
|
195
192
|
puts Paint["Successfully removed repository and all associated entries: #{repo_name}", :green]
|
|
@@ -199,39 +196,57 @@ module Icarus
|
|
|
199
196
|
end
|
|
200
197
|
end
|
|
201
198
|
|
|
202
|
-
def
|
|
203
|
-
#
|
|
204
|
-
begin
|
|
205
|
-
data = retrieve_from_url(url)
|
|
206
|
-
entities = data[type == :mod ? :mods : :tools] || []
|
|
207
|
-
|
|
208
|
-
entities.each do |entity_data|
|
|
209
|
-
# Find ALL matching entities in Firestore by name and author
|
|
210
|
-
collection = type == :mod ? firestore.mods : firestore.tools
|
|
211
|
-
matching_entities = collection.select do |e|
|
|
212
|
-
e.name == entity_data[:name] && e.author == entity_data[:author]
|
|
213
|
-
end
|
|
199
|
+
def delete_info_url(url, info_type, type)
|
|
200
|
+
puts Paint[" Removing #{info_type}: #{url}", :black] if verbose?
|
|
214
201
|
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
matching_entities.each do |entity|
|
|
222
|
-
puts Paint[" Removing #{type}: #{entity.name} (ID: #{entity.id})", :black] if verbose?
|
|
223
|
-
track_delete(type, "#{entity.name} (#{entity.id})") { firestore.delete(type, entity) }
|
|
224
|
-
end
|
|
202
|
+
failures_before = @delete_failures.size + @failed_entity_fetches.size
|
|
203
|
+
begin
|
|
204
|
+
entities_for_url(url, type).each do |entity|
|
|
205
|
+
puts Paint[" Removing #{type}: #{entity.name} (ID: #{entity.id})", :black] if verbose?
|
|
206
|
+
track_delete(type, "#{entity.name} (#{entity.id})") { firestore.delete(type, entity) }
|
|
225
207
|
end
|
|
226
208
|
rescue Tools::Sync::RequestFailed, SocketError, IOError, SystemCallError, Timeout::Error,
|
|
227
209
|
JSON::ParserError => e
|
|
228
|
-
@failed_entity_fetches ||= []
|
|
229
210
|
@failed_entity_fetches << { url: url, error: e.class.name, message: e.message }
|
|
230
211
|
warn Paint["Warning: Could not fetch #{url} to remove entities: #{e.message}", :yellow]
|
|
231
212
|
end
|
|
213
|
+
|
|
214
|
+
# Keep the URL listed while any of its entities remain
|
|
215
|
+
return if (@delete_failures.size + @failed_entity_fetches.size) > failures_before
|
|
216
|
+
|
|
217
|
+
track_delete(info_type, url) { firestore.delete(info_type, url) }
|
|
218
|
+
end
|
|
219
|
+
|
|
220
|
+
# Entities (mods or tools) that came from +url+: the docs the source map attributes to it, plus, for docs
|
|
221
|
+
# with no recorded source, those matching a name+author in the file. Docs mapped to another URL
|
|
222
|
+
# (e.g. a fork publishing the same name/author) are never touched.
|
|
223
|
+
def entities_for_url(url, type)
|
|
224
|
+
collection = type == :mod ? firestore.mods : firestore.tools
|
|
225
|
+
sources = firestore.sources(type == :mod ? :mods : :tools)
|
|
226
|
+
|
|
227
|
+
entities = collection.select { |e| sources[e.id.to_s] == url }
|
|
228
|
+
unmapped = collection.reject { |e| sources.key?(e.id.to_s) }
|
|
229
|
+
return entities if unmapped.empty?
|
|
230
|
+
|
|
231
|
+
begin
|
|
232
|
+
data = retrieve_from_url(url)
|
|
233
|
+
rescue Tools::Sync::RequestGone
|
|
234
|
+
# File removed by its author: only the mapped docs can be attributed to it
|
|
235
|
+
return entities
|
|
236
|
+
end
|
|
237
|
+
|
|
238
|
+
(data[type == :mod ? :mods : :tools] || []).each do |entity_data|
|
|
239
|
+
matching = unmapped.select { |e| e.name == entity_data[:name] && e.author == entity_data[:author] }
|
|
240
|
+
if matching.size > 1 && verbose?
|
|
241
|
+
warn Paint[" Note: Found #{matching.size} entities matching '#{entity_data[:name]}' by #{entity_data[:author]}", :yellow]
|
|
242
|
+
end
|
|
243
|
+
entities.concat(matching)
|
|
244
|
+
end
|
|
245
|
+
|
|
246
|
+
entities.uniq(&:id)
|
|
232
247
|
end
|
|
233
248
|
|
|
234
|
-
def preview_cascade_deletions(
|
|
249
|
+
def preview_cascade_deletions(_repo_name, modinfo_urls, toolinfo_urls)
|
|
235
250
|
mods = []
|
|
236
251
|
tools = []
|
|
237
252
|
|
|
@@ -254,20 +269,7 @@ module Icarus
|
|
|
254
269
|
end
|
|
255
270
|
|
|
256
271
|
def fetch_entities_from_url(url, type)
|
|
257
|
-
|
|
258
|
-
entity_data_list = data[type == :mod ? :mods : :tools] || []
|
|
259
|
-
|
|
260
|
-
collection = type == :mod ? firestore.mods : firestore.tools
|
|
261
|
-
entities = []
|
|
262
|
-
|
|
263
|
-
entity_data_list.each do |entity_data|
|
|
264
|
-
matching_entities = collection.select do |e|
|
|
265
|
-
e.name == entity_data[:name] && e.author == entity_data[:author]
|
|
266
|
-
end
|
|
267
|
-
entities.concat(matching_entities)
|
|
268
|
-
end
|
|
269
|
-
|
|
270
|
-
entities
|
|
272
|
+
entities_for_url(url, type)
|
|
271
273
|
rescue Tools::Sync::RequestFailed, SocketError, IOError, SystemCallError, Timeout::Error, JSON::ParserError => e
|
|
272
274
|
warn Paint["Warning: Could not fetch #{url}: #{e.message}", :yellow] if verbose?
|
|
273
275
|
nil
|
|
@@ -292,10 +294,10 @@ module Icarus
|
|
|
292
294
|
preview[:toolinfo_urls].each { |url| puts " • #{url}" }
|
|
293
295
|
end
|
|
294
296
|
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
297
|
+
return unless preview[:tools].any?
|
|
298
|
+
|
|
299
|
+
puts " - Tools: #{preview[:tools].size}"
|
|
300
|
+
preview[:tools].each { |tool| puts " • #{tool.name} by #{tool.author} (ID: #{tool.id})" }
|
|
299
301
|
end
|
|
300
302
|
|
|
301
303
|
def track_delete(type, identifier)
|
|
@@ -8,8 +8,8 @@ module Icarus
|
|
|
8
8
|
# Base class for all subcommands
|
|
9
9
|
class SubcommandBase < Base
|
|
10
10
|
class_option :verbose,
|
|
11
|
-
|
|
12
|
-
|
|
11
|
+
aliases: "-v", type: :boolean, repeatable: true, default: [true],
|
|
12
|
+
desc: "Increase verbosity. May be repeated for even more verbosity."
|
|
13
13
|
|
|
14
14
|
no_commands do
|
|
15
15
|
def check_false
|
data/lib/icarus/mod/cli/sync.rb
CHANGED
|
@@ -13,21 +13,26 @@ module Icarus
|
|
|
13
13
|
class Sync < SubcommandBase
|
|
14
14
|
$firestore = nil
|
|
15
15
|
|
|
16
|
+
# Stage 2 refuses to delete more than this share of a collection (or to empty it) without --force
|
|
17
|
+
MASS_DELETE_RATIO = 0.25
|
|
18
|
+
MASS_DELETE_MIN = 5
|
|
19
|
+
|
|
16
20
|
class_option :dry_run, type: :boolean, default: false, desc: "Dry run (no changes will be made)"
|
|
21
|
+
class_option :force, type: :boolean, default: false,
|
|
22
|
+
desc: "Let mods/tools sync delete over #{(MASS_DELETE_RATIO * 100).to_i}% of a collection"
|
|
17
23
|
|
|
18
24
|
desc "all", "Run all sync jobs"
|
|
19
25
|
def all
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
26
|
+
# Each stage exits 1 on failure; run the rest anyway, then exit 1 if any failed
|
|
27
|
+
failed = %i[toolinfo tools modinfo mods].map do |stage|
|
|
28
|
+
puts "Running #{stage.to_s.capitalize} Sync..." if verbose?
|
|
29
|
+
invoke stage
|
|
30
|
+
false
|
|
31
|
+
rescue SystemExit => e
|
|
32
|
+
!e.success?
|
|
33
|
+
end
|
|
28
34
|
|
|
29
|
-
|
|
30
|
-
invoke :mods
|
|
35
|
+
exit 1 if failed.any?
|
|
31
36
|
end
|
|
32
37
|
|
|
33
38
|
desc "modinfo", "Reads from 'meta/repos/list' and Syncs any modinfo files we find (github only for now)"
|
|
@@ -89,8 +94,8 @@ module Icarus
|
|
|
89
94
|
|
|
90
95
|
if options[:dry_run]
|
|
91
96
|
puts Paint["Dry run; no changes made. Would have deleted #{deleted_count} duplicate #{type}.", :yellow] if verbose?
|
|
92
|
-
|
|
93
|
-
puts "Deleted #{deleted_count} duplicate #{type}."
|
|
97
|
+
elsif verbose?
|
|
98
|
+
puts "Deleted #{deleted_count} duplicate #{type}."
|
|
94
99
|
end
|
|
95
100
|
end
|
|
96
101
|
|
|
@@ -174,7 +179,7 @@ module Icarus
|
|
|
174
179
|
kept.each do |list|
|
|
175
180
|
url = sources[list.id.to_s]
|
|
176
181
|
reason = url ? "its source could not be read (#{url})" : "no recorded source yet, and a source failed"
|
|
177
|
-
warn Paint[" #{list.author ||
|
|
182
|
+
warn Paint[" #{list.author || "NoOne"}/#{list.name || "Unnamed"} (#{list.id}): #{reason}", :yellow]
|
|
178
183
|
end
|
|
179
184
|
end
|
|
180
185
|
|
|
@@ -212,6 +217,7 @@ module Icarus
|
|
|
212
217
|
end
|
|
213
218
|
rescue Icarus::Mod::Tools::Error => e
|
|
214
219
|
warn e.message
|
|
220
|
+
exit 1
|
|
215
221
|
end
|
|
216
222
|
|
|
217
223
|
def sync_list(type)
|
|
@@ -223,10 +229,12 @@ module Icarus
|
|
|
223
229
|
info_array = sync.info_array
|
|
224
230
|
|
|
225
231
|
puts "Retrieving List Data..." if verbose?
|
|
226
|
-
|
|
232
|
+
# A snapshot: creating a doc appends it to the cached collection, which would dilute the delete guard
|
|
233
|
+
list_array = sync.send(type).dup
|
|
227
234
|
|
|
228
235
|
return if options[:check]
|
|
229
236
|
|
|
237
|
+
updating = !options[:dry_run]
|
|
230
238
|
puts "Updating List Data..." if verbose?
|
|
231
239
|
info_array.each do |list|
|
|
232
240
|
verb = "Creating"
|
|
@@ -248,7 +256,7 @@ module Icarus
|
|
|
248
256
|
|
|
249
257
|
if verbose > 1
|
|
250
258
|
print format("#{verb} %-<name>60s",
|
|
251
|
-
name: "'#{list.author ||
|
|
259
|
+
name: "'#{list.author || "NoOne"}/#{list.name || "Unnamed"}'")
|
|
252
260
|
end
|
|
253
261
|
|
|
254
262
|
if options[:dry_run]
|
|
@@ -257,31 +265,68 @@ module Icarus
|
|
|
257
265
|
end
|
|
258
266
|
|
|
259
267
|
response = sync.update(list)
|
|
260
|
-
puts success_or_failure(
|
|
268
|
+
puts success_or_failure(response) if verbose > 1
|
|
261
269
|
end
|
|
262
270
|
|
|
271
|
+
unless options[:dry_run]
|
|
272
|
+
puts "Created/Updated #{info_array.count} Items" if verbose?
|
|
273
|
+
sync.save_sources
|
|
274
|
+
end
|
|
275
|
+
|
|
276
|
+
delete_array = protect_transient_sources(sync, list_array.filter { |list| sync.find_info(list).nil? })
|
|
277
|
+
|
|
263
278
|
if options[:dry_run]
|
|
279
|
+
preview_deletions(type, delete_array, list_array, info_array)
|
|
264
280
|
puts Paint["Dry run; no changes will be made", :white] if verbose?
|
|
265
281
|
return
|
|
266
282
|
end
|
|
267
283
|
|
|
268
|
-
puts "Created/Updated #{info_array.count} Items" if verbose?
|
|
269
|
-
sync.save_sources
|
|
270
|
-
|
|
271
|
-
delete_array = protect_transient_sources(sync, list_array.filter { |list| sync.find_info(list).nil? })
|
|
272
|
-
|
|
273
284
|
return unless delete_array.any?
|
|
274
285
|
|
|
286
|
+
raise Icarus::Mod::Tools::Error, mass_delete_message("Refusing", type, delete_array, list_array) if
|
|
287
|
+
mass_delete?(delete_array, list_array, info_array)
|
|
288
|
+
|
|
275
289
|
puts "Deleting outdated items..." if verbose?
|
|
276
290
|
delete_array.each do |list|
|
|
277
|
-
print "Deleting '#{list.author ||
|
|
291
|
+
print "Deleting '#{list.author || "NoOne"}/#{list.name || "Unnamed"}'#{" " * 20}" if verbose > 1
|
|
278
292
|
response = sync.delete(list)
|
|
279
|
-
puts success_or_failure(
|
|
293
|
+
puts success_or_failure(response) if verbose > 1
|
|
280
294
|
end
|
|
281
295
|
|
|
282
296
|
puts "Deleted #{delete_array.count} outdated items" if verbose?
|
|
283
297
|
rescue Icarus::Mod::Tools::Error => e
|
|
284
298
|
warn e.message
|
|
299
|
+
exit 1
|
|
300
|
+
rescue StandardError
|
|
301
|
+
# Keep the sources of the docs written before a failure, so the next run can protect them (#16)
|
|
302
|
+
sync&.save_sources if updating
|
|
303
|
+
raise
|
|
304
|
+
end
|
|
305
|
+
|
|
306
|
+
# A wave of 404s (a GitHub outage, a mass rename) looks like every author removing their file, so stage 2
|
|
307
|
+
# will not delete more than MASS_DELETE_RATIO of a collection, or empty it, unless run with --force
|
|
308
|
+
def mass_delete?(delete_array, list_array, info_array)
|
|
309
|
+
return false if options[:force] || delete_array.empty?
|
|
310
|
+
|
|
311
|
+
emptying = info_array.empty? && delete_array.size >= list_array.size
|
|
312
|
+
too_many = delete_array.size >= MASS_DELETE_MIN && delete_array.size > list_array.size * MASS_DELETE_RATIO
|
|
313
|
+
emptying || too_many
|
|
314
|
+
end
|
|
315
|
+
|
|
316
|
+
def mass_delete_message(verb, type, delete_array, list_array)
|
|
317
|
+
Paint["#{verb} to delete #{delete_array.size} of #{list_array.size} #{type}; " \
|
|
318
|
+
"check the sources, then rerun with --force if this is intended", :red, :bright]
|
|
319
|
+
end
|
|
320
|
+
|
|
321
|
+
# What a real run would delete, and whether the mass-delete brake would stop it
|
|
322
|
+
def preview_deletions(type, delete_array, list_array, info_array)
|
|
323
|
+
return unless verbose?
|
|
324
|
+
|
|
325
|
+
puts "Would delete #{delete_array.count} outdated item(s)#{":" if delete_array.any?}"
|
|
326
|
+
delete_array.each { |list| puts Paint[" #{list.author || "NoOne"}/#{list.name || "Unnamed"} (#{list.id})", :yellow] }
|
|
327
|
+
return unless mass_delete?(delete_array, list_array, info_array)
|
|
328
|
+
|
|
329
|
+
warn mass_delete_message("Would refuse", type, delete_array, list_array)
|
|
285
330
|
end
|
|
286
331
|
end
|
|
287
332
|
end
|
|
@@ -23,9 +23,8 @@ module Icarus
|
|
|
23
23
|
validator = Icarus::Mod::Tools::Validator.new(type)
|
|
24
24
|
|
|
25
25
|
puts "Validating Entries..." if verbose?
|
|
26
|
-
max_length = validator.array.map { |info| info.uniq_name.length }.max + 1
|
|
26
|
+
max_length = (validator.array.map { |info| info.uniq_name.length }.max || 0) + 1
|
|
27
27
|
|
|
28
|
-
# rubocop:disable Lint/FormatParameterMismatch
|
|
29
28
|
validator.array.each do |info|
|
|
30
29
|
print Paint[format("%s %-#{max_length}s", "Running validation steps on", info.uniq_name), :cyan, :bright] if verbose > 1
|
|
31
30
|
|
|
@@ -47,8 +46,6 @@ module Icarus
|
|
|
47
46
|
puts info.warnings.map { |warning| Paint["#{warning} in #{info.uniq_name}", :yellow] }.join("\n")
|
|
48
47
|
puts "\n" if verbose > 1
|
|
49
48
|
end
|
|
50
|
-
# rubocop:enable Lint/FormatParameterMismatch
|
|
51
|
-
|
|
52
49
|
exit exit_code
|
|
53
50
|
end
|
|
54
51
|
end
|