Icarus-Mod-Tools 2.7.0 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +55 -0
- data/.rubocop_todo.yml +142 -0
- data/.ruby-version +1 -1
- data/CHANGELOG.md +34 -0
- data/CLAUDE.md +4 -4
- data/Gemfile +3 -5
- data/Gemfile.lock +18 -27
- data/Guardfile +2 -0
- data/README.md +5 -2
- data/icarus-mod-tools.gemspec +2 -1
- data/lib/icarus/mod/cli/add.rb +53 -19
- data/lib/icarus/mod/cli/base.rb +4 -4
- data/lib/icarus/mod/cli/command.rb +6 -1
- data/lib/icarus/mod/cli/list.rb +19 -7
- data/lib/icarus/mod/cli/remove.rb +64 -62
- data/lib/icarus/mod/cli/subcommand_base.rb +2 -2
- data/lib/icarus/mod/cli/sync.rb +47 -18
- data/lib/icarus/mod/cli/validate.rb +1 -4
- data/lib/icarus/mod/config.rb +2 -2
- data/lib/icarus/mod/firestore.rb +38 -21
- data/lib/icarus/mod/github.rb +39 -22
- data/lib/icarus/mod/tools/baseinfo.rb +84 -22
- data/lib/icarus/mod/tools/modinfo.rb +1 -1
- data/lib/icarus/mod/tools/sync/entities.rb +102 -0
- data/lib/icarus/mod/tools/sync/helpers.rb +6 -2
- data/lib/icarus/mod/tools/sync/mods.rb +6 -79
- data/lib/icarus/mod/tools/sync/tools.rb +6 -81
- data/lib/icarus/mod/tools/toolinfo.rb +12 -8
- data/lib/icarus/mod/tools/validator.rb +7 -7
- data/lib/icarus/mod/version.rb +1 -1
- metadata +20 -4
- data/sig/database/sync/sync.rbs +0 -15
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require "uri"
|
|
4
|
+
|
|
3
5
|
module Icarus
|
|
4
6
|
module Mod
|
|
5
7
|
module Tools
|
|
@@ -12,6 +14,12 @@ module Icarus
|
|
|
12
14
|
|
|
13
15
|
HASHKEYS = %i[name author version compatibility description files imageURL readmeURL].freeze
|
|
14
16
|
|
|
17
|
+
# Appended to warnings for input the stricter rules reject but the old ones accepted
|
|
18
|
+
FUTURE_ERROR = "(this will become an error in a future release)"
|
|
19
|
+
|
|
20
|
+
# Keys that may be read through method_missing even when absent from the data
|
|
21
|
+
EXTRA_KEYS = %i[files fileType fileURL].freeze
|
|
22
|
+
|
|
15
23
|
# Match github.com URLs with /blob/ or /raw/ and convert to raw.githubusercontent.com
|
|
16
24
|
GITHUB_BLOB_RAW_URL_PATTERN = %r{
|
|
17
25
|
(https?)://(?:www\.)?github\.com/ # Protocol and GitHub domain (capture group 1)
|
|
@@ -33,7 +41,7 @@ module Icarus
|
|
|
33
41
|
end
|
|
34
42
|
|
|
35
43
|
def author_id
|
|
36
|
-
author.downcase.gsub(/\s+/, "_")
|
|
44
|
+
author.to_s.downcase.gsub(/\s+/, "_")
|
|
37
45
|
end
|
|
38
46
|
|
|
39
47
|
def read(data)
|
|
@@ -70,11 +78,11 @@ module Icarus
|
|
|
70
78
|
end
|
|
71
79
|
|
|
72
80
|
def uniq_name
|
|
73
|
-
"#{author.strip}/#{name.strip}"
|
|
81
|
+
"#{author.to_s.strip}/#{name.to_s.strip}"
|
|
74
82
|
end
|
|
75
83
|
|
|
76
|
-
def to_json(*
|
|
77
|
-
JSON.generate(@data, *
|
|
84
|
+
def to_json(*)
|
|
85
|
+
JSON.generate(@data, *)
|
|
78
86
|
end
|
|
79
87
|
|
|
80
88
|
def to_h
|
|
@@ -92,11 +100,11 @@ module Icarus
|
|
|
92
100
|
validate_version
|
|
93
101
|
|
|
94
102
|
%w[name author description].each do |key|
|
|
95
|
-
|
|
103
|
+
validate_required_string(key, @data[key.to_sym])
|
|
96
104
|
end
|
|
97
105
|
|
|
98
106
|
%w[imageURL readmeURL].each do |key|
|
|
99
|
-
@
|
|
107
|
+
check_url(@data[key.to_sym], "Invalid URL #{key.capitalize}: #{@data[key.to_sym] || "blank"}")
|
|
100
108
|
end
|
|
101
109
|
|
|
102
110
|
@validated = true
|
|
@@ -109,62 +117,116 @@ module Icarus
|
|
|
109
117
|
end
|
|
110
118
|
|
|
111
119
|
def file_types
|
|
112
|
-
files
|
|
120
|
+
files.is_a?(Hash) ? files.keys : []
|
|
113
121
|
end
|
|
114
122
|
|
|
115
123
|
def file_urls
|
|
116
|
-
files
|
|
124
|
+
files.is_a?(Hash) ? files.values : []
|
|
117
125
|
end
|
|
118
126
|
|
|
119
|
-
|
|
120
|
-
|
|
127
|
+
# Firestore#update guards its payload with #empty?
|
|
128
|
+
def empty?
|
|
129
|
+
@data.nil? || @data.empty?
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
# Data keys and known fields read as nil when absent; any other name (a typo) raises
|
|
133
|
+
def method_missing(method_name, *args, &)
|
|
134
|
+
return super unless known_key?(method_name)
|
|
135
|
+
|
|
136
|
+
@data[method_name.to_sym]
|
|
121
137
|
end
|
|
122
138
|
|
|
123
139
|
def respond_to_missing?(method_name, include_private = false)
|
|
124
|
-
|
|
140
|
+
known_key?(method_name) || super
|
|
125
141
|
end
|
|
126
142
|
|
|
127
143
|
private
|
|
128
144
|
|
|
145
|
+
def known_key?(method_name)
|
|
146
|
+
return false if method_name.to_s.end_with?("=", "?", "!")
|
|
147
|
+
|
|
148
|
+
@data.key?(method_name.to_sym) || self.class::HASHKEYS.include?(method_name.to_sym) || EXTRA_KEYS.include?(method_name.to_sym)
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
# The file type must be exactly one of these, with an optional leading dot
|
|
129
152
|
def filetype_pattern
|
|
153
|
+
/\A\.?(zip|pak|exmodz?)\z/i
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
# The previous, unanchored rule; a match here but not above is only a warning for now
|
|
157
|
+
def legacy_filetype_pattern
|
|
130
158
|
/(zip|pak|exmodz?)/i
|
|
131
159
|
end
|
|
132
160
|
|
|
133
|
-
def
|
|
134
|
-
|
|
161
|
+
def validate_required_string(key, value)
|
|
162
|
+
if value.nil? || value == ""
|
|
163
|
+
@errors << "#{key.capitalize} cannot be blank"
|
|
164
|
+
elsif !value.is_a?(String)
|
|
165
|
+
@errors << "#{key.capitalize} must be a string"
|
|
166
|
+
end
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
# :ok, :legacy (passed the old check only, so warn) or :invalid
|
|
170
|
+
def url_status(url)
|
|
171
|
+
return :ok if url.nil? || url == ""
|
|
172
|
+
return :invalid unless url.is_a?(String)
|
|
173
|
+
return :ok if strict_url?(url)
|
|
174
|
+
|
|
175
|
+
url.match?(URI::DEFAULT_PARSER.make_regexp) ? :legacy : :invalid
|
|
135
176
|
end
|
|
136
177
|
|
|
137
|
-
def
|
|
138
|
-
|
|
178
|
+
def strict_url?(url)
|
|
179
|
+
uri = URI.parse(url)
|
|
180
|
+
%w[http https].include?(uri.scheme&.downcase) && !uri.host.to_s.empty?
|
|
181
|
+
rescue URI::Error
|
|
182
|
+
false
|
|
183
|
+
end
|
|
139
184
|
|
|
140
|
-
|
|
185
|
+
def check_url(url, message)
|
|
186
|
+
case url_status(url)
|
|
187
|
+
when :invalid then @errors << message
|
|
188
|
+
when :legacy then @warnings << "#{message} #{FUTURE_ERROR}"
|
|
189
|
+
end
|
|
141
190
|
end
|
|
142
191
|
|
|
143
192
|
def validate_files
|
|
144
193
|
@warnings << "This mod uses deprecated fields (fileType and fileURL)" if @data.key?(:fileType) || @data.key?(:fileURL)
|
|
194
|
+
|
|
195
|
+
if !files.nil? && !files.is_a?(Hash)
|
|
196
|
+
@errors << "Files must be an object"
|
|
197
|
+
return
|
|
198
|
+
end
|
|
199
|
+
|
|
145
200
|
@warnings << "files should not be empty" if file_types.empty?
|
|
146
201
|
|
|
147
202
|
file_types.each { |file_type| validate_filetype(file_type) }
|
|
148
203
|
|
|
149
|
-
file_urls.each
|
|
150
|
-
@errors << "Invalid URL: #{file_url}" unless validate_url(file_url)
|
|
151
|
-
end
|
|
204
|
+
file_urls.each { |file_url| check_url(file_url, "Invalid URL: #{file_url}") }
|
|
152
205
|
end
|
|
153
206
|
|
|
154
207
|
def validate_filetype(file_type)
|
|
155
|
-
|
|
208
|
+
name = file_type.to_s
|
|
209
|
+
return if name.match?(filetype_pattern)
|
|
210
|
+
|
|
211
|
+
if name.match?(legacy_filetype_pattern)
|
|
212
|
+
@warnings << "Invalid fileType: #{name.upcase} #{FUTURE_ERROR}"
|
|
213
|
+
else
|
|
214
|
+
@errors << "Invalid fileType: #{name.upcase}"
|
|
215
|
+
end
|
|
156
216
|
end
|
|
157
217
|
|
|
158
218
|
def validate_version
|
|
159
219
|
if version.nil?
|
|
160
220
|
@warnings << "Version was nil, it has been defaulted to 1.0"
|
|
221
|
+
elsif !version.is_a?(String)
|
|
222
|
+
@errors << "Version must be a string"
|
|
161
223
|
else
|
|
162
|
-
@warnings << "Version should be a version string" unless
|
|
224
|
+
@warnings << "Version should be a version string" unless /\A\d+(\.\d+)*\z/.match?(version)
|
|
163
225
|
end
|
|
164
226
|
end
|
|
165
227
|
|
|
166
228
|
def normalize_github_url(url)
|
|
167
|
-
return url
|
|
229
|
+
return url unless url.is_a?(String) && !url.empty?
|
|
168
230
|
|
|
169
231
|
if url.match?(GITHUB_BLOB_RAW_URL_PATTERN)
|
|
170
232
|
normalized = url.gsub(GITHUB_BLOB_RAW_URL_PATTERN, '\1://raw.githubusercontent.com/\2/\3/\4')
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "paint"
|
|
4
|
+
require "firestore"
|
|
5
|
+
require "tools/sync/helpers"
|
|
6
|
+
|
|
7
|
+
module Icarus
|
|
8
|
+
module Mod
|
|
9
|
+
module Tools
|
|
10
|
+
module Sync
|
|
11
|
+
# Shared meta/modinfo -> mods and meta/toolinfo -> tools sync. Subclasses set TYPE (:mod or :tool),
|
|
12
|
+
# INFO (:modinfo or :toolinfo) and ENTITY (the Modinfo or Toolinfo class).
|
|
13
|
+
class Entities
|
|
14
|
+
include Helpers
|
|
15
|
+
|
|
16
|
+
def initialize(client: nil)
|
|
17
|
+
@firestore = client || Firestore.new
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
# Every entry we could read from the info URLs this run. URLs that failed transiently
|
|
21
|
+
# (see #transient_failures) contribute nothing here, so their entries must not be deleted.
|
|
22
|
+
def info_array
|
|
23
|
+
return @info_array if @info_array
|
|
24
|
+
|
|
25
|
+
@transient_failures = []
|
|
26
|
+
@info_array = @firestore.send(self.class::INFO).map do |url|
|
|
27
|
+
next unless url
|
|
28
|
+
|
|
29
|
+
entries_from(url)
|
|
30
|
+
rescue Icarus::Mod::Tools::Sync::RequestFailed => e
|
|
31
|
+
@transient_failures << url if e.transient?
|
|
32
|
+
warn "Skipped; Failed to retrieve #{url}#{" (transient; its existing entries are kept)" if e.transient?}"
|
|
33
|
+
next
|
|
34
|
+
rescue JSON::ParserError => e
|
|
35
|
+
@transient_failures << url
|
|
36
|
+
warn Paint["WARNING: Skipped; Invalid JSON in #{url}: #{e.message}", :red, :bright]
|
|
37
|
+
warn Paint["WARNING: Existing entries from #{url} are kept until it parses again; " \
|
|
38
|
+
"remove or rename the file to remove them", :red, :bright]
|
|
39
|
+
next
|
|
40
|
+
end.flatten.compact.uniq { |entry| [entry.name, entry.author] }
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# The info URLs that could not be read this run for a reason that may clear up
|
|
44
|
+
# (network error, 5xx, rate limit, invalid JSON). Gone (404/410) URLs are not included.
|
|
45
|
+
def transient_failures
|
|
46
|
+
info_array
|
|
47
|
+
@transient_failures || []
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def find(info)
|
|
51
|
+
@firestore.find_by_type(type: plural.to_s, name: info.name, author: info.author)&.id
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def find_info(info)
|
|
55
|
+
@info_array.find { |entry| entry.name == info.name && entry.author == info.author }
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
def update(info)
|
|
59
|
+
success = @firestore.update(self.class::TYPE, info, merge: false)
|
|
60
|
+
@firestore.record_source(plural, info.id, info.source_url) if success && info.source_url
|
|
61
|
+
|
|
62
|
+
success
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
# doc ID => info URL it was last synced from (see Firestore#sources)
|
|
66
|
+
def sources
|
|
67
|
+
@firestore.sources(plural)
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def save_sources
|
|
71
|
+
@firestore.save_sources(plural)
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
def delete(info)
|
|
75
|
+
@firestore.delete(self.class::TYPE, info)
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
private
|
|
79
|
+
|
|
80
|
+
def plural
|
|
81
|
+
:"#{self.class::TYPE}s"
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
# A body that is not a JSON object, or an entry that is not one, is treated as invalid JSON
|
|
85
|
+
# (transient), so one malformed file cannot abort the whole sync
|
|
86
|
+
def entries_from(url)
|
|
87
|
+
body = retrieve_from_url(url)
|
|
88
|
+
entries = body[plural] if body.is_a?(Hash)
|
|
89
|
+
raise JSON::ParserError, "no '#{plural}' array" unless entries.is_a?(Array)
|
|
90
|
+
raise JSON::ParserError, "'#{plural}' entries must be objects" unless entries.all?(Hash)
|
|
91
|
+
|
|
92
|
+
entries.filter_map do |data|
|
|
93
|
+
next unless data[:name].is_a?(String) && /[a-z0-9]+/i.match?(data[:name])
|
|
94
|
+
|
|
95
|
+
self.class::ENTITY.new(data).tap { |entry| entry.source_url = url }
|
|
96
|
+
end
|
|
97
|
+
end
|
|
98
|
+
end
|
|
99
|
+
end
|
|
100
|
+
end
|
|
101
|
+
end
|
|
102
|
+
end
|
|
@@ -39,7 +39,11 @@ module Icarus
|
|
|
39
39
|
def retrieve_from_url(url)
|
|
40
40
|
raise RequestFailed, "Invalid URI: '#{url}'" unless url && url =~ URI::DEFAULT_PARSER.make_regexp
|
|
41
41
|
|
|
42
|
-
uri =
|
|
42
|
+
uri = begin
|
|
43
|
+
URI(url)
|
|
44
|
+
rescue URI::InvalidURIError
|
|
45
|
+
raise RequestFailed, "Invalid URI: '#{url}'"
|
|
46
|
+
end
|
|
43
47
|
http = Net::HTTP.new(uri.host, uri.port)
|
|
44
48
|
|
|
45
49
|
if uri.scheme == "https"
|
|
@@ -56,7 +60,7 @@ module Icarus
|
|
|
56
60
|
end
|
|
57
61
|
|
|
58
62
|
res = begin
|
|
59
|
-
http.get(uri.
|
|
63
|
+
http.get(uri.request_uri)
|
|
60
64
|
rescue *NETWORK_ERRORS => e
|
|
61
65
|
raise RequestTransient, "HTTP Request failed for #{url} (#{e.class}): #{e.message}"
|
|
62
66
|
end
|
|
@@ -1,93 +1,20 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require "
|
|
4
|
-
require "firestore"
|
|
5
|
-
require "tools/sync/helpers"
|
|
3
|
+
require "tools/sync/entities"
|
|
6
4
|
|
|
7
5
|
module Icarus
|
|
8
6
|
module Mod
|
|
9
7
|
module Tools
|
|
10
8
|
module Sync
|
|
11
|
-
#
|
|
12
|
-
class Mods
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
@firestore = client || Firestore.new
|
|
17
|
-
end
|
|
9
|
+
# Syncs meta/modinfo to the mods collection (see Entities)
|
|
10
|
+
class Mods < Entities
|
|
11
|
+
TYPE = :mod
|
|
12
|
+
INFO = :modinfo
|
|
13
|
+
ENTITY = Icarus::Mod::Tools::Modinfo
|
|
18
14
|
|
|
19
15
|
def mods
|
|
20
16
|
@firestore.mods
|
|
21
17
|
end
|
|
22
|
-
|
|
23
|
-
# Every mod we could read from the modinfo URLs this run. URLs that failed transiently
|
|
24
|
-
# (see #transient_failures) contribute nothing here, so their mods must not be deleted.
|
|
25
|
-
def info_array
|
|
26
|
-
return @info_array if @info_array
|
|
27
|
-
|
|
28
|
-
@transient_failures = []
|
|
29
|
-
@info_array = @firestore.modinfo.map do |url|
|
|
30
|
-
mods_from(url)
|
|
31
|
-
rescue Icarus::Mod::Tools::Sync::RequestFailed => e
|
|
32
|
-
@transient_failures << url if e.transient?
|
|
33
|
-
warn "Skipped; Failed to retrieve #{url}#{' (transient; its existing entries are kept)' if e.transient?}"
|
|
34
|
-
next
|
|
35
|
-
rescue JSON::ParserError => e
|
|
36
|
-
@transient_failures << url
|
|
37
|
-
warn Paint["WARNING: Skipped; Invalid JSON in #{url}: #{e.message}", :red, :bright]
|
|
38
|
-
warn Paint["WARNING: Existing entries from #{url} are kept until it parses again; " \
|
|
39
|
-
"remove or rename the file to remove them", :red, :bright]
|
|
40
|
-
next
|
|
41
|
-
end.flatten.compact.uniq { |mod| [mod.name, mod.author] }
|
|
42
|
-
end
|
|
43
|
-
|
|
44
|
-
# The modinfo URLs that could not be read this run for a reason that may clear up
|
|
45
|
-
# (network error, 5xx, rate limit, invalid JSON). Gone (404/410) URLs are not included.
|
|
46
|
-
def transient_failures
|
|
47
|
-
info_array
|
|
48
|
-
@transient_failures || []
|
|
49
|
-
end
|
|
50
|
-
|
|
51
|
-
def find(modinfo)
|
|
52
|
-
@firestore.find_by_type(type: "mods", name: modinfo.name, author: modinfo.author)&.id
|
|
53
|
-
end
|
|
54
|
-
|
|
55
|
-
def find_info(modinfo)
|
|
56
|
-
@info_array.find { |mod| mod.name == modinfo.name && mod.author == modinfo.author }
|
|
57
|
-
end
|
|
58
|
-
|
|
59
|
-
def update(modinfo)
|
|
60
|
-
success = @firestore.update(:mod, modinfo, merge: false)
|
|
61
|
-
@firestore.record_source(:mods, modinfo.id, modinfo.source_url) if success && modinfo.source_url
|
|
62
|
-
|
|
63
|
-
success
|
|
64
|
-
end
|
|
65
|
-
|
|
66
|
-
# doc ID => modinfo URL it was last synced from (see Firestore#sources)
|
|
67
|
-
def sources
|
|
68
|
-
@firestore.sources(:mods)
|
|
69
|
-
end
|
|
70
|
-
|
|
71
|
-
def save_sources
|
|
72
|
-
@firestore.save_sources(:mods)
|
|
73
|
-
end
|
|
74
|
-
|
|
75
|
-
def delete(modinfo)
|
|
76
|
-
@firestore.delete(:mod, modinfo)
|
|
77
|
-
end
|
|
78
|
-
|
|
79
|
-
private
|
|
80
|
-
|
|
81
|
-
def mods_from(url)
|
|
82
|
-
mods = retrieve_from_url(url)[:mods]
|
|
83
|
-
raise JSON::ParserError, "no 'mods' array" unless mods.is_a?(Array)
|
|
84
|
-
|
|
85
|
-
mods.filter_map do |data|
|
|
86
|
-
next unless /[a-z0-9]+/i.match?(data[:name])
|
|
87
|
-
|
|
88
|
-
Icarus::Mod::Tools::Modinfo.new(data).tap { |mod| mod.source_url = url }
|
|
89
|
-
end
|
|
90
|
-
end
|
|
91
18
|
end
|
|
92
19
|
end
|
|
93
20
|
end
|
|
@@ -1,95 +1,20 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require "
|
|
4
|
-
require "firestore"
|
|
5
|
-
require "tools/sync/helpers"
|
|
3
|
+
require "tools/sync/entities"
|
|
6
4
|
|
|
7
5
|
module Icarus
|
|
8
6
|
module Mod
|
|
9
7
|
module Tools
|
|
10
8
|
module Sync
|
|
11
|
-
#
|
|
12
|
-
class Tools
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
@firestore = client || Firestore.new
|
|
17
|
-
end
|
|
9
|
+
# Syncs meta/toolinfo to the tools collection (see Entities)
|
|
10
|
+
class Tools < Entities
|
|
11
|
+
TYPE = :tool
|
|
12
|
+
INFO = :toolinfo
|
|
13
|
+
ENTITY = Icarus::Mod::Tools::Toolinfo
|
|
18
14
|
|
|
19
15
|
def tools
|
|
20
16
|
@firestore.tools
|
|
21
17
|
end
|
|
22
|
-
|
|
23
|
-
# Every tool we could read from the toolinfo URLs this run. URLs that failed transiently
|
|
24
|
-
# (see #transient_failures) contribute nothing here, so their tools must not be deleted.
|
|
25
|
-
def info_array
|
|
26
|
-
return @info_array if @info_array
|
|
27
|
-
|
|
28
|
-
@transient_failures = []
|
|
29
|
-
@info_array = @firestore.toolinfo.map do |url|
|
|
30
|
-
next unless url
|
|
31
|
-
|
|
32
|
-
tools_from(url)
|
|
33
|
-
rescue Icarus::Mod::Tools::Sync::RequestFailed => e
|
|
34
|
-
@transient_failures << url if e.transient?
|
|
35
|
-
warn "Skipped; Failed to retrieve #{url}#{' (transient; its existing entries are kept)' if e.transient?}"
|
|
36
|
-
next
|
|
37
|
-
rescue JSON::ParserError => e
|
|
38
|
-
@transient_failures << url
|
|
39
|
-
warn Paint["WARNING: Skipped; Invalid JSON in #{url}: #{e.message}", :red, :bright]
|
|
40
|
-
warn Paint["WARNING: Existing entries from #{url} are kept until it parses again; " \
|
|
41
|
-
"remove or rename the file to remove them", :red, :bright]
|
|
42
|
-
next
|
|
43
|
-
end.flatten.compact.uniq { |tool| [tool.name, tool.author] }
|
|
44
|
-
end
|
|
45
|
-
|
|
46
|
-
# The toolinfo URLs that could not be read this run for a reason that may clear up
|
|
47
|
-
# (network error, 5xx, rate limit, invalid JSON). Gone (404/410) URLs are not included.
|
|
48
|
-
def transient_failures
|
|
49
|
-
info_array
|
|
50
|
-
@transient_failures || []
|
|
51
|
-
end
|
|
52
|
-
|
|
53
|
-
def find(toolinfo)
|
|
54
|
-
@firestore.find_by_type(type: "tools", name: toolinfo.name, author: toolinfo.author)&.id
|
|
55
|
-
end
|
|
56
|
-
|
|
57
|
-
def find_info(toolinfo)
|
|
58
|
-
@info_array.find { |tool| tool.name == toolinfo.name && tool.author == toolinfo.author }
|
|
59
|
-
end
|
|
60
|
-
|
|
61
|
-
def update(toolinfo)
|
|
62
|
-
success = @firestore.update(:tool, toolinfo, merge: false)
|
|
63
|
-
@firestore.record_source(:tools, toolinfo.id, toolinfo.source_url) if success && toolinfo.source_url
|
|
64
|
-
|
|
65
|
-
success
|
|
66
|
-
end
|
|
67
|
-
|
|
68
|
-
# doc ID => toolinfo URL it was last synced from (see Firestore#sources)
|
|
69
|
-
def sources
|
|
70
|
-
@firestore.sources(:tools)
|
|
71
|
-
end
|
|
72
|
-
|
|
73
|
-
def save_sources
|
|
74
|
-
@firestore.save_sources(:tools)
|
|
75
|
-
end
|
|
76
|
-
|
|
77
|
-
def delete(toolinfo)
|
|
78
|
-
@firestore.delete(:tool, toolinfo)
|
|
79
|
-
end
|
|
80
|
-
|
|
81
|
-
private
|
|
82
|
-
|
|
83
|
-
def tools_from(url)
|
|
84
|
-
tools = retrieve_from_url(url)[:tools]
|
|
85
|
-
raise JSON::ParserError, "no 'tools' array" unless tools.is_a?(Array)
|
|
86
|
-
|
|
87
|
-
tools.filter_map do |data|
|
|
88
|
-
next unless /[a-z0-9]+/i.match?(data[:name])
|
|
89
|
-
|
|
90
|
-
Icarus::Mod::Tools::Toolinfo.new(data).tap { |tool| tool.source_url = url }
|
|
91
|
-
end
|
|
92
|
-
end
|
|
93
18
|
end
|
|
94
19
|
end
|
|
95
20
|
end
|
|
@@ -19,6 +19,15 @@ module Icarus
|
|
|
19
19
|
HASHKEYS.each_with_object({}) { |key, hash| hash[key] = @data[key] }
|
|
20
20
|
end
|
|
21
21
|
|
|
22
|
+
def validate
|
|
23
|
+
return true if @validated
|
|
24
|
+
|
|
25
|
+
validate_filetype(fileType)
|
|
26
|
+
check_url(fileURL, "Invalid fileURL: #{@data[:fileURL] || "blank"}")
|
|
27
|
+
|
|
28
|
+
super
|
|
29
|
+
end
|
|
30
|
+
|
|
22
31
|
private
|
|
23
32
|
|
|
24
33
|
def normalize_github_urls_in_data
|
|
@@ -32,16 +41,11 @@ module Icarus
|
|
|
32
41
|
end
|
|
33
42
|
|
|
34
43
|
def filetype_pattern
|
|
35
|
-
|
|
44
|
+
/\A\.?(zip|exe)\z/i
|
|
36
45
|
end
|
|
37
46
|
|
|
38
|
-
def
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
validate_filetype(fileType)
|
|
42
|
-
@errors << "Invalid fileURL: #{@data[:fileURL] || "blank"}" unless validate_url(fileURL)
|
|
43
|
-
|
|
44
|
-
super
|
|
47
|
+
def legacy_filetype_pattern
|
|
48
|
+
/(zip|exe)/i
|
|
45
49
|
end
|
|
46
50
|
end
|
|
47
51
|
end
|
|
@@ -9,13 +9,13 @@ module Icarus
|
|
|
9
9
|
|
|
10
10
|
def initialize(type)
|
|
11
11
|
@array = case type
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
12
|
+
when :modinfo
|
|
13
|
+
Sync::Mods.new.info_array
|
|
14
|
+
when :toolinfo
|
|
15
|
+
Sync::Tools.new.info_array
|
|
16
|
+
else
|
|
17
|
+
raise ArgumentError, "Invalid type: #{type}"
|
|
18
|
+
end
|
|
19
19
|
end
|
|
20
20
|
end
|
|
21
21
|
end
|
data/lib/icarus/mod/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: Icarus-Mod-Tools
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version:
|
|
4
|
+
version: 3.0.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Donovan Young
|
|
@@ -37,6 +37,20 @@ dependencies:
|
|
|
37
37
|
- - "~>"
|
|
38
38
|
- !ruby/object:Gem::Version
|
|
39
39
|
version: '6.0'
|
|
40
|
+
- !ruby/object:Gem::Dependency
|
|
41
|
+
name: ostruct
|
|
42
|
+
requirement: !ruby/object:Gem::Requirement
|
|
43
|
+
requirements:
|
|
44
|
+
- - "~>"
|
|
45
|
+
- !ruby/object:Gem::Version
|
|
46
|
+
version: '0.6'
|
|
47
|
+
type: :runtime
|
|
48
|
+
prerelease: false
|
|
49
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
50
|
+
requirements:
|
|
51
|
+
- - "~>"
|
|
52
|
+
- !ruby/object:Gem::Version
|
|
53
|
+
version: '0.6'
|
|
40
54
|
- !ruby/object:Gem::Dependency
|
|
41
55
|
name: paint
|
|
42
56
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -75,6 +89,8 @@ extra_rdoc_files: []
|
|
|
75
89
|
files:
|
|
76
90
|
- ".claude-context.md"
|
|
77
91
|
- ".rspec"
|
|
92
|
+
- ".rubocop.yml"
|
|
93
|
+
- ".rubocop_todo.yml"
|
|
78
94
|
- ".ruby-version"
|
|
79
95
|
- CHANGELOG.md
|
|
80
96
|
- CLAUDE.md
|
|
@@ -101,6 +117,7 @@ files:
|
|
|
101
117
|
- lib/icarus/mod/tools.rb
|
|
102
118
|
- lib/icarus/mod/tools/baseinfo.rb
|
|
103
119
|
- lib/icarus/mod/tools/modinfo.rb
|
|
120
|
+
- lib/icarus/mod/tools/sync/entities.rb
|
|
104
121
|
- lib/icarus/mod/tools/sync/helpers.rb
|
|
105
122
|
- lib/icarus/mod/tools/sync/modinfo_list.rb
|
|
106
123
|
- lib/icarus/mod/tools/sync/mods.rb
|
|
@@ -109,7 +126,6 @@ files:
|
|
|
109
126
|
- lib/icarus/mod/tools/toolinfo.rb
|
|
110
127
|
- lib/icarus/mod/tools/validator.rb
|
|
111
128
|
- lib/icarus/mod/version.rb
|
|
112
|
-
- sig/database/sync/sync.rbs
|
|
113
129
|
homepage: https://github.com/DonovanMods/icarus-mod-tools
|
|
114
130
|
licenses: []
|
|
115
131
|
metadata:
|
|
@@ -125,14 +141,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
|
125
141
|
requirements:
|
|
126
142
|
- - ">="
|
|
127
143
|
- !ruby/object:Gem::Version
|
|
128
|
-
version: '
|
|
144
|
+
version: '4.0'
|
|
129
145
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
130
146
|
requirements:
|
|
131
147
|
- - ">="
|
|
132
148
|
- !ruby/object:Gem::Version
|
|
133
149
|
version: '0'
|
|
134
150
|
requirements: []
|
|
135
|
-
rubygems_version:
|
|
151
|
+
rubygems_version: 4.0.20
|
|
136
152
|
specification_version: 4
|
|
137
153
|
summary: Various tools for Icarus Modding
|
|
138
154
|
test_files: []
|
data/sig/database/sync/sync.rbs
DELETED
|
@@ -1,15 +0,0 @@
|
|
|
1
|
-
# See the writing guide of rbs: https://github.com/ruby/rbs#guides
|
|
2
|
-
#
|
|
3
|
-
module Icarus
|
|
4
|
-
module Mod
|
|
5
|
-
class Github
|
|
6
|
-
attr_reader :client: Octokit::Client
|
|
7
|
-
attr_accessor :repo: String
|
|
8
|
-
|
|
9
|
-
def initialize: (void) -> void
|
|
10
|
-
def all_files: (String) -> Array[String]
|
|
11
|
-
|
|
12
|
-
VERSION: String
|
|
13
|
-
end
|
|
14
|
-
end
|
|
15
|
-
end
|