Icarus-Mod-Tools 2.6.0 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +55 -0
- data/.rubocop_todo.yml +142 -0
- data/.ruby-version +1 -1
- data/CHANGELOG.md +48 -0
- data/CLAUDE.md +6 -4
- data/Gemfile +3 -5
- data/Gemfile.lock +18 -27
- data/Guardfile +2 -0
- data/README.md +9 -1
- data/icarus-mod-tools.gemspec +2 -1
- data/imtconfig.sample.json +2 -0
- data/lib/icarus/mod/cli/add.rb +53 -19
- data/lib/icarus/mod/cli/base.rb +4 -4
- data/lib/icarus/mod/cli/command.rb +6 -1
- data/lib/icarus/mod/cli/list.rb +19 -7
- data/lib/icarus/mod/cli/remove.rb +65 -62
- data/lib/icarus/mod/cli/subcommand_base.rb +2 -2
- data/lib/icarus/mod/cli/sync.rb +139 -35
- data/lib/icarus/mod/cli/validate.rb +1 -4
- data/lib/icarus/mod/config.rb +2 -2
- data/lib/icarus/mod/firestore.rb +132 -15
- data/lib/icarus/mod/github.rb +39 -22
- data/lib/icarus/mod/tools/baseinfo.rb +86 -22
- data/lib/icarus/mod/tools/modinfo.rb +1 -1
- data/lib/icarus/mod/tools/sync/entities.rb +102 -0
- data/lib/icarus/mod/tools/sync/helpers.rb +38 -4
- data/lib/icarus/mod/tools/sync/modinfo_list.rb +30 -10
- data/lib/icarus/mod/tools/sync/mods.rb +6 -37
- data/lib/icarus/mod/tools/sync/toolinfo_list.rb +30 -10
- data/lib/icarus/mod/tools/sync/tools.rb +6 -39
- data/lib/icarus/mod/tools/toolinfo.rb +12 -8
- data/lib/icarus/mod/tools/validator.rb +7 -7
- data/lib/icarus/mod/version.rb +1 -1
- metadata +20 -4
- data/sig/database/sync/sync.rbs +0 -15
data/lib/icarus/mod/firestore.rb
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require "digest"
|
|
3
4
|
require "google/cloud/firestore"
|
|
4
5
|
require "tools/modinfo"
|
|
5
6
|
require "tools/toolinfo"
|
|
@@ -40,19 +41,71 @@ module Icarus
|
|
|
40
41
|
@tools ||= list(:tools)
|
|
41
42
|
end
|
|
42
43
|
|
|
44
|
+
# Internal map of mod/tool doc ID => the modinfo/toolinfo URL it was last synced from, kept in
|
|
45
|
+
# meta/mod_sources and meta/tool_sources ({ sources: { "<doc id>" => "<url>" } }). Sync uses it to protect
|
|
46
|
+
# only the docs of a source that failed transiently (#16). Consumers never read it; mods/tools docs are unchanged.
|
|
47
|
+
def sources(type)
|
|
48
|
+
key = pluralize(type)
|
|
49
|
+
@sources ||= {}
|
|
50
|
+
@sources[key] ||= begin
|
|
51
|
+
data = @client.doc(sources_path(key)).get&.data
|
|
52
|
+
(data&.dig(:sources) || {}).transform_keys(&:to_s)
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
# Records (in memory) that doc +id+ came from +url+; #save_sources writes it
|
|
57
|
+
def record_source(type, id, url)
|
|
58
|
+
return if id.nil? || url.nil?
|
|
59
|
+
|
|
60
|
+
map = sources(type)
|
|
61
|
+
return if map[id.to_s] == url
|
|
62
|
+
|
|
63
|
+
map[id.to_s] = url
|
|
64
|
+
sources_changed(type) << id.to_s
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
# Writes the entries changed since the map was read or last saved. Only those keys are written (merged),
|
|
68
|
+
# so entries another sync host recorded meanwhile are kept.
|
|
69
|
+
def save_sources(type)
|
|
70
|
+
changed = sources_changed(type)
|
|
71
|
+
return true if changed.empty?
|
|
72
|
+
|
|
73
|
+
map = sources(type)
|
|
74
|
+
entries = changed.to_h { |id| [id, map.fetch(id) { Google::Cloud::Firestore::FieldValue.delete }] }
|
|
75
|
+
response = @client.doc(sources_path(pluralize(type))).set({ sources: entries }, merge: true)
|
|
76
|
+
success = response.is_a?(Google::Cloud::Firestore::CommitResponse::WriteResult)
|
|
77
|
+
changed.clear if success
|
|
78
|
+
|
|
79
|
+
success
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
# Gives doc +to+ the source of doc +from+ (cleanup copies +from+'s data onto +to+), and saves
|
|
83
|
+
def move_source(type, from:, to:)
|
|
84
|
+
url = sources(type)[from.to_s]
|
|
85
|
+
return true unless url
|
|
86
|
+
|
|
87
|
+
record_source(type, to, url)
|
|
88
|
+
save_sources(type)
|
|
89
|
+
end
|
|
90
|
+
|
|
43
91
|
def find_by_type(type:, name:, author:)
|
|
44
92
|
instance_variable_get(:"@#{type}").find { |obj| obj.name == name && obj.author == author }
|
|
45
93
|
end
|
|
46
94
|
|
|
95
|
+
# A deterministic document id for a new :mods or :tools document, so a mod that is deleted and later
|
|
96
|
+
# re-added gets the same id back (#16). Author and name are case- and whitespace-normalised, and the
|
|
97
|
+
# collection is part of the key so a mod and a tool with the same author and name do not collide.
|
|
98
|
+
def stable_id(type, author:, name:)
|
|
99
|
+
key = [pluralize(type), author.to_s.strip.downcase, name.to_s.strip.downcase].join("\0")
|
|
100
|
+
Digest::SHA256.hexdigest(key)[0, 20]
|
|
101
|
+
end
|
|
102
|
+
|
|
47
103
|
def update(type, payload, merge: false)
|
|
48
104
|
raise "You must specify a payload to update" if payload&.empty? || payload.nil?
|
|
49
105
|
|
|
50
106
|
response = case type.to_sym
|
|
51
|
-
when :modinfo, :toolinfo
|
|
52
|
-
|
|
53
|
-
@client.doc(collections.meta.send(type)).set({ list: update_array }, merge:) if update_array.any?
|
|
54
|
-
when :repositories
|
|
55
|
-
@client.doc(collections.meta.repositories).set({ list: payload }, merge:)
|
|
107
|
+
when :modinfo, :toolinfo, :repositories
|
|
108
|
+
update_list(type, [payload].flatten.uniq)
|
|
56
109
|
when :mod, :tool
|
|
57
110
|
create_or_update(pluralize(type), payload, merge:)
|
|
58
111
|
else
|
|
@@ -62,6 +115,19 @@ module Icarus
|
|
|
62
115
|
response.is_a?(Google::Cloud::Firestore::DocumentReference) || response.is_a?(Google::Cloud::Firestore::CommitResponse::WriteResult)
|
|
63
116
|
end
|
|
64
117
|
|
|
118
|
+
# Overwrites the :mod or :tool document at +id+ with +data+ (merge: false), keeping the document's id.
|
|
119
|
+
# Used by `imt sync cleanup` to copy the newest duplicate's data onto the oldest one (#16).
|
|
120
|
+
def replace_data(type, id:, data:)
|
|
121
|
+
raise "Invalid type: #{type}" unless %i[mod tool].include?(type.to_sym)
|
|
122
|
+
|
|
123
|
+
collection = pluralize(type)
|
|
124
|
+
response = @client.doc("#{collections.send(collection)}/#{id}").set(data, merge: false)
|
|
125
|
+
success = response.is_a?(Google::Cloud::Firestore::CommitResponse::WriteResult)
|
|
126
|
+
replace_cached(collection, id, data) if success
|
|
127
|
+
|
|
128
|
+
success
|
|
129
|
+
end
|
|
130
|
+
|
|
65
131
|
def delete(type, payload)
|
|
66
132
|
case type.to_sym
|
|
67
133
|
when :mod, :tool
|
|
@@ -71,13 +137,16 @@ module Icarus
|
|
|
71
137
|
cache_var = type == :mod ? :@mods : :@tools
|
|
72
138
|
cached_collection = instance_variable_get(cache_var)
|
|
73
139
|
cached_collection&.delete_if { |item| item.id == payload.id }
|
|
140
|
+
forget_source(type, payload.id)
|
|
74
141
|
end
|
|
75
142
|
when :modinfo, :toolinfo, :repositories
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
response = @client.doc(collections.meta.send(type)).set(
|
|
143
|
+
# Removed on the server, so entries another host added meanwhile are kept
|
|
144
|
+
values = [payload].flatten
|
|
145
|
+
response = @client.doc(collections.meta.send(type)).set(
|
|
146
|
+
{ list: Google::Cloud::Firestore::FieldValue.array_delete(*values) }, merge: true
|
|
147
|
+
)
|
|
79
148
|
# Invalidate cache if Firestore update was successful
|
|
80
|
-
instance_variable_set(:"@#{type}",
|
|
149
|
+
instance_variable_set(:"@#{type}", send(type) - values) if response.is_a?(Google::Cloud::Firestore::CommitResponse::WriteResult)
|
|
81
150
|
else
|
|
82
151
|
raise "Invalid type: #{type}"
|
|
83
152
|
end
|
|
@@ -90,7 +159,8 @@ module Icarus
|
|
|
90
159
|
def list(type)
|
|
91
160
|
case type.to_sym
|
|
92
161
|
when :modinfo, :toolinfo, :repositories
|
|
93
|
-
@client.doc(collections.meta.send(type)).get
|
|
162
|
+
snapshot = @client.doc(collections.meta.send(type)).get
|
|
163
|
+
snapshot.missing? ? [] : snapshot[:list] || []
|
|
94
164
|
when :mods, :tools
|
|
95
165
|
@client.col(collections.send(type)).get.map do |doc|
|
|
96
166
|
klass = type == :mods ? Icarus::Mod::Tools::Modinfo : Icarus::Mod::Tools::Toolinfo
|
|
@@ -101,20 +171,67 @@ module Icarus
|
|
|
101
171
|
end
|
|
102
172
|
end
|
|
103
173
|
|
|
174
|
+
# Adds +values+ to a meta list. The union happens on the server, so two sync hosts (or a sync and `imt add`)
|
|
175
|
+
# writing at once keep each other's additions. The cache is refreshed so a later stage of `sync all` sees them.
|
|
176
|
+
def update_list(type, values)
|
|
177
|
+
response = @client.doc(collections.meta.send(type)).set(
|
|
178
|
+
{ list: Google::Cloud::Firestore::FieldValue.array_union(*values) }, merge: true
|
|
179
|
+
)
|
|
180
|
+
if response.is_a?(Google::Cloud::Firestore::CommitResponse::WriteResult)
|
|
181
|
+
instance_variable_set(:"@#{type}", (send(type) + values).uniq)
|
|
182
|
+
end
|
|
183
|
+
|
|
184
|
+
response
|
|
185
|
+
end
|
|
186
|
+
|
|
104
187
|
def create_or_update(type, payload, merge:)
|
|
105
188
|
doc_id = payload.id || find_by_type(type:, name: payload.name, author: payload.author)&.id
|
|
106
189
|
|
|
107
190
|
return @client.doc("#{collections.send(type)}/#{doc_id}").set(payload.to_h, merge:) if doc_id
|
|
108
191
|
|
|
109
|
-
|
|
192
|
+
# New document: write it at its stable id (set creates it, or updates a doc a previous re-add left there).
|
|
193
|
+
# Only if that id is already held by a different mod (same author/name in another case) fall back to add.
|
|
194
|
+
cached_collection = instance_variable_get(:"@#{type}")
|
|
195
|
+
new_id = stable_id(type, author: payload.author, name: payload.name)
|
|
196
|
+
if cached_collection&.any? { |item| item.id == new_id }
|
|
197
|
+
response = @client.col(collections.send(type)).add(payload.to_h)
|
|
198
|
+
new_id = response.document_id
|
|
199
|
+
else
|
|
200
|
+
response = @client.doc("#{collections.send(type)}/#{new_id}").set(payload.to_h, merge:)
|
|
201
|
+
end
|
|
110
202
|
|
|
111
203
|
# Update cache to prevent duplicate creation within the same sync run
|
|
112
|
-
payload.id =
|
|
113
|
-
cache_var = :"@#{type}"
|
|
114
|
-
cached_collection = instance_variable_get(cache_var)
|
|
204
|
+
payload.id = new_id
|
|
115
205
|
cached_collection&.push(payload)
|
|
116
206
|
|
|
117
|
-
|
|
207
|
+
response
|
|
208
|
+
end
|
|
209
|
+
|
|
210
|
+
def forget_source(type, id)
|
|
211
|
+
return unless sources(type).delete(id.to_s)
|
|
212
|
+
|
|
213
|
+
sources_changed(type) << id.to_s
|
|
214
|
+
save_sources(type)
|
|
215
|
+
end
|
|
216
|
+
|
|
217
|
+
# Doc IDs whose source entry changed (or was removed) since the map was read or last saved
|
|
218
|
+
def sources_changed(type)
|
|
219
|
+
@sources_changed ||= Hash.new { |hash, key| hash[key] = Set.new }
|
|
220
|
+
@sources_changed[pluralize(type)]
|
|
221
|
+
end
|
|
222
|
+
|
|
223
|
+
def sources_path(collection)
|
|
224
|
+
name = :"#{collection.chomp("s")}_sources"
|
|
225
|
+
collections.meta&.send(name) || "meta/#{name}"
|
|
226
|
+
end
|
|
227
|
+
|
|
228
|
+
def replace_cached(collection, id, data)
|
|
229
|
+
cached_collection = instance_variable_get(:"@#{collection}")
|
|
230
|
+
index = cached_collection&.index { |item| item.id == id }
|
|
231
|
+
return unless index
|
|
232
|
+
|
|
233
|
+
old = cached_collection[index]
|
|
234
|
+
cached_collection[index] = old.class.new(data, id:, created: old.created_at, updated: Time.now)
|
|
118
235
|
end
|
|
119
236
|
|
|
120
237
|
def pluralize(type)
|
data/lib/icarus/mod/github.rb
CHANGED
|
@@ -12,6 +12,7 @@ module Icarus
|
|
|
12
12
|
self.repository = repo if repo
|
|
13
13
|
@client = Octokit::Client.new(access_token: Config.github.token)
|
|
14
14
|
@resources = []
|
|
15
|
+
@cache = {}
|
|
15
16
|
end
|
|
16
17
|
|
|
17
18
|
def repository
|
|
@@ -22,6 +23,7 @@ module Icarus
|
|
|
22
23
|
|
|
23
24
|
def repository=(repo)
|
|
24
25
|
@resources = [] # reset the resources cache
|
|
26
|
+
@cache = {}
|
|
25
27
|
@repository = repo.gsub(%r{https?://.*github\.com/}, "")
|
|
26
28
|
end
|
|
27
29
|
|
|
@@ -29,33 +31,48 @@ module Icarus
|
|
|
29
31
|
# path: the path to search in
|
|
30
32
|
# cache: whether to use the cached resources
|
|
31
33
|
# recursive: whether to recursively search subdirectories
|
|
34
|
+
# Results are cached per [path, recursive]; the top-level listing also lives in #resources
|
|
32
35
|
def all_files(path: nil, cache: true, recursive: false, &block)
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
end
|
|
45
|
-
|
|
46
|
-
block&.call(entry)
|
|
47
|
-
@resources << entry # cache the file
|
|
48
|
-
end
|
|
49
|
-
rescue Octokit::NotFound
|
|
50
|
-
warn "WARNING: Could not access #{repository}: 404 - not found"
|
|
51
|
-
end
|
|
36
|
+
key = [path, recursive]
|
|
37
|
+
cached = key == [nil, false] ? @resources : @cache[key]
|
|
38
|
+
|
|
39
|
+
# If we've already been called with these arguments for this repository, use the cached resources
|
|
40
|
+
files = (cached if cache && cached&.any?) || fetch_files(path, recursive).tap do |found|
|
|
41
|
+
key == [nil, false] ? @resources = found : @cache[key] = found
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
if block
|
|
45
|
+
files.each { |file| block.call(file) }
|
|
46
|
+
return nil
|
|
52
47
|
end
|
|
53
48
|
|
|
54
|
-
|
|
49
|
+
files
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
# Returns the first file whose name equals the given name (case-insensitive)
|
|
53
|
+
def find(name)
|
|
54
|
+
all_files { |file| return file if file[:name].to_s.casecmp?(name.to_s) }
|
|
55
55
|
end
|
|
56
56
|
|
|
57
|
-
|
|
58
|
-
|
|
57
|
+
private
|
|
58
|
+
|
|
59
|
+
# Fetches the files (not directories) under path from the API
|
|
60
|
+
def fetch_files(path, recursive)
|
|
61
|
+
found = []
|
|
62
|
+
|
|
63
|
+
@client.contents(repository, path:).each do |entry|
|
|
64
|
+
if entry[:type] == "dir"
|
|
65
|
+
found.concat(fetch_files(entry[:path], true)) if recursive
|
|
66
|
+
next # we don't need directories in our output
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
found << entry
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
found
|
|
73
|
+
rescue Octokit::NotFound
|
|
74
|
+
warn "WARNING: Could not access #{repository}: 404 - not found"
|
|
75
|
+
found
|
|
59
76
|
end
|
|
60
77
|
end
|
|
61
78
|
end
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require "uri"
|
|
4
|
+
|
|
3
5
|
module Icarus
|
|
4
6
|
module Mod
|
|
5
7
|
module Tools
|
|
@@ -7,9 +9,17 @@ module Icarus
|
|
|
7
9
|
class Baseinfo
|
|
8
10
|
attr_reader :data
|
|
9
11
|
attr_accessor :id, :created_at, :updated_at
|
|
12
|
+
# The modinfo/toolinfo URL this entry was read from during sync; never written to the entry's doc
|
|
13
|
+
attr_accessor :source_url
|
|
10
14
|
|
|
11
15
|
HASHKEYS = %i[name author version compatibility description files imageURL readmeURL].freeze
|
|
12
16
|
|
|
17
|
+
# Appended to warnings for input the stricter rules reject but the old ones accepted
|
|
18
|
+
FUTURE_ERROR = "(this will become an error in a future release)"
|
|
19
|
+
|
|
20
|
+
# Keys that may be read through method_missing even when absent from the data
|
|
21
|
+
EXTRA_KEYS = %i[files fileType fileURL].freeze
|
|
22
|
+
|
|
13
23
|
# Match github.com URLs with /blob/ or /raw/ and convert to raw.githubusercontent.com
|
|
14
24
|
GITHUB_BLOB_RAW_URL_PATTERN = %r{
|
|
15
25
|
(https?)://(?:www\.)?github\.com/ # Protocol and GitHub domain (capture group 1)
|
|
@@ -31,7 +41,7 @@ module Icarus
|
|
|
31
41
|
end
|
|
32
42
|
|
|
33
43
|
def author_id
|
|
34
|
-
author.downcase.gsub(/\s+/, "_")
|
|
44
|
+
author.to_s.downcase.gsub(/\s+/, "_")
|
|
35
45
|
end
|
|
36
46
|
|
|
37
47
|
def read(data)
|
|
@@ -68,11 +78,11 @@ module Icarus
|
|
|
68
78
|
end
|
|
69
79
|
|
|
70
80
|
def uniq_name
|
|
71
|
-
"#{author.strip}/#{name.strip}"
|
|
81
|
+
"#{author.to_s.strip}/#{name.to_s.strip}"
|
|
72
82
|
end
|
|
73
83
|
|
|
74
|
-
def to_json(*
|
|
75
|
-
JSON.generate(@data, *
|
|
84
|
+
def to_json(*)
|
|
85
|
+
JSON.generate(@data, *)
|
|
76
86
|
end
|
|
77
87
|
|
|
78
88
|
def to_h
|
|
@@ -90,11 +100,11 @@ module Icarus
|
|
|
90
100
|
validate_version
|
|
91
101
|
|
|
92
102
|
%w[name author description].each do |key|
|
|
93
|
-
|
|
103
|
+
validate_required_string(key, @data[key.to_sym])
|
|
94
104
|
end
|
|
95
105
|
|
|
96
106
|
%w[imageURL readmeURL].each do |key|
|
|
97
|
-
@
|
|
107
|
+
check_url(@data[key.to_sym], "Invalid URL #{key.capitalize}: #{@data[key.to_sym] || "blank"}")
|
|
98
108
|
end
|
|
99
109
|
|
|
100
110
|
@validated = true
|
|
@@ -107,62 +117,116 @@ module Icarus
|
|
|
107
117
|
end
|
|
108
118
|
|
|
109
119
|
def file_types
|
|
110
|
-
files
|
|
120
|
+
files.is_a?(Hash) ? files.keys : []
|
|
111
121
|
end
|
|
112
122
|
|
|
113
123
|
def file_urls
|
|
114
|
-
files
|
|
124
|
+
files.is_a?(Hash) ? files.values : []
|
|
115
125
|
end
|
|
116
126
|
|
|
117
|
-
|
|
118
|
-
|
|
127
|
+
# Firestore#update guards its payload with #empty?
|
|
128
|
+
def empty?
|
|
129
|
+
@data.nil? || @data.empty?
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
# Data keys and known fields read as nil when absent; any other name (a typo) raises
|
|
133
|
+
def method_missing(method_name, *args, &)
|
|
134
|
+
return super unless known_key?(method_name)
|
|
135
|
+
|
|
136
|
+
@data[method_name.to_sym]
|
|
119
137
|
end
|
|
120
138
|
|
|
121
139
|
def respond_to_missing?(method_name, include_private = false)
|
|
122
|
-
|
|
140
|
+
known_key?(method_name) || super
|
|
123
141
|
end
|
|
124
142
|
|
|
125
143
|
private
|
|
126
144
|
|
|
145
|
+
def known_key?(method_name)
|
|
146
|
+
return false if method_name.to_s.end_with?("=", "?", "!")
|
|
147
|
+
|
|
148
|
+
@data.key?(method_name.to_sym) || self.class::HASHKEYS.include?(method_name.to_sym) || EXTRA_KEYS.include?(method_name.to_sym)
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
# The file type must be exactly one of these, with an optional leading dot
|
|
127
152
|
def filetype_pattern
|
|
153
|
+
/\A\.?(zip|pak|exmodz?)\z/i
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
# The previous, unanchored rule; a match here but not above is only a warning for now
|
|
157
|
+
def legacy_filetype_pattern
|
|
128
158
|
/(zip|pak|exmodz?)/i
|
|
129
159
|
end
|
|
130
160
|
|
|
131
|
-
def
|
|
132
|
-
|
|
161
|
+
def validate_required_string(key, value)
|
|
162
|
+
if value.nil? || value == ""
|
|
163
|
+
@errors << "#{key.capitalize} cannot be blank"
|
|
164
|
+
elsif !value.is_a?(String)
|
|
165
|
+
@errors << "#{key.capitalize} must be a string"
|
|
166
|
+
end
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
# :ok, :legacy (passed the old check only, so warn) or :invalid
|
|
170
|
+
def url_status(url)
|
|
171
|
+
return :ok if url.nil? || url == ""
|
|
172
|
+
return :invalid unless url.is_a?(String)
|
|
173
|
+
return :ok if strict_url?(url)
|
|
174
|
+
|
|
175
|
+
url.match?(URI::DEFAULT_PARSER.make_regexp) ? :legacy : :invalid
|
|
133
176
|
end
|
|
134
177
|
|
|
135
|
-
def
|
|
136
|
-
|
|
178
|
+
def strict_url?(url)
|
|
179
|
+
uri = URI.parse(url)
|
|
180
|
+
%w[http https].include?(uri.scheme&.downcase) && !uri.host.to_s.empty?
|
|
181
|
+
rescue URI::Error
|
|
182
|
+
false
|
|
183
|
+
end
|
|
137
184
|
|
|
138
|
-
|
|
185
|
+
def check_url(url, message)
|
|
186
|
+
case url_status(url)
|
|
187
|
+
when :invalid then @errors << message
|
|
188
|
+
when :legacy then @warnings << "#{message} #{FUTURE_ERROR}"
|
|
189
|
+
end
|
|
139
190
|
end
|
|
140
191
|
|
|
141
192
|
def validate_files
|
|
142
193
|
@warnings << "This mod uses deprecated fields (fileType and fileURL)" if @data.key?(:fileType) || @data.key?(:fileURL)
|
|
194
|
+
|
|
195
|
+
if !files.nil? && !files.is_a?(Hash)
|
|
196
|
+
@errors << "Files must be an object"
|
|
197
|
+
return
|
|
198
|
+
end
|
|
199
|
+
|
|
143
200
|
@warnings << "files should not be empty" if file_types.empty?
|
|
144
201
|
|
|
145
202
|
file_types.each { |file_type| validate_filetype(file_type) }
|
|
146
203
|
|
|
147
|
-
file_urls.each
|
|
148
|
-
@errors << "Invalid URL: #{file_url}" unless validate_url(file_url)
|
|
149
|
-
end
|
|
204
|
+
file_urls.each { |file_url| check_url(file_url, "Invalid URL: #{file_url}") }
|
|
150
205
|
end
|
|
151
206
|
|
|
152
207
|
def validate_filetype(file_type)
|
|
153
|
-
|
|
208
|
+
name = file_type.to_s
|
|
209
|
+
return if name.match?(filetype_pattern)
|
|
210
|
+
|
|
211
|
+
if name.match?(legacy_filetype_pattern)
|
|
212
|
+
@warnings << "Invalid fileType: #{name.upcase} #{FUTURE_ERROR}"
|
|
213
|
+
else
|
|
214
|
+
@errors << "Invalid fileType: #{name.upcase}"
|
|
215
|
+
end
|
|
154
216
|
end
|
|
155
217
|
|
|
156
218
|
def validate_version
|
|
157
219
|
if version.nil?
|
|
158
220
|
@warnings << "Version was nil, it has been defaulted to 1.0"
|
|
221
|
+
elsif !version.is_a?(String)
|
|
222
|
+
@errors << "Version must be a string"
|
|
159
223
|
else
|
|
160
|
-
@warnings << "Version should be a version string" unless
|
|
224
|
+
@warnings << "Version should be a version string" unless /\A\d+(\.\d+)*\z/.match?(version)
|
|
161
225
|
end
|
|
162
226
|
end
|
|
163
227
|
|
|
164
228
|
def normalize_github_url(url)
|
|
165
|
-
return url
|
|
229
|
+
return url unless url.is_a?(String) && !url.empty?
|
|
166
230
|
|
|
167
231
|
if url.match?(GITHUB_BLOB_RAW_URL_PATTERN)
|
|
168
232
|
normalized = url.gsub(GITHUB_BLOB_RAW_URL_PATTERN, '\1://raw.githubusercontent.com/\2/\3/\4')
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "paint"
|
|
4
|
+
require "firestore"
|
|
5
|
+
require "tools/sync/helpers"
|
|
6
|
+
|
|
7
|
+
module Icarus
|
|
8
|
+
module Mod
|
|
9
|
+
module Tools
|
|
10
|
+
module Sync
|
|
11
|
+
# Shared meta/modinfo -> mods and meta/toolinfo -> tools sync. Subclasses set TYPE (:mod or :tool),
|
|
12
|
+
# INFO (:modinfo or :toolinfo) and ENTITY (the Modinfo or Toolinfo class).
|
|
13
|
+
class Entities
|
|
14
|
+
include Helpers
|
|
15
|
+
|
|
16
|
+
def initialize(client: nil)
|
|
17
|
+
@firestore = client || Firestore.new
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
# Every entry we could read from the info URLs this run. URLs that failed transiently
|
|
21
|
+
# (see #transient_failures) contribute nothing here, so their entries must not be deleted.
|
|
22
|
+
def info_array
|
|
23
|
+
return @info_array if @info_array
|
|
24
|
+
|
|
25
|
+
@transient_failures = []
|
|
26
|
+
@info_array = @firestore.send(self.class::INFO).map do |url|
|
|
27
|
+
next unless url
|
|
28
|
+
|
|
29
|
+
entries_from(url)
|
|
30
|
+
rescue Icarus::Mod::Tools::Sync::RequestFailed => e
|
|
31
|
+
@transient_failures << url if e.transient?
|
|
32
|
+
warn "Skipped; Failed to retrieve #{url}#{" (transient; its existing entries are kept)" if e.transient?}"
|
|
33
|
+
next
|
|
34
|
+
rescue JSON::ParserError => e
|
|
35
|
+
@transient_failures << url
|
|
36
|
+
warn Paint["WARNING: Skipped; Invalid JSON in #{url}: #{e.message}", :red, :bright]
|
|
37
|
+
warn Paint["WARNING: Existing entries from #{url} are kept until it parses again; " \
|
|
38
|
+
"remove or rename the file to remove them", :red, :bright]
|
|
39
|
+
next
|
|
40
|
+
end.flatten.compact.uniq { |entry| [entry.name, entry.author] }
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# The info URLs that could not be read this run for a reason that may clear up
|
|
44
|
+
# (network error, 5xx, rate limit, invalid JSON). Gone (404/410) URLs are not included.
|
|
45
|
+
def transient_failures
|
|
46
|
+
info_array
|
|
47
|
+
@transient_failures || []
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def find(info)
|
|
51
|
+
@firestore.find_by_type(type: plural.to_s, name: info.name, author: info.author)&.id
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def find_info(info)
|
|
55
|
+
@info_array.find { |entry| entry.name == info.name && entry.author == info.author }
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
def update(info)
|
|
59
|
+
success = @firestore.update(self.class::TYPE, info, merge: false)
|
|
60
|
+
@firestore.record_source(plural, info.id, info.source_url) if success && info.source_url
|
|
61
|
+
|
|
62
|
+
success
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
# doc ID => info URL it was last synced from (see Firestore#sources)
|
|
66
|
+
def sources
|
|
67
|
+
@firestore.sources(plural)
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def save_sources
|
|
71
|
+
@firestore.save_sources(plural)
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
def delete(info)
|
|
75
|
+
@firestore.delete(self.class::TYPE, info)
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
private
|
|
79
|
+
|
|
80
|
+
def plural
|
|
81
|
+
:"#{self.class::TYPE}s"
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
# A body that is not a JSON object, or an entry that is not one, is treated as invalid JSON
|
|
85
|
+
# (transient), so one malformed file cannot abort the whole sync
|
|
86
|
+
def entries_from(url)
|
|
87
|
+
body = retrieve_from_url(url)
|
|
88
|
+
entries = body[plural] if body.is_a?(Hash)
|
|
89
|
+
raise JSON::ParserError, "no '#{plural}' array" unless entries.is_a?(Array)
|
|
90
|
+
raise JSON::ParserError, "'#{plural}' entries must be objects" unless entries.all?(Hash)
|
|
91
|
+
|
|
92
|
+
entries.filter_map do |data|
|
|
93
|
+
next unless data[:name].is_a?(String) && /[a-z0-9]+/i.match?(data[:name])
|
|
94
|
+
|
|
95
|
+
self.class::ENTITY.new(data).tap { |entry| entry.source_url = url }
|
|
96
|
+
end
|
|
97
|
+
end
|
|
98
|
+
end
|
|
99
|
+
end
|
|
100
|
+
end
|
|
101
|
+
end
|
|
102
|
+
end
|
|
@@ -9,14 +9,41 @@ module Icarus
|
|
|
9
9
|
module Mod
|
|
10
10
|
module Tools
|
|
11
11
|
module Sync
|
|
12
|
-
|
|
12
|
+
# Raised when a source URL cannot be read. Subclasses say whether the failure is permanent
|
|
13
|
+
# (RequestGone: the author removed or renamed the file) or may clear up on a later run
|
|
14
|
+
# (RequestTransient). Sync only deletes catalog entries on a permanent failure (#16).
|
|
15
|
+
class RequestFailed < StandardError
|
|
16
|
+
def transient?
|
|
17
|
+
false
|
|
18
|
+
end
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
# The source is gone (HTTP 404 or 410): its entries may be removed from the catalog
|
|
22
|
+
class RequestGone < RequestFailed; end
|
|
23
|
+
|
|
24
|
+
# The source could not be read this run (timeout, network/DNS/TLS error, 5xx, 403/429 rate limits,
|
|
25
|
+
# or any other unexpected status): its entries must be kept until a run can read it again
|
|
26
|
+
class RequestTransient < RequestFailed
|
|
27
|
+
def transient?
|
|
28
|
+
true
|
|
29
|
+
end
|
|
30
|
+
end
|
|
13
31
|
|
|
14
32
|
# Sync helper methods
|
|
15
33
|
module Helpers
|
|
34
|
+
GONE_STATUS_CODES = %w[404 410].freeze
|
|
35
|
+
|
|
36
|
+
# Errors raised by Net::HTTP for a request that never got a usable response
|
|
37
|
+
NETWORK_ERRORS = [Timeout::Error, SocketError, SystemCallError, IOError, OpenSSL::SSL::SSLError, Net::ProtocolError].freeze
|
|
38
|
+
|
|
16
39
|
def retrieve_from_url(url)
|
|
17
40
|
raise RequestFailed, "Invalid URI: '#{url}'" unless url && url =~ URI::DEFAULT_PARSER.make_regexp
|
|
18
41
|
|
|
19
|
-
uri =
|
|
42
|
+
uri = begin
|
|
43
|
+
URI(url)
|
|
44
|
+
rescue URI::InvalidURIError
|
|
45
|
+
raise RequestFailed, "Invalid URI: '#{url}'"
|
|
46
|
+
end
|
|
20
47
|
http = Net::HTTP.new(uri.host, uri.port)
|
|
21
48
|
|
|
22
49
|
if uri.scheme == "https"
|
|
@@ -32,9 +59,16 @@ module Icarus
|
|
|
32
59
|
}
|
|
33
60
|
end
|
|
34
61
|
|
|
35
|
-
res =
|
|
62
|
+
res = begin
|
|
63
|
+
http.get(uri.request_uri)
|
|
64
|
+
rescue *NETWORK_ERRORS => e
|
|
65
|
+
raise RequestTransient, "HTTP Request failed for #{url} (#{e.class}): #{e.message}"
|
|
66
|
+
end
|
|
36
67
|
|
|
37
|
-
|
|
68
|
+
unless res&.code == "200"
|
|
69
|
+
error_class = GONE_STATUS_CODES.include?(res&.code) ? RequestGone : RequestTransient
|
|
70
|
+
raise error_class, "HTTP Request failed for #{url} (#{res&.code}): #{res&.message}"
|
|
71
|
+
end
|
|
38
72
|
|
|
39
73
|
JSON.parse(res.body, symbolize_names: true)
|
|
40
74
|
end
|