Icarus-Mod-Tools 2.7.0 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -45,7 +45,7 @@ module Icarus
45
45
 
46
46
  no_commands do
47
47
  def list_for_type(type)
48
- klass = (type == :mods) ? Icarus::Mod::Tools::Modinfo : Icarus::Mod::Tools::Toolinfo
48
+ klass = type == :mods ? Icarus::Mod::Tools::Modinfo : Icarus::Mod::Tools::Toolinfo
49
49
  valid_keys = klass::HASHKEYS + [:updated_at]
50
50
  sort_field = options[:sort]&.to_sym
51
51
  filter = !options[:filter].empty?
@@ -54,9 +54,15 @@ module Icarus
54
54
  filter_field = options[:filter].first&.to_sym
55
55
  filter_value = options[:filter].last&.to_s
56
56
 
57
- raise Icarus::Mod::Tools::Error, "Invalid filter option #{options[:filter]}" unless options[:filter].empty? || options[:filter]&.count == 2
57
+ unless options[:filter].empty? || options[:filter]&.count == 2
58
+ raise Icarus::Mod::Tools::Error,
59
+ "Invalid filter option #{options[:filter]}"
60
+ end
58
61
 
59
- raise Icarus::Mod::Tools::Error, "Invalid filter field '#{filter_field}'" unless filter_field && valid_keys.include?(filter_field)
62
+ unless filter_field && valid_keys.include?(filter_field)
63
+ raise Icarus::Mod::Tools::Error,
64
+ "Invalid filter field '#{filter_field}'"
65
+ end
60
66
  end
61
67
 
62
68
  raise Icarus::Mod::Tools::Error, "Invalid sort field '#{sort_field}'" unless valid_keys.include?(sort_field)
@@ -67,7 +73,8 @@ module Icarus
67
73
  items = Firestore.new.send(type)
68
74
 
69
75
  # Filter by field
70
- items.select! { |item| item.send(filter_field).downcase =~ /#{filter_value&.downcase}/ } if filter_field
76
+ # Case-insensitive substring match; the value is literal text, not a regex
77
+ items.select! { |item| item.send(filter_field).to_s.downcase.include?(filter_value.to_s.downcase) } if filter_field
71
78
 
72
79
  if items.empty?
73
80
  puts "no entries found" if verbose?
@@ -90,16 +97,21 @@ module Icarus
90
97
  end
91
98
 
92
99
  # Sort by field, optionally subsorting by name
93
- ((sort_field == :name) ? items.sort_by(&:name) : items.sort_by { |item| [item.send(sort_field), item.name] }).each do |item|
100
+ sorted = if sort_field == :name
101
+ items.sort_by(&:name)
102
+ else
103
+ items.sort_by { |item| [item.send(sort_field).to_s, item.name.to_s] }
104
+ end
105
+ sorted.each do |item|
94
106
  data_format = "%-<name>50s %-<author>20s v%-<version>10s%-<updated_at>20s"
95
107
  data_format += " %-<id>20s %<description>s" if verbose > 1
96
108
 
97
- puts format(data_format, item.to_h.merge(id: item.id, updated_at: item.updated_at.strftime("%Y-%m-%d %H:%M:%S")))
109
+ puts format(data_format, item.to_h.merge(id: item.id, updated_at: item.updated_at&.strftime("%Y-%m-%d %H:%M:%S")))
98
110
  end
99
111
 
100
112
  puts "Total: #{items.count}" if verbose?
101
113
  rescue Icarus::Mod::Tools::Error => e
102
- puts e.message
114
+ warn e.message
103
115
  exit 1
104
116
  end
105
117
  end
@@ -11,6 +11,7 @@ module Icarus
11
11
  # rubocop:disable Style/GlobalVars
12
12
  class Remove < SubcommandBase
13
13
  include Tools::Sync::Helpers
14
+
14
15
  class_option :dry_run, type: :boolean, default: false, desc: "Dry run (no changes will be made)"
15
16
 
16
17
  desc "repos REPO", "Removes an entry from 'meta/repos/list' and cascades to associated mods/tools"
@@ -152,10 +153,14 @@ module Icarus
152
153
 
153
154
  def cascade_delete_repo(repo_name)
154
155
  @delete_failures = []
156
+ @failed_entity_fetches = []
157
+
158
+ # The stored entry may be a full URL; match on the "owner/repo" part
159
+ owner_repo = repo_name.sub(%r{\Ahttps?://(www\.)?github\.com/}i, "").sub(%r{/+\z}, "")
155
160
 
156
- # Find all modinfo URLs belonging to this repository
157
- # Match full "owner/repo" path component, not substring
158
- repo_pattern = %r{/#{Regexp.escape(repo_name)}(?=/|$)}
161
+ # Find all modinfo/toolinfo URLs belonging to this repository. Anchored to the host and the full
162
+ # "owner/repo" path component so another repo's subdirectory or a similarly named repo can't match.
163
+ repo_pattern = %r{\Ahttps?://(raw\.githubusercontent\.com|github\.com)/#{Regexp.escape(owner_repo)}/}i
159
164
  modinfo_urls = firestore.modinfo.select { |url| url.match?(repo_pattern) }
160
165
  toolinfo_urls = firestore.toolinfo.select { |url| url.match?(repo_pattern) }
161
166
 
@@ -168,28 +173,20 @@ module Icarus
168
173
  return
169
174
  end
170
175
 
171
- # Delete modinfo URLs and their associated mods
172
- modinfo_urls.each do |url|
173
- puts Paint[" Removing modinfo: #{url}", :black] if verbose?
174
- track_delete(:modinfo, url) { firestore.delete(:modinfo, url) }
175
-
176
- # Find and delete associated mods
177
- delete_entities_from_url(url, :mod)
178
- end
179
-
180
- # Delete toolinfo URLs and their associated tools
181
- toolinfo_urls.each do |url|
182
- puts Paint[" Removing toolinfo: #{url}", :black] if verbose?
183
- track_delete(:toolinfo, url) { firestore.delete(:toolinfo, url) }
184
-
185
- # Find and delete associated tools
186
- delete_entities_from_url(url, :tool)
187
- end
176
+ # Delete each URL's entities, then the URL itself, but only if all of its entities went
177
+ modinfo_urls.each { |url| delete_info_url(url, :modinfo, :mod) }
178
+ toolinfo_urls.each { |url| delete_info_url(url, :toolinfo, :tool) }
188
179
 
189
180
  # Report any failures
190
181
  report_delete_failures
191
182
  report_fetch_failures
192
183
 
184
+ if @delete_failures.any? || @failed_entity_fetches.any?
185
+ # Keep the repo entry so the removal can be retried; entries still listed would otherwise be orphaned
186
+ warn Paint["Repository not removed (kept so the removal can be retried): #{repo_name}", :red]
187
+ exit 1
188
+ end
189
+
193
190
  # Finally, remove the repository
194
191
  if firestore.delete(:repositories, repo_name)
195
192
  puts Paint["Successfully removed repository and all associated entries: #{repo_name}", :green]
@@ -199,39 +196,57 @@ module Icarus
199
196
  end
200
197
  end
201
198
 
202
- def delete_entities_from_url(url, type)
203
- # Fetch the modinfo/toolinfo JSON
204
- begin
205
- data = retrieve_from_url(url)
206
- entities = data[type == :mod ? :mods : :tools] || []
207
-
208
- entities.each do |entity_data|
209
- # Find ALL matching entities in Firestore by name and author
210
- collection = type == :mod ? firestore.mods : firestore.tools
211
- matching_entities = collection.select do |e|
212
- e.name == entity_data[:name] && e.author == entity_data[:author]
213
- end
199
+ def delete_info_url(url, info_type, type)
200
+ puts Paint[" Removing #{info_type}: #{url}", :black] if verbose?
214
201
 
215
- next if matching_entities.empty?
216
-
217
- if matching_entities.size > 1 && verbose?
218
- warn Paint[" Note: Found #{matching_entities.size} entities matching '#{entity_data[:name]}' by #{entity_data[:author]}", :yellow]
219
- end
220
-
221
- matching_entities.each do |entity|
222
- puts Paint[" Removing #{type}: #{entity.name} (ID: #{entity.id})", :black] if verbose?
223
- track_delete(type, "#{entity.name} (#{entity.id})") { firestore.delete(type, entity) }
224
- end
202
+ failures_before = @delete_failures.size + @failed_entity_fetches.size
203
+ begin
204
+ entities_for_url(url, type).each do |entity|
205
+ puts Paint[" Removing #{type}: #{entity.name} (ID: #{entity.id})", :black] if verbose?
206
+ track_delete(type, "#{entity.name} (#{entity.id})") { firestore.delete(type, entity) }
225
207
  end
226
208
  rescue Tools::Sync::RequestFailed, SocketError, IOError, SystemCallError, Timeout::Error,
227
209
  JSON::ParserError => e
228
- @failed_entity_fetches ||= []
229
210
  @failed_entity_fetches << { url: url, error: e.class.name, message: e.message }
230
211
  warn Paint["Warning: Could not fetch #{url} to remove entities: #{e.message}", :yellow]
231
212
  end
213
+
214
+ # Keep the URL listed while any of its entities remain
215
+ return if (@delete_failures.size + @failed_entity_fetches.size) > failures_before
216
+
217
+ track_delete(info_type, url) { firestore.delete(info_type, url) }
218
+ end
219
+
220
+ # Entities (mods or tools) that came from +url+: the docs the source map attributes to it, plus, for docs
221
+ # with no recorded source, those matching a name+author in the file. Docs mapped to another URL
222
+ # (e.g. a fork publishing the same name/author) are never touched.
223
+ def entities_for_url(url, type)
224
+ collection = type == :mod ? firestore.mods : firestore.tools
225
+ sources = firestore.sources(type == :mod ? :mods : :tools)
226
+
227
+ entities = collection.select { |e| sources[e.id.to_s] == url }
228
+ unmapped = collection.reject { |e| sources.key?(e.id.to_s) }
229
+ return entities if unmapped.empty?
230
+
231
+ begin
232
+ data = retrieve_from_url(url)
233
+ rescue Tools::Sync::RequestGone
234
+ # File removed by its author: only the mapped docs can be attributed to it
235
+ return entities
236
+ end
237
+
238
+ (data[type == :mod ? :mods : :tools] || []).each do |entity_data|
239
+ matching = unmapped.select { |e| e.name == entity_data[:name] && e.author == entity_data[:author] }
240
+ if matching.size > 1 && verbose?
241
+ warn Paint[" Note: Found #{matching.size} entities matching '#{entity_data[:name]}' by #{entity_data[:author]}", :yellow]
242
+ end
243
+ entities.concat(matching)
244
+ end
245
+
246
+ entities.uniq(&:id)
232
247
  end
233
248
 
234
- def preview_cascade_deletions(repo_name, modinfo_urls, toolinfo_urls)
249
+ def preview_cascade_deletions(_repo_name, modinfo_urls, toolinfo_urls)
235
250
  mods = []
236
251
  tools = []
237
252
 
@@ -254,20 +269,7 @@ module Icarus
254
269
  end
255
270
 
256
271
  def fetch_entities_from_url(url, type)
257
- data = retrieve_from_url(url)
258
- entity_data_list = data[type == :mod ? :mods : :tools] || []
259
-
260
- collection = type == :mod ? firestore.mods : firestore.tools
261
- entities = []
262
-
263
- entity_data_list.each do |entity_data|
264
- matching_entities = collection.select do |e|
265
- e.name == entity_data[:name] && e.author == entity_data[:author]
266
- end
267
- entities.concat(matching_entities)
268
- end
269
-
270
- entities
272
+ entities_for_url(url, type)
271
273
  rescue Tools::Sync::RequestFailed, SocketError, IOError, SystemCallError, Timeout::Error, JSON::ParserError => e
272
274
  warn Paint["Warning: Could not fetch #{url}: #{e.message}", :yellow] if verbose?
273
275
  nil
@@ -292,10 +294,10 @@ module Icarus
292
294
  preview[:toolinfo_urls].each { |url| puts " • #{url}" }
293
295
  end
294
296
 
295
- if preview[:tools].any?
296
- puts " - Tools: #{preview[:tools].size}"
297
- preview[:tools].each { |tool| puts " • #{tool.name} by #{tool.author} (ID: #{tool.id})" }
298
- end
297
+ return unless preview[:tools].any?
298
+
299
+ puts " - Tools: #{preview[:tools].size}"
300
+ preview[:tools].each { |tool| puts " • #{tool.name} by #{tool.author} (ID: #{tool.id})" }
299
301
  end
300
302
 
301
303
  def track_delete(type, identifier)
@@ -8,8 +8,8 @@ module Icarus
8
8
  # Base class for all subcommands
9
9
  class SubcommandBase < Base
10
10
  class_option :verbose,
11
- aliases: "-v", type: :boolean, repeatable: true, default: [true],
12
- desc: "Increase verbosity. May be repeated for even more verbosity."
11
+ aliases: "-v", type: :boolean, repeatable: true, default: [true],
12
+ desc: "Increase verbosity. May be repeated for even more verbosity."
13
13
 
14
14
  no_commands do
15
15
  def check_false
@@ -13,21 +13,26 @@ module Icarus
13
13
  class Sync < SubcommandBase
14
14
  $firestore = nil
15
15
 
16
+ # Stage 2 refuses to delete more than this share of a collection (or to empty it) without --force
17
+ MASS_DELETE_RATIO = 0.25
18
+ MASS_DELETE_MIN = 5
19
+
16
20
  class_option :dry_run, type: :boolean, default: false, desc: "Dry run (no changes will be made)"
21
+ class_option :force, type: :boolean, default: false,
22
+ desc: "Let mods/tools sync delete over #{(MASS_DELETE_RATIO * 100).to_i}% of a collection"
17
23
 
18
24
  desc "all", "Run all sync jobs"
19
25
  def all
20
- puts "Running Toolinfo Sync..." if verbose?
21
- invoke :toolinfo
22
-
23
- puts "Running Tools Sync..." if verbose?
24
- invoke :tools
25
-
26
- puts "Running Modinfo Sync..." if verbose?
27
- invoke :modinfo
26
+ # Each stage exits 1 on failure; run the rest anyway, then exit 1 if any failed
27
+ failed = %i[toolinfo tools modinfo mods].map do |stage|
28
+ puts "Running #{stage.to_s.capitalize} Sync..." if verbose?
29
+ invoke stage
30
+ false
31
+ rescue SystemExit => e
32
+ !e.success?
33
+ end
28
34
 
29
- puts "Running Mods Sync..." if verbose?
30
- invoke :mods
35
+ exit 1 if failed.any?
31
36
  end
32
37
 
33
38
  desc "modinfo", "Reads from 'meta/repos/list' and Syncs any modinfo files we find (github only for now)"
@@ -89,8 +94,8 @@ module Icarus
89
94
 
90
95
  if options[:dry_run]
91
96
  puts Paint["Dry run; no changes made. Would have deleted #{deleted_count} duplicate #{type}.", :yellow] if verbose?
92
- else
93
- puts "Deleted #{deleted_count} duplicate #{type}." if verbose?
97
+ elsif verbose?
98
+ puts "Deleted #{deleted_count} duplicate #{type}."
94
99
  end
95
100
  end
96
101
 
@@ -174,7 +179,7 @@ module Icarus
174
179
  kept.each do |list|
175
180
  url = sources[list.id.to_s]
176
181
  reason = url ? "its source could not be read (#{url})" : "no recorded source yet, and a source failed"
177
- warn Paint[" #{list.author || 'NoOne'}/#{list.name || 'Unnamed'} (#{list.id}): #{reason}", :yellow]
182
+ warn Paint[" #{list.author || "NoOne"}/#{list.name || "Unnamed"} (#{list.id}): #{reason}", :yellow]
178
183
  end
179
184
  end
180
185
 
@@ -212,6 +217,7 @@ module Icarus
212
217
  end
213
218
  rescue Icarus::Mod::Tools::Error => e
214
219
  warn e.message
220
+ exit 1
215
221
  end
216
222
 
217
223
  def sync_list(type)
@@ -223,10 +229,12 @@ module Icarus
223
229
  info_array = sync.info_array
224
230
 
225
231
  puts "Retrieving List Data..." if verbose?
226
- list_array = sync.send(type)
232
+ # A snapshot: creating a doc appends it to the cached collection, which would dilute the delete guard
233
+ list_array = sync.send(type).dup
227
234
 
228
235
  return if options[:check]
229
236
 
237
+ updating = !options[:dry_run]
230
238
  puts "Updating List Data..." if verbose?
231
239
  info_array.each do |list|
232
240
  verb = "Creating"
@@ -248,7 +256,7 @@ module Icarus
248
256
 
249
257
  if verbose > 1
250
258
  print format("#{verb} %-<name>60s",
251
- name: "'#{list.author || 'NoOne'}/#{list.name || 'Unnamed'}'")
259
+ name: "'#{list.author || "NoOne"}/#{list.name || "Unnamed"}'")
252
260
  end
253
261
 
254
262
  if options[:dry_run]
@@ -257,7 +265,7 @@ module Icarus
257
265
  end
258
266
 
259
267
  response = sync.update(list)
260
- puts success_or_failure(status: response) if verbose > 1
268
+ puts success_or_failure(response) if verbose > 1
261
269
  end
262
270
 
263
271
  if options[:dry_run]
@@ -272,16 +280,37 @@ module Icarus
272
280
 
273
281
  return unless delete_array.any?
274
282
 
283
+ refuse_mass_delete(type, delete_array, list_array, info_array)
284
+
275
285
  puts "Deleting outdated items..." if verbose?
276
286
  delete_array.each do |list|
277
- print "Deleting '#{list.author || 'NoOne'}/#{list.name || 'Unnamed'}'#{' ' * 20}" if verbose > 1
287
+ print "Deleting '#{list.author || "NoOne"}/#{list.name || "Unnamed"}'#{" " * 20}" if verbose > 1
278
288
  response = sync.delete(list)
279
- puts success_or_failure(status: response) if verbose > 1
289
+ puts success_or_failure(response) if verbose > 1
280
290
  end
281
291
 
282
292
  puts "Deleted #{delete_array.count} outdated items" if verbose?
283
293
  rescue Icarus::Mod::Tools::Error => e
284
294
  warn e.message
295
+ exit 1
296
+ rescue StandardError
297
+ # Keep the sources of the docs written before a failure, so the next run can protect them (#16)
298
+ sync&.save_sources if updating
299
+ raise
300
+ end
301
+
302
+ # A wave of 404s (a GitHub outage, a mass rename) looks like every author removing their file, so stage 2
303
+ # will not delete more than MASS_DELETE_RATIO of a collection, or empty it, unless run with --force
304
+ def refuse_mass_delete(type, delete_array, list_array, info_array)
305
+ return if options[:force]
306
+
307
+ emptying = info_array.empty? && delete_array.size >= list_array.size
308
+ too_many = delete_array.size >= MASS_DELETE_MIN && delete_array.size > list_array.size * MASS_DELETE_RATIO
309
+ return unless emptying || too_many
310
+
311
+ raise Icarus::Mod::Tools::Error,
312
+ Paint["Refusing to delete #{delete_array.size} of #{list_array.size} #{type}; " \
313
+ "check the sources, then rerun with --force if this is intended", :red, :bright]
285
314
  end
286
315
  end
287
316
  end
@@ -23,9 +23,8 @@ module Icarus
23
23
  validator = Icarus::Mod::Tools::Validator.new(type)
24
24
 
25
25
  puts "Validating Entries..." if verbose?
26
- max_length = validator.array.map { |info| info.uniq_name.length }.max + 1
26
+ max_length = (validator.array.map { |info| info.uniq_name.length }.max || 0) + 1
27
27
 
28
- # rubocop:disable Lint/FormatParameterMismatch
29
28
  validator.array.each do |info|
30
29
  print Paint[format("%s %-#{max_length}s", "Running validation steps on", info.uniq_name), :cyan, :bright] if verbose > 1
31
30
 
@@ -47,8 +46,6 @@ module Icarus
47
46
  puts info.warnings.map { |warning| Paint["#{warning} in #{info.uniq_name}", :yellow] }.join("\n")
48
47
  puts "\n" if verbose > 1
49
48
  end
50
- # rubocop:enable Lint/FormatParameterMismatch
51
-
52
49
  exit exit_code
53
50
  end
54
51
  end
@@ -19,11 +19,11 @@ module Icarus
19
19
  end
20
20
 
21
21
  def firebase
22
- @config.firebase
22
+ config.firebase
23
23
  end
24
24
 
25
25
  def github
26
- @config.github
26
+ config.github
27
27
  end
28
28
 
29
29
  private
@@ -1,7 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require "digest"
4
- require "set"
5
4
  require "google/cloud/firestore"
6
5
  require "tools/modinfo"
7
6
  require "tools/toolinfo"
@@ -62,17 +61,20 @@ module Icarus
62
61
  return if map[id.to_s] == url
63
62
 
64
63
  map[id.to_s] = url
65
- sources_changed << pluralize(type)
64
+ sources_changed(type) << id.to_s
66
65
  end
67
66
 
68
- # Writes the source map if it changed since it was read or last saved
67
+ # Writes the entries changed since the map was read or last saved. Only those keys are written (merged),
68
+ # so entries another sync host recorded meanwhile are kept.
69
69
  def save_sources(type)
70
- key = pluralize(type)
71
- return true unless sources_changed.include?(key)
70
+ changed = sources_changed(type)
71
+ return true if changed.empty?
72
72
 
73
- response = @client.doc(sources_path(key)).set({ sources: sources(key) }, merge: false)
73
+ map = sources(type)
74
+ entries = changed.to_h { |id| [id, map.fetch(id) { Google::Cloud::Firestore::FieldValue.delete }] }
75
+ response = @client.doc(sources_path(pluralize(type))).set({ sources: entries }, merge: true)
74
76
  success = response.is_a?(Google::Cloud::Firestore::CommitResponse::WriteResult)
75
- sources_changed.delete(key) if success
77
+ changed.clear if success
76
78
 
77
79
  success
78
80
  end
@@ -102,11 +104,8 @@ module Icarus
102
104
  raise "You must specify a payload to update" if payload&.empty? || payload.nil?
103
105
 
104
106
  response = case type.to_sym
105
- when :modinfo, :toolinfo
106
- update_array = (send(type) + [payload]).flatten.uniq
107
- @client.doc(collections.meta.send(type)).set({ list: update_array }, merge:) if update_array.any?
108
- when :repositories
109
- @client.doc(collections.meta.repositories).set({ list: payload }, merge:)
107
+ when :modinfo, :toolinfo, :repositories
108
+ update_list(type, [payload].flatten.uniq)
110
109
  when :mod, :tool
111
110
  create_or_update(pluralize(type), payload, merge:)
112
111
  else
@@ -141,11 +140,13 @@ module Icarus
141
140
  forget_source(type, payload.id)
142
141
  end
143
142
  when :modinfo, :toolinfo, :repositories
144
- update_array = (send(type) - [payload]).flatten.uniq
145
-
146
- response = @client.doc(collections.meta.send(type)).set({ list: update_array })
143
+ # Removed on the server, so entries another host added meanwhile are kept
144
+ values = [payload].flatten
145
+ response = @client.doc(collections.meta.send(type)).set(
146
+ { list: Google::Cloud::Firestore::FieldValue.array_delete(*values) }, merge: true
147
+ )
147
148
  # Invalidate cache if Firestore update was successful
148
- instance_variable_set(:"@#{type}", update_array) if response.is_a?(Google::Cloud::Firestore::CommitResponse::WriteResult)
149
+ instance_variable_set(:"@#{type}", send(type) - values) if response.is_a?(Google::Cloud::Firestore::CommitResponse::WriteResult)
149
150
  else
150
151
  raise "Invalid type: #{type}"
151
152
  end
@@ -158,7 +159,8 @@ module Icarus
158
159
  def list(type)
159
160
  case type.to_sym
160
161
  when :modinfo, :toolinfo, :repositories
161
- @client.doc(collections.meta.send(type)).get[:list]
162
+ snapshot = @client.doc(collections.meta.send(type)).get
163
+ snapshot.missing? ? [] : snapshot[:list] || []
162
164
  when :mods, :tools
163
165
  @client.col(collections.send(type)).get.map do |doc|
164
166
  klass = type == :mods ? Icarus::Mod::Tools::Modinfo : Icarus::Mod::Tools::Toolinfo
@@ -169,6 +171,19 @@ module Icarus
169
171
  end
170
172
  end
171
173
 
174
+ # Adds +values+ to a meta list. The union happens on the server, so two sync hosts (or a sync and `imt add`)
175
+ # writing at once keep each other's additions. The cache is refreshed so a later stage of `sync all` sees them.
176
+ def update_list(type, values)
177
+ response = @client.doc(collections.meta.send(type)).set(
178
+ { list: Google::Cloud::Firestore::FieldValue.array_union(*values) }, merge: true
179
+ )
180
+ if response.is_a?(Google::Cloud::Firestore::CommitResponse::WriteResult)
181
+ instance_variable_set(:"@#{type}", (send(type) + values).uniq)
182
+ end
183
+
184
+ response
185
+ end
186
+
172
187
  def create_or_update(type, payload, merge:)
173
188
  doc_id = payload.id || find_by_type(type:, name: payload.name, author: payload.author)&.id
174
189
 
@@ -195,16 +210,18 @@ module Icarus
195
210
  def forget_source(type, id)
196
211
  return unless sources(type).delete(id.to_s)
197
212
 
198
- sources_changed << pluralize(type)
213
+ sources_changed(type) << id.to_s
199
214
  save_sources(type)
200
215
  end
201
216
 
202
- def sources_changed
203
- @sources_changed ||= Set.new
217
+ # Doc IDs whose source entry changed (or was removed) since the map was read or last saved
218
+ def sources_changed(type)
219
+ @sources_changed ||= Hash.new { |hash, key| hash[key] = Set.new }
220
+ @sources_changed[pluralize(type)]
204
221
  end
205
222
 
206
223
  def sources_path(collection)
207
- name = :"#{collection.chomp('s')}_sources"
224
+ name = :"#{collection.chomp("s")}_sources"
208
225
  collections.meta&.send(name) || "meta/#{name}"
209
226
  end
210
227
 
@@ -12,6 +12,7 @@ module Icarus
12
12
  self.repository = repo if repo
13
13
  @client = Octokit::Client.new(access_token: Config.github.token)
14
14
  @resources = []
15
+ @cache = {}
15
16
  end
16
17
 
17
18
  def repository
@@ -22,6 +23,7 @@ module Icarus
22
23
 
23
24
  def repository=(repo)
24
25
  @resources = [] # reset the resources cache
26
+ @cache = {}
25
27
  @repository = repo.gsub(%r{https?://.*github\.com/}, "")
26
28
  end
27
29
 
@@ -29,33 +31,48 @@ module Icarus
29
31
  # path: the path to search in
30
32
  # cache: whether to use the cached resources
31
33
  # recursive: whether to recursively search subdirectories
34
+ # Results are cached per [path, recursive]; the top-level listing also lives in #resources
32
35
  def all_files(path: nil, cache: true, recursive: false, &block)
33
- # If we've already been called for this repository, use the cached resources
34
- use_cache = @resources.any? && cache
35
-
36
- if use_cache
37
- @resources.each { |file| block.call(file) } if block
38
- else
39
- begin
40
- @client.contents(repository, path:).each do |entry|
41
- if entry[:type] == "dir"
42
- all_files(path: entry[:path], cache: false, recursive: true, &block) if recursive
43
- next # we don't need directories in our output
44
- end
45
-
46
- block&.call(entry)
47
- @resources << entry # cache the file
48
- end
49
- rescue Octokit::NotFound
50
- warn "WARNING: Could not access #{repository}: 404 - not found"
51
- end
36
+ key = [path, recursive]
37
+ cached = key == [nil, false] ? @resources : @cache[key]
38
+
39
+ # If we've already been called with these arguments for this repository, use the cached resources
40
+ files = (cached if cache && cached&.any?) || fetch_files(path, recursive).tap do |found|
41
+ key == [nil, false] ? @resources = found : @cache[key] = found
42
+ end
43
+
44
+ if block
45
+ files.each { |file| block.call(file) }
46
+ return nil
52
47
  end
53
48
 
54
- @resources unless block
49
+ files
50
+ end
51
+
52
+ # Returns the first file whose name equals the given name (case-insensitive)
53
+ def find(name)
54
+ all_files { |file| return file if file[:name].to_s.casecmp?(name.to_s) }
55
55
  end
56
56
 
57
- def find(pattern)
58
- all_files { |file| return file if /#{pattern}/i.match?(file[:name]) }
57
+ private
58
+
59
+ # Fetches the files (not directories) under path from the API
60
+ def fetch_files(path, recursive)
61
+ found = []
62
+
63
+ @client.contents(repository, path:).each do |entry|
64
+ if entry[:type] == "dir"
65
+ found.concat(fetch_files(entry[:path], true)) if recursive
66
+ next # we don't need directories in our output
67
+ end
68
+
69
+ found << entry
70
+ end
71
+
72
+ found
73
+ rescue Octokit::NotFound
74
+ warn "WARNING: Could not access #{repository}: 404 - not found"
75
+ found
59
76
  end
60
77
  end
61
78
  end