Icarus-Mod-Tools 3.0.0 → 3.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 17d3d3cdfeeb5953e405d4888f0c748b00d330af51a14714c51e04b927baedb9
4
- data.tar.gz: 29f015ac208933825b954bb33fda452cd684499f8e2e10950d241e0a4323fbac
3
+ metadata.gz: de06ee7fa7b4384bd296cd428c7946edc03c9dde4d8313f8cc1b9550095d89a0
4
+ data.tar.gz: cc5488cad0c60bc7c52eb535ae241d4ee3457446802affb70f2adf60d368829e
5
5
  SHA512:
6
- metadata.gz: fad56b2b02cfd1412d332f2d47be963ef557ce21bfa25f3af6b103ee99ab4cb632517e23a59d7d512cbc339746906392396133b238d004e9c0ef82a23e76fc3e
7
- data.tar.gz: bd460ed0255ede73b7cda973e0d0a39c22a2a285600e6f2399920905c1b626e2b40dc65599578f67e50b8f8c609238fde203eb7b3cd4090c3c3ef7e24d3fb974
6
+ metadata.gz: f3ad937f990a1b765c033af08dbef572c0e02db86138c95400ebf06b38c07c94ef41e3bd2bc692c816a906b302d2ad64da9b6a39cbdeb4f167afea315a8dcc9b
7
+ data.tar.gz: e2a1a5ede7f1733ef712d24e0c62f119757e382c27dbb31a5ef39a5ab85fdf582723c7578f5940894d42cf02d1ce9750efb37e22591152216f4eff772974970c
data/CHANGELOG.md CHANGED
@@ -4,6 +4,18 @@ All notable changes to this project will be documented in this file.
4
4
 
5
5
  ## History (reverse chronological order)
6
6
 
7
+ ### v3.2.0 - 2026-10-30
8
+
9
+ - Add `imt validate repo OWNER/REPO [--json]`, a read-only pre-check for a repo before it is added to `meta/repos` (repo state, info files, entry validation and link checks; exits 0/1/2)
10
+ - Link checks never request loopback, private, link-local or other non-public addresses (checked on every redirect, and the request goes to the checked address), since the links come from the repo being vetted
11
+
12
+ ### v3.1.0 - 2026-10-30
13
+
14
+ - `imt sync mods`/`tools --dry-run` now lists the entries a real run would delete, and warns when the mass-delete brake would refuse (a real run would exit 1 without `--force`)
15
+ - Document `imt add --dry-run` in the README
16
+ - Update `google-cloud-firestore` to 3.x and `octokit` to 10.x (no API changes affect `imt`)
17
+ - Update development tools (`pry` 0.16, `guard`, `rake`)
18
+
7
19
  ### v3.0.0 - 2026-10-30
8
20
 
9
21
  **Breaking:** requires Ruby 4.0 or greater. Sync commands now exit 1 on failure, and `imt sync mods`/`tools` refuse a mass delete without `--force`.
data/Gemfile CHANGED
@@ -11,7 +11,7 @@ group :development do
11
11
  gem "fuubar"
12
12
  gem "guard", "~> 2.18"
13
13
  gem "guard-rspec", "~> 4.7"
14
- gem "pry", "~> 0.14.1"
14
+ gem "pry", "~> 0.16"
15
15
  gem "rspec", "~> 3.12"
16
16
  gem "rubocop", "~> 1.92"
17
17
  gem "rubocop-rspec", "~> 3.10", require: false
data/Gemfile.lock CHANGED
@@ -1,9 +1,9 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- Icarus-Mod-Tools (3.0.0)
5
- google-cloud-firestore (~> 2.7)
6
- octokit (~> 6.0)
4
+ Icarus-Mod-Tools (3.2.0)
5
+ google-cloud-firestore (~> 3.3)
6
+ octokit (~> 10.0)
7
7
  ostruct (~> 0.6)
8
8
  paint (~> 2.3)
9
9
  thor (~> 1.2)
@@ -23,7 +23,7 @@ GEM
23
23
  faraday-net_http (>= 2.0, < 3.6)
24
24
  json
25
25
  logger
26
- faraday-net_http (3.4.2)
26
+ faraday-net_http (3.5.0)
27
27
  net-http (~> 0.5)
28
28
  faraday-retry (2.4.0)
29
29
  faraday (~> 2.0)
@@ -43,7 +43,7 @@ GEM
43
43
  fuubar (2.5.1)
44
44
  rspec-core (~> 3.0)
45
45
  ruby-progressbar (~> 1.4)
46
- gapic-common (1.2.0)
46
+ gapic-common (1.6.1)
47
47
  faraday (>= 1.9, < 3.a)
48
48
  faraday-retry (>= 1.0, < 3.a)
49
49
  google-cloud-env (~> 2.2)
@@ -52,73 +52,97 @@ GEM
52
52
  googleapis-common-protos (~> 1.6)
53
53
  googleapis-common-protos-types (~> 1.15)
54
54
  googleauth (~> 1.12)
55
- grpc (~> 1.66)
56
- google-cloud-core (1.8.0)
55
+ grpc (>= 1.84.1, < 2.a)
56
+ google-cloud-core (1.9.1)
57
57
  google-cloud-env (>= 1.0, < 3.a)
58
58
  google-cloud-errors (~> 1.0)
59
- google-cloud-env (2.3.1)
59
+ google-cloud-env (2.4.1)
60
60
  base64 (~> 0.2)
61
61
  faraday (>= 1.0, < 3.a)
62
- google-cloud-errors (1.5.0)
63
- google-cloud-firestore (2.16.1)
62
+ google-cloud-errors (1.7.1)
63
+ google-cloud-firestore (3.3.0)
64
64
  bigdecimal (~> 3.0)
65
65
  concurrent-ruby (~> 1.0)
66
- google-cloud-core (~> 1.5)
67
- google-cloud-firestore-v1 (>= 0.10, < 2.a)
66
+ google-cloud-core (~> 1.7)
67
+ google-cloud-firestore-v1 (~> 2.0)
68
68
  rbtree (~> 0.4.2)
69
- google-cloud-firestore-v1 (1.3.0)
70
- gapic-common (>= 0.25.0, < 2.a)
69
+ google-cloud-firestore-v1 (2.6.0)
70
+ gapic-common (~> 1.6)
71
71
  google-cloud-errors (~> 1.0)
72
- google-cloud-location (>= 0.7, < 2.a)
73
- google-cloud-location (1.2.1)
74
- gapic-common (~> 1.2)
72
+ google-cloud-location (~> 1.0)
73
+ google-cloud-location (1.6.0)
74
+ gapic-common (~> 1.6)
75
75
  google-cloud-errors (~> 1.0)
76
- google-logging-utils (0.2.0)
77
- google-protobuf (4.33.4)
76
+ google-logging-utils (0.2.2)
77
+ google-protobuf (4.36.2)
78
78
  bigdecimal
79
- rake (>= 13)
80
- googleapis-common-protos (1.9.0)
79
+ rake (~> 13.3)
80
+ google-protobuf (4.36.2-aarch64-linux-gnu)
81
+ bigdecimal
82
+ rake (~> 13.3)
83
+ google-protobuf (4.36.2-aarch64-linux-musl)
84
+ bigdecimal
85
+ rake (~> 13.3)
86
+ google-protobuf (4.36.2-arm64-darwin)
87
+ bigdecimal
88
+ rake (~> 13.3)
89
+ google-protobuf (4.36.2-x86-linux-gnu)
90
+ bigdecimal
91
+ rake (~> 13.3)
92
+ google-protobuf (4.36.2-x86-linux-musl)
93
+ bigdecimal
94
+ rake (~> 13.3)
95
+ google-protobuf (4.36.2-x86_64-darwin)
96
+ bigdecimal
97
+ rake (~> 13.3)
98
+ google-protobuf (4.36.2-x86_64-linux-gnu)
99
+ bigdecimal
100
+ rake (~> 13.3)
101
+ google-protobuf (4.36.2-x86_64-linux-musl)
102
+ bigdecimal
103
+ rake (~> 13.3)
104
+ googleapis-common-protos (1.11.0)
81
105
  google-protobuf (~> 4.26)
82
106
  googleapis-common-protos-types (~> 1.21)
83
- grpc (~> 1.41)
84
- googleapis-common-protos-types (1.22.0)
107
+ grpc (>= 1.84.1, < 2.a)
108
+ googleapis-common-protos-types (1.24.0)
85
109
  google-protobuf (~> 4.26)
86
- googleauth (1.16.1)
110
+ googleauth (1.17.7)
87
111
  faraday (>= 1.0, < 3.a)
88
112
  google-cloud-env (~> 2.2)
89
113
  google-logging-utils (~> 0.1)
90
114
  jwt (>= 1.4, < 4.0)
91
- multi_json (~> 1.11)
92
115
  os (>= 0.9, < 2.0)
116
+ pstore (~> 0.1)
93
117
  signet (>= 0.16, < 2.a)
94
- grpc (1.76.0)
118
+ grpc (1.85.0)
95
119
  google-protobuf (>= 3.25, < 5.0)
96
120
  googleapis-common-protos-types (~> 1.0)
97
- grpc (1.76.0-aarch64-linux-gnu)
121
+ grpc (1.85.0-aarch64-linux-gnu)
98
122
  google-protobuf (>= 3.25, < 5.0)
99
123
  googleapis-common-protos-types (~> 1.0)
100
- grpc (1.76.0-aarch64-linux-musl)
124
+ grpc (1.85.0-aarch64-linux-musl)
101
125
  google-protobuf (>= 3.25, < 5.0)
102
126
  googleapis-common-protos-types (~> 1.0)
103
- grpc (1.76.0-arm64-darwin)
127
+ grpc (1.85.0-arm64-darwin)
104
128
  google-protobuf (>= 3.25, < 5.0)
105
129
  googleapis-common-protos-types (~> 1.0)
106
- grpc (1.76.0-x86-linux-gnu)
130
+ grpc (1.85.0-x86-linux-gnu)
107
131
  google-protobuf (>= 3.25, < 5.0)
108
132
  googleapis-common-protos-types (~> 1.0)
109
- grpc (1.76.0-x86-linux-musl)
133
+ grpc (1.85.0-x86-linux-musl)
110
134
  google-protobuf (>= 3.25, < 5.0)
111
135
  googleapis-common-protos-types (~> 1.0)
112
- grpc (1.76.0-x86_64-darwin)
136
+ grpc (1.85.0-x86_64-darwin)
113
137
  google-protobuf (>= 3.25, < 5.0)
114
138
  googleapis-common-protos-types (~> 1.0)
115
- grpc (1.76.0-x86_64-linux-gnu)
139
+ grpc (1.85.0-x86_64-linux-gnu)
116
140
  google-protobuf (>= 3.25, < 5.0)
117
141
  googleapis-common-protos-types (~> 1.0)
118
- grpc (1.76.0-x86_64-linux-musl)
142
+ grpc (1.85.0-x86_64-linux-musl)
119
143
  google-protobuf (>= 3.25, < 5.0)
120
144
  googleapis-common-protos-types (~> 1.0)
121
- guard (2.20.0)
145
+ guard (2.20.2)
122
146
  formatador (>= 0.2.4)
123
147
  listen (>= 2.7, < 4.0)
124
148
  logger (~> 1.6)
@@ -146,14 +170,16 @@ GEM
146
170
  logger (1.7.0)
147
171
  lumberjack (1.4.2)
148
172
  method_source (1.1.0)
149
- multi_json (1.21.2)
150
173
  nenv (0.3.0)
151
- net-http (0.9.1)
174
+ net-http (0.10.0)
175
+ net-protocol (>= 0.3.0)
152
176
  uri (>= 0.11.1)
177
+ net-protocol (0.4.0)
178
+ timeout
153
179
  notiffany (0.1.3)
154
180
  nenv (~> 0.1)
155
181
  shellany (~> 0.0)
156
- octokit (6.1.1)
182
+ octokit (10.0.0)
157
183
  faraday (>= 1, < 3)
158
184
  sawyer (~> 0.9)
159
185
  os (1.1.4)
@@ -164,17 +190,19 @@ GEM
164
190
  ast (~> 2.4.1)
165
191
  racc
166
192
  prism (1.9.0)
167
- pry (0.14.2)
193
+ pry (0.16.1)
168
194
  coderay (~> 1.1)
169
195
  method_source (~> 1.0)
170
- public_suffix (7.0.2)
196
+ reline (>= 0.6.0)
197
+ pstore (0.3.0)
198
+ public_suffix (7.0.5)
171
199
  racc (1.8.1)
172
200
  rainbow (3.1.1)
173
- rake (13.3.1)
201
+ rake (13.4.2)
174
202
  rb-fsevent (0.11.2)
175
203
  rb-inotify (0.11.1)
176
204
  ffi (~> 1.0)
177
- rbtree (0.4.6)
205
+ rbtree (0.4.7)
178
206
  regexp_parser (2.11.3)
179
207
  reline (0.6.3)
180
208
  io-console (~> 0.5)
@@ -214,16 +242,16 @@ GEM
214
242
  addressable (>= 2.3.5)
215
243
  faraday (>= 0.17.3, < 3)
216
244
  shellany (0.0.1)
217
- signet (0.21.0)
245
+ signet (0.22.2)
218
246
  addressable (~> 2.8)
219
247
  faraday (>= 0.17.5, < 3.a)
220
248
  jwt (>= 1.5, < 4.0)
221
- multi_json (~> 1.10)
222
249
  thor (1.5.0)
250
+ timeout (0.6.1)
223
251
  unicode-display_width (3.2.0)
224
252
  unicode-emoji (~> 4.1)
225
253
  unicode-emoji (4.2.0)
226
- uri (1.1.1)
254
+ uri (1.1.2)
227
255
 
228
256
  PLATFORMS
229
257
  aarch64-linux-gnu
@@ -244,7 +272,7 @@ DEPENDENCIES
244
272
  fuubar
245
273
  guard (~> 2.18)
246
274
  guard-rspec (~> 4.7)
247
- pry (~> 0.14.1)
275
+ pry (~> 0.16)
248
276
  rake (~> 13.0)
249
277
  rspec (~> 3.12)
250
278
  rubocop (~> 1.92)
data/README.md CHANGED
@@ -84,8 +84,11 @@ Options:
84
84
  -V, [--version], [--no-version] # Print the version and exit
85
85
  -v, [--verbose], [--no-verbose] # Increase verbosity. May be repeated for even more verbosity.
86
86
  # Default: [true]
87
+ [--dry-run], [--no-dry-run] # Validate and show what would be added without writing
87
88
  ```
88
89
 
90
+ `imt add mod` validates every entry in the file before writing any of them, and each `add` command exits 1 on failure.
91
+
89
92
  #### `imt list`
90
93
 
91
94
  ```sh
@@ -149,7 +152,7 @@ Options:
149
152
  [--force] # Let mods/tools sync delete over 25% of a collection
150
153
  ```
151
154
 
152
- **Removal**: Mod and tool authors control their own entries. If a `modinfo.json` or `toolinfo.json` is removed or renamed (so it returns HTTP 404 or 410), `imt sync mods`/`tools` deletes its entries. If a file cannot be read for any other reason (timeout, network error, server error, rate limit, or invalid JSON), that file's entries are kept on that run and a warning names them; other sources are unaffected. Sync tracks which file each entry came from in the internal `meta/mod_sources` and `meta/tool_sources` documents (override with `collections.meta.mod_sources` / `tool_sources` in the config). An entry with no recorded source yet is kept while any source fails. As a brake on a wave of 404s (a GitHub outage, a mass rename), `imt sync mods`/`tools` refuses to delete more than 25% of a collection (at least 5 entries), or to empty it, and exits 1; rerun with `--force` once you have checked the sources.
155
+ **Removal**: Mod and tool authors control their own entries. If a `modinfo.json` or `toolinfo.json` is removed or renamed (so it returns HTTP 404 or 410), `imt sync mods`/`tools` deletes its entries. If a file cannot be read for any other reason (timeout, network error, server error, rate limit, or invalid JSON), that file's entries are kept on that run and a warning names them; other sources are unaffected. Sync tracks which file each entry came from in the internal `meta/mod_sources` and `meta/tool_sources` documents (override with `collections.meta.mod_sources` / `tool_sources` in the config). An entry with no recorded source yet is kept while any source fails. As a brake on a wave of 404s (a GitHub outage, a mass rename), `imt sync mods`/`tools` refuses to delete more than 25% of a collection (at least 5 entries), or to empty it, and exits 1; rerun with `--force` once you have checked the sources. `--dry-run` lists what a real run would delete and warns if the brake would stop it.
153
156
 
154
157
  **Exit status**: each `imt sync` command exits 1 when it fails. `imt sync all` runs every stage even if one fails, then exits 1.
155
158
 
@@ -157,6 +160,20 @@ Options:
157
160
 
158
161
  **Note**: Sync operations automatically convert GitHub URLs to the correct `raw.githubusercontent.com` format for direct downloads. If you're using GitHub URLs in `modinfo.json` or `toolinfo.json` files, you can use either `/blob/` or `/raw/` formats and they will be automatically converted. Warnings will be displayed when URLs are auto-fixed so you can update your source files.
159
162
 
163
+ #### `imt validate`
164
+
165
+ ```sh
166
+ Commands:
167
+ imt validate help [COMMAND] # Describe subcommands or one specific subcommand
168
+ imt validate modinfo # Reads modinfo data from 'meta/modinfo/list' and Validates syntax of modfiles
169
+ imt validate repo OWNER/REPO # Vets a GitHub repo before it is added to 'meta/repos/list'
170
+ imt validate toolinfo # Reads Toolinfo data from 'meta/toolinfo/list' and Validates syntax of toolfiles
171
+ ```
172
+
173
+ `imt validate repo OWNER/REPO [--json]` (a `github.com` URL also works) vets a repo someone asked to have added, before you add it. It is read-only, never touches Firestore, and needs only the GitHub token from the config. It checks that the repo exists, is public and is not archived; finds `modinfo.json` and/or `toolinfo.json` (at least one is required); validates every entry; and requests every file, image and readme link (following up to 5 redirects) to confirm it resolves. A broken download link is an error, a broken image or readme link is a warning, and a download hosted outside GitHub or a tool shipping an `.exe` is a warning. Use `-v`/`-vv` for more detail, or `--json` to print the machine-readable result (`repo`, `exists`, `private`, `archived`, `default_branch`, `info_files`, `problems`, `entries`, `valid`) and nothing else on stdout.
174
+
175
+ **Exit status**: 0 when the repo is valid, 1 when it is not, 2 on usage, config or GitHub API errors (for example a rate limit or bad token).
176
+
160
177
  ## Development
161
178
 
162
179
  After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
@@ -31,8 +31,8 @@ Gem::Specification.new do |spec|
31
31
 
32
32
  # Uncomment to register a new dependency of your gem
33
33
  # spec.add_dependency "example-gem", "~> 1.0"
34
- spec.add_dependency "google-cloud-firestore", "~> 2.7"
35
- spec.add_dependency "octokit", "~> 6.0"
34
+ spec.add_dependency "google-cloud-firestore", "~> 3.3"
35
+ spec.add_dependency "octokit", "~> 10.0"
36
36
  spec.add_dependency "ostruct", "~> 0.6"
37
37
  spec.add_dependency "paint", "~> 2.3"
38
38
  spec.add_dependency "thor", "~> 1.2"
@@ -268,19 +268,23 @@ module Icarus
268
268
  puts success_or_failure(response) if verbose > 1
269
269
  end
270
270
 
271
+ unless options[:dry_run]
272
+ puts "Created/Updated #{info_array.count} Items" if verbose?
273
+ sync.save_sources
274
+ end
275
+
276
+ delete_array = protect_transient_sources(sync, list_array.filter { |list| sync.find_info(list).nil? })
277
+
271
278
  if options[:dry_run]
279
+ preview_deletions(type, delete_array, list_array, info_array)
272
280
  puts Paint["Dry run; no changes will be made", :white] if verbose?
273
281
  return
274
282
  end
275
283
 
276
- puts "Created/Updated #{info_array.count} Items" if verbose?
277
- sync.save_sources
278
-
279
- delete_array = protect_transient_sources(sync, list_array.filter { |list| sync.find_info(list).nil? })
280
-
281
284
  return unless delete_array.any?
282
285
 
283
- refuse_mass_delete(type, delete_array, list_array, info_array)
286
+ raise Icarus::Mod::Tools::Error, mass_delete_message("Refusing", type, delete_array, list_array) if
287
+ mass_delete?(delete_array, list_array, info_array)
284
288
 
285
289
  puts "Deleting outdated items..." if verbose?
286
290
  delete_array.each do |list|
@@ -301,16 +305,28 @@ module Icarus
301
305
 
302
306
  # A wave of 404s (a GitHub outage, a mass rename) looks like every author removing their file, so stage 2
303
307
  # will not delete more than MASS_DELETE_RATIO of a collection, or empty it, unless run with --force
304
- def refuse_mass_delete(type, delete_array, list_array, info_array)
305
- return if options[:force]
308
+ def mass_delete?(delete_array, list_array, info_array)
309
+ return false if options[:force] || delete_array.empty?
306
310
 
307
311
  emptying = info_array.empty? && delete_array.size >= list_array.size
308
312
  too_many = delete_array.size >= MASS_DELETE_MIN && delete_array.size > list_array.size * MASS_DELETE_RATIO
309
- return unless emptying || too_many
313
+ emptying || too_many
314
+ end
315
+
316
+ def mass_delete_message(verb, type, delete_array, list_array)
317
+ Paint["#{verb} to delete #{delete_array.size} of #{list_array.size} #{type}; " \
318
+ "check the sources, then rerun with --force if this is intended", :red, :bright]
319
+ end
320
+
321
+ # What a real run would delete, and whether the mass-delete brake would stop it
322
+ def preview_deletions(type, delete_array, list_array, info_array)
323
+ return unless verbose?
324
+
325
+ puts "Would delete #{delete_array.count} outdated item(s)#{":" if delete_array.any?}"
326
+ delete_array.each { |list| puts Paint[" #{list.author || "NoOne"}/#{list.name || "Unnamed"} (#{list.id})", :yellow] }
327
+ return unless mass_delete?(delete_array, list_array, info_array)
310
328
 
311
- raise Icarus::Mod::Tools::Error,
312
- Paint["Refusing to delete #{delete_array.size} of #{list_array.size} #{type}; " \
313
- "check the sources, then rerun with --force if this is intended", :red, :bright]
329
+ warn mass_delete_message("Would refuse", type, delete_array, list_array)
314
330
  end
315
331
  end
316
332
  end
@@ -1,6 +1,8 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require "tools/validator"
4
+ require "tools/repo_check"
5
+ require "tools/repo_report"
4
6
 
5
7
  module Icarus
6
8
  module Mod
@@ -17,6 +19,26 @@ module Icarus
17
19
  validate(:toolinfo)
18
20
  end
19
21
 
22
+ desc "repo OWNER/REPO", "Vets a GitHub repo (and its modinfo/toolinfo links) before it is added to 'meta/repos/list'; read-only"
23
+ method_option :json, type: :boolean, default: false, desc: "Print the result as JSON"
24
+ long_desc <<~DESC
25
+ Exits 0 when the repo is valid, 1 when it is not, and 2 on usage, config or GitHub API errors.
26
+ Never writes to Firestore and does not need it; only the GitHub token from the config is used.
27
+ DESC
28
+ def repo(name)
29
+ result = Icarus::Mod::Tools::RepoCheck.new(name).call
30
+
31
+ if options[:json]
32
+ puts JSON.generate(result)
33
+ else
34
+ Icarus::Mod::Tools::RepoReport.new(result, verbose:).print
35
+ end
36
+ exit(result[:valid] ? 0 : 1)
37
+ rescue Icarus::Mod::Tools::RepoCheck::Error => e
38
+ warn "Error: #{e.message}"
39
+ exit 2
40
+ end
41
+
20
42
  no_commands do
21
43
  def validate(type)
22
44
  exit_code = 0
@@ -0,0 +1,32 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "uri"
4
+ require "tools/modinfo"
5
+
6
+ module Icarus
7
+ module Mod
8
+ module Tools
9
+ # The links inside a Modinfo/Toolinfo entry that RepoCheck probes
10
+ module EntryLinks
11
+ module_function
12
+
13
+ # [field, url, download?] for every non-blank string link in the entry
14
+ def for(info)
15
+ links = info.is_a?(Modinfo) ? mod_links(info) : [["fileURL", info.fileURL, true]]
16
+ links += [["imageURL", info.imageURL, false], ["readmeURL", info.readmeURL, false]]
17
+ links.select { |_, url, _| url.is_a?(String) && !url.strip.empty? }
18
+ end
19
+
20
+ def mod_links(info)
21
+ info.file_urls.zip(info.file_types).map { |url, name| ["files.#{name}", url, true] }
22
+ end
23
+
24
+ def windows_tool?(info)
25
+ info.fileType.to_s.match?(/\A\.?exe\z/i) || URI.parse(info.fileURL.to_s).path.to_s.match?(/\.exe\z/i)
26
+ rescue URI::InvalidURIError
27
+ false
28
+ end
29
+ end
30
+ end
31
+ end
32
+ end
@@ -0,0 +1,116 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "ipaddr"
4
+ require "resolv"
5
+ require "uri"
6
+ require "net/http"
7
+ require "tools/sync/helpers"
8
+
9
+ module Icarus
10
+ module Mod
11
+ module Tools
12
+ # Requests a URL (following redirects) without downloading it, to see where it ends up
13
+ class LinkProbe
14
+ MAX_REDIRECTS = 5
15
+ REDIRECT_CODES = [301, 302, 303, 307, 308].freeze
16
+ # Statuses that mean "HEAD is not allowed here", so try a ranged GET instead
17
+ HEAD_REFUSED = [403, 405, 501].freeze
18
+ OK_STATUSES = [200, 206].freeze
19
+
20
+ # Links come from a stranger's modinfo.json and are probed from the maintainer's network, and the status
21
+ # ends up in a public comment: never request loopback, private, link-local or other non-public addresses
22
+ NON_PUBLIC = %w[0.0.0.0/8 100.64.0.0/10 192.0.0.0/24 198.18.0.0/15 224.0.0.0/3 ::/128 64:ff9b::/96 ff00::/8]
23
+ .map { |range| IPAddr.new(range) }.freeze
24
+
25
+ # http: a callable (uri, verb, ip) -> [status, location]; resolver: a callable (host) -> [addresses].
26
+ # Both are replaceable so specs need no network.
27
+ def initialize(http: nil, resolver: nil)
28
+ @http = http || method(:net_request)
29
+ @resolver = resolver || Resolv.method(:getaddresses)
30
+ end
31
+
32
+ def self.github_host?(host)
33
+ host == "github.com" || host.to_s.end_with?(".github.com") || host == "githubusercontent.com" ||
34
+ host.to_s.end_with?(".githubusercontent.com")
35
+ end
36
+
37
+ # Follows up to MAX_REDIRECTS redirects; returns [status, final_host, problem]
38
+ def call(url)
39
+ uri = URI.parse(url)
40
+ status = nil
41
+ (MAX_REDIRECTS + 1).times do
42
+ status, location, refused = hop(uri)
43
+ return [status, uri.host, refused] if refused || !(REDIRECT_CODES.include?(status) && location)
44
+
45
+ uri = URI.join(uri, location)
46
+ end
47
+ [status, uri.host, "too many redirects (more than #{MAX_REDIRECTS})"]
48
+ rescue URI::Error, *Sync::Helpers::NETWORK_ERRORS => e
49
+ [nil, nil, "request failed (#{e.message})"]
50
+ end
51
+
52
+ private
53
+
54
+ # One request, after checking where it would go: [status, location] or [nil, nil, refusal]
55
+ def hop(uri)
56
+ ips, refused = public_addresses(uri)
57
+ refused ? [nil, nil, refused] : request(uri, ips)
58
+ end
59
+
60
+ # The addresses to connect to (IPv4 first), or a reason to refuse. Every address the host resolves to must be
61
+ # public, and the request then goes to a checked address, so a DNS answer that changes in between cannot
62
+ # redirect it.
63
+ def public_addresses(uri)
64
+ return [nil, "refused: only http and https links are checked"] unless %w[http https].include?(uri.scheme)
65
+
66
+ host = uri.hostname
67
+ addresses = ip_literal?(host) ? [host] : @resolver.call(host)
68
+ return [nil, "refused: #{uri.host} does not resolve"] if addresses.empty?
69
+ return [nil, "refused: #{uri.host} resolves to a non-public address"] unless addresses.all? { |a| public_ip?(a) }
70
+
71
+ [addresses.partition { |a| IPAddr.new(a).ipv4? }.flatten, nil]
72
+ end
73
+
74
+ def ip_literal?(host)
75
+ IPAddr.new(host)
76
+ true
77
+ rescue IPAddr::InvalidAddressError
78
+ false
79
+ end
80
+
81
+ def public_ip?(address)
82
+ ip = IPAddr.new(address)
83
+ ip = ip.native if ip.ipv4_mapped?
84
+ !(ip.private? || ip.loopback? || ip.link_local? || NON_PUBLIC.any? { |range| range.include?(ip) })
85
+ rescue IPAddr::InvalidAddressError
86
+ false
87
+ end
88
+
89
+ # HEAD, then a one-byte ranged GET if HEAD is refused. An unreachable address (no IPv6 route, say) falls back
90
+ # to the next checked one; the last failure propagates.
91
+ def request(uri, ips)
92
+ ips.each_with_index do |ip, index|
93
+ result = @http.call(uri, "HEAD", ip)
94
+ return HEAD_REFUSED.include?(result.first) ? @http.call(uri, "GET", ip) : result
95
+ rescue *Sync::Helpers::NETWORK_ERRORS
96
+ raise if index == ips.size - 1
97
+ end
98
+ end
99
+
100
+ def net_request(uri, verb, ip)
101
+ req = verb == "GET" ? Net::HTTP::Get.new(uri.request_uri, "Range" => "bytes=0-0") : Net::HTTP::Head.new(uri.request_uri)
102
+ # Yield from the block so a server that ignores Range never has its whole body downloaded
103
+ connection(uri, ip).start { |conn| conn.request(req) { |res| return [res.code.to_i, res["location"]] } }
104
+ end
105
+
106
+ def connection(uri, ip)
107
+ Net::HTTP.new(uri.host, uri.port).tap do |http|
108
+ http.ipaddr = ip
109
+ http.use_ssl = uri.scheme == "https"
110
+ http.open_timeout = http.read_timeout = 15
111
+ end
112
+ end
113
+ end
114
+ end
115
+ end
116
+ end
@@ -0,0 +1,140 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "github"
4
+ require "tools/modinfo"
5
+ require "tools/toolinfo"
6
+ require "tools/link_probe"
7
+ require "tools/entry_links"
8
+ require "tools/sync/helpers"
9
+
10
+ module Icarus
11
+ module Mod
12
+ module Tools
13
+ # Read-only vetting of a GitHub repo before it is added to meta/repos: checks the repo itself, its
14
+ # modinfo.json/toolinfo.json and every link they contain. Needs only the GitHub token, never Firestore.
15
+ class RepoCheck
16
+ include Sync::Helpers
17
+
18
+ # Usage or GitHub API failure (as opposed to a repo that simply fails validation)
19
+ class Error < StandardError; end
20
+
21
+ TYPES = {
22
+ modinfo: { file: "modinfo.json", type: "mod", key: :mods, entity: Modinfo },
23
+ toolinfo: { file: "toolinfo.json", type: "tool", key: :tools, entity: Toolinfo }
24
+ }.freeze
25
+
26
+ # http/resolver: see LinkProbe; replaceable so specs need no network
27
+ def initialize(repo, github: nil, http: nil, resolver: nil)
28
+ @github = github || Github.new
29
+ @github.repository = repo
30
+ @probe = LinkProbe.new(http:, resolver:)
31
+ end
32
+
33
+ def call
34
+ result = blank_result(repo_name)
35
+ check_repo(result[:repo], result)
36
+ result[:valid] = valid?(result)
37
+ result
38
+ rescue Octokit::Error => e
39
+ raise Error, "GitHub API error: #{e.class.name.split("::").last}: #{e.message.lines.first&.strip}"
40
+ end
41
+
42
+ private
43
+
44
+ def repo_name
45
+ repo = @github.repository.to_s.delete_suffix("/").delete_suffix(".git")
46
+ raise Error, "Repository must be given as OWNER/REPO" unless %r{\A[\w.-]+/[\w.-]+\z}.match?(repo)
47
+
48
+ repo
49
+ end
50
+
51
+ def blank_result(repo)
52
+ { repo:, exists: false, private: false, archived: false, default_branch: nil,
53
+ info_files: { modinfo: nil, toolinfo: nil }, problems: [], entries: [], valid: false }
54
+ end
55
+
56
+ def check_repo(repo, result)
57
+ record_repo(@github.client.repository(repo), result)
58
+ check_info_files(result) unless result[:private] || result[:archived]
59
+ rescue Octokit::NotFound
60
+ result[:problems] << "repository not found"
61
+ end
62
+
63
+ def record_repo(data, result)
64
+ result.merge!(exists: true, private: data[:private] ? true : false, archived: data[:archived] ? true : false,
65
+ default_branch: data[:default_branch])
66
+ result[:problems] << "repository is private" if result[:private]
67
+ result[:problems] << "repository is archived" if result[:archived]
68
+ end
69
+
70
+ def check_info_files(result)
71
+ TYPES.each do |kind, meta|
72
+ file = @github.find(meta[:file])
73
+ next unless file
74
+
75
+ result[:info_files][kind] = file[:download_url]
76
+ check_info_file(meta, file[:download_url], result)
77
+ end
78
+
79
+ result[:problems] << "no modinfo.json or toolinfo.json found" if result[:info_files].values.none?
80
+ end
81
+
82
+ def check_info_file(meta, url, result)
83
+ entries = entries_from(meta, retrieve_from_url(url))
84
+ entries.each_with_index do |data, index|
85
+ if usable_name?(data)
86
+ result[:entries] << check_entry(meta, data)
87
+ else
88
+ result[:problems] << "#{meta[:file]}: entry #{index + 1} has no usable name (sync would skip it)"
89
+ end
90
+ end
91
+ rescue Sync::RequestFailed, JSON::ParserError => e
92
+ result[:problems] << "#{meta[:file]}: #{e.message}"
93
+ end
94
+
95
+ # Same rules as Sync::Entities#entries_from
96
+ def entries_from(meta, body)
97
+ entries = body[meta[:key]] if body.is_a?(Hash)
98
+ raise JSON::ParserError, "no '#{meta[:key]}' array" unless entries.is_a?(Array)
99
+ raise JSON::ParserError, "'#{meta[:key]}' entries must be objects" unless entries.all?(Hash)
100
+
101
+ entries
102
+ end
103
+
104
+ def usable_name?(data)
105
+ data[:name].is_a?(String) && /[a-z0-9]+/i.match?(data[:name])
106
+ end
107
+
108
+ def check_entry(meta, data)
109
+ info = meta[:entity].new(data)
110
+ entry = entry_for(meta[:type], info)
111
+
112
+ entry[:warnings] << "tool ships a Windows executable" if meta[:type] == "tool" && EntryLinks.windows_tool?(info)
113
+ EntryLinks.for(info).each { |field, url, download| check_link(entry, field, url, download) }
114
+ entry
115
+ end
116
+
117
+ def entry_for(type, info)
118
+ { type:, name: info.name, author: info.author, version: info.version,
119
+ errors: info.errors.dup, warnings: info.warnings.dup, links: [] }
120
+ end
121
+
122
+ def check_link(entry, field, url, download)
123
+ status, host, problem = @probe.call(url)
124
+ entry[:links] << { field:, url:, status:, final_host: host }
125
+
126
+ if problem || !LinkProbe::OK_STATUSES.include?(status)
127
+ entry[download ? :errors : :warnings] << "#{field}: #{problem || "HTTP #{status}"} (#{url})"
128
+ elsif download && !LinkProbe.github_host?(host)
129
+ entry[:warnings] << "download hosted off GitHub: #{host}"
130
+ end
131
+ end
132
+
133
+ def valid?(result)
134
+ result[:exists] && result[:info_files].values.any? && result[:problems].empty? &&
135
+ result[:entries].none? { |entry| entry[:errors].any? }
136
+ end
137
+ end
138
+ end
139
+ end
140
+ end
@@ -0,0 +1,51 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "paint"
4
+
5
+ module Icarus
6
+ module Mod
7
+ module Tools
8
+ # Human-readable output for a RepoCheck result, in the style of `imt validate modinfo`
9
+ class RepoReport
10
+ def initialize(result, verbose: 1)
11
+ @result = result
12
+ @verbose = verbose
13
+ end
14
+
15
+ def print
16
+ print_header
17
+ @result[:problems].each { |problem| puts Paint["#{problem} in #{@result[:repo]}", :red] }
18
+ @result[:entries].each { |entry| print_entry(entry) }
19
+ print_verdict if @verbose.positive?
20
+ end
21
+
22
+ private
23
+
24
+ def print_header
25
+ puts Paint["Validating #{@result[:repo]}", :cyan, :bright] if @verbose.positive?
26
+ return unless @verbose > 1
27
+
28
+ puts Paint["Default branch: #{@result[:default_branch]}", :cyan]
29
+ @result[:info_files].each { |kind, url| puts "#{kind}: #{url || "not found"}" }
30
+ end
31
+
32
+ def print_verdict
33
+ valid = @result[:valid]
34
+ puts Paint[valid ? "VALID" : "NOT VALID", valid ? :green : :red, :bright]
35
+ end
36
+
37
+ def print_links(entry, name)
38
+ puts Paint["Checked #{entry[:type]} #{name}", :cyan, :bright]
39
+ entry[:links].each { |link| puts " #{link[:field]}: #{link[:status] || "failed"} #{link[:final_host]}" }
40
+ end
41
+
42
+ def print_entry(entry)
43
+ name = "#{entry[:author].to_s.strip}/#{entry[:name].to_s.strip}"
44
+ print_links(entry, name) if @verbose > 1
45
+ entry[:errors].each { |error| puts Paint["#{error} in #{name}", :red] }
46
+ entry[:warnings].each { |warning| puts Paint["#{warning} in #{name}", :yellow] }
47
+ end
48
+ end
49
+ end
50
+ end
51
+ end
@@ -2,6 +2,6 @@
2
2
 
3
3
  module Icarus
4
4
  module Mod
5
- VERSION = "3.0.0"
5
+ VERSION = "3.2.0"
6
6
  end
7
7
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: Icarus-Mod-Tools
3
3
  version: !ruby/object:Gem::Version
4
- version: 3.0.0
4
+ version: 3.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Donovan Young
@@ -15,28 +15,28 @@ dependencies:
15
15
  requirements:
16
16
  - - "~>"
17
17
  - !ruby/object:Gem::Version
18
- version: '2.7'
18
+ version: '3.3'
19
19
  type: :runtime
20
20
  prerelease: false
21
21
  version_requirements: !ruby/object:Gem::Requirement
22
22
  requirements:
23
23
  - - "~>"
24
24
  - !ruby/object:Gem::Version
25
- version: '2.7'
25
+ version: '3.3'
26
26
  - !ruby/object:Gem::Dependency
27
27
  name: octokit
28
28
  requirement: !ruby/object:Gem::Requirement
29
29
  requirements:
30
30
  - - "~>"
31
31
  - !ruby/object:Gem::Version
32
- version: '6.0'
32
+ version: '10.0'
33
33
  type: :runtime
34
34
  prerelease: false
35
35
  version_requirements: !ruby/object:Gem::Requirement
36
36
  requirements:
37
37
  - - "~>"
38
38
  - !ruby/object:Gem::Version
39
- version: '6.0'
39
+ version: '10.0'
40
40
  - !ruby/object:Gem::Dependency
41
41
  name: ostruct
42
42
  requirement: !ruby/object:Gem::Requirement
@@ -116,7 +116,11 @@ files:
116
116
  - lib/icarus/mod/github.rb
117
117
  - lib/icarus/mod/tools.rb
118
118
  - lib/icarus/mod/tools/baseinfo.rb
119
+ - lib/icarus/mod/tools/entry_links.rb
120
+ - lib/icarus/mod/tools/link_probe.rb
119
121
  - lib/icarus/mod/tools/modinfo.rb
122
+ - lib/icarus/mod/tools/repo_check.rb
123
+ - lib/icarus/mod/tools/repo_report.rb
120
124
  - lib/icarus/mod/tools/sync/entities.rb
121
125
  - lib/icarus/mod/tools/sync/helpers.rb
122
126
  - lib/icarus/mod/tools/sync/modinfo_list.rb