Icarus-Mod-Tools 2.6.0 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: f02e414cd84c6117e8ff650a1048c16eac751dae9cfe99a52a17a3e58682d1b0
4
- data.tar.gz: f14a5bf74a4fcd74be9c71aba1debf378d45af6bc54290b35c2d20236e17d3af
3
+ metadata.gz: 17d3d3cdfeeb5953e405d4888f0c748b00d330af51a14714c51e04b927baedb9
4
+ data.tar.gz: 29f015ac208933825b954bb33fda452cd684499f8e2e10950d241e0a4323fbac
5
5
  SHA512:
6
- metadata.gz: 636dfd397c992d7dd15d464584734f7e599417e1145f0cf139ffcf2f8eec9e409b473c4deae9abae453a42abbfe1f04138b36440669ead6db5c4386b6e33b4ca
7
- data.tar.gz: b5085429a592dfc1bd8bbf0fde991ade384afcd88aedd50c6f0969a8a786b4c3e98c740fda3053aa48c52ea734dfe74381d2d8797dde0b0e563abc467ec3e75d
6
+ metadata.gz: fad56b2b02cfd1412d332f2d47be963ef557ce21bfa25f3af6b103ee99ab4cb632517e23a59d7d512cbc339746906392396133b238d004e9c0ef82a23e76fc3e
7
+ data.tar.gz: bd460ed0255ede73b7cda973e0d0a39c22a2a285600e6f2399920905c1b626e2b40dc65599578f67e50b8f8c609238fde203eb7b3cd4090c3c3ef7e24d3fb974
data/.rubocop.yml ADDED
@@ -0,0 +1,55 @@
1
+ inherit_from: .rubocop_todo.yml
2
+
3
+ plugins:
4
+ - rubocop-rspec
5
+
6
+ AllCops:
7
+ TargetRubyVersion: 4.0
8
+ NewCops: enable
9
+ SuggestExtensions: false
10
+ Exclude:
11
+ - "pkg/**/*"
12
+ - "vendor/**/*"
13
+
14
+ Style/StringLiterals:
15
+ EnforcedStyle: double_quotes
16
+
17
+ Style/StringLiteralsInInterpolation:
18
+ EnforcedStyle: double_quotes
19
+
20
+ Layout/LineLength:
21
+ Max: 140
22
+
23
+ # The specs deliberately share many `let`s, group several expectations per example, mix `have_received`
24
+ # and `receive`, stub with `allow_any_instance_of` for the HTTP helpers, and build configs from OpenStruct
25
+ # (as Config does). These cops would force a rewrite of the suite for no behavior gain.
26
+ RSpec/MultipleMemoizedHelpers:
27
+ Enabled: false
28
+ RSpec/MultipleExpectations:
29
+ Enabled: false
30
+ RSpec/ExampleLength:
31
+ Enabled: false
32
+ RSpec/NestedGroups:
33
+ Enabled: false
34
+ RSpec/MessageSpies:
35
+ Enabled: false
36
+ RSpec/AnyInstance:
37
+ Enabled: false
38
+ RSpec/IndexedLet:
39
+ Enabled: false
40
+ RSpec/ReceiveMessages:
41
+ Enabled: false
42
+ RSpec/VerifiedDoubles:
43
+ Enabled: false
44
+ RSpec/SubjectStub:
45
+ Enabled: false
46
+ RSpec/ContextWording:
47
+ Enabled: false
48
+ RSpec/ExpectOutput:
49
+ Enabled: false
50
+ RSpec/InstanceVariable:
51
+ Enabled: false
52
+
53
+ # Config parses the JSON config into OpenStruct on purpose (dot access to nested settings)
54
+ Style/OpenStructUse:
55
+ Enabled: false
data/.rubocop_todo.yml ADDED
@@ -0,0 +1,142 @@
1
+ # This configuration was generated by
2
+ # `rubocop --auto-gen-config --no-exclude-limit`
3
+ # on 2026-10-31 01:35:19 UTC using RuboCop version 1.92.2.
4
+ # The point is for the user to remove these configuration records
5
+ # one by one as the offenses are removed from the code base.
6
+ # Note that changes in the inspected code, or installation of new
7
+ # versions of RuboCop, may require this file to be generated again.
8
+
9
+ # Offense count: 3
10
+ # This cop supports safe autocorrection (--autocorrect).
11
+ # Configuration parameters: Max, AllowHeredoc, AllowURI, AllowQualifiedName, URISchemes, AllowRBSInlineAnnotation, AllowCopDirectives, AllowedPatterns, SplitStrings.
12
+ # URISchemes: http, https
13
+ Layout/LineLength:
14
+ Exclude:
15
+ - 'lib/icarus/mod/cli/sync.rb'
16
+ - 'spec/icarus/mod/cli/remove_spec.rb'
17
+ - 'spec/icarus/mod/tools/baseinfo_spec.rb'
18
+
19
+ # Offense count: 24
20
+ # Configuration parameters: AllowedMethods, AllowedPatterns, CountRepeatedAttributes, Max.
21
+ Metrics/AbcSize:
22
+ Exclude:
23
+ - 'lib/icarus/mod/cli/add.rb'
24
+ - 'lib/icarus/mod/cli/command.rb'
25
+ - 'lib/icarus/mod/cli/list.rb'
26
+ - 'lib/icarus/mod/cli/remove.rb'
27
+ - 'lib/icarus/mod/cli/sync.rb'
28
+ - 'lib/icarus/mod/cli/validate.rb'
29
+ - 'lib/icarus/mod/firestore.rb'
30
+ - 'lib/icarus/mod/tools/baseinfo.rb'
31
+ - 'lib/icarus/mod/tools/sync/entities.rb'
32
+ - 'lib/icarus/mod/tools/sync/helpers.rb'
33
+
34
+ # Offense count: 2
35
+ # Configuration parameters: CountComments, Max, CountAsOne, AllowedMethods, AllowedPatterns, inherit_mode.
36
+ # AllowedMethods: refine
37
+ Metrics/BlockLength:
38
+ Exclude:
39
+ - 'lib/icarus/mod/cli/list.rb'
40
+ - 'lib/icarus/mod/cli/sync.rb'
41
+
42
+ # Offense count: 4
43
+ # Configuration parameters: CountComments, Max, CountAsOne.
44
+ Metrics/ClassLength:
45
+ Exclude:
46
+ - 'lib/icarus/mod/cli/remove.rb'
47
+ - 'lib/icarus/mod/cli/sync.rb'
48
+ - 'lib/icarus/mod/firestore.rb'
49
+ - 'lib/icarus/mod/tools/baseinfo.rb'
50
+
51
+ # Offense count: 16
52
+ # Configuration parameters: AllowedMethods, AllowedPatterns, Max.
53
+ Metrics/CyclomaticComplexity:
54
+ Exclude:
55
+ - 'lib/icarus/mod/cli/add.rb'
56
+ - 'lib/icarus/mod/cli/list.rb'
57
+ - 'lib/icarus/mod/cli/remove.rb'
58
+ - 'lib/icarus/mod/cli/sync.rb'
59
+ - 'lib/icarus/mod/cli/validate.rb'
60
+ - 'lib/icarus/mod/firestore.rb'
61
+ - 'lib/icarus/mod/github.rb'
62
+ - 'lib/icarus/mod/tools/baseinfo.rb'
63
+ - 'lib/icarus/mod/tools/sync/entities.rb'
64
+ - 'lib/icarus/mod/tools/sync/helpers.rb'
65
+
66
+ # Offense count: 25
67
+ # Configuration parameters: CountComments, Max, CountAsOne, AllowedMethods, AllowedPatterns.
68
+ Metrics/MethodLength:
69
+ Exclude:
70
+ - 'lib/icarus/mod/cli/add.rb'
71
+ - 'lib/icarus/mod/cli/command.rb'
72
+ - 'lib/icarus/mod/cli/list.rb'
73
+ - 'lib/icarus/mod/cli/remove.rb'
74
+ - 'lib/icarus/mod/cli/sync.rb'
75
+ - 'lib/icarus/mod/cli/validate.rb'
76
+ - 'lib/icarus/mod/firestore.rb'
77
+ - 'lib/icarus/mod/github.rb'
78
+ - 'lib/icarus/mod/tools/sync/entities.rb'
79
+ - 'lib/icarus/mod/tools/sync/helpers.rb'
80
+ - 'lib/icarus/mod/tools/sync/modinfo_list.rb'
81
+ - 'lib/icarus/mod/tools/sync/toolinfo_list.rb'
82
+
83
+ # Offense count: 2
84
+ # Configuration parameters: Max, CountKeywordArgs, MaxOptionalParameters.
85
+ Metrics/ParameterLists:
86
+ Exclude:
87
+ - 'lib/icarus/mod/cli/remove.rb'
88
+
89
+ # Offense count: 12
90
+ # Configuration parameters: AllowedMethods, AllowedPatterns, Max.
91
+ Metrics/PerceivedComplexity:
92
+ Exclude:
93
+ - 'lib/icarus/mod/cli/add.rb'
94
+ - 'lib/icarus/mod/cli/list.rb'
95
+ - 'lib/icarus/mod/cli/remove.rb'
96
+ - 'lib/icarus/mod/cli/sync.rb'
97
+ - 'lib/icarus/mod/cli/validate.rb'
98
+ - 'lib/icarus/mod/github.rb'
99
+ - 'lib/icarus/mod/tools/sync/helpers.rb'
100
+
101
+ # Offense count: 3
102
+ # Configuration parameters: Mode, AllowedMethods, AllowedPatterns, AllowBangMethods, WaywardPredicates.
103
+ # AllowedMethods: call
104
+ # WaywardPredicates: infinite?, nonzero?
105
+ Naming/PredicateMethod:
106
+ Exclude:
107
+ - 'lib/icarus/mod/cli/sync.rb'
108
+ - 'lib/icarus/mod/firestore.rb'
109
+
110
+ # Offense count: 4
111
+ # This cop supports unsafe autocorrection (--autocorrect-all).
112
+ Style/DirectiveScope:
113
+ Exclude:
114
+ - 'lib/icarus/mod/cli/remove.rb'
115
+ - 'lib/icarus/mod/cli/subcommand_base.rb'
116
+ - 'lib/icarus/mod/cli/sync.rb'
117
+ - 'lib/icarus/mod/tools/toolinfo.rb'
118
+
119
+ # Offense count: 4
120
+ # Configuration parameters: AllowedVariables.
121
+ Style/GlobalVars:
122
+ Exclude:
123
+ - 'spec/icarus/mod/cli/remove_spec.rb'
124
+ - 'spec/icarus/mod/cli/sync_spec.rb'
125
+
126
+ # Offense count: 1
127
+ Style/MultilineBlockChain:
128
+ Exclude:
129
+ - 'lib/icarus/mod/tools/sync/entities.rb'
130
+
131
+ # Offense count: 2
132
+ # This cop supports unsafe autocorrection (--autocorrect-all).
133
+ Style/ReduceToHash:
134
+ Exclude:
135
+ - 'lib/icarus/mod/tools/baseinfo.rb'
136
+ - 'lib/icarus/mod/tools/toolinfo.rb'
137
+
138
+ # Offense count: 2
139
+ # This cop supports unsafe autocorrection (--autocorrect-all).
140
+ Style/SelectByRegexp:
141
+ Exclude:
142
+ - 'lib/icarus/mod/cli/remove.rb'
data/.ruby-version CHANGED
@@ -1 +1 @@
1
- 3.4
1
+ 4.0
data/CHANGELOG.md CHANGED
@@ -4,6 +4,54 @@ All notable changes to this project will be documented in this file.
4
4
 
5
5
  ## History (reverse chronological order)
6
6
 
7
+ ### v3.0.0 - 2026-10-30
8
+
9
+ **Breaking:** requires Ruby 4.0 or greater. Sync commands now exit 1 on failure, and `imt sync mods`/`tools` refuse a mass delete without `--force`.
10
+
11
+ - Require Ruby 4.0 or greater (was 3.1)
12
+ - Fixes `imt sync` crashing during Firestore authentication with `wrong number of arguments (given 2, expected 1)` on Ruby 3.1 once `json` 3.x is installed; `multi_json` releases that support `json` 3.x (1.21.2+) need Ruby 3.2 or newer
13
+ - Update `multi_json` to 1.21.2
14
+ - Declare `ostruct` as a runtime dependency; it is no longer a default gem on Ruby 4.0, so a plain `gem install` could fail to load the config
15
+ - Update `faraday`, `jwt`, `addressable` and `concurrent-ruby` to clear known security advisories
16
+ - Remove the unused, stale RBS signatures under `sig/`
17
+ - Fix `imt sync all` not creating mods/tools for a `modinfo.json`/`toolinfo.json` found earlier in the same run; the new URL was only picked up on the next run
18
+ - Fix two sync hosts (or a sync and `imt add`/`remove`) running at once overwriting each other's changes to `meta/modinfo`, `meta/toolinfo`, `meta/repos` and the source maps
19
+ - Add a mass-delete brake: `imt sync mods`/`tools` refuses to delete more than 25% of a collection (at least 5 entries), or to empty it, unless run with `--force`
20
+ - Fix `imt sync` commands exiting 0 on failure; they now exit 1, and `imt sync all` runs every stage before exiting 1
21
+ - Fix `imt sync mods`/`tools` always printing "Success" at `-vv`, even when an update or delete failed
22
+ - Fix one `modinfo.json`/`toolinfo.json` whose body is not a JSON object (or whose entries are not objects) aborting the whole sync; it is now treated like invalid JSON and its existing entries are kept
23
+ - Fix a crash on a first run against an empty database (missing `meta` documents)
24
+ - Fix source URLs losing their query string, and a malformed URL crashing the sync instead of being skipped
25
+ - Save recorded sources even when a sync fails partway through, so the next run can still protect those entries
26
+ - Fix `imt remove repos` removing the wrong entries
27
+ - Matches a repo's `modinfo.json`/`toolinfo.json` URLs only on the GitHub host and exact `owner/repo` (case-insensitive); before, a same-named subdirectory in someone else's repo matched
28
+ - Deletes only the mods and tools that `meta/mod_sources`/`meta/tool_sources` attribute to the repo's URLs (falling back to name and author for entries with no recorded source), so a fork publishing the same name and author keeps its entry
29
+ - Cascades for repos stored as full URLs too
30
+ - Keeps the repo entry and exits 1 when any delete or fetch fails, so the removal can be retried
31
+ - Tighten validation: file types, URLs (http/https with a host) and versions are now matched exactly; input the old checks accepted (`notzipx`, `javascript:` URLs, `1.0 garbage`) is a warning for now and will become an error in a future release
32
+ - Fix non-string values (`"version": 1.0`, `"name": 5`, numeric URLs, `files` that is not an object) crashing validation; they are now reported as errors
33
+ - Fix `Toolinfo#validate` not being public, and unknown attribute names on mod/tool entries silently returning nil instead of raising
34
+ - Fix `imt list mods`/`tools` crashing on `--filter` values containing regex characters, on empty or non-text fields, on `--sort files` or fields missing from some entries, and on entries with no update time; filters are now case-insensitive substring matches and errors go to stderr
35
+ - Fix `imt validate` crashing on an empty list, and `imt` crashing on a malformed config file (it now prints an error and exits 1)
36
+ - Fix the GitHub file cache returning a top-level listing for recursive or sub-path lookups, and `find("modinfo.json")` matching names like `my_modinfo.json.bak`
37
+ - CI now also runs RuboCop and `bundler-audit`, and pins its GitHub Actions to commit SHAs; add a `.rubocop.yml` matching the project's style (existing size/complexity offenses are recorded in `.rubocop_todo.yml`)
38
+ - Update `rubocop-rspec` to 3.x (2.x no longer loads with current RuboCop)
39
+ - Fix `imt add`: `mod` validates every entry before writing anything and reports bad JSON or a missing `mods` list; every `add` command exits 1 on failure; add `--dry-run`
40
+
41
+ ### v2.7.0 - 2026-10-30
42
+
43
+ - Fix sync deleting a source's mods and tools when its `modinfo.json`/`toolinfo.json` could not be read (#16)
44
+ - Authors still control removal: if a file is removed or renamed (HTTP 404 or 410), its entries are deleted as before
45
+ - A transient failure (timeout, network/DNS/TLS error, 5xx, 403/429 rate limit, any other unexpected status) or invalid JSON no longer deletes that source's entries; deletions are blocked per source, so other authors' removals still go through, and a warning lists every kept entry and why
46
+ - Sync records which file each mod and tool came from in new internal Firestore docs, `meta/mod_sources` and `meta/tool_sources` (the mods and tools docs are unchanged). An entry with no recorded source yet, such as on the first run after upgrading, is kept while any source fails transiently
47
+ - Invalid JSON is reported with a loud warning so an author mid-edit is not wiped
48
+ - Fix a repository whose GitHub search fails transiently (rate limit, 5xx, network error) aborting `imt sync modinfo`/`toolinfo`; it is now skipped with a warning and its existing list entries are kept (#16)
49
+ - Change `imt sync cleanup` to keep the oldest duplicate, whose ID clients already store, instead of the most recently updated one (#16)
50
+ - The most recently updated duplicate's data is copied onto the kept entry before the others are deleted
51
+ - `--dry-run` shows what would be kept, copied and deleted
52
+ - Give new mods and tools deterministic document IDs derived from type, author and name, so an entry that is deleted and re-added keeps its ID (#16)
53
+ - Existing documents keep their current IDs
54
+
7
55
  ### v2.6.0 - 2026-03-04
8
56
 
9
57
  - Add `imt sync cleanup` command to remove duplicate entries from mods and tools collections
data/CLAUDE.md CHANGED
@@ -24,6 +24,7 @@ bundle exec rspec # Run tests only (with Fuubar formatter)
24
24
  bundle exec rspec spec/path/to/file_spec.rb # Run single test file
25
25
  bundle exec rspec spec/path/to/file_spec.rb:42 # Run single test at line 42
26
26
  guard # Auto-run tests on file changes
27
+ bundle exec rubocop # Lint (CI runs it; .rubocop_todo.yml holds the existing metrics offenses)
27
28
  ```
28
29
 
29
30
  ### Console & Debugging
@@ -45,11 +46,13 @@ imt [command] # Run installed gem
45
46
 
46
47
  The tool implements a two-stage sync pipeline:
47
48
 
48
- 1. **GitHub → Meta Collections**: Scan repositories for metadata files and sync to `meta/modinfo`, `meta/toolinfo`, `meta/repos`
49
+ 1. **GitHub → Meta Collections**: Scan repositories for metadata files and sync to `meta/modinfo`, `meta/toolinfo`, `meta/repos` (`meta/mod_sources` and `meta/tool_sources` are internal, written by stage 2)
49
50
  2. **Meta → Entity Collections**: Process meta lists to populate/update `mods` and `tools` collections
50
51
 
51
52
  All operations support `--dry-run` for safe testing.
52
53
 
54
+ Deletion contract (#16): stage 2 deletes a doc only when no info file still lists it. `Sync::Helpers#retrieve_from_url` raises `RequestGone` for 404/410 (author removed the file: delete) and `RequestTransient` for everything else; invalid JSON also counts as transient. Docs do not record their source URL; instead `meta/mod_sources` / `meta/tool_sources` (Firestore, shared across sync hosts) map doc ID => source URL, written by `Sync::Mods/Tools#update` + `save_sources` and pruned by `Firestore#delete`. A transient failure protects only the docs mapped to that URL; unmapped docs are kept while any URL failed. Stage 1 never drops list entries (`Firestore#update` unions server-side with `FieldValue.array_union`; source maps are written per key with `merge: true`, so concurrent hosts don't overwrite each other). Stage 2 refuses to delete more than 25% of a collection (min 5) or empty it without `--force`, and a repo whose GitHub search fails transiently is skipped. New docs get a stable ID (`Firestore#stable_id`); `imt sync cleanup` keeps the oldest duplicate and copies the newest data onto it.
55
+
53
56
  ### Core Components
54
57
 
55
58
  **CLI Layer** (`lib/icarus/mod/cli/`)
@@ -70,8 +73,7 @@ All operations support `--dry-run` for safe testing.
70
73
 
71
74
  - `ModinfoList`: GitHub → meta/modinfo sync
72
75
  - `ToolinfoList`: GitHub → meta/toolinfo sync
73
- - `Mods`: meta/modinfo → mods sync
74
- - `Tools`: meta/toolinfo → tools sync
76
+ - `Entities`: shared meta → entity sync; `Mods` (meta/modinfo → mods) and `Tools` (meta/toolinfo → tools) only set `TYPE`/`INFO`/`ENTITY`
75
77
  - `Helpers`: HTTP utilities with custom SSL verification for Ruby 3.4+ CRL issues
76
78
 
77
79
  **External Services** (`lib/icarus/mod/`)
@@ -146,7 +148,7 @@ Ruby 3.4+ with newer OpenSSL may fail CRL checking. The codebase handles this in
146
148
 
147
149
  - **Firestore credentials**: Must be the full JSON object in config, not a file path
148
150
  - **GitHub repo format**: `owner/repo` format (URL prefixes are stripped automatically)
149
- - **Ruby version**: 3.1+ required for pattern matching and modern features
151
+ - **Ruby version**: 4.0+ required
150
152
  - **Platform**: WSL2/Linux recommended; not tested on Windows
151
153
  - **Require paths**: The gemspec sets `require_paths` to both `lib` and `lib/icarus/mod`, so files use relative requires like `require "cli/base"` instead of `require_relative`
152
154
  - **Verbosity**: `-v` and `-vv` flags control output detail in subcommands
data/Gemfile CHANGED
@@ -7,14 +7,12 @@ gemspec
7
7
 
8
8
  gem "rake", "~> 13.0"
9
9
 
10
- group :develop do
10
+ group :development do
11
11
  gem "fuubar"
12
12
  gem "guard", "~> 2.18"
13
13
  gem "guard-rspec", "~> 4.7"
14
14
  gem "pry", "~> 0.14.1"
15
15
  gem "rspec", "~> 3.12"
16
- gem "rubocop", "~> 1.41"
17
- gem "rubocop-rspec", "~> 2.16", require: false
16
+ gem "rubocop", "~> 1.92"
17
+ gem "rubocop-rspec", "~> 3.10", require: false
18
18
  end
19
-
20
- gem "ostruct", "~> 0.6.3"
data/Gemfile.lock CHANGED
@@ -1,25 +1,26 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- Icarus-Mod-Tools (2.5.3)
4
+ Icarus-Mod-Tools (3.0.0)
5
5
  google-cloud-firestore (~> 2.7)
6
6
  octokit (~> 6.0)
7
+ ostruct (~> 0.6)
7
8
  paint (~> 2.3)
8
9
  thor (~> 1.2)
9
10
 
10
11
  GEM
11
12
  remote: https://rubygems.org/
12
13
  specs:
13
- addressable (2.8.8)
14
+ addressable (2.9.1)
14
15
  public_suffix (>= 2.0.2, < 8.0)
15
16
  ast (2.4.3)
16
17
  base64 (0.3.0)
17
18
  bigdecimal (3.3.1)
18
19
  coderay (1.1.3)
19
- concurrent-ruby (1.3.6)
20
+ concurrent-ruby (1.3.8)
20
21
  diff-lcs (1.6.2)
21
- faraday (2.14.0)
22
- faraday-net_http (>= 2.0, < 3.5)
22
+ faraday (2.15.1)
23
+ faraday-net_http (>= 2.0, < 3.6)
23
24
  json
24
25
  logger
25
26
  faraday-net_http (3.4.2)
@@ -133,8 +134,8 @@ GEM
133
134
  guard-compat (~> 1.1)
134
135
  rspec (>= 2.99.0, < 4.0)
135
136
  io-console (0.8.2)
136
- json (2.18.0)
137
- jwt (3.1.2)
137
+ json (3.1.0)
138
+ jwt (3.3.0)
138
139
  base64
139
140
  language_server-protocol (3.17.0.5)
140
141
  lint_roller (1.1.0)
@@ -145,7 +146,7 @@ GEM
145
146
  logger (1.7.0)
146
147
  lumberjack (1.4.2)
147
148
  method_source (1.1.0)
148
- multi_json (1.19.1)
149
+ multi_json (1.21.2)
149
150
  nenv (0.3.0)
150
151
  net-http (0.9.1)
151
152
  uri (>= 0.11.1)
@@ -190,33 +191,24 @@ GEM
190
191
  diff-lcs (>= 1.2.0, < 2.0)
191
192
  rspec-support (~> 3.13.0)
192
193
  rspec-support (3.13.7)
193
- rubocop (1.84.0)
194
- json (~> 2.3)
194
+ rubocop (1.92.2)
195
+ json (>= 2.3)
195
196
  language_server-protocol (~> 3.17.0.2)
196
197
  lint_roller (~> 1.1.0)
197
- parallel (~> 1.10)
198
+ parallel (>= 1.10)
198
199
  parser (>= 3.3.0.2)
199
200
  rainbow (>= 2.2.2, < 4.0)
200
- regexp_parser (>= 2.9.3, < 3.0)
201
+ regexp_parser (>= 2.11.3, < 3.0)
201
202
  rubocop-ast (>= 1.49.0, < 2.0)
202
203
  ruby-progressbar (~> 1.7)
203
204
  unicode-display_width (>= 2.4.0, < 4.0)
204
205
  rubocop-ast (1.49.0)
205
206
  parser (>= 3.3.7.2)
206
207
  prism (~> 1.7)
207
- rubocop-capybara (2.22.1)
208
+ rubocop-rspec (3.10.2)
208
209
  lint_roller (~> 1.1)
209
- rubocop (~> 1.72, >= 1.72.1)
210
- rubocop-factory_bot (2.28.0)
211
- lint_roller (~> 1.1)
212
- rubocop (~> 1.72, >= 1.72.1)
213
- rubocop-rspec (2.31.0)
214
- rubocop (~> 1.40)
215
- rubocop-capybara (~> 2.17)
216
- rubocop-factory_bot (~> 2.22)
217
- rubocop-rspec_rails (~> 2.28)
218
- rubocop-rspec_rails (2.29.1)
219
- rubocop (~> 1.61)
210
+ regexp_parser (>= 2.0)
211
+ rubocop (~> 1.86, >= 1.86.2)
220
212
  ruby-progressbar (1.13.0)
221
213
  sawyer (0.9.3)
222
214
  addressable (>= 2.3.5)
@@ -252,12 +244,11 @@ DEPENDENCIES
252
244
  fuubar
253
245
  guard (~> 2.18)
254
246
  guard-rspec (~> 4.7)
255
- ostruct (~> 0.6.3)
256
247
  pry (~> 0.14.1)
257
248
  rake (~> 13.0)
258
249
  rspec (~> 3.12)
259
- rubocop (~> 1.41)
260
- rubocop-rspec (~> 2.16)
250
+ rubocop (~> 1.92)
251
+ rubocop-rspec (~> 3.10)
261
252
 
262
253
  BUNDLED WITH
263
254
  4.0.5
data/Guardfile CHANGED
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  # More info at https://github.com/guard/guard#readme
2
4
 
3
5
  guard :rspec, cmd: "bundle exec rspec" do
data/README.md CHANGED
@@ -8,7 +8,7 @@ To use this app, you'll need to obtain the following:
8
8
 
9
9
  - A Github ACCESS_TOKEN (doesn't need access to any repos, this is used purely to make API calls)
10
10
  - A Google Cloud Platform credentials `keyfile.json`
11
- - Ruby 3.1 (or greater)
11
+ - Ruby 4.0 (or greater)
12
12
 
13
13
  If you aren't sure how to obtain these credentials, please see:
14
14
 
@@ -132,6 +132,7 @@ Options:
132
132
  ```sh
133
133
  Commands:
134
134
  imt sync all # Run all sync jobs
135
+ imt sync cleanup # Remove duplicate entries from mods and tools collections
135
136
  imt sync help [COMMAND] # Describe subcommands or one specific subcommand
136
137
  imt sync modinfo # Reads from 'meta/repos/list' and Syncs any modinfo files we find (github only for now)
137
138
  imt sync mods # Reads from 'meta/modinfo/list' and updates the 'mods' database accordingly
@@ -145,8 +146,15 @@ Options:
145
146
  -v, [--verbose], [--no-verbose] # Increase verbosity. May be repeated for even more verbosity.
146
147
  # Default: [true]
147
148
  [--dry-run], [--no-dry-run] # Dry run (no changes will be made)
149
+ [--force] # Let mods/tools sync delete over 25% of a collection
148
150
  ```
149
151
 
152
+ **Removal**: Mod and tool authors control their own entries. If a `modinfo.json` or `toolinfo.json` is removed or renamed (so it returns HTTP 404 or 410), `imt sync mods`/`tools` deletes its entries. If a file cannot be read for any other reason (timeout, network error, server error, rate limit, or invalid JSON), that file's entries are kept on that run and a warning names them; other sources are unaffected. Sync tracks which file each entry came from in the internal `meta/mod_sources` and `meta/tool_sources` documents (override with `collections.meta.mod_sources` / `tool_sources` in the config). An entry with no recorded source yet is kept while any source fails. As a brake on a wave of 404s (a GitHub outage, a mass rename), `imt sync mods`/`tools` refuses to delete more than 25% of a collection (at least 5 entries), or to empty it, and exits 1; rerun with `--force` once you have checked the sources.
153
+
154
+ **Exit status**: each `imt sync` command exits 1 when it fails. `imt sync all` runs every stage even if one fails, then exits 1.
155
+
156
+ **IDs**: New mods and tools get a document ID derived from their type, author and name, so an entry that is deleted and later re-added gets the same ID. `imt sync cleanup` keeps the oldest of any duplicates (the ID clients already have), copies the newest duplicate's data onto it, and deletes the rest.
157
+
150
158
  **Note**: Sync operations automatically convert GitHub URLs to the correct `raw.githubusercontent.com` format for direct downloads. If you're using GitHub URLs in `modinfo.json` or `toolinfo.json` files, you can use either `/blob/` or `/raw/` formats and they will be automatically converted. Warnings will be displayed when URLs are auto-fixed so you can update your source files.
151
159
 
152
160
  ## Development
@@ -11,7 +11,7 @@ Gem::Specification.new do |spec|
11
11
  spec.summary = "Various tools for Icarus Modding"
12
12
  spec.description = spec.summary
13
13
  spec.homepage = "https://github.com/DonovanMods/icarus-mod-tools"
14
- spec.required_ruby_version = ">= 3.1"
14
+ spec.required_ruby_version = ">= 4.0"
15
15
 
16
16
  spec.metadata["homepage_uri"] = spec.homepage
17
17
  spec.metadata["source_code_uri"] = spec.homepage
@@ -33,6 +33,7 @@ Gem::Specification.new do |spec|
33
33
  # spec.add_dependency "example-gem", "~> 1.0"
34
34
  spec.add_dependency "google-cloud-firestore", "~> 2.7"
35
35
  spec.add_dependency "octokit", "~> 6.0"
36
+ spec.add_dependency "ostruct", "~> 0.6"
36
37
  spec.add_dependency "paint", "~> 2.3"
37
38
  spec.add_dependency "thor", "~> 1.2"
38
39
  # For more information and examples about making a new gem, check out our
@@ -15,8 +15,10 @@
15
15
  "collections": {
16
16
  "meta": {
17
17
  "modinfo": "meta/modinfo",
18
+ "mod_sources": "meta/mod_sources",
18
19
  "repositories": "meta/repos",
19
20
  "status": "meta/status",
21
+ "tool_sources": "meta/tool_sources",
20
22
  "toolinfo": "meta/toolinfo"
21
23
  },
22
24
  "mods": "mods",
@@ -8,34 +8,26 @@ module Icarus
8
8
  module CLI
9
9
  # Sync CLI command definitions
10
10
  class Add < SubcommandBase
11
+ class_option :dry_run, type: :boolean, default: false, desc: "Dry run (validate and show what would be added, but change nothing)"
12
+
11
13
  desc "modinfo", "Adds an entry to 'meta/modinfo/list'"
12
14
  def modinfo(item)
13
- firestore = Firestore.new
14
- payload = [firestore.modinfo, item].flatten.compact
15
-
16
- puts firestore.update(:modinfo, payload, merge: true) ? "Success" : "Failure"
15
+ add_to_list(:modinfo, :modinfo, item)
17
16
  end
18
17
 
19
18
  desc "toolinfo", "Adds an entry to 'meta/toolinfo/list'"
20
19
  def toolinfo(item)
21
- firestore = Firestore.new
22
- payload = [firestore.toolinfo, item].flatten.compact
23
-
24
- puts firestore.update(:toolinfo, payload, merge: true) ? "Success" : "Failure"
20
+ add_to_list(:toolinfo, :toolinfo, item)
25
21
  end
26
22
 
27
23
  desc "repos", "Adds an entry to 'meta/repos/list'"
28
24
  def repos(item)
29
- firestore = Firestore.new
30
- payload = [firestore.repositories, item].flatten.compact
31
-
32
- puts firestore.update(:repositories, payload, merge: true) ? "Success" : "Failure"
25
+ add_to_list(:repositories, :repositories, item)
33
26
  end
34
27
 
35
28
  desc "mod", "Adds an entry to 'mods'"
36
29
  method_option :modinfo, type: :string, required: true, default: "modinfo.json", desc: "Path to the modinfo.json file"
37
30
  def mod
38
- firestore = Firestore.new
39
31
  data = options[:modinfo]
40
32
 
41
33
  if data.nil? || !File.exist?(data)
@@ -43,15 +35,57 @@ module Icarus
43
35
  exit 1
44
36
  end
45
37
 
46
- JSON.parse(File.read(data), symbolize_names: true)[:mods].each do |mod|
47
- modinfo = Icarus::Mod::Tools::Modinfo.new(mod)
38
+ begin
39
+ parsed = JSON.parse(File.read(data), symbolize_names: true)
40
+ rescue JSON::ParserError => e
41
+ warn "Invalid JSON in #{data}: #{e.message.lines.first.strip}"
42
+ exit 1
43
+ end
44
+
45
+ mods = parsed[:mods] if parsed.is_a?(Hash)
46
+
47
+ unless mods.is_a?(Array)
48
+ warn "Invalid data file: no 'mods' list found in #{data}"
49
+ exit 1
50
+ end
48
51
 
49
- unless modinfo.valid?
50
- warn "Invalid modinfo: #{modinfo.errors}"
51
- exit 1
52
+ # Validate everything up front so a bad entry never leaves a partial add behind
53
+ modinfos = mods.map { |mod| Icarus::Mod::Tools::Modinfo.new(mod.is_a?(Hash) ? mod : {}) }
54
+ invalid = modinfos.reject(&:valid?)
55
+
56
+ if invalid.any?
57
+ invalid.each { |modinfo| warn "Invalid modinfo: #{modinfo.errors}" }
58
+ exit 1
59
+ end
60
+
61
+ if options[:dry_run]
62
+ modinfos.each { |modinfo| puts Paint["Would add: #{modinfo.name}", :yellow] }
63
+ return
64
+ end
65
+
66
+ firestore = Firestore.new
67
+ results = modinfos.map { |modinfo| report(firestore.update(:mod, modinfo, merge: true)) }
68
+
69
+ exit 1 unless results.all?
70
+ end
71
+
72
+ no_commands do
73
+ # Appends item to one of the meta lists, exiting 1 if the update fails
74
+ def add_to_list(type, collection, item)
75
+ if options[:dry_run]
76
+ puts Paint["Would add to #{type}: #{item}", :yellow]
77
+ return
52
78
  end
53
79
 
54
- puts firestore.update(:mod, modinfo, merge: true) ? "Success" : "Failure"
80
+ firestore = Firestore.new
81
+ payload = [firestore.send(collection), item].flatten.compact
82
+
83
+ exit 1 unless report(firestore.update(type, payload, merge: true))
84
+ end
85
+
86
+ def report(success)
87
+ puts success ? "Success" : "Failure"
88
+ success
55
89
  end
56
90
  end
57
91
  end
@@ -11,12 +11,12 @@ module Icarus
11
11
  # The Base CLI class for Icarus Mod Tools. This is inherited by all subcommands.
12
12
  class Base < Thor
13
13
  class_option :config,
14
- aliases: "-C", type: :string, default: File.join(Dir.home, "/.imtconfig.json"),
15
- desc: "Path to the config file"
14
+ aliases: "-C", type: :string, default: File.join(Dir.home, "/.imtconfig.json"),
15
+ desc: "Path to the config file"
16
16
 
17
17
  class_option :version,
18
- aliases: "-V", type: :boolean,
19
- desc: "Print the version and exit"
18
+ aliases: "-V", type: :boolean,
19
+ desc: "Print the version and exit"
20
20
 
21
21
  def self.exit_on_failure?
22
22
  true
@@ -26,7 +26,12 @@ module Icarus
26
26
  exit 1
27
27
  end
28
28
 
29
- Icarus::Mod::Config.read(options[:config])
29
+ begin
30
+ Icarus::Mod::Config.read(options[:config])
31
+ rescue JSON::ParserError => e
32
+ warn "Could not find or read Config from '#{options[:config]}' - the file is malformed JSON (#{e.message.lines.first.strip})"
33
+ exit 1
34
+ end
30
35
  end
31
36
 
32
37
  desc "sync", "Syncs the databases"