Icarus-Mod-Tools 2.6.0 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +55 -0
- data/.rubocop_todo.yml +142 -0
- data/.ruby-version +1 -1
- data/CHANGELOG.md +48 -0
- data/CLAUDE.md +6 -4
- data/Gemfile +3 -5
- data/Gemfile.lock +18 -27
- data/Guardfile +2 -0
- data/README.md +9 -1
- data/icarus-mod-tools.gemspec +2 -1
- data/imtconfig.sample.json +2 -0
- data/lib/icarus/mod/cli/add.rb +53 -19
- data/lib/icarus/mod/cli/base.rb +4 -4
- data/lib/icarus/mod/cli/command.rb +6 -1
- data/lib/icarus/mod/cli/list.rb +19 -7
- data/lib/icarus/mod/cli/remove.rb +65 -62
- data/lib/icarus/mod/cli/subcommand_base.rb +2 -2
- data/lib/icarus/mod/cli/sync.rb +139 -35
- data/lib/icarus/mod/cli/validate.rb +1 -4
- data/lib/icarus/mod/config.rb +2 -2
- data/lib/icarus/mod/firestore.rb +132 -15
- data/lib/icarus/mod/github.rb +39 -22
- data/lib/icarus/mod/tools/baseinfo.rb +86 -22
- data/lib/icarus/mod/tools/modinfo.rb +1 -1
- data/lib/icarus/mod/tools/sync/entities.rb +102 -0
- data/lib/icarus/mod/tools/sync/helpers.rb +38 -4
- data/lib/icarus/mod/tools/sync/modinfo_list.rb +30 -10
- data/lib/icarus/mod/tools/sync/mods.rb +6 -37
- data/lib/icarus/mod/tools/sync/toolinfo_list.rb +30 -10
- data/lib/icarus/mod/tools/sync/tools.rb +6 -39
- data/lib/icarus/mod/tools/toolinfo.rb +12 -8
- data/lib/icarus/mod/tools/validator.rb +7 -7
- data/lib/icarus/mod/version.rb +1 -1
- metadata +20 -4
- data/sig/database/sync/sync.rbs +0 -15
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 17d3d3cdfeeb5953e405d4888f0c748b00d330af51a14714c51e04b927baedb9
|
|
4
|
+
data.tar.gz: 29f015ac208933825b954bb33fda452cd684499f8e2e10950d241e0a4323fbac
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: fad56b2b02cfd1412d332f2d47be963ef557ce21bfa25f3af6b103ee99ab4cb632517e23a59d7d512cbc339746906392396133b238d004e9c0ef82a23e76fc3e
|
|
7
|
+
data.tar.gz: bd460ed0255ede73b7cda973e0d0a39c22a2a285600e6f2399920905c1b626e2b40dc65599578f67e50b8f8c609238fde203eb7b3cd4090c3c3ef7e24d3fb974
|
data/.rubocop.yml
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
inherit_from: .rubocop_todo.yml
|
|
2
|
+
|
|
3
|
+
plugins:
|
|
4
|
+
- rubocop-rspec
|
|
5
|
+
|
|
6
|
+
AllCops:
|
|
7
|
+
TargetRubyVersion: 4.0
|
|
8
|
+
NewCops: enable
|
|
9
|
+
SuggestExtensions: false
|
|
10
|
+
Exclude:
|
|
11
|
+
- "pkg/**/*"
|
|
12
|
+
- "vendor/**/*"
|
|
13
|
+
|
|
14
|
+
Style/StringLiterals:
|
|
15
|
+
EnforcedStyle: double_quotes
|
|
16
|
+
|
|
17
|
+
Style/StringLiteralsInInterpolation:
|
|
18
|
+
EnforcedStyle: double_quotes
|
|
19
|
+
|
|
20
|
+
Layout/LineLength:
|
|
21
|
+
Max: 140
|
|
22
|
+
|
|
23
|
+
# The specs deliberately share many `let`s, group several expectations per example, mix `have_received`
|
|
24
|
+
# and `receive`, stub with `allow_any_instance_of` for the HTTP helpers, and build configs from OpenStruct
|
|
25
|
+
# (as Config does). These cops would force a rewrite of the suite for no behavior gain.
|
|
26
|
+
RSpec/MultipleMemoizedHelpers:
|
|
27
|
+
Enabled: false
|
|
28
|
+
RSpec/MultipleExpectations:
|
|
29
|
+
Enabled: false
|
|
30
|
+
RSpec/ExampleLength:
|
|
31
|
+
Enabled: false
|
|
32
|
+
RSpec/NestedGroups:
|
|
33
|
+
Enabled: false
|
|
34
|
+
RSpec/MessageSpies:
|
|
35
|
+
Enabled: false
|
|
36
|
+
RSpec/AnyInstance:
|
|
37
|
+
Enabled: false
|
|
38
|
+
RSpec/IndexedLet:
|
|
39
|
+
Enabled: false
|
|
40
|
+
RSpec/ReceiveMessages:
|
|
41
|
+
Enabled: false
|
|
42
|
+
RSpec/VerifiedDoubles:
|
|
43
|
+
Enabled: false
|
|
44
|
+
RSpec/SubjectStub:
|
|
45
|
+
Enabled: false
|
|
46
|
+
RSpec/ContextWording:
|
|
47
|
+
Enabled: false
|
|
48
|
+
RSpec/ExpectOutput:
|
|
49
|
+
Enabled: false
|
|
50
|
+
RSpec/InstanceVariable:
|
|
51
|
+
Enabled: false
|
|
52
|
+
|
|
53
|
+
# Config parses the JSON config into OpenStruct on purpose (dot access to nested settings)
|
|
54
|
+
Style/OpenStructUse:
|
|
55
|
+
Enabled: false
|
data/.rubocop_todo.yml
ADDED
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
# This configuration was generated by
|
|
2
|
+
# `rubocop --auto-gen-config --no-exclude-limit`
|
|
3
|
+
# on 2026-10-31 01:35:19 UTC using RuboCop version 1.92.2.
|
|
4
|
+
# The point is for the user to remove these configuration records
|
|
5
|
+
# one by one as the offenses are removed from the code base.
|
|
6
|
+
# Note that changes in the inspected code, or installation of new
|
|
7
|
+
# versions of RuboCop, may require this file to be generated again.
|
|
8
|
+
|
|
9
|
+
# Offense count: 3
|
|
10
|
+
# This cop supports safe autocorrection (--autocorrect).
|
|
11
|
+
# Configuration parameters: Max, AllowHeredoc, AllowURI, AllowQualifiedName, URISchemes, AllowRBSInlineAnnotation, AllowCopDirectives, AllowedPatterns, SplitStrings.
|
|
12
|
+
# URISchemes: http, https
|
|
13
|
+
Layout/LineLength:
|
|
14
|
+
Exclude:
|
|
15
|
+
- 'lib/icarus/mod/cli/sync.rb'
|
|
16
|
+
- 'spec/icarus/mod/cli/remove_spec.rb'
|
|
17
|
+
- 'spec/icarus/mod/tools/baseinfo_spec.rb'
|
|
18
|
+
|
|
19
|
+
# Offense count: 24
|
|
20
|
+
# Configuration parameters: AllowedMethods, AllowedPatterns, CountRepeatedAttributes, Max.
|
|
21
|
+
Metrics/AbcSize:
|
|
22
|
+
Exclude:
|
|
23
|
+
- 'lib/icarus/mod/cli/add.rb'
|
|
24
|
+
- 'lib/icarus/mod/cli/command.rb'
|
|
25
|
+
- 'lib/icarus/mod/cli/list.rb'
|
|
26
|
+
- 'lib/icarus/mod/cli/remove.rb'
|
|
27
|
+
- 'lib/icarus/mod/cli/sync.rb'
|
|
28
|
+
- 'lib/icarus/mod/cli/validate.rb'
|
|
29
|
+
- 'lib/icarus/mod/firestore.rb'
|
|
30
|
+
- 'lib/icarus/mod/tools/baseinfo.rb'
|
|
31
|
+
- 'lib/icarus/mod/tools/sync/entities.rb'
|
|
32
|
+
- 'lib/icarus/mod/tools/sync/helpers.rb'
|
|
33
|
+
|
|
34
|
+
# Offense count: 2
|
|
35
|
+
# Configuration parameters: CountComments, Max, CountAsOne, AllowedMethods, AllowedPatterns, inherit_mode.
|
|
36
|
+
# AllowedMethods: refine
|
|
37
|
+
Metrics/BlockLength:
|
|
38
|
+
Exclude:
|
|
39
|
+
- 'lib/icarus/mod/cli/list.rb'
|
|
40
|
+
- 'lib/icarus/mod/cli/sync.rb'
|
|
41
|
+
|
|
42
|
+
# Offense count: 4
|
|
43
|
+
# Configuration parameters: CountComments, Max, CountAsOne.
|
|
44
|
+
Metrics/ClassLength:
|
|
45
|
+
Exclude:
|
|
46
|
+
- 'lib/icarus/mod/cli/remove.rb'
|
|
47
|
+
- 'lib/icarus/mod/cli/sync.rb'
|
|
48
|
+
- 'lib/icarus/mod/firestore.rb'
|
|
49
|
+
- 'lib/icarus/mod/tools/baseinfo.rb'
|
|
50
|
+
|
|
51
|
+
# Offense count: 16
|
|
52
|
+
# Configuration parameters: AllowedMethods, AllowedPatterns, Max.
|
|
53
|
+
Metrics/CyclomaticComplexity:
|
|
54
|
+
Exclude:
|
|
55
|
+
- 'lib/icarus/mod/cli/add.rb'
|
|
56
|
+
- 'lib/icarus/mod/cli/list.rb'
|
|
57
|
+
- 'lib/icarus/mod/cli/remove.rb'
|
|
58
|
+
- 'lib/icarus/mod/cli/sync.rb'
|
|
59
|
+
- 'lib/icarus/mod/cli/validate.rb'
|
|
60
|
+
- 'lib/icarus/mod/firestore.rb'
|
|
61
|
+
- 'lib/icarus/mod/github.rb'
|
|
62
|
+
- 'lib/icarus/mod/tools/baseinfo.rb'
|
|
63
|
+
- 'lib/icarus/mod/tools/sync/entities.rb'
|
|
64
|
+
- 'lib/icarus/mod/tools/sync/helpers.rb'
|
|
65
|
+
|
|
66
|
+
# Offense count: 25
|
|
67
|
+
# Configuration parameters: CountComments, Max, CountAsOne, AllowedMethods, AllowedPatterns.
|
|
68
|
+
Metrics/MethodLength:
|
|
69
|
+
Exclude:
|
|
70
|
+
- 'lib/icarus/mod/cli/add.rb'
|
|
71
|
+
- 'lib/icarus/mod/cli/command.rb'
|
|
72
|
+
- 'lib/icarus/mod/cli/list.rb'
|
|
73
|
+
- 'lib/icarus/mod/cli/remove.rb'
|
|
74
|
+
- 'lib/icarus/mod/cli/sync.rb'
|
|
75
|
+
- 'lib/icarus/mod/cli/validate.rb'
|
|
76
|
+
- 'lib/icarus/mod/firestore.rb'
|
|
77
|
+
- 'lib/icarus/mod/github.rb'
|
|
78
|
+
- 'lib/icarus/mod/tools/sync/entities.rb'
|
|
79
|
+
- 'lib/icarus/mod/tools/sync/helpers.rb'
|
|
80
|
+
- 'lib/icarus/mod/tools/sync/modinfo_list.rb'
|
|
81
|
+
- 'lib/icarus/mod/tools/sync/toolinfo_list.rb'
|
|
82
|
+
|
|
83
|
+
# Offense count: 2
|
|
84
|
+
# Configuration parameters: Max, CountKeywordArgs, MaxOptionalParameters.
|
|
85
|
+
Metrics/ParameterLists:
|
|
86
|
+
Exclude:
|
|
87
|
+
- 'lib/icarus/mod/cli/remove.rb'
|
|
88
|
+
|
|
89
|
+
# Offense count: 12
|
|
90
|
+
# Configuration parameters: AllowedMethods, AllowedPatterns, Max.
|
|
91
|
+
Metrics/PerceivedComplexity:
|
|
92
|
+
Exclude:
|
|
93
|
+
- 'lib/icarus/mod/cli/add.rb'
|
|
94
|
+
- 'lib/icarus/mod/cli/list.rb'
|
|
95
|
+
- 'lib/icarus/mod/cli/remove.rb'
|
|
96
|
+
- 'lib/icarus/mod/cli/sync.rb'
|
|
97
|
+
- 'lib/icarus/mod/cli/validate.rb'
|
|
98
|
+
- 'lib/icarus/mod/github.rb'
|
|
99
|
+
- 'lib/icarus/mod/tools/sync/helpers.rb'
|
|
100
|
+
|
|
101
|
+
# Offense count: 3
|
|
102
|
+
# Configuration parameters: Mode, AllowedMethods, AllowedPatterns, AllowBangMethods, WaywardPredicates.
|
|
103
|
+
# AllowedMethods: call
|
|
104
|
+
# WaywardPredicates: infinite?, nonzero?
|
|
105
|
+
Naming/PredicateMethod:
|
|
106
|
+
Exclude:
|
|
107
|
+
- 'lib/icarus/mod/cli/sync.rb'
|
|
108
|
+
- 'lib/icarus/mod/firestore.rb'
|
|
109
|
+
|
|
110
|
+
# Offense count: 4
|
|
111
|
+
# This cop supports unsafe autocorrection (--autocorrect-all).
|
|
112
|
+
Style/DirectiveScope:
|
|
113
|
+
Exclude:
|
|
114
|
+
- 'lib/icarus/mod/cli/remove.rb'
|
|
115
|
+
- 'lib/icarus/mod/cli/subcommand_base.rb'
|
|
116
|
+
- 'lib/icarus/mod/cli/sync.rb'
|
|
117
|
+
- 'lib/icarus/mod/tools/toolinfo.rb'
|
|
118
|
+
|
|
119
|
+
# Offense count: 4
|
|
120
|
+
# Configuration parameters: AllowedVariables.
|
|
121
|
+
Style/GlobalVars:
|
|
122
|
+
Exclude:
|
|
123
|
+
- 'spec/icarus/mod/cli/remove_spec.rb'
|
|
124
|
+
- 'spec/icarus/mod/cli/sync_spec.rb'
|
|
125
|
+
|
|
126
|
+
# Offense count: 1
|
|
127
|
+
Style/MultilineBlockChain:
|
|
128
|
+
Exclude:
|
|
129
|
+
- 'lib/icarus/mod/tools/sync/entities.rb'
|
|
130
|
+
|
|
131
|
+
# Offense count: 2
|
|
132
|
+
# This cop supports unsafe autocorrection (--autocorrect-all).
|
|
133
|
+
Style/ReduceToHash:
|
|
134
|
+
Exclude:
|
|
135
|
+
- 'lib/icarus/mod/tools/baseinfo.rb'
|
|
136
|
+
- 'lib/icarus/mod/tools/toolinfo.rb'
|
|
137
|
+
|
|
138
|
+
# Offense count: 2
|
|
139
|
+
# This cop supports unsafe autocorrection (--autocorrect-all).
|
|
140
|
+
Style/SelectByRegexp:
|
|
141
|
+
Exclude:
|
|
142
|
+
- 'lib/icarus/mod/cli/remove.rb'
|
data/.ruby-version
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
|
|
1
|
+
4.0
|
data/CHANGELOG.md
CHANGED
|
@@ -4,6 +4,54 @@ All notable changes to this project will be documented in this file.
|
|
|
4
4
|
|
|
5
5
|
## History (reverse chronological order)
|
|
6
6
|
|
|
7
|
+
### v3.0.0 - 2026-10-30
|
|
8
|
+
|
|
9
|
+
**Breaking:** requires Ruby 4.0 or greater. Sync commands now exit 1 on failure, and `imt sync mods`/`tools` refuse a mass delete without `--force`.
|
|
10
|
+
|
|
11
|
+
- Require Ruby 4.0 or greater (was 3.1)
|
|
12
|
+
- Fixes `imt sync` crashing during Firestore authentication with `wrong number of arguments (given 2, expected 1)` on Ruby 3.1 once `json` 3.x is installed; `multi_json` releases that support `json` 3.x (1.21.2+) need Ruby 3.2 or newer
|
|
13
|
+
- Update `multi_json` to 1.21.2
|
|
14
|
+
- Declare `ostruct` as a runtime dependency; it is no longer a default gem on Ruby 4.0, so a plain `gem install` could fail to load the config
|
|
15
|
+
- Update `faraday`, `jwt`, `addressable` and `concurrent-ruby` to clear known security advisories
|
|
16
|
+
- Remove the unused, stale RBS signatures under `sig/`
|
|
17
|
+
- Fix `imt sync all` not creating mods/tools for a `modinfo.json`/`toolinfo.json` found earlier in the same run; the new URL was only picked up on the next run
|
|
18
|
+
- Fix two sync hosts (or a sync and `imt add`/`remove`) running at once overwriting each other's changes to `meta/modinfo`, `meta/toolinfo`, `meta/repos` and the source maps
|
|
19
|
+
- Add a mass-delete brake: `imt sync mods`/`tools` refuses to delete more than 25% of a collection (at least 5 entries), or to empty it, unless run with `--force`
|
|
20
|
+
- Fix `imt sync` commands exiting 0 on failure; they now exit 1, and `imt sync all` runs every stage before exiting 1
|
|
21
|
+
- Fix `imt sync mods`/`tools` always printing "Success" at `-vv`, even when an update or delete failed
|
|
22
|
+
- Fix one `modinfo.json`/`toolinfo.json` whose body is not a JSON object (or whose entries are not objects) aborting the whole sync; it is now treated like invalid JSON and its existing entries are kept
|
|
23
|
+
- Fix a crash on a first run against an empty database (missing `meta` documents)
|
|
24
|
+
- Fix source URLs losing their query string, and a malformed URL crashing the sync instead of being skipped
|
|
25
|
+
- Save recorded sources even when a sync fails partway through, so the next run can still protect those entries
|
|
26
|
+
- Fix `imt remove repos` removing the wrong entries
|
|
27
|
+
- Matches a repo's `modinfo.json`/`toolinfo.json` URLs only on the GitHub host and exact `owner/repo` (case-insensitive); before, a same-named subdirectory in someone else's repo matched
|
|
28
|
+
- Deletes only the mods and tools that `meta/mod_sources`/`meta/tool_sources` attribute to the repo's URLs (falling back to name and author for entries with no recorded source), so a fork publishing the same name and author keeps its entry
|
|
29
|
+
- Cascades for repos stored as full URLs too
|
|
30
|
+
- Keeps the repo entry and exits 1 when any delete or fetch fails, so the removal can be retried
|
|
31
|
+
- Tighten validation: file types, URLs (http/https with a host) and versions are now matched exactly; input the old checks accepted (`notzipx`, `javascript:` URLs, `1.0 garbage`) is a warning for now and will become an error in a future release
|
|
32
|
+
- Fix non-string values (`"version": 1.0`, `"name": 5`, numeric URLs, `files` that is not an object) crashing validation; they are now reported as errors
|
|
33
|
+
- Fix `Toolinfo#validate` not being public, and unknown attribute names on mod/tool entries silently returning nil instead of raising
|
|
34
|
+
- Fix `imt list mods`/`tools` crashing on `--filter` values containing regex characters, on empty or non-text fields, on `--sort files` or fields missing from some entries, and on entries with no update time; filters are now case-insensitive substring matches and errors go to stderr
|
|
35
|
+
- Fix `imt validate` crashing on an empty list, and `imt` crashing on a malformed config file (it now prints an error and exits 1)
|
|
36
|
+
- Fix the GitHub file cache returning a top-level listing for recursive or sub-path lookups, and `find("modinfo.json")` matching names like `my_modinfo.json.bak`
|
|
37
|
+
- CI now also runs RuboCop and `bundler-audit`, and pins its GitHub Actions to commit SHAs; add a `.rubocop.yml` matching the project's style (existing size/complexity offenses are recorded in `.rubocop_todo.yml`)
|
|
38
|
+
- Update `rubocop-rspec` to 3.x (2.x no longer loads with current RuboCop)
|
|
39
|
+
- Fix `imt add`: `mod` validates every entry before writing anything and reports bad JSON or a missing `mods` list; every `add` command exits 1 on failure; add `--dry-run`
|
|
40
|
+
|
|
41
|
+
### v2.7.0 - 2026-10-30
|
|
42
|
+
|
|
43
|
+
- Fix sync deleting a source's mods and tools when its `modinfo.json`/`toolinfo.json` could not be read (#16)
|
|
44
|
+
- Authors still control removal: if a file is removed or renamed (HTTP 404 or 410), its entries are deleted as before
|
|
45
|
+
- A transient failure (timeout, network/DNS/TLS error, 5xx, 403/429 rate limit, any other unexpected status) or invalid JSON no longer deletes that source's entries; deletions are blocked per source, so other authors' removals still go through, and a warning lists every kept entry and why
|
|
46
|
+
- Sync records which file each mod and tool came from in new internal Firestore docs, `meta/mod_sources` and `meta/tool_sources` (the mods and tools docs are unchanged). An entry with no recorded source yet, such as on the first run after upgrading, is kept while any source fails transiently
|
|
47
|
+
- Invalid JSON is reported with a loud warning so an author mid-edit is not wiped
|
|
48
|
+
- Fix a repository whose GitHub search fails transiently (rate limit, 5xx, network error) aborting `imt sync modinfo`/`toolinfo`; it is now skipped with a warning and its existing list entries are kept (#16)
|
|
49
|
+
- Change `imt sync cleanup` to keep the oldest duplicate, whose ID clients already store, instead of the most recently updated one (#16)
|
|
50
|
+
- The most recently updated duplicate's data is copied onto the kept entry before the others are deleted
|
|
51
|
+
- `--dry-run` shows what would be kept, copied and deleted
|
|
52
|
+
- Give new mods and tools deterministic document IDs derived from type, author and name, so an entry that is deleted and re-added keeps its ID (#16)
|
|
53
|
+
- Existing documents keep their current IDs
|
|
54
|
+
|
|
7
55
|
### v2.6.0 - 2026-03-04
|
|
8
56
|
|
|
9
57
|
- Add `imt sync cleanup` command to remove duplicate entries from mods and tools collections
|
data/CLAUDE.md
CHANGED
|
@@ -24,6 +24,7 @@ bundle exec rspec # Run tests only (with Fuubar formatter)
|
|
|
24
24
|
bundle exec rspec spec/path/to/file_spec.rb # Run single test file
|
|
25
25
|
bundle exec rspec spec/path/to/file_spec.rb:42 # Run single test at line 42
|
|
26
26
|
guard # Auto-run tests on file changes
|
|
27
|
+
bundle exec rubocop # Lint (CI runs it; .rubocop_todo.yml holds the existing metrics offenses)
|
|
27
28
|
```
|
|
28
29
|
|
|
29
30
|
### Console & Debugging
|
|
@@ -45,11 +46,13 @@ imt [command] # Run installed gem
|
|
|
45
46
|
|
|
46
47
|
The tool implements a two-stage sync pipeline:
|
|
47
48
|
|
|
48
|
-
1. **GitHub → Meta Collections**: Scan repositories for metadata files and sync to `meta/modinfo`, `meta/toolinfo`, `meta/repos`
|
|
49
|
+
1. **GitHub → Meta Collections**: Scan repositories for metadata files and sync to `meta/modinfo`, `meta/toolinfo`, `meta/repos` (`meta/mod_sources` and `meta/tool_sources` are internal, written by stage 2)
|
|
49
50
|
2. **Meta → Entity Collections**: Process meta lists to populate/update `mods` and `tools` collections
|
|
50
51
|
|
|
51
52
|
All operations support `--dry-run` for safe testing.
|
|
52
53
|
|
|
54
|
+
Deletion contract (#16): stage 2 deletes a doc only when no info file still lists it. `Sync::Helpers#retrieve_from_url` raises `RequestGone` for 404/410 (author removed the file: delete) and `RequestTransient` for everything else; invalid JSON also counts as transient. Docs do not record their source URL; instead `meta/mod_sources` / `meta/tool_sources` (Firestore, shared across sync hosts) map doc ID => source URL, written by `Sync::Mods/Tools#update` + `save_sources` and pruned by `Firestore#delete`. A transient failure protects only the docs mapped to that URL; unmapped docs are kept while any URL failed. Stage 1 never drops list entries (`Firestore#update` unions server-side with `FieldValue.array_union`; source maps are written per key with `merge: true`, so concurrent hosts don't overwrite each other). Stage 2 refuses to delete more than 25% of a collection (min 5) or empty it without `--force`, and a repo whose GitHub search fails transiently is skipped. New docs get a stable ID (`Firestore#stable_id`); `imt sync cleanup` keeps the oldest duplicate and copies the newest data onto it.
|
|
55
|
+
|
|
53
56
|
### Core Components
|
|
54
57
|
|
|
55
58
|
**CLI Layer** (`lib/icarus/mod/cli/`)
|
|
@@ -70,8 +73,7 @@ All operations support `--dry-run` for safe testing.
|
|
|
70
73
|
|
|
71
74
|
- `ModinfoList`: GitHub → meta/modinfo sync
|
|
72
75
|
- `ToolinfoList`: GitHub → meta/toolinfo sync
|
|
73
|
-
- `
|
|
74
|
-
- `Tools`: meta/toolinfo → tools sync
|
|
76
|
+
- `Entities`: shared meta → entity sync; `Mods` (meta/modinfo → mods) and `Tools` (meta/toolinfo → tools) only set `TYPE`/`INFO`/`ENTITY`
|
|
75
77
|
- `Helpers`: HTTP utilities with custom SSL verification for Ruby 3.4+ CRL issues
|
|
76
78
|
|
|
77
79
|
**External Services** (`lib/icarus/mod/`)
|
|
@@ -146,7 +148,7 @@ Ruby 3.4+ with newer OpenSSL may fail CRL checking. The codebase handles this in
|
|
|
146
148
|
|
|
147
149
|
- **Firestore credentials**: Must be the full JSON object in config, not a file path
|
|
148
150
|
- **GitHub repo format**: `owner/repo` format (URL prefixes are stripped automatically)
|
|
149
|
-
- **Ruby version**:
|
|
151
|
+
- **Ruby version**: 4.0+ required
|
|
150
152
|
- **Platform**: WSL2/Linux recommended; not tested on Windows
|
|
151
153
|
- **Require paths**: The gemspec sets `require_paths` to both `lib` and `lib/icarus/mod`, so files use relative requires like `require "cli/base"` instead of `require_relative`
|
|
152
154
|
- **Verbosity**: `-v` and `-vv` flags control output detail in subcommands
|
data/Gemfile
CHANGED
|
@@ -7,14 +7,12 @@ gemspec
|
|
|
7
7
|
|
|
8
8
|
gem "rake", "~> 13.0"
|
|
9
9
|
|
|
10
|
-
group :
|
|
10
|
+
group :development do
|
|
11
11
|
gem "fuubar"
|
|
12
12
|
gem "guard", "~> 2.18"
|
|
13
13
|
gem "guard-rspec", "~> 4.7"
|
|
14
14
|
gem "pry", "~> 0.14.1"
|
|
15
15
|
gem "rspec", "~> 3.12"
|
|
16
|
-
gem "rubocop", "~> 1.
|
|
17
|
-
gem "rubocop-rspec", "~>
|
|
16
|
+
gem "rubocop", "~> 1.92"
|
|
17
|
+
gem "rubocop-rspec", "~> 3.10", require: false
|
|
18
18
|
end
|
|
19
|
-
|
|
20
|
-
gem "ostruct", "~> 0.6.3"
|
data/Gemfile.lock
CHANGED
|
@@ -1,25 +1,26 @@
|
|
|
1
1
|
PATH
|
|
2
2
|
remote: .
|
|
3
3
|
specs:
|
|
4
|
-
Icarus-Mod-Tools (
|
|
4
|
+
Icarus-Mod-Tools (3.0.0)
|
|
5
5
|
google-cloud-firestore (~> 2.7)
|
|
6
6
|
octokit (~> 6.0)
|
|
7
|
+
ostruct (~> 0.6)
|
|
7
8
|
paint (~> 2.3)
|
|
8
9
|
thor (~> 1.2)
|
|
9
10
|
|
|
10
11
|
GEM
|
|
11
12
|
remote: https://rubygems.org/
|
|
12
13
|
specs:
|
|
13
|
-
addressable (2.
|
|
14
|
+
addressable (2.9.1)
|
|
14
15
|
public_suffix (>= 2.0.2, < 8.0)
|
|
15
16
|
ast (2.4.3)
|
|
16
17
|
base64 (0.3.0)
|
|
17
18
|
bigdecimal (3.3.1)
|
|
18
19
|
coderay (1.1.3)
|
|
19
|
-
concurrent-ruby (1.3.
|
|
20
|
+
concurrent-ruby (1.3.8)
|
|
20
21
|
diff-lcs (1.6.2)
|
|
21
|
-
faraday (2.
|
|
22
|
-
faraday-net_http (>= 2.0, < 3.
|
|
22
|
+
faraday (2.15.1)
|
|
23
|
+
faraday-net_http (>= 2.0, < 3.6)
|
|
23
24
|
json
|
|
24
25
|
logger
|
|
25
26
|
faraday-net_http (3.4.2)
|
|
@@ -133,8 +134,8 @@ GEM
|
|
|
133
134
|
guard-compat (~> 1.1)
|
|
134
135
|
rspec (>= 2.99.0, < 4.0)
|
|
135
136
|
io-console (0.8.2)
|
|
136
|
-
json (
|
|
137
|
-
jwt (3.
|
|
137
|
+
json (3.1.0)
|
|
138
|
+
jwt (3.3.0)
|
|
138
139
|
base64
|
|
139
140
|
language_server-protocol (3.17.0.5)
|
|
140
141
|
lint_roller (1.1.0)
|
|
@@ -145,7 +146,7 @@ GEM
|
|
|
145
146
|
logger (1.7.0)
|
|
146
147
|
lumberjack (1.4.2)
|
|
147
148
|
method_source (1.1.0)
|
|
148
|
-
multi_json (1.
|
|
149
|
+
multi_json (1.21.2)
|
|
149
150
|
nenv (0.3.0)
|
|
150
151
|
net-http (0.9.1)
|
|
151
152
|
uri (>= 0.11.1)
|
|
@@ -190,33 +191,24 @@ GEM
|
|
|
190
191
|
diff-lcs (>= 1.2.0, < 2.0)
|
|
191
192
|
rspec-support (~> 3.13.0)
|
|
192
193
|
rspec-support (3.13.7)
|
|
193
|
-
rubocop (1.
|
|
194
|
-
json (
|
|
194
|
+
rubocop (1.92.2)
|
|
195
|
+
json (>= 2.3)
|
|
195
196
|
language_server-protocol (~> 3.17.0.2)
|
|
196
197
|
lint_roller (~> 1.1.0)
|
|
197
|
-
parallel (
|
|
198
|
+
parallel (>= 1.10)
|
|
198
199
|
parser (>= 3.3.0.2)
|
|
199
200
|
rainbow (>= 2.2.2, < 4.0)
|
|
200
|
-
regexp_parser (>= 2.
|
|
201
|
+
regexp_parser (>= 2.11.3, < 3.0)
|
|
201
202
|
rubocop-ast (>= 1.49.0, < 2.0)
|
|
202
203
|
ruby-progressbar (~> 1.7)
|
|
203
204
|
unicode-display_width (>= 2.4.0, < 4.0)
|
|
204
205
|
rubocop-ast (1.49.0)
|
|
205
206
|
parser (>= 3.3.7.2)
|
|
206
207
|
prism (~> 1.7)
|
|
207
|
-
rubocop-
|
|
208
|
+
rubocop-rspec (3.10.2)
|
|
208
209
|
lint_roller (~> 1.1)
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
lint_roller (~> 1.1)
|
|
212
|
-
rubocop (~> 1.72, >= 1.72.1)
|
|
213
|
-
rubocop-rspec (2.31.0)
|
|
214
|
-
rubocop (~> 1.40)
|
|
215
|
-
rubocop-capybara (~> 2.17)
|
|
216
|
-
rubocop-factory_bot (~> 2.22)
|
|
217
|
-
rubocop-rspec_rails (~> 2.28)
|
|
218
|
-
rubocop-rspec_rails (2.29.1)
|
|
219
|
-
rubocop (~> 1.61)
|
|
210
|
+
regexp_parser (>= 2.0)
|
|
211
|
+
rubocop (~> 1.86, >= 1.86.2)
|
|
220
212
|
ruby-progressbar (1.13.0)
|
|
221
213
|
sawyer (0.9.3)
|
|
222
214
|
addressable (>= 2.3.5)
|
|
@@ -252,12 +244,11 @@ DEPENDENCIES
|
|
|
252
244
|
fuubar
|
|
253
245
|
guard (~> 2.18)
|
|
254
246
|
guard-rspec (~> 4.7)
|
|
255
|
-
ostruct (~> 0.6.3)
|
|
256
247
|
pry (~> 0.14.1)
|
|
257
248
|
rake (~> 13.0)
|
|
258
249
|
rspec (~> 3.12)
|
|
259
|
-
rubocop (~> 1.
|
|
260
|
-
rubocop-rspec (~>
|
|
250
|
+
rubocop (~> 1.92)
|
|
251
|
+
rubocop-rspec (~> 3.10)
|
|
261
252
|
|
|
262
253
|
BUNDLED WITH
|
|
263
254
|
4.0.5
|
data/Guardfile
CHANGED
data/README.md
CHANGED
|
@@ -8,7 +8,7 @@ To use this app, you'll need to obtain the following:
|
|
|
8
8
|
|
|
9
9
|
- A Github ACCESS_TOKEN (doesn't need access to any repos, this is used purely to make API calls)
|
|
10
10
|
- A Google Cloud Platform credentials `keyfile.json`
|
|
11
|
-
- Ruby
|
|
11
|
+
- Ruby 4.0 (or greater)
|
|
12
12
|
|
|
13
13
|
If you aren't sure how to obtain these credentials, please see:
|
|
14
14
|
|
|
@@ -132,6 +132,7 @@ Options:
|
|
|
132
132
|
```sh
|
|
133
133
|
Commands:
|
|
134
134
|
imt sync all # Run all sync jobs
|
|
135
|
+
imt sync cleanup # Remove duplicate entries from mods and tools collections
|
|
135
136
|
imt sync help [COMMAND] # Describe subcommands or one specific subcommand
|
|
136
137
|
imt sync modinfo # Reads from 'meta/repos/list' and Syncs any modinfo files we find (github only for now)
|
|
137
138
|
imt sync mods # Reads from 'meta/modinfo/list' and updates the 'mods' database accordingly
|
|
@@ -145,8 +146,15 @@ Options:
|
|
|
145
146
|
-v, [--verbose], [--no-verbose] # Increase verbosity. May be repeated for even more verbosity.
|
|
146
147
|
# Default: [true]
|
|
147
148
|
[--dry-run], [--no-dry-run] # Dry run (no changes will be made)
|
|
149
|
+
[--force] # Let mods/tools sync delete over 25% of a collection
|
|
148
150
|
```
|
|
149
151
|
|
|
152
|
+
**Removal**: Mod and tool authors control their own entries. If a `modinfo.json` or `toolinfo.json` is removed or renamed (so it returns HTTP 404 or 410), `imt sync mods`/`tools` deletes its entries. If a file cannot be read for any other reason (timeout, network error, server error, rate limit, or invalid JSON), that file's entries are kept on that run and a warning names them; other sources are unaffected. Sync tracks which file each entry came from in the internal `meta/mod_sources` and `meta/tool_sources` documents (override with `collections.meta.mod_sources` / `tool_sources` in the config). An entry with no recorded source yet is kept while any source fails. As a brake on a wave of 404s (a GitHub outage, a mass rename), `imt sync mods`/`tools` refuses to delete more than 25% of a collection (at least 5 entries), or to empty it, and exits 1; rerun with `--force` once you have checked the sources.
|
|
153
|
+
|
|
154
|
+
**Exit status**: each `imt sync` command exits 1 when it fails. `imt sync all` runs every stage even if one fails, then exits 1.
|
|
155
|
+
|
|
156
|
+
**IDs**: New mods and tools get a document ID derived from their type, author and name, so an entry that is deleted and later re-added gets the same ID. `imt sync cleanup` keeps the oldest of any duplicates (the ID clients already have), copies the newest duplicate's data onto it, and deletes the rest.
|
|
157
|
+
|
|
150
158
|
**Note**: Sync operations automatically convert GitHub URLs to the correct `raw.githubusercontent.com` format for direct downloads. If you're using GitHub URLs in `modinfo.json` or `toolinfo.json` files, you can use either `/blob/` or `/raw/` formats and they will be automatically converted. Warnings will be displayed when URLs are auto-fixed so you can update your source files.
|
|
151
159
|
|
|
152
160
|
## Development
|
data/icarus-mod-tools.gemspec
CHANGED
|
@@ -11,7 +11,7 @@ Gem::Specification.new do |spec|
|
|
|
11
11
|
spec.summary = "Various tools for Icarus Modding"
|
|
12
12
|
spec.description = spec.summary
|
|
13
13
|
spec.homepage = "https://github.com/DonovanMods/icarus-mod-tools"
|
|
14
|
-
spec.required_ruby_version = ">=
|
|
14
|
+
spec.required_ruby_version = ">= 4.0"
|
|
15
15
|
|
|
16
16
|
spec.metadata["homepage_uri"] = spec.homepage
|
|
17
17
|
spec.metadata["source_code_uri"] = spec.homepage
|
|
@@ -33,6 +33,7 @@ Gem::Specification.new do |spec|
|
|
|
33
33
|
# spec.add_dependency "example-gem", "~> 1.0"
|
|
34
34
|
spec.add_dependency "google-cloud-firestore", "~> 2.7"
|
|
35
35
|
spec.add_dependency "octokit", "~> 6.0"
|
|
36
|
+
spec.add_dependency "ostruct", "~> 0.6"
|
|
36
37
|
spec.add_dependency "paint", "~> 2.3"
|
|
37
38
|
spec.add_dependency "thor", "~> 1.2"
|
|
38
39
|
# For more information and examples about making a new gem, check out our
|
data/imtconfig.sample.json
CHANGED
|
@@ -15,8 +15,10 @@
|
|
|
15
15
|
"collections": {
|
|
16
16
|
"meta": {
|
|
17
17
|
"modinfo": "meta/modinfo",
|
|
18
|
+
"mod_sources": "meta/mod_sources",
|
|
18
19
|
"repositories": "meta/repos",
|
|
19
20
|
"status": "meta/status",
|
|
21
|
+
"tool_sources": "meta/tool_sources",
|
|
20
22
|
"toolinfo": "meta/toolinfo"
|
|
21
23
|
},
|
|
22
24
|
"mods": "mods",
|
data/lib/icarus/mod/cli/add.rb
CHANGED
|
@@ -8,34 +8,26 @@ module Icarus
|
|
|
8
8
|
module CLI
|
|
9
9
|
# Sync CLI command definitions
|
|
10
10
|
class Add < SubcommandBase
|
|
11
|
+
class_option :dry_run, type: :boolean, default: false, desc: "Dry run (validate and show what would be added, but change nothing)"
|
|
12
|
+
|
|
11
13
|
desc "modinfo", "Adds an entry to 'meta/modinfo/list'"
|
|
12
14
|
def modinfo(item)
|
|
13
|
-
|
|
14
|
-
payload = [firestore.modinfo, item].flatten.compact
|
|
15
|
-
|
|
16
|
-
puts firestore.update(:modinfo, payload, merge: true) ? "Success" : "Failure"
|
|
15
|
+
add_to_list(:modinfo, :modinfo, item)
|
|
17
16
|
end
|
|
18
17
|
|
|
19
18
|
desc "toolinfo", "Adds an entry to 'meta/toolinfo/list'"
|
|
20
19
|
def toolinfo(item)
|
|
21
|
-
|
|
22
|
-
payload = [firestore.toolinfo, item].flatten.compact
|
|
23
|
-
|
|
24
|
-
puts firestore.update(:toolinfo, payload, merge: true) ? "Success" : "Failure"
|
|
20
|
+
add_to_list(:toolinfo, :toolinfo, item)
|
|
25
21
|
end
|
|
26
22
|
|
|
27
23
|
desc "repos", "Adds an entry to 'meta/repos/list'"
|
|
28
24
|
def repos(item)
|
|
29
|
-
|
|
30
|
-
payload = [firestore.repositories, item].flatten.compact
|
|
31
|
-
|
|
32
|
-
puts firestore.update(:repositories, payload, merge: true) ? "Success" : "Failure"
|
|
25
|
+
add_to_list(:repositories, :repositories, item)
|
|
33
26
|
end
|
|
34
27
|
|
|
35
28
|
desc "mod", "Adds an entry to 'mods'"
|
|
36
29
|
method_option :modinfo, type: :string, required: true, default: "modinfo.json", desc: "Path to the modinfo.json file"
|
|
37
30
|
def mod
|
|
38
|
-
firestore = Firestore.new
|
|
39
31
|
data = options[:modinfo]
|
|
40
32
|
|
|
41
33
|
if data.nil? || !File.exist?(data)
|
|
@@ -43,15 +35,57 @@ module Icarus
|
|
|
43
35
|
exit 1
|
|
44
36
|
end
|
|
45
37
|
|
|
46
|
-
|
|
47
|
-
|
|
38
|
+
begin
|
|
39
|
+
parsed = JSON.parse(File.read(data), symbolize_names: true)
|
|
40
|
+
rescue JSON::ParserError => e
|
|
41
|
+
warn "Invalid JSON in #{data}: #{e.message.lines.first.strip}"
|
|
42
|
+
exit 1
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
mods = parsed[:mods] if parsed.is_a?(Hash)
|
|
46
|
+
|
|
47
|
+
unless mods.is_a?(Array)
|
|
48
|
+
warn "Invalid data file: no 'mods' list found in #{data}"
|
|
49
|
+
exit 1
|
|
50
|
+
end
|
|
48
51
|
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
+
# Validate everything up front so a bad entry never leaves a partial add behind
|
|
53
|
+
modinfos = mods.map { |mod| Icarus::Mod::Tools::Modinfo.new(mod.is_a?(Hash) ? mod : {}) }
|
|
54
|
+
invalid = modinfos.reject(&:valid?)
|
|
55
|
+
|
|
56
|
+
if invalid.any?
|
|
57
|
+
invalid.each { |modinfo| warn "Invalid modinfo: #{modinfo.errors}" }
|
|
58
|
+
exit 1
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
if options[:dry_run]
|
|
62
|
+
modinfos.each { |modinfo| puts Paint["Would add: #{modinfo.name}", :yellow] }
|
|
63
|
+
return
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
firestore = Firestore.new
|
|
67
|
+
results = modinfos.map { |modinfo| report(firestore.update(:mod, modinfo, merge: true)) }
|
|
68
|
+
|
|
69
|
+
exit 1 unless results.all?
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
no_commands do
|
|
73
|
+
# Appends item to one of the meta lists, exiting 1 if the update fails
|
|
74
|
+
def add_to_list(type, collection, item)
|
|
75
|
+
if options[:dry_run]
|
|
76
|
+
puts Paint["Would add to #{type}: #{item}", :yellow]
|
|
77
|
+
return
|
|
52
78
|
end
|
|
53
79
|
|
|
54
|
-
|
|
80
|
+
firestore = Firestore.new
|
|
81
|
+
payload = [firestore.send(collection), item].flatten.compact
|
|
82
|
+
|
|
83
|
+
exit 1 unless report(firestore.update(type, payload, merge: true))
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
def report(success)
|
|
87
|
+
puts success ? "Success" : "Failure"
|
|
88
|
+
success
|
|
55
89
|
end
|
|
56
90
|
end
|
|
57
91
|
end
|
data/lib/icarus/mod/cli/base.rb
CHANGED
|
@@ -11,12 +11,12 @@ module Icarus
|
|
|
11
11
|
# The Base CLI class for Icarus Mod Tools. This is inherited by all subcommands.
|
|
12
12
|
class Base < Thor
|
|
13
13
|
class_option :config,
|
|
14
|
-
|
|
15
|
-
|
|
14
|
+
aliases: "-C", type: :string, default: File.join(Dir.home, "/.imtconfig.json"),
|
|
15
|
+
desc: "Path to the config file"
|
|
16
16
|
|
|
17
17
|
class_option :version,
|
|
18
|
-
|
|
19
|
-
|
|
18
|
+
aliases: "-V", type: :boolean,
|
|
19
|
+
desc: "Print the version and exit"
|
|
20
20
|
|
|
21
21
|
def self.exit_on_failure?
|
|
22
22
|
true
|
|
@@ -26,7 +26,12 @@ module Icarus
|
|
|
26
26
|
exit 1
|
|
27
27
|
end
|
|
28
28
|
|
|
29
|
-
|
|
29
|
+
begin
|
|
30
|
+
Icarus::Mod::Config.read(options[:config])
|
|
31
|
+
rescue JSON::ParserError => e
|
|
32
|
+
warn "Could not find or read Config from '#{options[:config]}' - the file is malformed JSON (#{e.message.lines.first.strip})"
|
|
33
|
+
exit 1
|
|
34
|
+
end
|
|
30
35
|
end
|
|
31
36
|
|
|
32
37
|
desc "sync", "Syncs the databases"
|