gemkeeper 0.7.2 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +18 -1
- data/README.md +11 -11
- data/lib/gemkeeper/bundler_mirror_configurator.rb +1 -1
- data/lib/gemkeeper/cli/commands/list.rb +2 -2
- data/lib/gemkeeper/cli/commands/server/start.rb +4 -4
- data/lib/gemkeeper/cli/commands/server/status.rb +3 -3
- data/lib/gemkeeper/cli/commands/server/stop.rb +3 -3
- data/lib/gemkeeper/cli/commands/sync.rb +1 -1
- data/lib/gemkeeper/compact_index_server/cache_meta.rb +34 -0
- data/lib/gemkeeper/compact_index_server/cache_store.rb +64 -0
- data/lib/gemkeeper/compact_index_server/gem_cache.rb +88 -0
- data/lib/gemkeeper/compact_index_server/gem_index.rb +78 -0
- data/lib/gemkeeper/compact_index_server/index_merger.rb +81 -0
- data/lib/gemkeeper/compact_index_server/response.rb +12 -0
- data/lib/gemkeeper/compact_index_server/response_builder.rb +63 -0
- data/lib/gemkeeper/compact_index_server/rubygems_client.rb +59 -0
- data/lib/gemkeeper/compact_index_server/spec_mapper.rb +38 -0
- data/lib/gemkeeper/compact_index_server/upload_handler.rb +36 -0
- data/lib/gemkeeper/compact_index_server/upstream_cache.rb +26 -0
- data/lib/gemkeeper/compact_index_server.rb +131 -0
- data/lib/gemkeeper/configuration.rb +1 -1
- data/lib/gemkeeper/gem_syncer.rb +53 -84
- data/lib/gemkeeper/gem_uploader.rb +26 -18
- data/lib/gemkeeper/rackup_process.rb +12 -7
- data/lib/gemkeeper/repo_fetcher.rb +80 -0
- data/lib/gemkeeper/server_manager.rb +1 -1
- data/lib/gemkeeper/version.rb +1 -1
- data/lib/gemkeeper.rb +2 -0
- data/specs/20260529-091429-replace-geminabox-compact-proxy/critique-consolidated-v-1.md +168 -0
- data/specs/20260529-091429-replace-geminabox-compact-proxy/critique-v-1-claude.md +124 -0
- data/specs/20260529-091429-replace-geminabox-compact-proxy/critique-v-1-codex.md +125 -0
- data/specs/20260529-091429-replace-geminabox-compact-proxy/critique-v-1-copilot.md +261 -0
- data/specs/20260529-091429-replace-geminabox-compact-proxy/spec.md +360 -0
- data/specs/20260529-131354-sync-serve-cache-contract/critique-consolidated-v-1.md +95 -0
- data/specs/20260529-131354-sync-serve-cache-contract/critique-v-1-claude.md +47 -0
- data/specs/20260529-131354-sync-serve-cache-contract/critique-v-1-codex.md +112 -0
- data/specs/20260529-131354-sync-serve-cache-contract/critique-v-1-copilot.md +169 -0
- data/specs/20260529-131354-sync-serve-cache-contract/implementation-summary.md +59 -0
- data/specs/20260529-131354-sync-serve-cache-contract/spec.md +169 -0
- metadata +38 -28
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 88d9b763777c56c907c9c945fb35f7a782b6be529d7af8f38b5d14a899cda825
|
|
4
|
+
data.tar.gz: 201987b7a1d4307548b8d7f020fb25a2e7ecf274f85e37f5156a26065fde8fa4
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 79924f19d7da544373cdd0d213e818a1b4fc1411651bdc1819d331fb16a9361200e20a2de52b4319ebac064bf64bc14503abae89d8463e2d522e04d9f840c4d1
|
|
7
|
+
data.tar.gz: 3e175013111f44149a4c3273fb21fa1fe1785763713fd14eb698e33dcd9b898933a5e6a485d76c5fe0e23ef1f3ed5a4d8f92aff5fea85793e94dda4b1a7e3b9a
|
data/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,21 @@
|
|
|
1
1
|
## [Unreleased]
|
|
2
2
|
|
|
3
|
+
## [0.8.0] - 2026-05-29
|
|
4
|
+
|
|
5
|
+
### Changed
|
|
6
|
+
|
|
7
|
+
- Refactored `CompactIndexServer` into focused collaborators — `RubygemsClient`, `IndexMerger`, `GemCache`, `ResponseBuilder`, `UploadHandler`, `SpecMapper`, plus `Response` and `CacheMeta` value objects.
|
|
8
|
+
No behavior change; this splits the responsibilities that the initial single-file server bundled together and restores the rubycritic score above the 90 gate.
|
|
9
|
+
- Renamed leftover "Geminabox" references to the Gemkeeper server, reflecting the compact index server that replaced it.
|
|
10
|
+
CLI output and `Configuration#server_url` (was `geminabox_url`) are updated; no behavior change.
|
|
11
|
+
|
|
12
|
+
### Fixed
|
|
13
|
+
|
|
14
|
+
- `gemkeeper server start` no longer fails with a `LoadError` when run from source.
|
|
15
|
+
The spawned `rackup` process does not inherit the executable's `$LOAD_PATH`, so it could not require `gemkeeper/compact_index_server`; the rackup command now passes the gem's `lib/` via `-I`.
|
|
16
|
+
- `gemkeeper sync` now repopulates a fresh or repointed server instead of skipping every gem.
|
|
17
|
+
The skip decision was based on a local build artifact, which could diverge from the server's store; it now asks the server's private store directly (new read-only `GET /gemkeeper/has/<name>/<version>` endpoint) and re-uploads an existing artifact without rebuilding when the server is missing it.
|
|
18
|
+
|
|
3
19
|
## [0.7.2] - 2026-05-28
|
|
4
20
|
|
|
5
21
|
### Fixed
|
|
@@ -170,7 +186,8 @@
|
|
|
170
186
|
|
|
171
187
|
- Initial release
|
|
172
188
|
|
|
173
|
-
[Unreleased]: https://github.com/danhorst/gemkeeper/compare/v0.
|
|
189
|
+
[Unreleased]: https://github.com/danhorst/gemkeeper/compare/v0.8.0...HEAD
|
|
190
|
+
[0.8.0]: https://github.com/danhorst/gemkeeper/compare/0.7.2...0.8.0
|
|
174
191
|
[0.7.2]: https://github.com/danhorst/gemkeeper/compare/0.7.1...0.7.2
|
|
175
192
|
[0.7.1]: https://github.com/danhorst/gemkeeper/compare/0.7.0...0.7.1
|
|
176
193
|
[0.7.0]: https://github.com/danhorst/gemkeeper/compare/0.6.7...0.7.0
|
data/README.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|

|
|
2
2
|
|
|
3
|
-
This project
|
|
3
|
+
This project manages private gem dependencies in an offline development environment, building internal gems from source and serving them through a local compact index server.
|
|
4
4
|
|
|
5
5
|
## Installation
|
|
6
6
|
|
|
@@ -17,14 +17,14 @@ brew tap danhorst/tap
|
|
|
17
17
|
brew install danhorst/tap/gemkeeper
|
|
18
18
|
```
|
|
19
19
|
|
|
20
|
-
Forumla: [`danhorst/homebrew-tap`][
|
|
20
|
+
Forumla: [`danhorst/homebrew-tap`][1]
|
|
21
21
|
|
|
22
22
|
## Workstation Setup
|
|
23
23
|
|
|
24
24
|
If you cannot reach your organization's private gem server, follow these steps to use gemkeeper as a local proxy.
|
|
25
25
|
|
|
26
26
|
**Prerequisites:** You must have HTTPS access to the internal gem repositories on GitHub.
|
|
27
|
-
Configure GitHub credentials before step 4 — see [GitHub authentication docs][
|
|
27
|
+
Configure GitHub credentials before step 4 — see [GitHub authentication docs][2].
|
|
28
28
|
|
|
29
29
|
1. Install gemkeeper:
|
|
30
30
|
|
|
@@ -65,12 +65,12 @@ bundle config set --local mirror.<your-private-gem-source-url> http://localhost:
|
|
|
65
65
|
```
|
|
66
66
|
|
|
67
67
|
Replace `<your-private-gem-source-url>` with the gem source URL declared in your `Gemfile` (the one that requires VPN or private credentials).
|
|
68
|
-
The mirror approach redirects gem resolution to your local
|
|
68
|
+
The mirror approach redirects gem resolution to your local Gemkeeper server without modifying the committed `Gemfile` or `Gemfile.lock`.
|
|
69
69
|
Public gems are proxied from RubyGems.org automatically.
|
|
70
70
|
|
|
71
71
|
## Quick Start
|
|
72
72
|
|
|
73
|
-
1. Ensure your org manifest is present at `~/.config/gemkeeper/manifest.yml` (see [Workstation Setup]
|
|
73
|
+
1. Ensure your org manifest is present at `~/.config/gemkeeper/manifest.yml` (see [Workstation Setup][3]), then create a project `gemkeeper.yml`:
|
|
74
74
|
|
|
75
75
|
```yaml
|
|
76
76
|
port: 9292
|
|
@@ -115,7 +115,7 @@ Gemkeeper looks for configuration files in these locations (in order):
|
|
|
115
115
|
### Configuration Options
|
|
116
116
|
|
|
117
117
|
```yaml
|
|
118
|
-
# Port for the
|
|
118
|
+
# Port for the Gemkeeper server (default: 9292)
|
|
119
119
|
port: 9292
|
|
120
120
|
|
|
121
121
|
# Where to clone gem repositories (default: ./cache/repos)
|
|
@@ -293,8 +293,8 @@ gemkeeper server stop
|
|
|
293
293
|
|
|
294
294
|
1. **Clone/Pull**: Gemkeeper clones (or pulls) gem repositories to a local cache.
|
|
295
295
|
2. **Build**: Builds `.gem` files from the source at the specified version/tag.
|
|
296
|
-
3. **Upload**: Uploads built gems to
|
|
297
|
-
4. **Proxy**:
|
|
296
|
+
3. **Upload**: Uploads built gems to the local Gemkeeper server.
|
|
297
|
+
4. **Proxy**: The server proxies public gems from RubyGems.org, so you only need one gem source.
|
|
298
298
|
|
|
299
299
|
This lets you use a combination of public and private gems from a single gem source.
|
|
300
300
|
|
|
@@ -306,6 +306,6 @@ bundle exec rake test # Run tests
|
|
|
306
306
|
bundle exec rubocop # Run linter
|
|
307
307
|
```
|
|
308
308
|
|
|
309
|
-
[1]: https://github.com/
|
|
310
|
-
[2]: https://github.com/
|
|
311
|
-
[3]:
|
|
309
|
+
[1]: https://github.com/danhorst/homebrew-tap
|
|
310
|
+
[2]: https://docs.github.com/en/authentication
|
|
311
|
+
[3]: #workstation-setup
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
module Gemkeeper
|
|
4
|
-
# Configures Bundler mirror settings so private gem registries proxy through local
|
|
4
|
+
# Configures Bundler mirror settings so private gem registries proxy through the local Gemkeeper server.
|
|
5
5
|
class BundlerMirrorConfigurator
|
|
6
6
|
def initialize(candidates, port:, global:)
|
|
7
7
|
@remotes = candidates.filter_map { |c| c[:remote] if c[:source_type] == :private_gem }.uniq
|
|
@@ -4,7 +4,7 @@ module Gemkeeper
|
|
|
4
4
|
module CLI
|
|
5
5
|
module Commands
|
|
6
6
|
class List < Dry::CLI::Command
|
|
7
|
-
desc "List gems cached
|
|
7
|
+
desc "List gems cached locally"
|
|
8
8
|
|
|
9
9
|
option :config, type: :string, desc: "Path to config file"
|
|
10
10
|
|
|
@@ -15,7 +15,7 @@ module Gemkeeper
|
|
|
15
15
|
gem_files = Dir.glob(File.join(gems_path, "gems", "*.gem"))
|
|
16
16
|
|
|
17
17
|
if gem_files.empty?
|
|
18
|
-
puts "No gems cached
|
|
18
|
+
puts "No gems cached locally"
|
|
19
19
|
puts " Gems directory: #{gems_path}"
|
|
20
20
|
return
|
|
21
21
|
end
|
|
@@ -5,7 +5,7 @@ module Gemkeeper
|
|
|
5
5
|
module Commands
|
|
6
6
|
module Server
|
|
7
7
|
class Start < Dry::CLI::Command
|
|
8
|
-
desc "Start the
|
|
8
|
+
desc "Start the Gemkeeper server"
|
|
9
9
|
|
|
10
10
|
option :port, type: :integer, desc: "Port to run server on"
|
|
11
11
|
option :config, type: :string, desc: "Path to config file"
|
|
@@ -18,15 +18,15 @@ module Gemkeeper
|
|
|
18
18
|
config = override_port(config, port) if port
|
|
19
19
|
|
|
20
20
|
manager = ServerManager.new(config)
|
|
21
|
-
url = config.
|
|
21
|
+
url = config.server_url
|
|
22
22
|
|
|
23
23
|
if options[:foreground]
|
|
24
|
-
puts "Starting
|
|
24
|
+
puts "Starting Gemkeeper server at #{url}"
|
|
25
25
|
puts "Press Ctrl+C to stop"
|
|
26
26
|
manager.start_foreground
|
|
27
27
|
else
|
|
28
28
|
manager.start
|
|
29
|
-
puts "
|
|
29
|
+
puts "Gemkeeper server started at #{url}"
|
|
30
30
|
puts "PID: #{File.read(config.pid_file).strip}"
|
|
31
31
|
end
|
|
32
32
|
rescue ServerAlreadyRunningError => error
|
|
@@ -5,7 +5,7 @@ module Gemkeeper
|
|
|
5
5
|
module Commands
|
|
6
6
|
module Server
|
|
7
7
|
class Status < Dry::CLI::Command
|
|
8
|
-
desc "Check
|
|
8
|
+
desc "Check Gemkeeper server status"
|
|
9
9
|
|
|
10
10
|
option :config, type: :string, desc: "Path to config file"
|
|
11
11
|
|
|
@@ -15,11 +15,11 @@ module Gemkeeper
|
|
|
15
15
|
status = manager.status
|
|
16
16
|
|
|
17
17
|
if status[:running]
|
|
18
|
-
puts "
|
|
18
|
+
puts "Gemkeeper server is running"
|
|
19
19
|
puts " PID: #{status[:pid]}" if status[:pid]
|
|
20
20
|
puts " URL: #{status[:url]}"
|
|
21
21
|
else
|
|
22
|
-
puts "
|
|
22
|
+
puts "Gemkeeper server is not running"
|
|
23
23
|
end
|
|
24
24
|
end
|
|
25
25
|
end
|
|
@@ -5,7 +5,7 @@ module Gemkeeper
|
|
|
5
5
|
module Commands
|
|
6
6
|
module Server
|
|
7
7
|
class Stop < Dry::CLI::Command
|
|
8
|
-
desc "Stop the
|
|
8
|
+
desc "Stop the Gemkeeper server"
|
|
9
9
|
|
|
10
10
|
option :config, type: :string, desc: "Path to config file"
|
|
11
11
|
|
|
@@ -14,9 +14,9 @@ module Gemkeeper
|
|
|
14
14
|
manager = ServerManager.new(config)
|
|
15
15
|
manager.stop
|
|
16
16
|
|
|
17
|
-
puts "
|
|
17
|
+
puts "Gemkeeper server stopped"
|
|
18
18
|
rescue ServerNotRunningError
|
|
19
|
-
puts "
|
|
19
|
+
puts "Gemkeeper server is not running"
|
|
20
20
|
rescue ServerError => error
|
|
21
21
|
warn "Error stopping server: #{error.message}"
|
|
22
22
|
exit 1
|
|
@@ -12,7 +12,7 @@ module Gemkeeper
|
|
|
12
12
|
def call(gem_name: nil, **options)
|
|
13
13
|
config = Configuration.load(options[:config])
|
|
14
14
|
gems_to_sync = select_gems(config, gem_name)
|
|
15
|
-
syncer = GemSyncer.new(config, GemUploader.new(config.
|
|
15
|
+
syncer = GemSyncer.new(config, GemUploader.new(config.server_url), manifest: ManifestReader.load)
|
|
16
16
|
counts, failures = run_sync(gems_to_sync, syncer)
|
|
17
17
|
report_results(counts, failures, gems_to_sync.size)
|
|
18
18
|
end
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "time"
|
|
4
|
+
|
|
5
|
+
module Gemkeeper
|
|
6
|
+
class CompactIndexServer
|
|
7
|
+
# The sidecar metadata for a cached upstream document: its ETag and when it
|
|
8
|
+
# was fetched. Knows whether it has aged past a TTL and how to serialize.
|
|
9
|
+
class CacheMeta
|
|
10
|
+
attr_reader :etag
|
|
11
|
+
|
|
12
|
+
def self.load(hash)
|
|
13
|
+
return nil unless hash
|
|
14
|
+
|
|
15
|
+
new(hash["etag"], hash["fetched_at"])
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def initialize(etag, fetched_at)
|
|
19
|
+
@etag = etag
|
|
20
|
+
@fetched_at = fetched_at
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def expired?(ttl)
|
|
24
|
+
return true unless @fetched_at
|
|
25
|
+
|
|
26
|
+
Time.now - Time.parse(@fetched_at.to_s) > ttl
|
|
27
|
+
rescue ArgumentError
|
|
28
|
+
true
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def to_h = { "etag" => @etag, "fetched_at" => @fetched_at }
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
end
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "digest"
|
|
4
|
+
require "fileutils"
|
|
5
|
+
require "yaml"
|
|
6
|
+
|
|
7
|
+
require_relative "cache_meta"
|
|
8
|
+
|
|
9
|
+
module Gemkeeper
|
|
10
|
+
class CompactIndexServer
|
|
11
|
+
# Handles on-disk cache I/O: atomic writes, sidecar metadata, TTL checks.
|
|
12
|
+
# All paths are resolved relative to a base directory.
|
|
13
|
+
class CacheStore
|
|
14
|
+
ENTRIES = {
|
|
15
|
+
versions: "versions",
|
|
16
|
+
versions_merged: "versions.merged",
|
|
17
|
+
versions_meta: "versions.meta",
|
|
18
|
+
names: "names",
|
|
19
|
+
names_merged: "names.merged",
|
|
20
|
+
names_meta: "names.meta"
|
|
21
|
+
}.freeze
|
|
22
|
+
|
|
23
|
+
def initialize(base_dir)
|
|
24
|
+
@base_dir = base_dir
|
|
25
|
+
FileUtils.mkdir_p([base_dir, File.join(base_dir, "info"), File.join(base_dir, "gems")])
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def path(key_or_filename)
|
|
29
|
+
filename = ENTRIES.fetch(key_or_filename, key_or_filename)
|
|
30
|
+
File.join(@base_dir, filename.to_s)
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def read_meta(filename)
|
|
34
|
+
full_path = File.join(@base_dir, "#{filename}.meta")
|
|
35
|
+
return nil unless File.exist?(full_path)
|
|
36
|
+
|
|
37
|
+
CacheMeta.load(YAML.safe_load_file(full_path))
|
|
38
|
+
rescue StandardError
|
|
39
|
+
nil
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def write_meta(filename, etag:)
|
|
43
|
+
meta = CacheMeta.new(etag, Time.now.utc.iso8601)
|
|
44
|
+
atomic_write(File.join(@base_dir, "#{filename}.meta"), meta.to_h.to_yaml)
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def atomic_write(full_path, content)
|
|
48
|
+
tmp = "#{full_path}.tmp.#{Process.pid}.#{Thread.current.object_id}"
|
|
49
|
+
File.binwrite(tmp, content)
|
|
50
|
+
File.rename(tmp, full_path)
|
|
51
|
+
rescue StandardError
|
|
52
|
+
File.unlink(tmp) if tmp && File.exist?(tmp)
|
|
53
|
+
raise
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def etag_for(key)
|
|
57
|
+
full_path = path(key)
|
|
58
|
+
return nil unless File.exist?(full_path)
|
|
59
|
+
|
|
60
|
+
Digest::SHA256.hexdigest(File.binread(full_path))
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
end
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "digest"
|
|
4
|
+
require "rubygems"
|
|
5
|
+
|
|
6
|
+
module Gemkeeper
|
|
7
|
+
class CompactIndexServer
|
|
8
|
+
# Per-gem caching of /info documents and .gem binaries from RubyGems.org.
|
|
9
|
+
# Serves from the local RubyGems cache, then the disk cache, then upstream.
|
|
10
|
+
class GemCache
|
|
11
|
+
INFO_TTL = 3600 # 60 minutes
|
|
12
|
+
|
|
13
|
+
def initialize(store, client)
|
|
14
|
+
@store = store
|
|
15
|
+
@client = client
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
# Returns {body:, etag:} or nil (not found).
|
|
19
|
+
# Raises UpstreamUnavailableError when unreachable with no cache.
|
|
20
|
+
def info(gemname)
|
|
21
|
+
path = @store.path("info/#{gemname}")
|
|
22
|
+
meta = @store.read_meta("info/#{gemname}")
|
|
23
|
+
return cached_entry(path, meta) if fresh?(path, meta)
|
|
24
|
+
|
|
25
|
+
fetch_info(gemname, path, meta)
|
|
26
|
+
rescue UpstreamUnavailableError
|
|
27
|
+
raise unless File.exist?(path)
|
|
28
|
+
|
|
29
|
+
cached_entry(path, meta)
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
# Returns a local path to the gem binary, or nil (not found).
|
|
33
|
+
# Raises UpstreamUnavailableError when unreachable with no cached copy.
|
|
34
|
+
def binary(filename)
|
|
35
|
+
system_gem_path(filename) ||
|
|
36
|
+
disk_cache_path(filename) ||
|
|
37
|
+
fetch_binary(filename, @store.path("gems/#{filename}"))
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
private
|
|
41
|
+
|
|
42
|
+
def system_gem_path(filename)
|
|
43
|
+
Gem.path.each do |gem_path|
|
|
44
|
+
candidate = File.join(gem_path, "cache", filename)
|
|
45
|
+
return candidate if File.exist?(candidate)
|
|
46
|
+
end
|
|
47
|
+
nil
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def disk_cache_path(filename)
|
|
51
|
+
path = @store.path("gems/#{filename}")
|
|
52
|
+
File.exist?(path) ? path : nil
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
def fetch_info(gemname, path, meta)
|
|
56
|
+
response = @client.get("/info/#{gemname}", meta&.etag)
|
|
57
|
+
return store_info(gemname, path, response) unless response.not_modified?
|
|
58
|
+
|
|
59
|
+
@store.write_meta("info/#{gemname}", etag: meta.etag)
|
|
60
|
+
cached_entry(path, meta)
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
def store_info(gemname, path, response)
|
|
64
|
+
return nil unless response.success?
|
|
65
|
+
|
|
66
|
+
body = response.body
|
|
67
|
+
etag = response.etag || Digest::SHA256.hexdigest(body)
|
|
68
|
+
@store.atomic_write(path, body)
|
|
69
|
+
@store.write_meta("info/#{gemname}", etag: etag)
|
|
70
|
+
{ body: body, etag: etag }
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
def fetch_binary(filename, cache_path)
|
|
74
|
+
response = @client.get("/gems/#{filename}", nil)
|
|
75
|
+
return nil unless response.success?
|
|
76
|
+
|
|
77
|
+
@store.atomic_write(cache_path, response.body) && cache_path
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
def fresh?(path, meta) = File.exist?(path) && meta && !meta.expired?(INFO_TTL)
|
|
81
|
+
|
|
82
|
+
def cached_entry(path, meta)
|
|
83
|
+
body = File.binread(path)
|
|
84
|
+
{ body:, etag: meta&.etag || Digest::SHA256.hexdigest(body) }
|
|
85
|
+
end
|
|
86
|
+
end
|
|
87
|
+
end
|
|
88
|
+
end
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "compact_index"
|
|
4
|
+
require "digest"
|
|
5
|
+
require "fileutils"
|
|
6
|
+
require "rubygems/package"
|
|
7
|
+
|
|
8
|
+
require_relative "spec_mapper"
|
|
9
|
+
|
|
10
|
+
module Gemkeeper
|
|
11
|
+
class CompactIndexServer
|
|
12
|
+
# Builds and maintains the in-memory index of privately-hosted gems.
|
|
13
|
+
# Scans gems_dir on construction and after each successful upload.
|
|
14
|
+
class GemIndex
|
|
15
|
+
def initialize(gems_dir)
|
|
16
|
+
@gems_dir = gems_dir
|
|
17
|
+
@gems = {}
|
|
18
|
+
FileUtils.mkdir_p(@gems_dir)
|
|
19
|
+
rebuild
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def [](name) = @gems[name]
|
|
23
|
+
def keys = @gems.keys
|
|
24
|
+
def values = @gems.values
|
|
25
|
+
|
|
26
|
+
# True when the private store holds this exact name and bare-semver version.
|
|
27
|
+
def serves?(name, version)
|
|
28
|
+
gem = @gems[name]
|
|
29
|
+
gem ? gem.versions.any? { |gem_version| gem_version.number == version } : false
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def gem_path(filename)
|
|
33
|
+
path = File.join(@gems_dir, filename)
|
|
34
|
+
File.exist?(path) ? path : nil
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
# Copies source_path into gems_dir, derives the filename from spec, and rebuilds.
|
|
38
|
+
# Raises Errno::EEXIST if the gem already exists. Returns the target filename.
|
|
39
|
+
def add(source_path, spec)
|
|
40
|
+
filename = SpecMapper.filename(spec)
|
|
41
|
+
target = File.join(@gems_dir, filename)
|
|
42
|
+
raise Errno::EEXIST, target if File.exist?(target)
|
|
43
|
+
|
|
44
|
+
copy_into_place(source_path, target)
|
|
45
|
+
rebuild
|
|
46
|
+
filename
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def rebuild
|
|
50
|
+
gems = {}
|
|
51
|
+
Dir.glob(File.join(@gems_dir, "*.gem")).each { |gem_file| index_gem(gems, gem_file) }
|
|
52
|
+
gems.each_value { |gem| stamp_checksum(gem) }
|
|
53
|
+
@gems = gems
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
private
|
|
57
|
+
|
|
58
|
+
def index_gem(gems, gem_file)
|
|
59
|
+
spec = Gem::Package.new(gem_file).spec
|
|
60
|
+
name = spec.name
|
|
61
|
+
(gems[name] ||= CompactIndex::Gem.new(name, [])).versions << SpecMapper.gem_version(spec, gem_file)
|
|
62
|
+
rescue StandardError => error
|
|
63
|
+
warn "gemkeeper: skipping #{File.basename(gem_file)}: #{error.message}"
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
def stamp_checksum(gem)
|
|
67
|
+
versions = gem.versions
|
|
68
|
+
versions.last.info_checksum = Digest::MD5.hexdigest(CompactIndex.info(versions))
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
def copy_into_place(source_path, target)
|
|
72
|
+
tmp = "#{target}.tmp.#{Process.pid}.#{Thread.current.object_id}"
|
|
73
|
+
FileUtils.cp(source_path, tmp)
|
|
74
|
+
File.rename(tmp, target)
|
|
75
|
+
end
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
end
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "compact_index"
|
|
4
|
+
require "digest"
|
|
5
|
+
require "time"
|
|
6
|
+
|
|
7
|
+
module Gemkeeper
|
|
8
|
+
class CompactIndexServer
|
|
9
|
+
# Generates the merged /versions and /names index files, combining upstream
|
|
10
|
+
# RubyGems.org data (refreshed on a TTL) with the locally-hosted private gems.
|
|
11
|
+
class IndexMerger
|
|
12
|
+
VERSIONS_TTL = 1800 # 30 minutes
|
|
13
|
+
NAMES_TTL = 3600 # 60 minutes
|
|
14
|
+
|
|
15
|
+
def initialize(store, client)
|
|
16
|
+
@store = store
|
|
17
|
+
@client = client
|
|
18
|
+
@versions_etag = store.etag_for(:versions_merged)
|
|
19
|
+
@names_etag = store.etag_for(:names_merged)
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
# Returns {path:, etag:} for the merged /versions file.
|
|
23
|
+
def versions(private_gems)
|
|
24
|
+
refresh(:versions, "/versions", VERSIONS_TTL)
|
|
25
|
+
regenerate_versions(private_gems)
|
|
26
|
+
{ path: @store.path(:versions_merged), etag: @versions_etag }
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
# Returns {path:, etag:} for the merged /names file.
|
|
30
|
+
def names(private_names)
|
|
31
|
+
refresh(:names, "/names", NAMES_TTL)
|
|
32
|
+
regenerate_names(private_names)
|
|
33
|
+
{ path: @store.path(:names_merged), etag: @names_etag }
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
private
|
|
37
|
+
|
|
38
|
+
def refresh(key, upstream_path, ttl)
|
|
39
|
+
meta = @store.read_meta(key.to_s)
|
|
40
|
+
return if meta && !meta.expired?(ttl) && File.exist?(@store.path(key))
|
|
41
|
+
|
|
42
|
+
apply(key, meta, @client.get(upstream_path, meta&.etag))
|
|
43
|
+
rescue UpstreamUnavailableError
|
|
44
|
+
nil # regeneration proceeds with whatever is cached
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def apply(key, meta, response)
|
|
48
|
+
if response.not_modified?
|
|
49
|
+
etag = meta.etag
|
|
50
|
+
else
|
|
51
|
+
@store.atomic_write(@store.path(key), response.body)
|
|
52
|
+
etag = response.etag
|
|
53
|
+
end
|
|
54
|
+
@store.write_meta(key.to_s, etag: etag)
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
def regenerate_versions(private_gems)
|
|
58
|
+
versions_path = @store.path(:versions)
|
|
59
|
+
unless File.exist?(versions_path)
|
|
60
|
+
@store.atomic_write(versions_path, "created_at: #{Time.now.utc.iso8601}\n---\n")
|
|
61
|
+
end
|
|
62
|
+
merged = CompactIndex.versions(CompactIndex::VersionsFile.new(versions_path), private_gems)
|
|
63
|
+
@store.atomic_write(@store.path(:versions_merged), merged)
|
|
64
|
+
@versions_etag = Digest::SHA256.hexdigest(merged)
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
def regenerate_names(private_names)
|
|
68
|
+
merged = CompactIndex.names((upstream_names + private_names).uniq.sort)
|
|
69
|
+
@store.atomic_write(@store.path(:names_merged), merged)
|
|
70
|
+
@names_etag = Digest::SHA256.hexdigest(merged)
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
def upstream_names
|
|
74
|
+
names_path = @store.path(:names)
|
|
75
|
+
return [] unless File.exist?(names_path)
|
|
76
|
+
|
|
77
|
+
File.read(names_path).lines.map(&:strip).reject { |line| line.empty? || line == "---" }
|
|
78
|
+
end
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
end
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Gemkeeper
|
|
4
|
+
class CompactIndexServer
|
|
5
|
+
# Outcome of a compact-index request: a not-modified marker (304) or a
|
|
6
|
+
# body+etag (200), or a non-success status. Callers ask, never inspect.
|
|
7
|
+
Response = Struct.new(:status, :body, :etag) do
|
|
8
|
+
def not_modified? = status == 304
|
|
9
|
+
def success? = status == 200
|
|
10
|
+
end
|
|
11
|
+
end
|
|
12
|
+
end
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "digest"
|
|
4
|
+
|
|
5
|
+
module Gemkeeper
|
|
6
|
+
class CompactIndexServer
|
|
7
|
+
# Builds Rack responses for compact index payloads: conditional GET (ETag),
|
|
8
|
+
# byte-range requests, and the digest/accept-ranges headers Bundler expects.
|
|
9
|
+
class ResponseBuilder
|
|
10
|
+
def self.file(path)
|
|
11
|
+
[200,
|
|
12
|
+
{ "content-type" => "application/octet-stream",
|
|
13
|
+
"content-length" => File.size(path).to_s },
|
|
14
|
+
File.open(path, "rb")]
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def initialize(req)
|
|
18
|
+
@req = req
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
# 200 / 206 / 304 for an in-memory index body.
|
|
22
|
+
def index(body, etag)
|
|
23
|
+
quoted = %("#{etag}")
|
|
24
|
+
return [304, { "etag" => quoted }, []] if @req.env["HTTP_IF_NONE_MATCH"] == quoted
|
|
25
|
+
|
|
26
|
+
range(body, etag) || [200, index_headers(body, etag), [body]]
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
private
|
|
30
|
+
|
|
31
|
+
def range(body, etag)
|
|
32
|
+
header = @req.env["HTTP_RANGE"]
|
|
33
|
+
return nil unless header
|
|
34
|
+
|
|
35
|
+
size = body.bytesize
|
|
36
|
+
match = header.match(/\Abytes=(\d+)-(\d*)\z/)
|
|
37
|
+
return unsatisfiable(size) unless match
|
|
38
|
+
|
|
39
|
+
start_byte = match[1].to_i
|
|
40
|
+
return unsatisfiable(size) if start_byte >= size
|
|
41
|
+
|
|
42
|
+
partial(body, etag, start_byte, match[2], size)
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def partial(body, etag, start_byte, raw_end, size)
|
|
46
|
+
end_byte = raw_end.empty? ? size - 1 : [raw_end.to_i, size - 1].min
|
|
47
|
+
slice = body.byteslice(start_byte, end_byte - start_byte + 1)
|
|
48
|
+
[206,
|
|
49
|
+
index_headers(body, etag).merge("content-range" => "bytes #{start_byte}-#{end_byte}/#{size}"),
|
|
50
|
+
[slice]]
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def unsatisfiable(size) = [416, { "content-range" => "bytes */#{size}" }, []]
|
|
54
|
+
|
|
55
|
+
def index_headers(body, etag)
|
|
56
|
+
{ "content-type" => "text/plain",
|
|
57
|
+
"etag" => %("#{etag}"),
|
|
58
|
+
"repr-digest" => "sha-256=:#{Digest::SHA256.base64digest(body)}:",
|
|
59
|
+
"accept-ranges" => "bytes" }
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
end
|