gemkeeper 0.7.2 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +18 -1
  3. data/README.md +11 -11
  4. data/lib/gemkeeper/bundler_mirror_configurator.rb +1 -1
  5. data/lib/gemkeeper/cli/commands/list.rb +2 -2
  6. data/lib/gemkeeper/cli/commands/server/start.rb +4 -4
  7. data/lib/gemkeeper/cli/commands/server/status.rb +3 -3
  8. data/lib/gemkeeper/cli/commands/server/stop.rb +3 -3
  9. data/lib/gemkeeper/cli/commands/sync.rb +1 -1
  10. data/lib/gemkeeper/compact_index_server/cache_meta.rb +34 -0
  11. data/lib/gemkeeper/compact_index_server/cache_store.rb +64 -0
  12. data/lib/gemkeeper/compact_index_server/gem_cache.rb +88 -0
  13. data/lib/gemkeeper/compact_index_server/gem_index.rb +78 -0
  14. data/lib/gemkeeper/compact_index_server/index_merger.rb +81 -0
  15. data/lib/gemkeeper/compact_index_server/response.rb +12 -0
  16. data/lib/gemkeeper/compact_index_server/response_builder.rb +63 -0
  17. data/lib/gemkeeper/compact_index_server/rubygems_client.rb +59 -0
  18. data/lib/gemkeeper/compact_index_server/spec_mapper.rb +38 -0
  19. data/lib/gemkeeper/compact_index_server/upload_handler.rb +36 -0
  20. data/lib/gemkeeper/compact_index_server/upstream_cache.rb +26 -0
  21. data/lib/gemkeeper/compact_index_server.rb +131 -0
  22. data/lib/gemkeeper/configuration.rb +1 -1
  23. data/lib/gemkeeper/gem_syncer.rb +53 -84
  24. data/lib/gemkeeper/gem_uploader.rb +26 -18
  25. data/lib/gemkeeper/rackup_process.rb +12 -7
  26. data/lib/gemkeeper/repo_fetcher.rb +80 -0
  27. data/lib/gemkeeper/server_manager.rb +1 -1
  28. data/lib/gemkeeper/version.rb +1 -1
  29. data/lib/gemkeeper.rb +2 -0
  30. data/specs/20260529-091429-replace-geminabox-compact-proxy/critique-consolidated-v-1.md +168 -0
  31. data/specs/20260529-091429-replace-geminabox-compact-proxy/critique-v-1-claude.md +124 -0
  32. data/specs/20260529-091429-replace-geminabox-compact-proxy/critique-v-1-codex.md +125 -0
  33. data/specs/20260529-091429-replace-geminabox-compact-proxy/critique-v-1-copilot.md +261 -0
  34. data/specs/20260529-091429-replace-geminabox-compact-proxy/spec.md +360 -0
  35. data/specs/20260529-131354-sync-serve-cache-contract/critique-consolidated-v-1.md +95 -0
  36. data/specs/20260529-131354-sync-serve-cache-contract/critique-v-1-claude.md +47 -0
  37. data/specs/20260529-131354-sync-serve-cache-contract/critique-v-1-codex.md +112 -0
  38. data/specs/20260529-131354-sync-serve-cache-contract/critique-v-1-copilot.md +169 -0
  39. data/specs/20260529-131354-sync-serve-cache-contract/implementation-summary.md +59 -0
  40. data/specs/20260529-131354-sync-serve-cache-contract/spec.md +169 -0
  41. metadata +38 -28
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 79e1b77de1b2e65913e3e6c5c779ec2da40de77248d3f19023638f6bde8de52d
4
- data.tar.gz: 3ab3353aeacb8a191b390130e15b69226e1f81955a88175183d25a21b0d150b3
3
+ metadata.gz: 88d9b763777c56c907c9c945fb35f7a782b6be529d7af8f38b5d14a899cda825
4
+ data.tar.gz: 201987b7a1d4307548b8d7f020fb25a2e7ecf274f85e37f5156a26065fde8fa4
5
5
  SHA512:
6
- metadata.gz: 0a55f01e490978fbb972e095407a7f18ad915e0c9a35bfacd54e577a1a7285cf7703d5bca056ab5ce7fa4d792385ca64ddc842b93028d763879ec186b6bc86b0
7
- data.tar.gz: 4318c10b34fb9a183bcd4d08f9a78b0295dc83bd7d90246a6711cf69a0c1c614cda600713f2e917b64de35024c159f7d3943f365c77639fc01126420afe13a47
6
+ metadata.gz: 79924f19d7da544373cdd0d213e818a1b4fc1411651bdc1819d331fb16a9361200e20a2de52b4319ebac064bf64bc14503abae89d8463e2d522e04d9f840c4d1
7
+ data.tar.gz: 3e175013111f44149a4c3273fb21fa1fe1785763713fd14eb698e33dcd9b898933a5e6a485d76c5fe0e23ef1f3ed5a4d8f92aff5fea85793e94dda4b1a7e3b9a
data/CHANGELOG.md CHANGED
@@ -1,5 +1,21 @@
1
1
  ## [Unreleased]
2
2
 
3
+ ## [0.8.0] - 2026-05-29
4
+
5
+ ### Changed
6
+
7
+ - Refactored `CompactIndexServer` into focused collaborators — `RubygemsClient`, `IndexMerger`, `GemCache`, `ResponseBuilder`, `UploadHandler`, `SpecMapper`, plus `Response` and `CacheMeta` value objects.
8
+ No behavior change; this splits the responsibilities that the initial single-file server bundled together and restores the rubycritic score above the 90 gate.
9
+ - Renamed leftover "Geminabox" references to the Gemkeeper server, reflecting the compact index server that replaced it.
10
+ CLI output and `Configuration#server_url` (was `geminabox_url`) are updated; no behavior change.
11
+
12
+ ### Fixed
13
+
14
+ - `gemkeeper server start` no longer fails with a `LoadError` when run from source.
15
+ The spawned `rackup` process does not inherit the executable's `$LOAD_PATH`, so it could not require `gemkeeper/compact_index_server`; the rackup command now passes the gem's `lib/` via `-I`.
16
+ - `gemkeeper sync` now repopulates a fresh or repointed server instead of skipping every gem.
17
+ The skip decision was based on a local build artifact, which could diverge from the server's store; it now asks the server's private store directly (new read-only `GET /gemkeeper/has/<name>/<version>` endpoint) and re-uploads an existing artifact without rebuilding when the server is missing it.
18
+
3
19
  ## [0.7.2] - 2026-05-28
4
20
 
5
21
  ### Fixed
@@ -170,7 +186,8 @@
170
186
 
171
187
  - Initial release
172
188
 
173
- [Unreleased]: https://github.com/danhorst/gemkeeper/compare/v0.7.2...HEAD
189
+ [Unreleased]: https://github.com/danhorst/gemkeeper/compare/v0.8.0...HEAD
190
+ [0.8.0]: https://github.com/danhorst/gemkeeper/compare/0.7.2...0.8.0
174
191
  [0.7.2]: https://github.com/danhorst/gemkeeper/compare/0.7.1...0.7.2
175
192
  [0.7.1]: https://github.com/danhorst/gemkeeper/compare/0.7.0...0.7.1
176
193
  [0.7.0]: https://github.com/danhorst/gemkeeper/compare/0.6.7...0.7.0
data/README.md CHANGED
@@ -1,6 +1,6 @@
1
1
  ![Gemkeeper](./img/gemkeeper.jpeg)
2
2
 
3
- This project is an opinionated wrapper around [Gem in a Box][1] for managing private gem dependencies in an offline development environment.
3
+ This project manages private gem dependencies in an offline development environment, building internal gems from source and serving them through a local compact index server.
4
4
 
5
5
  ## Installation
6
6
 
@@ -17,14 +17,14 @@ brew tap danhorst/tap
17
17
  brew install danhorst/tap/gemkeeper
18
18
  ```
19
19
 
20
- Forumla: [`danhorst/homebrew-tap`][2]
20
+ Forumla: [`danhorst/homebrew-tap`][1]
21
21
 
22
22
  ## Workstation Setup
23
23
 
24
24
  If you cannot reach your organization's private gem server, follow these steps to use gemkeeper as a local proxy.
25
25
 
26
26
  **Prerequisites:** You must have HTTPS access to the internal gem repositories on GitHub.
27
- Configure GitHub credentials before step 4 — see [GitHub authentication docs][3].
27
+ Configure GitHub credentials before step 4 — see [GitHub authentication docs][2].
28
28
 
29
29
  1. Install gemkeeper:
30
30
 
@@ -65,12 +65,12 @@ bundle config set --local mirror.<your-private-gem-source-url> http://localhost:
65
65
  ```
66
66
 
67
67
  Replace `<your-private-gem-source-url>` with the gem source URL declared in your `Gemfile` (the one that requires VPN or private credentials).
68
- The mirror approach redirects gem resolution to your local Geminabox without modifying the committed `Gemfile` or `Gemfile.lock`.
68
+ The mirror approach redirects gem resolution to your local Gemkeeper server without modifying the committed `Gemfile` or `Gemfile.lock`.
69
69
  Public gems are proxied from RubyGems.org automatically.
70
70
 
71
71
  ## Quick Start
72
72
 
73
- 1. Ensure your org manifest is present at `~/.config/gemkeeper/manifest.yml` (see [Workstation Setup](#workstation-setup)), then create a project `gemkeeper.yml`:
73
+ 1. Ensure your org manifest is present at `~/.config/gemkeeper/manifest.yml` (see [Workstation Setup][3]), then create a project `gemkeeper.yml`:
74
74
 
75
75
  ```yaml
76
76
  port: 9292
@@ -115,7 +115,7 @@ Gemkeeper looks for configuration files in these locations (in order):
115
115
  ### Configuration Options
116
116
 
117
117
  ```yaml
118
- # Port for the Geminabox server (default: 9292)
118
+ # Port for the Gemkeeper server (default: 9292)
119
119
  port: 9292
120
120
 
121
121
  # Where to clone gem repositories (default: ./cache/repos)
@@ -293,8 +293,8 @@ gemkeeper server stop
293
293
 
294
294
  1. **Clone/Pull**: Gemkeeper clones (or pulls) gem repositories to a local cache.
295
295
  2. **Build**: Builds `.gem` files from the source at the specified version/tag.
296
- 3. **Upload**: Uploads built gems to a local Geminabox server.
297
- 4. **Proxy**: Geminabox proxies public gems from RubyGems.org, so you only need one gem source.
296
+ 3. **Upload**: Uploads built gems to the local Gemkeeper server.
297
+ 4. **Proxy**: The server proxies public gems from RubyGems.org, so you only need one gem source.
298
298
 
299
299
  This lets you use a combination of public and private gems from a single gem source.
300
300
 
@@ -306,6 +306,6 @@ bundle exec rake test # Run tests
306
306
  bundle exec rubocop # Run linter
307
307
  ```
308
308
 
309
- [1]: https://github.com/geminabox/geminabox
310
- [2]: https://github.com/danhorst/homebrew-tap
311
- [3]: https://docs.github.com/en/authentication
309
+ [1]: https://github.com/danhorst/homebrew-tap
310
+ [2]: https://docs.github.com/en/authentication
311
+ [3]: #workstation-setup
@@ -1,7 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Gemkeeper
4
- # Configures Bundler mirror settings so private gem registries proxy through local Geminabox.
4
+ # Configures Bundler mirror settings so private gem registries proxy through the local Gemkeeper server.
5
5
  class BundlerMirrorConfigurator
6
6
  def initialize(candidates, port:, global:)
7
7
  @remotes = candidates.filter_map { |c| c[:remote] if c[:source_type] == :private_gem }.uniq
@@ -4,7 +4,7 @@ module Gemkeeper
4
4
  module CLI
5
5
  module Commands
6
6
  class List < Dry::CLI::Command
7
- desc "List gems cached in Geminabox"
7
+ desc "List gems cached locally"
8
8
 
9
9
  option :config, type: :string, desc: "Path to config file"
10
10
 
@@ -15,7 +15,7 @@ module Gemkeeper
15
15
  gem_files = Dir.glob(File.join(gems_path, "gems", "*.gem"))
16
16
 
17
17
  if gem_files.empty?
18
- puts "No gems cached in Geminabox"
18
+ puts "No gems cached locally"
19
19
  puts " Gems directory: #{gems_path}"
20
20
  return
21
21
  end
@@ -5,7 +5,7 @@ module Gemkeeper
5
5
  module Commands
6
6
  module Server
7
7
  class Start < Dry::CLI::Command
8
- desc "Start the Geminabox server"
8
+ desc "Start the Gemkeeper server"
9
9
 
10
10
  option :port, type: :integer, desc: "Port to run server on"
11
11
  option :config, type: :string, desc: "Path to config file"
@@ -18,15 +18,15 @@ module Gemkeeper
18
18
  config = override_port(config, port) if port
19
19
 
20
20
  manager = ServerManager.new(config)
21
- url = config.geminabox_url
21
+ url = config.server_url
22
22
 
23
23
  if options[:foreground]
24
- puts "Starting Geminabox server at #{url}"
24
+ puts "Starting Gemkeeper server at #{url}"
25
25
  puts "Press Ctrl+C to stop"
26
26
  manager.start_foreground
27
27
  else
28
28
  manager.start
29
- puts "Geminabox server started at #{url}"
29
+ puts "Gemkeeper server started at #{url}"
30
30
  puts "PID: #{File.read(config.pid_file).strip}"
31
31
  end
32
32
  rescue ServerAlreadyRunningError => error
@@ -5,7 +5,7 @@ module Gemkeeper
5
5
  module Commands
6
6
  module Server
7
7
  class Status < Dry::CLI::Command
8
- desc "Check Geminabox server status"
8
+ desc "Check Gemkeeper server status"
9
9
 
10
10
  option :config, type: :string, desc: "Path to config file"
11
11
 
@@ -15,11 +15,11 @@ module Gemkeeper
15
15
  status = manager.status
16
16
 
17
17
  if status[:running]
18
- puts "Geminabox server is running"
18
+ puts "Gemkeeper server is running"
19
19
  puts " PID: #{status[:pid]}" if status[:pid]
20
20
  puts " URL: #{status[:url]}"
21
21
  else
22
- puts "Geminabox server is not running"
22
+ puts "Gemkeeper server is not running"
23
23
  end
24
24
  end
25
25
  end
@@ -5,7 +5,7 @@ module Gemkeeper
5
5
  module Commands
6
6
  module Server
7
7
  class Stop < Dry::CLI::Command
8
- desc "Stop the Geminabox server"
8
+ desc "Stop the Gemkeeper server"
9
9
 
10
10
  option :config, type: :string, desc: "Path to config file"
11
11
 
@@ -14,9 +14,9 @@ module Gemkeeper
14
14
  manager = ServerManager.new(config)
15
15
  manager.stop
16
16
 
17
- puts "Geminabox server stopped"
17
+ puts "Gemkeeper server stopped"
18
18
  rescue ServerNotRunningError
19
- puts "Geminabox server is not running"
19
+ puts "Gemkeeper server is not running"
20
20
  rescue ServerError => error
21
21
  warn "Error stopping server: #{error.message}"
22
22
  exit 1
@@ -12,7 +12,7 @@ module Gemkeeper
12
12
  def call(gem_name: nil, **options)
13
13
  config = Configuration.load(options[:config])
14
14
  gems_to_sync = select_gems(config, gem_name)
15
- syncer = GemSyncer.new(config, GemUploader.new(config.geminabox_url), manifest: ManifestReader.load)
15
+ syncer = GemSyncer.new(config, GemUploader.new(config.server_url), manifest: ManifestReader.load)
16
16
  counts, failures = run_sync(gems_to_sync, syncer)
17
17
  report_results(counts, failures, gems_to_sync.size)
18
18
  end
@@ -0,0 +1,34 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "time"
4
+
5
+ module Gemkeeper
6
+ class CompactIndexServer
7
+ # The sidecar metadata for a cached upstream document: its ETag and when it
8
+ # was fetched. Knows whether it has aged past a TTL and how to serialize.
9
+ class CacheMeta
10
+ attr_reader :etag
11
+
12
+ def self.load(hash)
13
+ return nil unless hash
14
+
15
+ new(hash["etag"], hash["fetched_at"])
16
+ end
17
+
18
+ def initialize(etag, fetched_at)
19
+ @etag = etag
20
+ @fetched_at = fetched_at
21
+ end
22
+
23
+ def expired?(ttl)
24
+ return true unless @fetched_at
25
+
26
+ Time.now - Time.parse(@fetched_at.to_s) > ttl
27
+ rescue ArgumentError
28
+ true
29
+ end
30
+
31
+ def to_h = { "etag" => @etag, "fetched_at" => @fetched_at }
32
+ end
33
+ end
34
+ end
@@ -0,0 +1,64 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "digest"
4
+ require "fileutils"
5
+ require "yaml"
6
+
7
+ require_relative "cache_meta"
8
+
9
+ module Gemkeeper
10
+ class CompactIndexServer
11
+ # Handles on-disk cache I/O: atomic writes, sidecar metadata, TTL checks.
12
+ # All paths are resolved relative to a base directory.
13
+ class CacheStore
14
+ ENTRIES = {
15
+ versions: "versions",
16
+ versions_merged: "versions.merged",
17
+ versions_meta: "versions.meta",
18
+ names: "names",
19
+ names_merged: "names.merged",
20
+ names_meta: "names.meta"
21
+ }.freeze
22
+
23
+ def initialize(base_dir)
24
+ @base_dir = base_dir
25
+ FileUtils.mkdir_p([base_dir, File.join(base_dir, "info"), File.join(base_dir, "gems")])
26
+ end
27
+
28
+ def path(key_or_filename)
29
+ filename = ENTRIES.fetch(key_or_filename, key_or_filename)
30
+ File.join(@base_dir, filename.to_s)
31
+ end
32
+
33
+ def read_meta(filename)
34
+ full_path = File.join(@base_dir, "#{filename}.meta")
35
+ return nil unless File.exist?(full_path)
36
+
37
+ CacheMeta.load(YAML.safe_load_file(full_path))
38
+ rescue StandardError
39
+ nil
40
+ end
41
+
42
+ def write_meta(filename, etag:)
43
+ meta = CacheMeta.new(etag, Time.now.utc.iso8601)
44
+ atomic_write(File.join(@base_dir, "#{filename}.meta"), meta.to_h.to_yaml)
45
+ end
46
+
47
+ def atomic_write(full_path, content)
48
+ tmp = "#{full_path}.tmp.#{Process.pid}.#{Thread.current.object_id}"
49
+ File.binwrite(tmp, content)
50
+ File.rename(tmp, full_path)
51
+ rescue StandardError
52
+ File.unlink(tmp) if tmp && File.exist?(tmp)
53
+ raise
54
+ end
55
+
56
+ def etag_for(key)
57
+ full_path = path(key)
58
+ return nil unless File.exist?(full_path)
59
+
60
+ Digest::SHA256.hexdigest(File.binread(full_path))
61
+ end
62
+ end
63
+ end
64
+ end
@@ -0,0 +1,88 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "digest"
4
+ require "rubygems"
5
+
6
+ module Gemkeeper
7
+ class CompactIndexServer
8
+ # Per-gem caching of /info documents and .gem binaries from RubyGems.org.
9
+ # Serves from the local RubyGems cache, then the disk cache, then upstream.
10
+ class GemCache
11
+ INFO_TTL = 3600 # 60 minutes
12
+
13
+ def initialize(store, client)
14
+ @store = store
15
+ @client = client
16
+ end
17
+
18
+ # Returns {body:, etag:} or nil (not found).
19
+ # Raises UpstreamUnavailableError when unreachable with no cache.
20
+ def info(gemname)
21
+ path = @store.path("info/#{gemname}")
22
+ meta = @store.read_meta("info/#{gemname}")
23
+ return cached_entry(path, meta) if fresh?(path, meta)
24
+
25
+ fetch_info(gemname, path, meta)
26
+ rescue UpstreamUnavailableError
27
+ raise unless File.exist?(path)
28
+
29
+ cached_entry(path, meta)
30
+ end
31
+
32
+ # Returns a local path to the gem binary, or nil (not found).
33
+ # Raises UpstreamUnavailableError when unreachable with no cached copy.
34
+ def binary(filename)
35
+ system_gem_path(filename) ||
36
+ disk_cache_path(filename) ||
37
+ fetch_binary(filename, @store.path("gems/#{filename}"))
38
+ end
39
+
40
+ private
41
+
42
+ def system_gem_path(filename)
43
+ Gem.path.each do |gem_path|
44
+ candidate = File.join(gem_path, "cache", filename)
45
+ return candidate if File.exist?(candidate)
46
+ end
47
+ nil
48
+ end
49
+
50
+ def disk_cache_path(filename)
51
+ path = @store.path("gems/#{filename}")
52
+ File.exist?(path) ? path : nil
53
+ end
54
+
55
+ def fetch_info(gemname, path, meta)
56
+ response = @client.get("/info/#{gemname}", meta&.etag)
57
+ return store_info(gemname, path, response) unless response.not_modified?
58
+
59
+ @store.write_meta("info/#{gemname}", etag: meta.etag)
60
+ cached_entry(path, meta)
61
+ end
62
+
63
+ def store_info(gemname, path, response)
64
+ return nil unless response.success?
65
+
66
+ body = response.body
67
+ etag = response.etag || Digest::SHA256.hexdigest(body)
68
+ @store.atomic_write(path, body)
69
+ @store.write_meta("info/#{gemname}", etag: etag)
70
+ { body: body, etag: etag }
71
+ end
72
+
73
+ def fetch_binary(filename, cache_path)
74
+ response = @client.get("/gems/#{filename}", nil)
75
+ return nil unless response.success?
76
+
77
+ @store.atomic_write(cache_path, response.body) && cache_path
78
+ end
79
+
80
+ def fresh?(path, meta) = File.exist?(path) && meta && !meta.expired?(INFO_TTL)
81
+
82
+ def cached_entry(path, meta)
83
+ body = File.binread(path)
84
+ { body:, etag: meta&.etag || Digest::SHA256.hexdigest(body) }
85
+ end
86
+ end
87
+ end
88
+ end
@@ -0,0 +1,78 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "compact_index"
4
+ require "digest"
5
+ require "fileutils"
6
+ require "rubygems/package"
7
+
8
+ require_relative "spec_mapper"
9
+
10
+ module Gemkeeper
11
+ class CompactIndexServer
12
+ # Builds and maintains the in-memory index of privately-hosted gems.
13
+ # Scans gems_dir on construction and after each successful upload.
14
+ class GemIndex
15
+ def initialize(gems_dir)
16
+ @gems_dir = gems_dir
17
+ @gems = {}
18
+ FileUtils.mkdir_p(@gems_dir)
19
+ rebuild
20
+ end
21
+
22
+ def [](name) = @gems[name]
23
+ def keys = @gems.keys
24
+ def values = @gems.values
25
+
26
+ # True when the private store holds this exact name and bare-semver version.
27
+ def serves?(name, version)
28
+ gem = @gems[name]
29
+ gem ? gem.versions.any? { |gem_version| gem_version.number == version } : false
30
+ end
31
+
32
+ def gem_path(filename)
33
+ path = File.join(@gems_dir, filename)
34
+ File.exist?(path) ? path : nil
35
+ end
36
+
37
+ # Copies source_path into gems_dir, derives the filename from spec, and rebuilds.
38
+ # Raises Errno::EEXIST if the gem already exists. Returns the target filename.
39
+ def add(source_path, spec)
40
+ filename = SpecMapper.filename(spec)
41
+ target = File.join(@gems_dir, filename)
42
+ raise Errno::EEXIST, target if File.exist?(target)
43
+
44
+ copy_into_place(source_path, target)
45
+ rebuild
46
+ filename
47
+ end
48
+
49
+ def rebuild
50
+ gems = {}
51
+ Dir.glob(File.join(@gems_dir, "*.gem")).each { |gem_file| index_gem(gems, gem_file) }
52
+ gems.each_value { |gem| stamp_checksum(gem) }
53
+ @gems = gems
54
+ end
55
+
56
+ private
57
+
58
+ def index_gem(gems, gem_file)
59
+ spec = Gem::Package.new(gem_file).spec
60
+ name = spec.name
61
+ (gems[name] ||= CompactIndex::Gem.new(name, [])).versions << SpecMapper.gem_version(spec, gem_file)
62
+ rescue StandardError => error
63
+ warn "gemkeeper: skipping #{File.basename(gem_file)}: #{error.message}"
64
+ end
65
+
66
+ def stamp_checksum(gem)
67
+ versions = gem.versions
68
+ versions.last.info_checksum = Digest::MD5.hexdigest(CompactIndex.info(versions))
69
+ end
70
+
71
+ def copy_into_place(source_path, target)
72
+ tmp = "#{target}.tmp.#{Process.pid}.#{Thread.current.object_id}"
73
+ FileUtils.cp(source_path, tmp)
74
+ File.rename(tmp, target)
75
+ end
76
+ end
77
+ end
78
+ end
@@ -0,0 +1,81 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "compact_index"
4
+ require "digest"
5
+ require "time"
6
+
7
+ module Gemkeeper
8
+ class CompactIndexServer
9
+ # Generates the merged /versions and /names index files, combining upstream
10
+ # RubyGems.org data (refreshed on a TTL) with the locally-hosted private gems.
11
+ class IndexMerger
12
+ VERSIONS_TTL = 1800 # 30 minutes
13
+ NAMES_TTL = 3600 # 60 minutes
14
+
15
+ def initialize(store, client)
16
+ @store = store
17
+ @client = client
18
+ @versions_etag = store.etag_for(:versions_merged)
19
+ @names_etag = store.etag_for(:names_merged)
20
+ end
21
+
22
+ # Returns {path:, etag:} for the merged /versions file.
23
+ def versions(private_gems)
24
+ refresh(:versions, "/versions", VERSIONS_TTL)
25
+ regenerate_versions(private_gems)
26
+ { path: @store.path(:versions_merged), etag: @versions_etag }
27
+ end
28
+
29
+ # Returns {path:, etag:} for the merged /names file.
30
+ def names(private_names)
31
+ refresh(:names, "/names", NAMES_TTL)
32
+ regenerate_names(private_names)
33
+ { path: @store.path(:names_merged), etag: @names_etag }
34
+ end
35
+
36
+ private
37
+
38
+ def refresh(key, upstream_path, ttl)
39
+ meta = @store.read_meta(key.to_s)
40
+ return if meta && !meta.expired?(ttl) && File.exist?(@store.path(key))
41
+
42
+ apply(key, meta, @client.get(upstream_path, meta&.etag))
43
+ rescue UpstreamUnavailableError
44
+ nil # regeneration proceeds with whatever is cached
45
+ end
46
+
47
+ def apply(key, meta, response)
48
+ if response.not_modified?
49
+ etag = meta.etag
50
+ else
51
+ @store.atomic_write(@store.path(key), response.body)
52
+ etag = response.etag
53
+ end
54
+ @store.write_meta(key.to_s, etag: etag)
55
+ end
56
+
57
+ def regenerate_versions(private_gems)
58
+ versions_path = @store.path(:versions)
59
+ unless File.exist?(versions_path)
60
+ @store.atomic_write(versions_path, "created_at: #{Time.now.utc.iso8601}\n---\n")
61
+ end
62
+ merged = CompactIndex.versions(CompactIndex::VersionsFile.new(versions_path), private_gems)
63
+ @store.atomic_write(@store.path(:versions_merged), merged)
64
+ @versions_etag = Digest::SHA256.hexdigest(merged)
65
+ end
66
+
67
+ def regenerate_names(private_names)
68
+ merged = CompactIndex.names((upstream_names + private_names).uniq.sort)
69
+ @store.atomic_write(@store.path(:names_merged), merged)
70
+ @names_etag = Digest::SHA256.hexdigest(merged)
71
+ end
72
+
73
+ def upstream_names
74
+ names_path = @store.path(:names)
75
+ return [] unless File.exist?(names_path)
76
+
77
+ File.read(names_path).lines.map(&:strip).reject { |line| line.empty? || line == "---" }
78
+ end
79
+ end
80
+ end
81
+ end
@@ -0,0 +1,12 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Gemkeeper
4
+ class CompactIndexServer
5
+ # Outcome of a compact-index request: a not-modified marker (304) or a
6
+ # body+etag (200), or a non-success status. Callers ask, never inspect.
7
+ Response = Struct.new(:status, :body, :etag) do
8
+ def not_modified? = status == 304
9
+ def success? = status == 200
10
+ end
11
+ end
12
+ end
@@ -0,0 +1,63 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "digest"
4
+
5
+ module Gemkeeper
6
+ class CompactIndexServer
7
+ # Builds Rack responses for compact index payloads: conditional GET (ETag),
8
+ # byte-range requests, and the digest/accept-ranges headers Bundler expects.
9
+ class ResponseBuilder
10
+ def self.file(path)
11
+ [200,
12
+ { "content-type" => "application/octet-stream",
13
+ "content-length" => File.size(path).to_s },
14
+ File.open(path, "rb")]
15
+ end
16
+
17
+ def initialize(req)
18
+ @req = req
19
+ end
20
+
21
+ # 200 / 206 / 304 for an in-memory index body.
22
+ def index(body, etag)
23
+ quoted = %("#{etag}")
24
+ return [304, { "etag" => quoted }, []] if @req.env["HTTP_IF_NONE_MATCH"] == quoted
25
+
26
+ range(body, etag) || [200, index_headers(body, etag), [body]]
27
+ end
28
+
29
+ private
30
+
31
+ def range(body, etag)
32
+ header = @req.env["HTTP_RANGE"]
33
+ return nil unless header
34
+
35
+ size = body.bytesize
36
+ match = header.match(/\Abytes=(\d+)-(\d*)\z/)
37
+ return unsatisfiable(size) unless match
38
+
39
+ start_byte = match[1].to_i
40
+ return unsatisfiable(size) if start_byte >= size
41
+
42
+ partial(body, etag, start_byte, match[2], size)
43
+ end
44
+
45
+ def partial(body, etag, start_byte, raw_end, size)
46
+ end_byte = raw_end.empty? ? size - 1 : [raw_end.to_i, size - 1].min
47
+ slice = body.byteslice(start_byte, end_byte - start_byte + 1)
48
+ [206,
49
+ index_headers(body, etag).merge("content-range" => "bytes #{start_byte}-#{end_byte}/#{size}"),
50
+ [slice]]
51
+ end
52
+
53
+ def unsatisfiable(size) = [416, { "content-range" => "bytes */#{size}" }, []]
54
+
55
+ def index_headers(body, etag)
56
+ { "content-type" => "text/plain",
57
+ "etag" => %("#{etag}"),
58
+ "repr-digest" => "sha-256=:#{Digest::SHA256.base64digest(body)}:",
59
+ "accept-ranges" => "bytes" }
60
+ end
61
+ end
62
+ end
63
+ end