gemkeeper 0.7.2 → 0.8.0

data/lib/gemkeeper/compact_index_server/rubygems_client.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "zlib"
+
+require_relative "response"
+
+module Gemkeeper
+  class CompactIndexServer
+    # HTTP client for RubyGems.org compact index endpoints.
+    # Speaks conditional GET (ETag) and gzip; raises UpstreamUnavailableError
+    # on any network failure so callers can fall back to cache.
+    class RubygemsClient
+      HOST         = "rubygems.org"
+      OPEN_TIMEOUT = 5
+      READ_TIMEOUT = 10
+
+      # Returns :not_modified, or {status:, body:, etag:}.
+      def get(path, if_none_match = nil)
+        uri = URI("https://#{HOST}#{path}")
+        req = build_request(uri, if_none_match)
+
+        Net::HTTP.start(uri.host, uri.port, use_ssl: true,
+                                            open_timeout: OPEN_TIMEOUT,
+                                            read_timeout: READ_TIMEOUT) do |http|
+          interpret(http.request(req))
+        end
+      rescue Errno::ECONNREFUSED, Errno::ECONNRESET, Errno::ETIMEDOUT,
+             Net::OpenTimeout, Net::ReadTimeout, SocketError, OpenSSL::SSL::SSLError => error
+        raise UpstreamUnavailableError, error.message
+      end
+
+      private
+
+      def build_request(uri, if_none_match)
+        req = Net::HTTP::Get.new(uri)
+        req["If-None-Match"]   = if_none_match if if_none_match
+        req["Accept-Encoding"] = "gzip"
+        req
+      end
+
+      def interpret(res)
+        case res.code.to_i
+        when 304 then Response.new(304, nil, nil)
+        when 200 then Response.new(200, decompress(res.body.to_s, res["Content-Encoding"]), res["ETag"])
+        else          Response.new(res.code.to_i, res.body.to_s, nil)
+        end
+      end
+
+      def decompress(body, encoding)
+        return body unless encoding&.include?("gzip")
+
+        Zlib::GzipReader.new(StringIO.new(body)).read
+      rescue Zlib::Error
+        body
+      end
+    end
+  end
+end

data/lib/gemkeeper/compact_index_server/spec_mapper.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require "compact_index"
+require "digest"
+
+module Gemkeeper
+  class CompactIndexServer
+    # Maps a Gem::Specification to the compact-index representations the index is
+    # built from: the on-disk gem filename and a CompactIndex::GemVersion entry.
+    module SpecMapper
+      module_function
+
+      def filename(spec)
+        platform = spec.platform.to_s
+        suffix   = platform.empty? || platform == "ruby" ? "" : "-#{platform}"
+        "#{spec.name}-#{spec.version}#{suffix}.gem"
+      end
+
+      def gem_version(spec, gem_file)
+        CompactIndex::GemVersion.new(
+          spec.version.to_s,
+          spec.platform.to_s,
+          Digest::SHA256.file(gem_file).hexdigest,
+          nil,
+          dependencies(spec),
+          spec.required_ruby_version&.to_s,
+          spec.required_rubygems_version&.to_s
+        )
+      end
+
+      def dependencies(spec)
+        (spec.runtime_dependencies || []).map do |dep|
+          CompactIndex::Dependency.new(dep.name, dep.requirement.to_s)
+        end
+      end
+    end
+  end
+end

data/lib/gemkeeper/compact_index_server/upload_handler.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require "rubygems/package"
+require "zlib"
+
+module Gemkeeper
+  class CompactIndexServer
+    # Handles POST /upload: reads the multipart gem file, parses its spec,
+    # and adds it to the index. Maps failures to compact-index HTTP responses.
+    class UploadHandler
+      def initialize(index)
+        @index = index
+      end
+
+      def call(req)
+        upload = req.params["file"]
+        return text(400, "Missing file parameter") unless upload
+
+        text(201, "Uploaded #{add(upload[:tempfile].path)}")
+      rescue Errno::EEXIST
+        text(409, "Gem already exists")
+      rescue Gem::Exception, Gem::Package::FormatError, Zlib::Error, TypeError, ArgumentError => error
+        text(422, "Invalid gem: #{error.message}")
+      end
+
+      private
+
+      def add(tempfile_path)
+        spec = Gem::Package.new(tempfile_path).spec
+        @index.add(tempfile_path, spec)
+      end
+
+      def text(status, message) = [status, { "content-type" => "text/plain" }, [message]]
+    end
+  end
+end

data/lib/gemkeeper/compact_index_server/upstream_cache.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require_relative "cache_store"
+require_relative "gem_cache"
+require_relative "index_merger"
+require_relative "rubygems_client"
+
+module Gemkeeper
+  class CompactIndexServer
+    # Composition root for upstream RubyGems.org caching. Wires a shared on-disk
+    # store and HTTP client into the merged-index and per-gem cache collaborators.
+    class UpstreamCache
+      def initialize(cache_dir)
+        store   = CacheStore.new(File.join(cache_dir, "rubygems_cache"))
+        client  = RubygemsClient.new
+        @merger = IndexMerger.new(store, client)
+        @gems   = GemCache.new(store, client)
+      end
+
+      def merged_versions(private_gems) = @merger.versions(private_gems)
+      def merged_names(private_names)   = @merger.names(private_names)
+      def info(gemname)                 = @gems.info(gemname)
+      def gem_binary(filename)          = @gems.binary(filename)
+    end
+  end
+end

data/lib/gemkeeper/compact_index_server.rb

@@ -0,0 +1,131 @@
+# frozen_string_literal: true
+
+require "compact_index"
+require "digest"
+require "rack"
+
+require_relative "compact_index_server/cache_store"
+require_relative "compact_index_server/gem_index"
+require_relative "compact_index_server/response_builder"
+require_relative "compact_index_server/upload_handler"
+require_relative "compact_index_server/upstream_cache"
+
+module Gemkeeper
+  # Rack application implementing the Bundler compact index protocol.
+  # Delegates private gem state to GemIndex and upstream caching to UpstreamCache.
+  class CompactIndexServer
+    VALID_NAME = /\A[a-zA-Z0-9._-]+\z/
+    RESOURCE_ROUTES = {
+      info: %r{\A/info/([^/]+)\z},
+      gem: %r{\A/gems/([^/]+\.gem)\z}
+    }.freeze
+    # Private-store presence check for `gemkeeper sync`; reads the index only,
+    # never the upstream proxy (unlike /info), so it can't be fooled by a public gem.
+    PRESENCE_ROUTE = %r{\A/gemkeeper/has/([^/]+)/([^/]+)\z}
+
+    def initialize(gems_path:, cache_dir:)
+      @index  = GemIndex.new(File.join(gems_path, "gems"))
+      @cache  = UpstreamCache.new(cache_dir)
+      @upload = UploadHandler.new(@index)
+    end
+
+    def call(env)
+      req  = Rack::Request.new(env)
+      path = req.path_info
+      case [req.request_method, path]
+      in ["GET", "/"]         then health
+      in ["GET", "/names"]    then serve_names(req)
+      in ["GET", "/versions"] then serve_versions(req)
+      in ["POST", "/upload"]  then @upload.call(req)
+      in ["GET", String => p] if p.start_with?("/gemkeeper/") then serve_presence(p)
+      in ["GET", _]           then serve_resource(path, req)
+      else not_found
+      end
+    end
+
+    private
+
+    # ── Routing ──────────────────────────────────────────────────────────────
+
+    def serve_presence(path)
+      match = path.match(PRESENCE_ROUTE)
+      return not_found unless match
+
+      name, version = match.captures
+      return invalid_name unless VALID_NAME.match?(name) && VALID_NAME.match?(version)
+
+      @index.serves?(name, version) ? present : not_found
+    end
+
+    def serve_resource(path, req)
+      type, name = match_resource(path)
+      return not_found unless type
+      return invalid_name unless VALID_NAME.match?(name)
+
+      type == :info ? serve_info(name, req) : serve_gem_file(name)
+    end
+
+    def match_resource(path)
+      RESOURCE_ROUTES.each do |type, pattern|
+        match = path.match(pattern)
+        return [type, match[1]] if match
+      end
+      nil
+    end
+
+    # ── Serving ───────────────────────────────────────────────────────────────
+
+    def serve_names(req)
+      result = @cache.merged_names(@index.keys)
+      serve_index_file(result[:path], result[:etag], req)
+    end
+
+    def serve_versions(req)
+      result = @cache.merged_versions(@index.values)
+      serve_index_file(result[:path], result[:etag], req)
+    end
+
+    def serve_info(gemname, req)
+      gem = @index[gemname]
+      gem ? serve_private_info(gem, req) : serve_upstream_info(gemname, req)
+    rescue UpstreamUnavailableError
+      upstream_unavailable
+    end
+
+    def serve_private_info(gem, req)
+      body = CompactIndex.info(gem.versions)
+      ResponseBuilder.new(req).index(body, Digest::SHA256.hexdigest(body))
+    end
+
+    def serve_upstream_info(gemname, req)
+      result = @cache.info(gemname)
+      result ? ResponseBuilder.new(req).index(result[:body], result[:etag]) : not_found
+    end
+
+    def serve_gem_file(filename)
+      path = @index.gem_path(filename) || @cache.gem_binary(filename)
+      path ? ResponseBuilder.file(path) : not_found
+    rescue UpstreamUnavailableError
+      upstream_unavailable
+    end
+
+    # ── HTTP response helpers ─────────────────────────────────────────────────
+
+    def serve_index_file(file_path, etag, req)
+      return upstream_unavailable unless file_path && File.exist?(file_path)
+
+      ResponseBuilder.new(req).index(File.binread(file_path), etag)
+    end
+
+    def health       = [200, { "content-type" => "text/plain" }, ["OK"]]
+    def present      = [200, { "content-type" => "text/plain" }, ["present"]]
+    def not_found    = [404, { "content-type" => "text/plain" }, ["Not Found"]]
+    def invalid_name = [400, { "content-type" => "text/plain" }, ["Invalid name"]]
+
+    def upstream_unavailable
+      [503, { "content-type" => "text/plain" },
+       ["Upstream unavailable and no local cache. " \
+        "Connect to the internet and run bundle install to warm the cache."]]
+    end
+  end
+end

data/lib/gemkeeper/configuration.rb

@@ -62,7 +62,7 @@ module Gemkeeper
       apply_config
     end
 
-    def geminabox_url
+    def server_url
       "http://localhost:#{port}"
     end
 

data/lib/gemkeeper/gem_syncer.rb

@@ -1,73 +1,45 @@
 # frozen_string_literal: true
 
+require "rubygems/package"
+
 module Gemkeeper
-  # Syncs a single gem: resolves version, checks cache, clones/pulls, builds, uploads.
+  # Syncs a single gem: checks the server, reuses a built artifact, or builds
+  # from source, then uploads. Repo acquisition is delegated to RepoFetcher.
   class GemSyncer
-    AUTH_ERROR_PATTERNS = [
-      /authentication failed/i,
-      /could not read from remote repository/i,
-      /permission denied \(publickey\)/i,
-      /repository not found/i,
-      /fatal: credential/i
-    ].freeze
-
     def initialize(config, uploader, manifest:)
-      @config = config
+      @config   = config
       @uploader = uploader
-      @manifest = manifest
+      @fetcher  = RepoFetcher.new(manifest, config.repos_path)
     end
 
     def sync(gem_def)
-      repo_url = resolve_repo(gem_def)
-      version = resolve_version(gem_def)
-      name = gem_def.name
-      gems_path = @config.gems_path
-
-      return :skipped if !gem_def.latest? && cached?(name, version, gems_path)
-
-      puts "Syncing #{name} @ #{version}..."
-      local_path = File.join(@config.repos_path, name)
-      repo = fetch_repo(repo_url, local_path)
-
-      if gem_def.latest?
-        version = latest_version!(repo, name, gems_path, repo_url)
-        return :skipped unless version
-      else
-        Output.step("Checking out #{version}...")
-        repo.checkout_version(version)
-      end
-
-      build_and_upload(local_path, gems_path)
-      :synced
+      gem_def.latest? ? sync_latest(gem_def) : sync_pinned(gem_def)
     end
 
     private
 
-    # Explicit repo: in gemkeeper.yml wins, but warns on divergence from the manifest.
-    # Otherwise the repo is resolved from the manifest by gem name.
-    def resolve_repo(gem_def)
-      manifest_repo = @manifest.repo_for(gem_def.name)
-      return manifest_repo || missing_repo!(gem_def.name) unless gem_def.repo
+    # Pinned / from_lockfile: version is known without the repo, so check the
+    # server first, then reuse a local artifact, and only build as a last resort.
+    def sync_pinned(gem_def)
+      name    = gem_def.name
+      version = resolve_version(gem_def)
+      return skip(name, version) if @uploader.serves?(name, version)
 
-      warn_if_divergent(gem_def.name, gem_def.repo, manifest_repo)
-      gem_def.repo
-    end
+      artifact = File.join(@config.gems_path, "#{name}-#{version}.gem")
+      return reupload(artifact, name, version) if reusable_artifact?(artifact, name, version)
 
-    def missing_repo!(name)
-      unless File.exist?(@manifest.path)
-        raise InvalidConfigError,
-              "No manifest found at #{@manifest.path} — run 'gemkeeper manifest generate' to create one"
-      end
-
-      raise InvalidConfigError,
-            "No repo configured for #{name.inspect} — add it to the manifest with 'gemkeeper manifest generate'"
+      build_and_upload(gem_def, version)
     end
 
-    def warn_if_divergent(name, config_repo, manifest_repo)
-      return unless manifest_repo && manifest_repo != config_repo
+    # latest: the version is only known after checkout, so the repo is always
+    # fetched; the server check then decides whether the resolved version uploads.
+    def sync_latest(gem_def)
+      repo, local_path = @fetcher.fetch(gem_def)
+      version = repo.current_version or
+        raise BuildError, "Could not read version from gemspec for #{gem_def.name}"
+      return skip(gem_def.name, version) if @uploader.serves?(gem_def.name, version)
 
-      warn "Warning: repo for #{name} in gemkeeper.yml (#{config_repo}) " \
-           "differs from manifest (#{manifest_repo}) — using gemkeeper.yml"
+      build_gem(local_path)
     end
 
     def resolve_version(gem_def)
@@ -84,49 +56,46 @@ module Gemkeeper
       versions[gem_def.name] or raise GitError, "#{gem_def.name} not found in #{lockfile_path}"
     end
 
-    def cached?(name, version, gems_path)
-      bare = version.delete_prefix("v")
-      gem_file = File.join(gems_path, "#{name}-#{bare}.gem")
-      return false unless File.exist?(gem_file)
+    def build_and_upload(gem_def, version)
+      repo, local_path = @fetcher.fetch(gem_def)
+      Output.step("Checking out #{version}...")
+      repo.checkout_version(version)
+      build_gem(local_path)
+    end
 
-      Output.skip("Skipping #{name} @ #{bare} (already cached)")
-      true
+    def build_gem(local_path)
+      Output.step("Building gem...")
+      gem_path = GemBuilder.new(local_path, @config.gems_path).build
+      upload(gem_path)
+      :synced
     end
 
-    def fetch_repo(repo_url, local_path)
-      repo = GitRepository.new(repo_url, local_path)
-      Output.step("Fetching from #{repo_url}...")
-      repo.clone_or_pull
-      repo
-    rescue GitError => git_error
-      raise auth_error?(git_error) ? auth_failure_error(repo_url, git_error) : git_error
+    # Reuse a previously built artifact only if its embedded spec matches the
+    # requested gem — never upload the wrong file because a name collided.
+    def reusable_artifact?(path, name, version)
+      return false unless File.exist?(path)
+
+      spec = Gem::Package.new(path).spec
+      spec.name == name && spec.version.to_s == version
+    rescue StandardError
+      false
     end
 
-    def latest_version!(repo, name, gems_path, repo_url)
-      version = repo.current_version or
-        raise BuildError, "Could not read version from gemspec in #{repo_url}"
-      cached?(name, version, gems_path) ? nil : version
+    def reupload(path, name, version)
+      Output.step("Uploading cached #{name} @ #{version} (no rebuild)...")
+      upload(path)
+      :synced
     end
 
-    def build_and_upload(local_path, gems_path)
-      Output.step("Building gem...")
-      gem_path = GemBuilder.new(local_path, gems_path).build
-      Output.step("Uploading to Geminabox...")
-      result = @uploader.upload(gem_path)
+    def upload(path)
+      result = @uploader.upload(path)
       Output.step(result[:message])
       Output.success("  Done!")
     end
 
-    def auth_error?(error)
-      AUTH_ERROR_PATTERNS.any? { |pat| error.message.match?(pat) }
-    end
-
-    def auth_failure_error(repo_url, original_error)
-      GitError.new(
-        "Git authentication failed for #{repo_url}.\n" \
-        "#{original_error.message}\n" \
-        "Configure GitHub credentials: https://docs.github.com/en/authentication"
-      )
+    def skip(name, version)
+      Output.skip("Skipping #{name} @ #{version} (already on server)")
+      :skipped
     end
   end
 end

data/lib/gemkeeper/gem_uploader.rb

@@ -4,12 +4,12 @@ require "faraday"
 require "faraday/multipart"
 
 module Gemkeeper
-  # Encapsulates Geminabox's HTTP API so callers never construct multipart requests directly.
+  # Encapsulates the Gemkeeper server's upload HTTP API so callers never construct multipart requests directly.
   class GemUploader
-    attr_reader :geminabox_url
+    attr_reader :server_url
 
-    def initialize(geminabox_url)
-      @geminabox_url = geminabox_url
+    def initialize(server_url)
+      @server_url = server_url
     end
 
     def upload(gem_path)
@@ -27,9 +27,20 @@ module Gemkeeper
 
       handle_response(response, gem_path)
     rescue Faraday::ConnectionFailed, Faraday::TimeoutError
-      raise ServerNotReachableError,
-            "Geminabox server is not reachable at #{@geminabox_url} — " \
-            "run 'gemkeeper server start' or check 'gemkeeper server status'"
+      not_reachable!
+    end
+
+    # True when the running server's private store already serves name@version.
+    # Hits the private-store endpoint, never /info, so a public gem can't fool it.
+    def serves?(name, version)
+      status = connection.get("/gemkeeper/has/#{name}/#{version}").status
+      case status
+      when 200 then true
+      when 404 then false
+      else raise ServerError, "Unexpected status #{status} checking #{name} #{version} on #{@server_url}"
+      end
+    rescue Faraday::ConnectionFailed, Faraday::TimeoutError
+      not_reachable!
     end
 
     def reachable?
@@ -40,22 +51,19 @@ module Gemkeeper
     end
 
     def list_gems
-      response = connection.get("/api/v1/gems.json")
-
-      body = response.body
-      raise UploadError, "Failed to list gems: #{response.status} #{body}" unless response.success?
-
-      JSON.parse(body)
-    rescue JSON::ParserError => parse_error
-      raise UploadError, "Invalid JSON response: #{parse_error.message}"
-    rescue Faraday::Error => connection_error
-      raise UploadError, "Connection error: #{connection_error.message}"
+      raise NotImplementedError, "list_gems is not supported by CompactIndexServer; use gemkeeper list instead"
     end
 
     private
 
+    def not_reachable!
+      raise ServerNotReachableError,
+            "Gemkeeper server is not reachable at #{@server_url} — " \
+            "run 'gemkeeper server start' or check 'gemkeeper server status'"
+    end
+
     def connection
-      @connection ||= Faraday.new(url: @geminabox_url) do |f|
+      @connection ||= Faraday.new(url: @server_url) do |f|
         f.request :multipart
         f.request :url_encoded
         f.adapter Faraday::Adapter::NetHttp

data/lib/gemkeeper/rackup_process.rb

@@ -34,24 +34,29 @@ module Gemkeeper
 
     def config_ru_content
       gems_path = @config.gems_path
+      cache_dir = @config.cache_dir
       <<~RUBY
         # frozen_string_literal: true
         # Auto-generated by Gemkeeper
 
-        require "geminabox"
+        require "gemkeeper/compact_index_server"
 
-        Geminabox.data = #{gems_path.inspect}
-        Geminabox.rubygems_proxy = true
-
-        run Geminabox::Server
+        run Gemkeeper::CompactIndexServer.new(
+          gems_path: #{gems_path.inspect},
+          cache_dir: #{cache_dir.inspect}
+        )
       RUBY
     end
 
     def build_rackup_cmd(*extra)
-      [rackup_bin, @config.config_ru_path, "--host", "127.0.0.1",
+      [rackup_bin, "-I", lib_dir, @config.config_ru_path, "--host", "127.0.0.1",
        "-p", @config.port.to_s, "-s", "puma", *extra]
     end
 
+    # The gem's own lib/, so the spawned rackup process can require gemkeeper
+    # even when running from source (it does not inherit the exe's $LOAD_PATH).
+    def lib_dir = File.expand_path("..", __dir__)
+
     def rackup_bin
       gem_home = ENV.fetch("GEM_HOME", nil)
       return "rackup" unless gem_home
@@ -66,7 +71,7 @@ module Gemkeeper
     end
 
     def wait_for_server
-      ServerReadinessProbe.new(@config.geminabox_url).wait
+      ServerReadinessProbe.new(@config.server_url).wait
     end
   end
 end

data/lib/gemkeeper/repo_fetcher.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+module Gemkeeper
+  # Resolves a gem's source repo URL (manifest lookup, with an optional
+  # gemkeeper.yml override) and clones or pulls it, mapping git authentication
+  # failures to actionable guidance.
+  class RepoFetcher
+    AUTH_ERROR_PATTERNS = [
+      /authentication failed/i,
+      /could not read from remote repository/i,
+      /permission denied \(publickey\)/i,
+      /repository not found/i,
+      /fatal: credential/i
+    ].freeze
+
+    def initialize(manifest, repos_path)
+      @manifest   = manifest
+      @repos_path = repos_path
+    end
+
+    # Resolves the repo URL, clones/pulls it, and returns [GitRepository, local_path].
+    def fetch(gem_def)
+      local_path = File.join(@repos_path, gem_def.name)
+      [clone_or_pull(resolve(gem_def), local_path), local_path]
+    end
+
+    # Explicit repo: in gemkeeper.yml wins, but warns on divergence from the
+    # manifest. Otherwise the repo is resolved from the manifest by gem name.
+    def resolve(gem_def)
+      name = gem_def.name
+      override = gem_def.repo
+      manifest_repo = @manifest.repo_for(name)
+      return manifest_repo || missing_repo!(name) unless override
+
+      warn_if_divergent(name, override, manifest_repo)
+      override
+    end
+
+    private
+
+    def missing_repo!(name)
+      path = @manifest.path
+      unless File.exist?(path)
+        raise InvalidConfigError,
+              "No manifest found at #{path} — run 'gemkeeper manifest generate' to create one"
+      end
+
+      raise InvalidConfigError,
+            "No repo configured for #{name.inspect} — add it to the manifest with 'gemkeeper manifest generate'"
+    end
+
+    def warn_if_divergent(name, override, manifest_repo)
+      return unless manifest_repo && manifest_repo != override
+
+      warn "Warning: repo for #{name} in gemkeeper.yml (#{override}) " \
+           "differs from manifest (#{manifest_repo}) — using gemkeeper.yml"
+    end
+
+    def clone_or_pull(repo_url, local_path)
+      repo = GitRepository.new(repo_url, local_path)
+      Output.step("Fetching from #{repo_url}...")
+      repo.clone_or_pull
+      repo
+    rescue GitError => git_error
+      raise auth_error?(git_error) ? auth_failure_error(repo_url, git_error) : git_error
+    end
+
+    def auth_error?(error)
+      AUTH_ERROR_PATTERNS.any? { |pat| error.message.match?(pat) }
+    end
+
+    def auth_failure_error(repo_url, original_error)
+      GitError.new(
+        "Git authentication failed for #{repo_url}.\n" \
+        "#{original_error.message}\n" \
+        "Configure GitHub credentials: https://docs.github.com/en/authentication"
+      )
+    end
+  end
+end