gem_server_conformance 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: cc996f39ef2b36f10b6888858981fff19bb5a06e547981643d3a8486a24b2aeb
+  data.tar.gz: da7ab819c31d6e1d1c6915f91cc2b31148a7cd19747ad45d9fa0d4905bfef38f
+SHA512:
+  metadata.gz: 1ce259e3b57721291668320ecf6f547809aceb5f41d7f5dce41624724d38fad03011c839e749cc4b369630a435683d65e91d0d3b07e277a7a9c2dc4806930ea4
+  data.tar.gz: 14ba296d864bebcdb5b4d0d5f857def2d33d3285b76f74816e2736afd828a1117660829e87bd5a8fc11917c58e4223d5062b5667c18b8592da94b786bc1f3d24

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.rubocop.yml

@@ -0,0 +1,65 @@
+AllCops:
+  TargetRubyVersion: 3.0
+  NewCops: enable
+
+require:
+  - rubocop-performance
+  - rubocop-rake
+  - rubocop-rspec
+
+Metrics:
+  Enabled: false
+
+Style/MultilineBlockChain:
+  Enabled: false
+
+Style/StringLiterals:
+  EnforcedStyle: double_quotes
+
+Style/StringLiteralsInInterpolation:
+  EnforcedStyle: double_quotes
+
+Style/SpecialGlobalVars:
+  Enabled: false
+
+Style/Documentation:
+  Enabled: false
+
+Style/StringConcatenation:
+  Enabled: false
+
+Naming/VariableNumber:
+  EnforcedStyle: snake_case
+  AllowedIdentifiers:
+    - sha256
+
+RSpec/ImplicitSubject:
+  Enabled: false
+
+RSpec/RepeatedExample:
+  Enabled: false
+
+Security/MarshalLoad:
+  Enabled: false
+
+RSpec/ExampleLength:
+  Enabled: false
+
+RSpec/MultipleExpectations:
+  Enabled: false
+
+RSpec/InstanceVariable:
+  Enabled: false
+
+RSpec/BeforeAfterAll:
+  Enabled: false
+
+RSpec/ExpectInHook:
+  Enabled: false
+
+RSpec/ContextWording:
+  Prefixes:
+    - after
+
+RSpec/NestedGroups:
+  Enabled: false

data/.ruby-version

@@ -0,0 +1 @@
+3.3.3

data/CHANGELOG.md

@@ -0,0 +1,5 @@
+## [Unreleased]
+
+## [0.1.0] - 2024-07-09
+
+- Initial release

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2024 Samuel Giddins
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,27 @@
+# GemServerConformance
+
+A conformance test suite for RubyGems servers.
+
+## Usage
+
+Run `gem_server_conformance` to run the test suite against a server. The endpoint to test can be specified with the `UPSTREAM` environment variable. Make sure to set the `GEM_HOST_API_KEY` environment variable to an API key with push/yank permissions if authentication is required.
+
+In addition to the standard interface,
+you will also need to define the following endpoints:
+
+- `POST /set_time` - Set the server's time to the value of the body's iso8601 value.
+- `POST /rebuild_versions_list` - Rebuild the base /versions file.
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/rubygems/gem_server_conformance.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/exe/gem_server_conformance

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "rspec"
+
+root = File.expand_path("..", __dir__)
+Dir.chdir(root)
+exit RSpec::Core::Runner.run(ARGV + [
+  File.join(root, "spec/")
+])

data/exe/gem_server_example

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require_relative "../lib/gem_server_conformance/server"
+require "rackup"
+
+GemServerConformance::Server::Application.run!

data/lib/gem_server_conformance/server.rb

@@ -0,0 +1,324 @@
+# frozen_string_literal: true
+
+require "compact_index"
+require "rubygems/package"
+require "sinatra/base"
+
+module GemServerConformance
+  class Server
+    class Application < Sinatra::Base
+      Sinatra::ShowExceptions.class_eval do
+        undef_method :prefers_plain_text?
+        def prefers_plain_text?(_)
+          true
+        end
+      end
+      def initialize(server = Server.new)
+        @server = server
+        super()
+      end
+
+      set :lock, true
+      set :raise_errors, true
+
+      post "/api/v1/gems" do
+        @server.push(request.body.read)
+      end
+
+      delete "/api/v1/gems/yank" do
+        @server.yank(params["gem_name"], params["version"], params["platform"])
+      end
+
+      after(%r{/(?:names|versions|info/[^/]+)}) do
+        headers "Content-Type" => "text/plain; charset=utf-8"
+        break unless response.status == 200
+
+        headers "Accept-Ranges" => "bytes"
+        etag Digest::MD5.hexdigest(response.body.join)
+        sha256 = Digest::SHA256.base64digest(response.body.join)
+        headers "Digest" => "sha-256=#{sha256}", "Repr-Digest" => "sha-256=:#{sha256}:"
+      end
+
+      get "/versions" do
+        @server.versions
+      end
+
+      get "/info/:name" do
+        @server.info(params[:name])
+      end
+
+      get "/names" do
+        @server.names
+      end
+
+      get "/quick/Marshal.4.8/:id.gemspec.rz" do
+        @server.quick_spec(params[:id])
+      end
+
+      get "/specs.4.8.gz" do
+        @server.specs_gz
+      end
+
+      get "/prerelease_specs.4.8.gz" do
+        @server.specs_gz("prerelease")
+      end
+
+      get "/latest_specs.4.8.gz" do
+        @server.specs_gz("latest")
+      end
+
+      get "/gems/:id.gem" do
+        @server.gem(params[:id])
+      end
+
+      # This is not part of the Gemstash API, but is used to set the time for the server
+
+      post "/set_time" do
+        @server.set_time Time.iso8601(request.body.read)
+      end
+
+      post "/rebuild_versions_list" do
+        @server.rebuild_versions_list
+      end
+    end
+
+    Version = Struct.new(:rubygem_name, :number, :platform, :info_checksum, :indexed, :prerelease, :sha256, :yanked_at,
+                         :yanked_info_checksum,
+                         :pushed_at, :package,
+                         :position, :latest)
+
+    def initialize
+      @log = []
+      @versions = []
+      @versions_tempfile = Tempfile.create("versions.list")
+      @versions_file = CompactIndex::VersionsFile.new(@versions_tempfile.path)
+      @time = Time.at(0).utc
+    end
+
+    def reorder_versions
+      @versions.group_by(&:rubygem_name).each_value do |versions|
+        numbers = versions.map(&:number).sort.reverse
+
+        versions.each do |version|
+          version.position = numbers.index(version.number)
+          version.latest = false
+        end
+
+        versions.select(&:indexed).reject(&:prerelease).group_by(&:platform).transform_values do |platform_versions|
+          platform_versions.max_by(&:number).latest = true
+        end
+      end
+    end
+
+    def push(gem)
+      package = Gem::Package.new(StringIO.new(gem))
+      log "Pushed #{package.spec.full_name}"
+      if @versions.any? { |v| v.package.spec.full_name == package.spec.full_name }
+        return [409, {}, ["Conflict: #{package.spec.full_name} already exists"]]
+      end
+
+      version = Version.new(package.spec.name, package.spec.version.to_s, package.spec.platform, nil,
+                            true, package.spec.version.prerelease?, Digest::SHA256.hexdigest(gem), nil, nil,
+                            @time, package)
+      @versions << version
+      reorder_versions
+      version.info_checksum = Digest::MD5.hexdigest(info(version.rubygem_name).last.join)
+
+      [200, {}, [@log.last]]
+    end
+
+    def yank(name, version, platform)
+      full_name = [name, version, platform].compact.-(["ruby"]).join("-")
+      yank = @versions.find do |v|
+        v.package.spec.full_name == full_name && v.indexed
+      end
+      return [404, {}, [""]] unless yank
+
+      log "Yanked #{yank.package.spec.full_name}"
+
+      yank.indexed = false
+      reorder_versions
+      yank.yanked_at = @time
+      yank.yanked_info_checksum = Digest::MD5.hexdigest(info(yank.rubygem_name).last.join)
+
+      [200, {}, [""]]
+    end
+
+    def set_time(time) # rubocop:disable Naming/AccessorMethodName
+      @time = time
+      log "Time set to #{time}"
+      [200, {}, [@log.last]]
+    end
+
+    def rebuild_versions_list
+      gems = compact_index_gem_versions(separate_yanks: true,
+                                        before: @time).group_by(&:name).transform_values do |gems_with_name|
+               versions = gems_with_name.flat_map(&:versions)
+               info_checksum = versions.last.info_checksum
+               versions.reject! { _1.number.start_with?("-") }
+               versions.each { |version| version.info_checksum = info_checksum }
+             end.map do |name, versions|
+        CompactIndex::Gem.new(name, versions)
+      end.sort_by(&:name)
+      @versions_file.create(gems, @time.iso8601)
+      @log << "Rebuilt versions list"
+      [200, {}, [@log.last]]
+    end
+
+    def versions
+      gems = compact_index_gem_versions(separate_yanks: true, after: @versions_file.updated_at.to_time)
+      [200, {},
+       # calculate_info_checksums: true breaks with yanks
+       [@versions_file.contents(gems)]]
+    end
+
+    def info(name)
+      versions = compact_index_gem_versions
+                 .select { _1.name == name }
+                 .flat_map(&:versions)
+      if versions.empty?
+        return [404, { "Content-Type" => "text/plain; charset=utf-8" },
+                ["This gem could not be found"]]
+      end
+
+      versions.reject! { _1.number.start_with?("-") }
+
+      [200, {}, [CompactIndex.info(versions)]]
+    end
+
+    def names
+      names = @versions.select(&:indexed).map!(&:rubygem_name).tap(&:sort!).tap(&:uniq!)
+      [200, {}, [CompactIndex.names(names)]]
+    end
+
+    class Asc
+      def initialize(obj)
+        @obj = obj
+      end
+
+      attr_reader :obj
+      protected :obj
+
+      def <=>(other)
+        return other.obj <=> obj if other.is_a?(self.class)
+
+        other <=> obj
+      end
+    end
+
+    def specs_gz(name = nil)
+      case name
+      when nil
+        specs = @versions.select(&:indexed).reject(&:prerelease)
+      when "latest"
+        specs = @versions.select(&:indexed).select(&:latest)
+      when "prerelease"
+        specs = @versions.select(&:indexed).select(&:prerelease)
+      else
+        return [404, { "Content-Type" => "text/plain" }, ["File not found: /specs.4.8.gz\n"]]
+      end
+
+      specs.sort_by! do |v|
+        [v.rubygem_name, Asc.new(v.position), Asc.new(v.platform.to_s)]
+      end
+      specs.map! do |v|
+        [v.rubygem_name, Gem::Version.new(v.number), v.platform.to_s]
+      end
+
+      [200, { "Content-Type" => "application/octet-stream" }, [Zlib.gzip(Marshal.dump(specs))]]
+    end
+
+    def quick_spec(original_name)
+      version = @versions.find do |v|
+        v.indexed && v.package.spec.original_name == original_name
+      end
+
+      if version
+        spec = version.package.spec.dup
+        spec.abbreviate
+        spec.sanitize
+        # TODO
+        if ENV["UPSTREAM"]
+          [200, { "Content-Type" => "text/plain" }, [Gem.deflate(Marshal.dump(spec))]]
+        else
+          [200, { "Content-Type" => "application/octet-stream" }, [Gem.deflate(Marshal.dump(spec))]]
+        end
+      else
+        [404, { "Content-Type" => "text/plain" }, ["File not found: /quick/Marshal.4.8/#{original_name}.gemspec.rz\n"]]
+      end
+    end
+
+    def gem(full_name)
+      version = @versions.find do |v|
+        v.indexed && v.package.spec.full_name == full_name
+      end
+
+      if version
+        contents = version.package.gem.io.string
+        [200, { "Content-Type" => "application/octet-stream" }, [contents]]
+      else
+        [404, { "Content-Type" => "text/plain" }, ["File not found: /gems/#{full_name}.gem\n"]]
+      end
+    end
+
+    private
+
+    def log(message)
+      @log << "[#{@time}] #{message}"
+      # warn @log.last
+    end
+
+    def compact_index_gem_versions(separate_yanks: false, before: nil, after: nil)
+      raise ArgumentError, "before and after cannot be used together" if before && after
+
+      if separate_yanks
+        yanks = @versions.select(&:yanked_at)
+        yanks.select! { _1.yanked_at > after } if after
+        yanks.reject! { _1.yanked_at <= before } if before
+
+        yanks.map! do |version|
+          version = version.dup
+          version.indexed = true
+          version.yanked_at = nil
+          version.yanked_info_checksum = nil
+          version
+        end
+      else
+        yanks = []
+      end
+
+      all_versions = @versions + yanks
+      all_versions.select! { (_1.yanked_at || _1.pushed_at) <= before } if before
+      all_versions.select! { _1.pushed_at > after || (_1.yanked_at && _1.yanked_at > after) } if after
+
+      all_versions
+        # use index to keep order stable
+        .sort_by.with_index { |version, idx| [version.yanked_at || version.pushed_at, idx] }
+        .map do |version|
+        spec = version.package.spec
+        CompactIndex::Gem.new(
+          version.rubygem_name,
+          [
+            CompactIndex::GemVersion.new(
+              version.number.then do |n|
+                version.indexed ? n : "-#{n}"
+              end,
+              version.platform,
+              version.sha256,
+              version.yanked_info_checksum || version.info_checksum,
+              spec.runtime_dependencies.map do |dep|
+                CompactIndex::Dependency.new(dep.name, dep.requirement.to_s)
+              end,
+              spec.required_ruby_version&.to_s, spec.required_rubygems_version&.to_s
+            )
+          ]
+        )
+      end
+    end
+  end
+end
+
+if __FILE__ == $0
+  require "rackup"
+  GemServerConformance::Server::Application.run!
+end

data/lib/gem_server_conformance/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module GemServerConformance
+  VERSION = "0.1.0"
+end