gemvault 0.1.4 → 0.2.0

data/lib/gemvault/vault.rb

@@ -1,193 +1,91 @@
-require "sqlite3"
-require "rubygems/package"
-require "fileutils"
-require "tempfile"
-require_relative "gem_entry"
-require_relative "gem_reference"
+require "forwardable"
+require "pathname"
+require_relative "vault_session"
 
 module Gemvault
-  # SQLite-backed archive of .gem blobs; supports add/remove/list/extract.
+  # The public vault interface. Delegates storage to a backend chosen by file
+  # format: a Dbvault (SQLite) for existing SQLite files, a Tarvault (tarball)
+  # otherwise. New vaults are Tarvaults. Only the selected backend is loaded,
+  # so the tar path never requires sqlite3.
   class Vault
+    extend VaultSession
+    extend Forwardable
+
     class Error < StandardError; end
     class NotFoundError < Error; end
     class DuplicateGemError < Error; end
     class InvalidGemError < Error; end
+    class UnsupportedVersionError < Error; end
+    class ReadOnlyError < Error; end
 
-    SCHEMA_VERSION = "1".freeze
     SQLITE_MAGIC = "SQLite format 3#{0.chr}".freeze
+    TAR_MAGIC = "ustar".freeze
+    TAR_MAGIC_OFFSET = 257
 
-    attr_reader :path
+    CURRENT_FORMAT = 2
+    MIN_READABLE_FORMAT = 1
 
-    def self.open(path, **opts, &block)
-      raise ArgumentError, "#{name}.open requires a block" unless block
+    def_delegators :@backend,
+                   :add, :remove, :gem_data, :gem_entries, :specs,
+                   :spec_from_blob, :with_gem_file, :size, :close, :closed?,
+                   :path, :format_version
 
-      vault = new(path, **opts)
-      begin
-        yield vault
-      ensure
-        vault.close
-      end
-    end
+    def self.assert_readable!(version:, path:)
+      return if version.between?(MIN_READABLE_FORMAT, CURRENT_FORMAT)
 
-    def initialize(path, create: false)
-      @path = File.expand_path(path)
-      create ? create_vault! : open_vault!
+      raise UnsupportedVersionError,
+            "Vault #{path} is format #{version}; this gemvault reads up to #{CURRENT_FORMAT}. Upgrade gemvault."
     end
 
-    def add(gem_path)
-      gem_path = File.expand_path(gem_path)
-      raise NotFoundError, "Gem file not found: #{gem_path}" unless File.file?(gem_path)
+    def self.backend_for(path, create:)
+      return build_tarvault(path, create: true) if create
+      raise NotFoundError, "Vault not found: #{path}" unless path.exist?
 
-      spec = load_gem_spec(gem_path)
-      raise_if_duplicate(spec)
-      insert_gem(gem_path, spec)
-    end
-
-    def remove(reference)
-      case reference
-      in GemReference::AnyVersion[name:]
-        @db.execute("DELETE FROM gems WHERE name = ?", [name])
-      in GemReference::SpecificVersion[name:, version:]
-        @db.execute(
-          "DELETE FROM gems WHERE name = ? AND version = ?",
-          [name, version.to_s],
-        )
+      case container_kind(path)
+      when :sqlite then build_dbvault(path)
+      when :tar then build_tarvault(path, create: false)
+      else
+        raise Error, "Unrecognized vault format: #{path} (not a Dbvault or Tarvault; it may require a newer gemvault)"
       end
-      @db.changes
     end
 
-    def gem_data(name, version, platform: "ruby")
-      row = @db.execute(
-        "SELECT data FROM gems WHERE name = ? AND version = ? AND platform = ?",
-        [name, version, platform],
-      ).first
+    def self.container_kind(path)
+      return :sqlite if sqlite?(path)
+      return :tar if tar?(path)
 
-      raise NotFoundError, "Gem not found: #{name}-#{version} (#{platform})" unless row
-
-      row["data"]
+      :unknown
     end
 
-    def specs
-      gem_entries.map { |entry| spec_from_blob(entry.name, entry.version, entry.platform) }
+    def self.sqlite?(path)
+      path.exist? && path.binread(SQLITE_MAGIC.bytesize) == SQLITE_MAGIC
     end
 
-    def gem_entries
-      @db.execute(
-        "SELECT name, version, platform, created_at FROM gems ORDER BY name, version",
-      ).map { |row| GemEntry.new(**row.transform_keys(&:to_sym)) }
-    end
+    def self.tar?(path)
+      return false unless path.exist?
 
-    def size
-      @db.execute("SELECT COUNT(*) AS count FROM gems").first["count"]
+      header = path.binread(TAR_MAGIC_OFFSET + TAR_MAGIC.bytesize)
+      header.to_s[TAR_MAGIC_OFFSET, TAR_MAGIC.bytesize] == TAR_MAGIC
     end
 
-    def close
-      @db.close if @db && !@db.closed?
-    end
-
-    def with_gem_file(name, version, platform: "ruby")
-      data = gem_data(name, version, platform: platform)
-      tmpfile = write_tempfile(data)
+    def self.build_dbvault(path)
       begin
-        yield tmpfile.path
-      ensure
-        tmpfile.close unless tmpfile.closed?
-        tmpfile.unlink
-      end
-    end
-
-    def spec_from_blob(name, version, platform = "ruby")
-      with_gem_file(name, version, platform: platform) do |path|
-        Gem::Package.new(path).spec
+        require_relative "dbvault"
+        Dbvault.new(path)
+      rescue LoadError => e
+        raise Error,
+              "#{path} is a legacy SQLite vault; it needs the sqlite3 gem (#{e.message}). " \
+              "Install sqlite3, or upgrade the vault with a gemvault that includes it."
       end
     end
 
-    private
-
-    def create_vault!
-      raise Error, "Vault already exists: #{@path}" if File.exist?(@path)
-
-      @db = new_database
-      create_schema
-    end
-
-    def open_vault!
-      raise NotFoundError, "Vault not found: #{@path}" unless File.exist?(@path)
-
-      validate_sqlite!
-      @db = new_database
-    end
-
-    def new_database
-      db = SQLite3::Database.new(@path)
-      db.results_as_hash = true
-      db
-    end
-
-    def load_gem_spec(gem_path)
-      Gem::Package.new(gem_path).spec
-    rescue StandardError => e
-      raise InvalidGemError, "Invalid gem file #{gem_path}: #{e.message}"
-    end
-
-    def raise_if_duplicate(spec)
-      existing = @db.execute(
-        "SELECT 1 FROM gems WHERE name = ? AND version = ? AND platform = ?",
-        [spec.name, spec.version.to_s, spec.platform.to_s],
-      )
-      return if existing.empty?
-
-      raise DuplicateGemError,
-            "Gem already in vault: #{spec.name}-#{spec.version} (#{spec.platform})"
+    def self.build_tarvault(path, create:)
+      require_relative "tarvault"
+      Tarvault.new(path, create:)
     end
 
-    def insert_gem(gem_path, spec)
-      data = File.binread(gem_path)
-      @db.execute(
-        "INSERT INTO gems (name, version, platform, data) VALUES (?, ?, ?, ?)",
-        [spec.name, spec.version.to_s, spec.platform.to_s, SQLite3::Blob.new(data)],
-      )
-    end
-
-    def write_tempfile(data)
-      tmpfile = Tempfile.new(["vault_gem", ".gem"])
-      tmpfile.binmode
-      tmpfile.write(data)
-      tmpfile.close
-      tmpfile
-    end
-
-    def create_schema
-      @db.execute_batch(<<~SQL)
-        CREATE TABLE metadata (
-          key TEXT PRIMARY KEY,
-          value TEXT NOT NULL
-        );
-
-        CREATE TABLE gems (
-          name TEXT NOT NULL,
-          version TEXT NOT NULL,
-          platform TEXT NOT NULL DEFAULT 'ruby',
-          data BLOB NOT NULL,
-          created_at TEXT NOT NULL DEFAULT (datetime('now')),
-          PRIMARY KEY (name, version, platform)
-        );
-      SQL
-
-      @db.execute(
-        "INSERT INTO metadata (key, value) VALUES (?, ?)",
-        ["vault_version", SCHEMA_VERSION],
-      )
-      @db.execute(
-        "INSERT INTO metadata (key, value) VALUES (?, ?)",
-        ["created_at", Time.now.utc.strftime("%Y-%m-%d %H:%M:%S")],
-      )
-    end
-
-    def validate_sqlite!
-      return if File.binread(@path, SQLITE_MAGIC.bytesize) == SQLITE_MAGIC
-
-      raise Error, "Not a valid vault file (not SQLite): #{@path}"
+    def initialize(path, create: false)
+      absolute_path = Pathname(path).expand_path
+      @backend = self.class.backend_for(absolute_path, create:)
     end
   end
 end

data/lib/gemvault/vault_path.rb

@@ -0,0 +1,33 @@
+require "uri"
+require "pathname"
+
+module Gemvault
+  ##
+  # Translates vault locators into filesystem paths.
+  #
+  # A locator is whatever a user hands the CLI or a package manager as "the
+  # vault": a plain filesystem path, a <tt>file://</tt> URI, or a
+  # <tt>vault://</tt> URI. Anything else passes through unchanged.
+  module VaultPath
+    SCHEMES = ["file", "vault"].freeze
+
+    module_function
+
+    # :call-seq:
+    #   resolve(locator) -> Pathname
+    #
+    # Resolves +locator+ to a Pathname. Two-slash relative forms such as
+    # <tt>file://vault.gemv</tt> parse their first segment as a URI host, so
+    # host and path are rejoined.
+    def resolve(locator)
+      begin
+        uri = URI.parse(locator.to_s)
+        return Pathname(locator.to_s) unless SCHEMES.include?(uri.scheme)
+
+        Pathname([uri.host, uri.path].compact.join)
+      rescue URI::InvalidURIError
+        Pathname(locator.to_s)
+      end
+    end
+  end
+end

data/lib/gemvault/vault_session.rb

@@ -0,0 +1,16 @@
+module Gemvault
+  # Class-method mixin giving vault types a block-scoped `open` that news up
+  # an instance, yields it, and guarantees `close`.
+  module VaultSession
+    def open(path, **opts)
+      raise ArgumentError, "#{name}.open requires a block" unless block_given?
+
+      vault = new(path, **opts)
+      begin
+        yield vault
+      ensure
+        vault.close
+      end
+    end
+  end
+end

data/lib/gemvault/vault_upgrade.rb

@@ -0,0 +1,94 @@
+require "pathname"
+require "fileutils"
+require_relative "vault"
+require_relative "deprecation"
+
+module Gemvault
+  # Migrates a vault to the current storage format by reading it through its
+  # existing backend and rewriting it through the current-format writer, then
+  # atomically swapping the file into place. Preserves each gem's created_at.
+  class VaultUpgrade
+    include FileUtils
+
+    # The from/to/gem-count summary of an upgrade; also detects an
+    # already-current (no-op) vault.
+    class Plan < Data.define(:from_version, :to_version, :gem_count)
+      def no_op? = from_version == to_version
+
+      def to_s = "format #{from_version} -> #{to_version}"
+
+      def deconstruct_keys(_keys)
+        { from_version:, to_version:, gem_count:, no_op: no_op? }
+      end
+    end
+
+    # Copies gems from a source vault into a target vault, preserving each
+    # gem's stored timestamp. Holds the two endpoints so the per-gem call
+    # takes only the entry.
+    class GemCopy < Data.define(:source, :target)
+      def call(entry)
+        source.with_gem_file(entry) do |gem_path|
+          target.add(gem_path, created_at: entry.created_at)
+        end
+      end
+    end
+
+    def initialize(path, backup: true)
+      @path = Pathname(path).expand_path
+      @backup = backup
+    end
+
+    def plan
+      Deprecation.silence do
+        Vault.open(@path) do |vault|
+          Plan.new(from_version: vault.format_version, to_version: Vault::CURRENT_FORMAT, gem_count: vault.size)
+        end
+      end
+    end
+
+    def call
+      summary = plan
+      return summary if summary.no_op?
+
+      backup! if @backup
+      rebuild(summary.gem_count)
+      summary
+    end
+
+    private
+
+    def backup!
+      backup_path = Pathname("#{@path}.bak")
+      raise Vault::Error, "Backup already exists: #{backup_path} (remove it or pass --no-backup)" if backup_path.exist?
+
+      cp(@path, backup_path)
+    end
+
+    def rebuild(expected_count)
+      tmp = Pathname("#{@path}.upgrading")
+      rm_f(tmp)
+      copy_current_format(tmp)
+      verify_count!(tmp:, expected_count:)
+      tmp.rename(@path)
+    end
+
+    def copy_current_format(tmp)
+      Deprecation.silence do
+        Vault.open(@path) do |old|
+          target = Vault.new(tmp, create: true)
+          copy = GemCopy.new(source: old, target:)
+          old.gem_entries.each { |entry| copy.call(entry) }
+          target.close
+        end
+      end
+    end
+
+    def verify_count!(tmp:, expected_count:)
+      actual = Vault.open(tmp, &:size)
+      return if actual == expected_count
+
+      rm_f(tmp)
+      raise Vault::Error, "Upgrade verification failed: expected #{expected_count} gems, got #{actual}"
+    end
+  end
+end

data/lib/gemvault/version.rb

@@ -1,3 +1,3 @@
 module Gemvault
-  VERSION = "0.1.4".freeze
+  VERSION = "0.2.0".freeze
 end

data/lib/gemvault.rb

@@ -1,5 +1,7 @@
 require_relative "gemvault/version"
+require_relative "gemvault/deprecation"
 require_relative "gemvault/vault"
+require_relative "gemvault/vault_path"
 
 ##
 # Top-level namespace for the gemvault gem.

data/lib/rubygems/resolver/vault_set.rb

@@ -11,7 +11,7 @@ class Gem::Resolver::VaultSet < Gem::Resolver::Set
   end
 
   def find_all(req)
-    @specs.filter_map { |tuple| index_spec_for(req, tuple) }
+    @specs.filter_map { |candidate| index_spec_for(candidate) if req.match?(candidate) }
   end
 
   def prefetch(reqs); end
@@ -22,19 +22,17 @@ class Gem::Resolver::VaultSet < Gem::Resolver::Set
 
       pp.breakable
 
-      pp.seplist @specs do |tuple|
-        pp.text tuple.full_name
+      pp.seplist @specs do |candidate|
+        pp.text candidate.full_name
       end
     end
   end
 
   private
 
-  def index_spec_for(req, tuple)
-    return unless req.match?(tuple)
-
+  def index_spec_for(candidate)
     Gem::Resolver::IndexSpecification.new(
-      self, tuple.name, tuple.version, @source, tuple.platform
+      self, candidate.name, candidate.version, @source, candidate.platform
     )
   end
 end

data/lib/rubygems/source/vault.rb

@@ -1,8 +1,11 @@
 require "gemvault/vault"
-require "uri"
+require "gemvault/vault_path"
+require "gemvault/gem_entry"
+require "pathname"
 
 ##
-# A source backed by a .gemv vault file (SQLite archive of .gem blobs).
+# A source backed by a .gemv vault file: a tar archive of .gem files indexed
+# by a manifest. Legacy SQLite vaults are read transparently.
 #
 # Used by the gemvault RubyGems plugin to support:
 #
@@ -10,12 +13,11 @@ require "uri"
 class Gem::Source::Vault < Gem::Source
   include Gem::UserInteraction
 
-  VAULT_URI_SCHEMES = %w[file vault].freeze
-
   attr_reader :path
 
   def initialize(path)
-    @path = File.expand_path(filesystem_path(path))
+    resolved = Gemvault::VaultPath.resolve(path)
+    @path = Pathname(resolved).expand_path.to_s
     super(@path)
     @uri = @path
     @specs = nil
@@ -24,7 +26,7 @@ class Gem::Source::Vault < Gem::Source
   def load_specs(type)
     verbose "Loading #{type} specs from vault at #{@path}"
     ensure_specs_loaded
-    select_tuples(type)
+    select_candidates(type)
   end
 
   def fetch_spec(name_tuple)
@@ -38,17 +40,15 @@ class Gem::Source::Vault < Gem::Source
 
   def download(spec, dir = Dir.pwd)
     verbose "Extracting #{spec.file_name} from vault at #{@path}"
-    cache_dir = File.join(dir, "cache")
-    FileUtils.mkdir_p(cache_dir)
-
-    dest = File.join(cache_dir, spec.file_name)
+    dest = Pathname(dir).join("cache").tap(&:mkpath).join(spec.file_name)
 
     Gemvault::Vault.open(@path) do |vault|
-      data = vault.gem_data(spec.name, spec.version.to_s, platform: spec.platform.to_s)
-      File.binwrite(dest, data)
+      entry = entry_for(spec)
+      bytes = vault.gem_data(entry)
+      dest.binwrite(bytes)
     end
 
-    dest
+    dest.to_s
   end
 
   def dependency_resolver_set(prerelease = nil)
@@ -91,35 +91,32 @@ class Gem::Source::Vault < Gem::Source
 
   private
 
-  def filesystem_path(path)
-    uri = URI.parse(path.to_s)
-    VAULT_URI_SCHEMES.include?(uri.scheme) ? uri.path : path.to_s
-  rescue URI::InvalidURIError
-    path.to_s
+  def entry_for(spec)
+    Gemvault::GemEntry.from_spec(spec)
   end
 
-  def select_tuples(type)
+  def select_candidates(type)
     case type
-    when :released then released_tuples
-    when :prerelease then prerelease_tuples
-    when :latest then latest_tuples
+    when :released then released_candidates
+    when :prerelease then prerelease_candidates
+    when :latest then latest_candidates
     else @specs.keys
     end
   end
 
-  def released_tuples
-    @specs.keys.reject { |tuple| tuple.version.prerelease? }
+  def released_candidates
+    @specs.keys.reject { |candidate| candidate.version.prerelease? }
   end
 
-  def prerelease_tuples
-    @specs.keys.select { |tuple| tuple.version.prerelease? }
+  def prerelease_candidates
+    @specs.keys.select { |candidate| candidate.version.prerelease? }
   end
 
-  def latest_tuples
+  def latest_candidates
     @specs.keys
-          .group_by { |tuple| [tuple.name, tuple.platform] }
+          .group_by { |candidate| [candidate.name, candidate.platform] }
           .values
-          .map { |tuples| tuples.max_by(&:version) }
+          .map { |versions| versions.max_by(&:version) }
   end
 
   def ensure_specs_loaded
@@ -128,9 +125,8 @@ class Gem::Source::Vault < Gem::Source
     @specs = {}
     Gemvault::Vault.open(@path) do |vault|
       vault.gem_entries.each do |entry|
-        spec = vault.spec_from_blob(entry.name, entry.version, entry.platform)
-        tuple = spec.name_tuple
-        @specs[tuple] = spec
+        spec = vault.spec_from_blob(entry)
+        @specs[spec.name_tuple] = spec
       end
     end
   end

data/lib/rubygems_plugin.rb

@@ -1,3 +1,10 @@
+# RubyGems can evaluate this plugin twice under Bundler: once through the
+# install-generated stub (via require) and again through Gem.load_env_plugins
+# (via load, which ignores $LOADED_FEATURES) once the gem's lib/ is on
+# $LOAD_PATH. Skip the redundant second evaluation so the constants and
+# monkey-patches below are installed exactly once.
+return if defined?(Gemvault::PLUGIN_LOADED)
+
 require_relative "rubygems/source/vault"
 require "rubygems/source_list"
 
@@ -14,6 +21,8 @@ require "rubygems/source_list"
 # 3. SourceList#<< -- route .gemv strings to Gem::Source::Vault
 
 module Gemvault
+  PLUGIN_LOADED = true
+
   # Lets .gemv paths bypass RubyGems' URI scheme validation.
   module AcceptVaultURI
     VALID_URI_SCHEMES = ["http", "https", "file", "s3"].freeze
@@ -22,8 +31,7 @@ module Gemvault
       Gem::OptionParser.accept Gem::URI::HTTP do |value|
         next value if value.to_s.end_with?(".gemv")
 
-        uri = parse_uri(value)
-        validate_scheme!(uri, value)
+        validate_uri!(value)
         value
       end
     end
@@ -31,12 +39,15 @@ module Gemvault
     private
 
     def parse_uri(value)
-      Gem::URI.parse value
-    rescue Gem::URI::InvalidURIError
-      raise Gem::OptionParser::InvalidArgument, value
+      begin
+        Gem::URI.parse value
+      rescue Gem::URI::InvalidURIError
+        raise Gem::OptionParser::InvalidArgument, value
+      end
     end
 
-    def validate_scheme!(uri, value)
+    def validate_uri!(value)
+      uri = parse_uri(value)
       return if VALID_URI_SCHEMES.include?(uri.scheme)
 
       schemes = VALID_URI_SCHEMES.map { |scheme| "#{scheme}://" }