gemvault 0.1.3 → 0.1.5

data/lib/gemvault/gem_extraction.rb

@@ -0,0 +1,45 @@
+require "rubygems/package"
+require "tempfile"
+require "stringio"
+require_relative "vault"
+
+module Gemvault
+  # Shared vault behavior: turn gem bytes (stored or on disk) into gemspecs and
+  # temporary files. Host classes must implement #gem_data(name, version,
+  # platform:) and #gem_entries.
+  module GemExtraction
+    def with_gem_file(name, version, platform: "ruby")
+      tmpfile = write_tempfile(gem_data(name, version, platform: platform))
+      begin
+        yield tmpfile.path
+      ensure
+        tmpfile.close unless tmpfile.closed?
+        tmpfile.unlink
+      end
+    end
+
+    def spec_from_blob(name, version, platform = "ruby")
+      Gem::Package.new(StringIO.new(gem_data(name, version, platform: platform))).spec
+    end
+
+    def specs
+      gem_entries.map { |entry| spec_from_blob(entry.name, entry.version, entry.platform) }
+    end
+
+    private
+
+    def spec_from_gem_file(gem_path)
+      Gem::Package.new(gem_path).spec
+    rescue StandardError => e
+      raise Vault::InvalidGemError, "Invalid gem file #{gem_path}: #{e.message}"
+    end
+
+    def write_tempfile(data)
+      tmpfile = Tempfile.new(["vault_gem", ".gem"])
+      tmpfile.binmode
+      tmpfile.write(data)
+      tmpfile.close
+      tmpfile
+    end
+  end
+end

data/lib/gemvault/manifest.rb

@@ -0,0 +1,76 @@
+require "json"
+require "openssl"
+require_relative "gem_entry"
+
+module Gemvault
+  # The manifest.json stored as the first entry of a Tarvault. Records each
+  # gem's identity, timestamp, and SHA256 digest so listing and integrity
+  # checks never require reading every gem blob.
+  class Manifest
+    FILENAME = "manifest.json".freeze
+    FORMAT_VERSION = 2
+    DIGEST = "SHA256".freeze
+
+    # One gem's metadata inside a manifest.
+    Record = Struct.new(:name, :version, :platform, :created_at, :sha256, :encrypted, keyword_init: true) do
+      def to_gem_entry
+        GemEntry.new(name: name, version: version, platform: platform, created_at: created_at)
+      end
+
+      def filename
+        to_gem_entry.filename
+      end
+    end
+
+    def self.digest(bytes)
+      OpenSSL::Digest.new(DIGEST).hexdigest(bytes)
+    end
+
+    def self.empty(created_at:)
+      new(created_at: created_at, records: [])
+    end
+
+    def self.parse(json)
+      data = JSON.parse(json)
+      records = data.fetch("gems", []).map { |g| Record.new(**g.transform_keys(&:to_sym)) }
+      new(
+        created_at: data["created_at"],
+        records: records,
+        format_version: (data["vault_version"] || FORMAT_VERSION).to_i,
+      )
+    end
+
+    attr_reader :records, :created_at, :format_version
+
+    def initialize(created_at:, records:, format_version: FORMAT_VERSION)
+      @created_at = created_at
+      @records = records
+      @format_version = format_version
+    end
+
+    def with(record)
+      self.class.new(created_at: created_at, records: records + [record], format_version: format_version)
+    end
+
+    def without(dropped)
+      self.class.new(created_at: created_at, records: records - dropped, format_version: format_version)
+    end
+
+    def find(name, version, platform)
+      records.find { |r| r.name == name && r.version == version && r.platform == platform }
+    end
+
+    def gem_entries
+      records.sort_by { |r| [r.name, r.version] }.map(&:to_gem_entry)
+    end
+
+    def to_json(*_args)
+      JSON.pretty_generate(
+        "vault_version" => FORMAT_VERSION,
+        "format" => "tarvault",
+        "created_at" => created_at,
+        "gems" => records.map(&:to_h),
+      )
+    end
+  end
+end

data/lib/gemvault/tar_archive.rb

@@ -0,0 +1,66 @@
+require "rubygems/package"
+require "tempfile"
+require "fileutils"
+require_relative "manifest"
+
+module Gemvault
+  # The tar file backing a Tarvault. Reads named entries and rewrites the
+  # whole archive atomically: tar has no index and the leading manifest entry
+  # changes size on every mutation, so mutation means a full rewrite.
+  class TarArchive
+    def initialize(path)
+      @path = path
+    end
+
+    def read(name)
+      bytes = nil
+      each_entry { |entry| bytes = entry.read if entry.full_name == name }
+      bytes
+    end
+
+    def gem_pairs
+      pairs = []
+      each_entry do |entry|
+        pairs << [entry.full_name, entry.read] unless entry.full_name == Manifest::FILENAME
+      end
+      pairs
+    end
+
+    def write(pairs)
+      tmp = Tempfile.create(["tarvault", ".tar"], File.dirname(@path))
+      write_entries(tmp, pairs)
+      tmp.flush
+      tmp.fsync
+      tmp.close
+      File.rename(tmp.path, @path)
+    rescue StandardError
+      cleanup(tmp)
+      raise
+    end
+
+    private
+
+    def each_entry(&)
+      File.open(@path, "rb") do |io|
+        Gem::Package::TarReader.new(io) { |reader| reader.each_entry(&) }
+      end
+    end
+
+    def write_entries(io, pairs)
+      Gem::Package::TarWriter.new(io) do |writer|
+        pairs.each { |name, bytes| add_entry(writer, name, bytes) }
+      end
+    end
+
+    def add_entry(writer, name, bytes)
+      writer.add_file_simple(name, 0o644, bytes.bytesize) { |dest| dest.write(bytes) }
+    end
+
+    def cleanup(tmp)
+      return unless tmp
+
+      tmp.close unless tmp.closed?
+      FileUtils.rm_f(tmp.path)
+    end
+  end
+end

data/lib/gemvault/tarvault.rb

@@ -0,0 +1,148 @@
+require "time"
+require "json"
+require_relative "vault"
+require_relative "vault_session"
+require_relative "gem_extraction"
+require_relative "manifest"
+require_relative "tar_archive"
+require_relative "gem_reference"
+
+module Gemvault
+  # A Tarvault: a tarball whose first entry is manifest.json and whose
+  # remaining entries are .gem files. Portable, dependency-free storage.
+  class Tarvault
+    extend VaultSession
+    include GemExtraction
+
+    attr_reader :path
+
+    def initialize(path, create: false)
+      @path = File.expand_path(path)
+      @archive = TarArchive.new(@path)
+      @closed = false
+      create ? create_vault! : load_manifest!
+    end
+
+    def add(gem_path, created_at: nil)
+      gem_path = File.expand_path(gem_path)
+      raise Vault::NotFoundError, "Gem file not found: #{gem_path}" unless File.file?(gem_path)
+
+      spec = spec_from_gem_file(gem_path)
+      raise_if_duplicate(spec)
+      store(spec, File.binread(gem_path), created_at || timestamp)
+    end
+
+    def remove(reference)
+      dropped = matching_records(reference)
+      return 0 if dropped.empty?
+
+      @manifest = @manifest.without(dropped)
+      rewrite(survivors_excluding(dropped))
+      dropped.size
+    end
+
+    def gem_data(name, version, platform: "ruby")
+      record = @manifest.find(name, version, platform)
+      raise Vault::NotFoundError, "Gem not found: #{name}-#{version} (#{platform})" unless record
+
+      verify(@archive.read(record.filename), record)
+    end
+
+    def gem_entries
+      @manifest.gem_entries
+    end
+
+    def format_version
+      @manifest.format_version
+    end
+
+    def size
+      @manifest.records.size
+    end
+
+    def close
+      @closed = true
+    end
+
+    def closed?
+      @closed
+    end
+
+    private
+
+    def create_vault!
+      raise Vault::Error, "Vault already exists: #{@path}" if File.exist?(@path)
+
+      @manifest = Manifest.empty(created_at: timestamp)
+      rewrite([])
+    end
+
+    def load_manifest!
+      raise Vault::NotFoundError, "Vault not found: #{@path}" unless File.exist?(@path)
+
+      @manifest = Manifest.parse(read_manifest_json)
+      Vault.assert_readable!(@manifest.format_version, @path)
+    rescue JSON::ParserError, Gem::Package::TarInvalidError, ArgumentError, Errno::EINVAL
+      raise Vault::Error, "Not a valid Tarvault: #{@path}"
+    end
+
+    def read_manifest_json
+      json = @archive.read(Manifest::FILENAME)
+      raise Vault::Error, "Not a valid Tarvault (missing manifest): #{@path}" unless json
+
+      json
+    end
+
+    def store(spec, bytes, created_at)
+      record = build_record(spec, bytes, created_at)
+      @manifest = @manifest.with(record)
+      rewrite(@archive.gem_pairs + [[record.filename, bytes]])
+    end
+
+    def rewrite(gem_pairs)
+      @archive.write([[Manifest::FILENAME, @manifest.to_json]] + gem_pairs)
+    end
+
+    def survivors_excluding(dropped)
+      names = dropped.map(&:filename)
+      @archive.gem_pairs.reject { |pair| names.include?(pair.first) }
+    end
+
+    def build_record(spec, bytes, created_at)
+      Manifest::Record.new(
+        name: spec.name,
+        version: spec.version.to_s,
+        platform: spec.platform.to_s,
+        created_at: created_at,
+        sha256: Manifest.digest(bytes),
+        encrypted: false,
+      )
+    end
+
+    def matching_records(reference)
+      case reference
+      in GemReference::AnyVersion[name:]
+        @manifest.records.select { |r| r.name == name }
+      in GemReference::SpecificVersion[name:, version:]
+        @manifest.records.select { |r| r.name == name && r.version == version.to_s }
+      end
+    end
+
+    def raise_if_duplicate(spec)
+      return unless @manifest.find(spec.name, spec.version.to_s, spec.platform.to_s)
+
+      raise Vault::DuplicateGemError,
+            "Gem already in vault: #{spec.name}-#{spec.version} (#{spec.platform})"
+    end
+
+    def verify(bytes, record)
+      return bytes if Manifest.digest(bytes) == record.sha256
+
+      raise Vault::Error, "Integrity check failed for #{record.filename}"
+    end
+
+    def timestamp
+      Time.now.utc.strftime("%Y-%m-%d %H:%M:%S")
+    end
+  end
+end

data/lib/gemvault/vault.rb

@@ -1,193 +1,87 @@
-require "sqlite3"
-require "rubygems/package"
-require "fileutils"
-require "tempfile"
-require_relative "gem_entry"
-require_relative "gem_reference"
+require "forwardable"
+require_relative "vault_session"
 
 module Gemvault
-  # SQLite-backed archive of .gem blobs; supports add/remove/list/extract.
+  # The public vault interface. Delegates storage to a backend chosen by file
+  # format: a Dbvault (SQLite) for existing SQLite files, a Tarvault (tarball)
+  # otherwise. New vaults are Tarvaults. Only the selected backend is loaded,
+  # so the tar path never requires sqlite3.
   class Vault
+    extend VaultSession
+    extend Forwardable
+
     class Error < StandardError; end
     class NotFoundError < Error; end
     class DuplicateGemError < Error; end
     class InvalidGemError < Error; end
+    class UnsupportedVersionError < Error; end
+    class ReadOnlyError < Error; end
 
-    SCHEMA_VERSION = "1".freeze
     SQLITE_MAGIC = "SQLite format 3#{0.chr}".freeze
+    TAR_MAGIC = "ustar".freeze
+    TAR_MAGIC_OFFSET = 257
 
-    attr_reader :path
-
-    def self.open(path, **opts, &block)
-      raise ArgumentError, "#{name}.open requires a block" unless block
-
-      vault = new(path, **opts)
-      begin
-        yield vault
-      ensure
-        vault.close
-      end
-    end
-
-    def initialize(path, create: false)
-      @path = File.expand_path(path)
-      create ? create_vault! : open_vault!
-    end
-
-    def add(gem_path)
-      gem_path = File.expand_path(gem_path)
-      raise NotFoundError, "Gem file not found: #{gem_path}" unless File.file?(gem_path)
-
-      spec = load_gem_spec(gem_path)
-      raise_if_duplicate(spec)
-      insert_gem(gem_path, spec)
-    end
-
-    def remove(reference)
-      case reference
-      in GemReference::AnyVersion[name:]
-        @db.execute("DELETE FROM gems WHERE name = ?", [name])
-      in GemReference::SpecificVersion[name:, version:]
-        @db.execute(
-          "DELETE FROM gems WHERE name = ? AND version = ?",
-          [name, version.to_s],
-        )
-      end
-      @db.changes
-    end
-
-    def gem_data(name, version, platform: "ruby")
-      row = @db.execute(
-        "SELECT data FROM gems WHERE name = ? AND version = ? AND platform = ?",
-        [name, version, platform],
-      ).first
-
-      raise NotFoundError, "Gem not found: #{name}-#{version} (#{platform})" unless row
-
-      row["data"]
-    end
+    CURRENT_FORMAT = 2
+    MIN_READABLE_FORMAT = 1
 
-    def specs
-      gem_entries.map { |entry| spec_from_blob(entry.name, entry.version, entry.platform) }
-    end
+    def_delegators :@backend,
+                   :add, :remove, :gem_data, :gem_entries, :specs,
+                   :spec_from_blob, :with_gem_file, :size, :close, :closed?,
+                   :path, :format_version
 
-    def gem_entries
-      @db.execute(
-        "SELECT name, version, platform, created_at FROM gems ORDER BY name, version",
-      ).map { |row| GemEntry.new(**row.transform_keys(&:to_sym)) }
-    end
+    def self.assert_readable!(version, path)
+      return if version.between?(MIN_READABLE_FORMAT, CURRENT_FORMAT)
 
-    def size
-      @db.execute("SELECT COUNT(*) AS count FROM gems").first["count"]
+      raise UnsupportedVersionError,
+            "Vault #{path} is format #{version}; this gemvault reads up to #{CURRENT_FORMAT}. Upgrade gemvault."
     end
 
-    def close
-      @db.close if @db && !@db.closed?
-    end
+    def self.backend_for(path, create:)
+      return build_tarvault(path, create: true) if create
+      raise NotFoundError, "Vault not found: #{path}" unless File.exist?(path)
 
-    def with_gem_file(name, version, platform: "ruby")
-      data = gem_data(name, version, platform: platform)
-      tmpfile = write_tempfile(data)
-      begin
-        yield tmpfile.path
-      ensure
-        tmpfile.close unless tmpfile.closed?
-        tmpfile.unlink
+      case container_kind(path)
+      when :sqlite then build_dbvault(path)
+      when :tar then build_tarvault(path, create: false)
+      else
+        raise Error, "Unrecognized vault format: #{path} (not a Dbvault or Tarvault; it may require a newer gemvault)"
       end
     end
 
-    def spec_from_blob(name, version, platform = "ruby")
-      with_gem_file(name, version, platform: platform) do |path|
-        Gem::Package.new(path).spec
-      end
-    end
-
-    private
+    def self.container_kind(path)
+      return :sqlite if sqlite?(path)
+      return :tar if tar?(path)
 
-    def create_vault!
-      raise Error, "Vault already exists: #{@path}" if File.exist?(@path)
-
-      @db = new_database
-      create_schema
+      :unknown
     end
 
-    def open_vault!
-      raise NotFoundError, "Vault not found: #{@path}" unless File.exist?(@path)
-
-      validate_sqlite!
-      @db = new_database
+    def self.sqlite?(path)
+      File.exist?(path) && File.binread(path, SQLITE_MAGIC.bytesize) == SQLITE_MAGIC
     end
 
-    def new_database
-      db = SQLite3::Database.new(@path)
-      db.results_as_hash = true
-      db
-    end
-
-    def load_gem_spec(gem_path)
-      Gem::Package.new(gem_path).spec
-    rescue StandardError => e
-      raise InvalidGemError, "Invalid gem file #{gem_path}: #{e.message}"
-    end
-
-    def raise_if_duplicate(spec)
-      existing = @db.execute(
-        "SELECT 1 FROM gems WHERE name = ? AND version = ? AND platform = ?",
-        [spec.name, spec.version.to_s, spec.platform.to_s],
-      )
-      return if existing.empty?
+    def self.tar?(path)
+      return false unless File.exist?(path)
 
-      raise DuplicateGemError,
-            "Gem already in vault: #{spec.name}-#{spec.version} (#{spec.platform})"
+      header = File.binread(path, TAR_MAGIC_OFFSET + TAR_MAGIC.bytesize)
+      header.to_s[TAR_MAGIC_OFFSET, TAR_MAGIC.bytesize] == TAR_MAGIC
     end
 
-    def insert_gem(gem_path, spec)
-      data = File.binread(gem_path)
-      @db.execute(
-        "INSERT INTO gems (name, version, platform, data) VALUES (?, ?, ?, ?)",
-        [spec.name, spec.version.to_s, spec.platform.to_s, SQLite3::Blob.new(data)],
-      )
+    def self.build_dbvault(path)
+      require_relative "dbvault"
+      Dbvault.new(path)
+    rescue LoadError => e
+      raise Error,
+            "#{path} is a legacy SQLite vault; it needs the sqlite3 gem (#{e.message}). " \
+            "Install sqlite3, or upgrade the vault with a gemvault that includes it."
     end
 
-    def write_tempfile(data)
-      tmpfile = Tempfile.new(["vault_gem", ".gem"])
-      tmpfile.binmode
-      tmpfile.write(data)
-      tmpfile.close
-      tmpfile
+    def self.build_tarvault(path, create:)
+      require_relative "tarvault"
+      Tarvault.new(path, create: create)
     end
 
-    def create_schema
-      @db.execute_batch(<<~SQL)
-        CREATE TABLE metadata (
-          key TEXT PRIMARY KEY,
-          value TEXT NOT NULL
-        );
-
-        CREATE TABLE gems (
-          name TEXT NOT NULL,
-          version TEXT NOT NULL,
-          platform TEXT NOT NULL DEFAULT 'ruby',
-          data BLOB NOT NULL,
-          created_at TEXT NOT NULL DEFAULT (datetime('now')),
-          PRIMARY KEY (name, version, platform)
-        );
-      SQL
-
-      @db.execute(
-        "INSERT INTO metadata (key, value) VALUES (?, ?)",
-        ["vault_version", SCHEMA_VERSION],
-      )
-      @db.execute(
-        "INSERT INTO metadata (key, value) VALUES (?, ?)",
-        ["created_at", Time.now.utc.strftime("%Y-%m-%d %H:%M:%S")],
-      )
-    end
-
-    def validate_sqlite!
-      return if File.binread(@path, SQLITE_MAGIC.bytesize) == SQLITE_MAGIC
-
-      raise Error, "Not a valid vault file (not SQLite): #{@path}"
+    def initialize(path, create: false)
+      @backend = self.class.backend_for(File.expand_path(path), create: create)
     end
   end
 end

data/lib/gemvault/vault_session.rb

@@ -0,0 +1,16 @@
+module Gemvault
+  # Class-method mixin giving vault types a block-scoped `open` that news up
+  # an instance, yields it, and guarantees `close`.
+  module VaultSession
+    def open(path, **opts)
+      raise ArgumentError, "#{name}.open requires a block" unless block_given?
+
+      vault = new(path, **opts)
+      begin
+        yield vault
+      ensure
+        vault.close
+      end
+    end
+  end
+end

data/lib/gemvault/vault_upgrade.rb

@@ -0,0 +1,87 @@
+require "fileutils"
+require_relative "vault"
+require_relative "deprecation"
+
+module Gemvault
+  # Migrates a vault to the current storage format by reading it through its
+  # existing backend and rewriting it through the current-format writer, then
+  # atomically swapping the file into place. Preserves each gem's created_at.
+  class VaultUpgrade
+    # The from/to/gem-count summary of an upgrade; also detects an
+    # already-current (no-op) vault.
+    Plan = Struct.new(:from_version, :to_version, :gem_count, keyword_init: true) do
+      def no_op?
+        from_version == to_version
+      end
+
+      def to_s
+        "format #{from_version} -> #{to_version}"
+      end
+    end
+
+    def initialize(path, backup: true)
+      @path = File.expand_path(path)
+      @backup = backup
+    end
+
+    def plan
+      Deprecation.silence do
+        Vault.open(@path) do |vault|
+          Plan.new(from_version: vault.format_version, to_version: Vault::CURRENT_FORMAT, gem_count: vault.size)
+        end
+      end
+    end
+
+    def call
+      summary = plan
+      return summary if summary.no_op?
+
+      backup! if @backup
+      rebuild(summary.gem_count)
+      summary
+    end
+
+    private
+
+    def backup!
+      backup_path = "#{@path}.bak"
+      if File.exist?(backup_path)
+        raise Vault::Error, "Backup already exists: #{backup_path} (remove it or pass --no-backup)"
+      end
+
+      FileUtils.cp(@path, backup_path)
+    end
+
+    def rebuild(expected_count)
+      tmp = "#{@path}.upgrading"
+      FileUtils.rm_f(tmp)
+      copy_current_format(tmp)
+      verify_count!(tmp, expected_count)
+      File.rename(tmp, @path)
+    end
+
+    def copy_current_format(tmp)
+      Deprecation.silence do
+        Vault.open(@path) do |old|
+          target = Vault.new(tmp, create: true)
+          old.gem_entries.each { |entry| copy_gem(old, target, entry) }
+          target.close
+        end
+      end
+    end
+
+    def copy_gem(old, target, entry)
+      old.with_gem_file(entry.name, entry.version, platform: entry.platform) do |gem_path|
+        target.add(gem_path, created_at: entry.created_at)
+      end
+    end
+
+    def verify_count!(tmp, expected_count)
+      actual = Vault.open(tmp, &:size)
+      return if actual == expected_count
+
+      FileUtils.rm_f(tmp)
+      raise Vault::Error, "Upgrade verification failed: expected #{expected_count} gems, got #{actual}"
+    end
+  end
+end

data/lib/gemvault/version.rb

@@ -1,3 +1,3 @@
 module Gemvault
-  VERSION = "0.1.3".freeze
+  VERSION = "0.1.5".freeze
 end