rubydb 0.1.0

data/lib/rubydb/backup/snapshot.rb

@@ -0,0 +1,267 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "time"
+require "json"
+require "digest"
+require "monitor"
+
+module RubyDB
+  module Backup
+    class SnapshotError < StandardError; end unless const_defined?(:SnapshotError)
+    # Snapshot - Creates point-in-time snapshots
+    class Snapshot
+      attr_reader :stats
+
+      def initialize(engine, config = {})
+        @engine = engine
+        @config = config
+        @snapshot_dir = config[:snapshot_dir] || "snapshots"
+        @max_snapshots = config[:max_snapshots] || 10
+        @retention_days = config[:retention_days] || 30
+        @snapshots = {}
+        @stats = {
+          snapshots_created: 0,
+          snapshots_restored: 0,
+          snapshots_deleted: 0,
+          total_size_bytes: 0,
+          last_snapshot: nil
+        }
+        @lock = Monitor.new
+
+        FileUtils.mkdir_p(@snapshot_dir)
+        load_snapshots
+      end
+
+      def create_snapshot(name = nil)
+        @lock.synchronize do
+          start_time = Time.now
+          snapshot_name = name || "snapshot_#{Time.now.strftime('%Y%m%d_%H%M%S')}"
+          snapshot_path = snapshot_path_for_name(snapshot_name)
+          return { success: false, error: "Invalid snapshot name" } unless snapshot_path
+
+          if Dir.exist?(snapshot_path)
+            return { success: false, error: "Snapshot already exists" }
+          end
+
+          begin
+            # Create snapshot
+            FileUtils.mkdir_p(snapshot_path)
+
+            # Get current database state
+            state = capture_database_state
+
+            # Store snapshot metadata
+            metadata = {
+              name: snapshot_name,
+              created_at: Time.now.iso8601,
+              engine_version: RubyDB::VERSION,
+              database_name: @engine.current_database_name,
+              state: state,
+              size: 0
+            }
+
+            # Create copy-on-write snapshot
+            create_cow_snapshot(snapshot_path)
+
+            # Write metadata
+            File.write(File.join(snapshot_path, "snapshot.json"), JSON.generate(metadata))
+
+            # Calculate size
+            size = calculate_snapshot_size(snapshot_path)
+            metadata[:size] = size
+            metadata[:files] = snapshot_files(snapshot_path).to_h do |file|
+              [File.basename(file), Digest::SHA256.file(file).hexdigest]
+            end
+            File.write(File.join(snapshot_path, "snapshot.json"), JSON.generate(metadata))
+
+            # Update stats
+            @snapshots[snapshot_name] = metadata
+            @stats[:snapshots_created] += 1
+            @stats[:total_size_bytes] += size
+            @stats[:last_snapshot] = Time.now
+
+            # Clean old snapshots
+            clean_old_snapshots
+
+            { success: true, snapshot_name: snapshot_name, metadata: metadata }
+
+          rescue => e
+            FileUtils.rm_rf(snapshot_path) if Dir.exist?(snapshot_path)
+            { success: false, error: e.message }
+          end
+        end
+      end
+
+      def restore_snapshot(snapshot_name, options = {})
+        @lock.synchronize do
+          snapshot_path = snapshot_path_for_name(snapshot_name)
+          return { success: false, error: "Invalid snapshot name" } unless snapshot_path
+
+          unless Dir.exist?(snapshot_path)
+            return { success: false, error: "Snapshot does not exist" }
+          end
+
+          metadata_path = File.join(snapshot_path, "snapshot.json")
+          unless File.exist?(metadata_path)
+            return { success: false, error: "Snapshot metadata not found" }
+          end
+
+          begin
+            metadata = JSON.parse(File.read(metadata_path), symbolize_names: true)
+            verify_snapshot!(snapshot_path, metadata)
+
+            # Restore from snapshot
+            restore_cow_snapshot(snapshot_path)
+
+            @stats[:snapshots_restored] += 1
+
+            { success: true, snapshot_name: snapshot_name, metadata: metadata }
+
+          rescue => e
+            { success: false, error: e.message }
+          end
+        end
+      end
+
+      def list_snapshots
+        @lock.synchronize do
+          @snapshots.values.sort_by { |s| s[:created_at] }.reverse
+        end
+      end
+
+      def delete_snapshot(snapshot_name)
+        @lock.synchronize do
+          snapshot_path = snapshot_path_for_name(snapshot_name)
+          return { success: false, error: "Invalid snapshot name" } unless snapshot_path
+
+          unless Dir.exist?(snapshot_path)
+            return { success: false, error: "Snapshot does not exist" }
+          end
+
+          size = calculate_snapshot_size(snapshot_path)
+          FileUtils.rm_rf(snapshot_path)
+
+          @snapshots.delete(snapshot_name)
+          @stats[:snapshots_deleted] += 1
+          @stats[:total_size_bytes] -= size
+
+          { success: true }
+        end
+      end
+
+      def stats
+        @lock.synchronize do
+          @stats.merge({
+            snapshots: @snapshots.size,
+            snapshot_dir: @snapshot_dir,
+            max_snapshots: @max_snapshots
+          })
+        end
+      end
+
+      private
+
+      def snapshot_path_for_name(snapshot_name)
+        name = snapshot_name.to_s
+        return nil if name.empty? || name != File.basename(name)
+
+        root = File.expand_path(@snapshot_dir)
+        path = File.expand_path(File.join(root, name))
+        path == root || path.start_with?("#{root}#{File::SEPARATOR}") ? path : nil
+      end
+
+      def load_snapshots
+        Dir.glob(File.join(@snapshot_dir, "*")).each do |path|
+          next unless Dir.exist?(path)
+          metadata_path = File.join(path, "snapshot.json")
+          next unless File.exist?(metadata_path)
+
+          begin
+            metadata = JSON.parse(File.read(metadata_path), symbolize_names: true)
+            @snapshots[metadata[:name]] = metadata
+          rescue
+            raise SnapshotError, "Invalid snapshot metadata: #{metadata_path}"
+          end
+        end
+      end
+
+      def capture_database_state
+        {
+          tables: @engine.list_tables,
+          sequences: @engine.respond_to?(:list_sequences) ? @engine.list_sequences : [],
+          current_lsn: @engine.respond_to?(:current_lsn) ? @engine.current_lsn : nil,
+          wal_position: @engine.respond_to?(:wal_position) ? @engine.wal_position : nil,
+          active_transactions: @engine.respond_to?(:active_transactions) ? @engine.active_transactions : []
+        }
+      end
+
+      def create_cow_snapshot(snapshot_path)
+        data_files = @engine.respond_to?(:data_files) ? @engine.data_files : []
+        raise SnapshotError, "Engine returned no data files" if data_files.empty?
+        data_files.each do |file|
+          dest = File.join(snapshot_path, File.basename(file))
+          begin
+            File.link(file, dest)
+          rescue Errno::EEXIST
+            raise
+          rescue SystemCallError
+            FileUtils.cp(file, dest)
+          end
+          raise SnapshotError, "Snapshot file was not created: #{dest}" unless File.file?(dest)
+        end
+      end
+
+      def restore_cow_snapshot(snapshot_path)
+        metadata = JSON.parse(File.read(File.join(snapshot_path, "snapshot.json")), symbolize_names: true)
+        verify_snapshot!(snapshot_path, metadata)
+        data_files = snapshot_files(snapshot_path)
+        data_files.each do |file|
+          dest = File.join(@engine.data_dir, File.basename(file))
+          FileUtils.cp(file, dest)
+          raise SnapshotError, "Snapshot restore failed: #{dest}" unless FileUtils.compare_file(file, dest)
+        end
+        @engine.reload_from_disk if @engine.respond_to?(:reload_from_disk)
+      end
+
+      def snapshot_files(snapshot_path)
+        Dir.glob(File.join(snapshot_path, "*")).select { |file| File.file?(file) && File.basename(file) != "snapshot.json" }
+      end
+
+      def verify_snapshot!(snapshot_path, metadata)
+        files = metadata[:files]
+        raise SnapshotError, "Snapshot has no file manifest" unless files.is_a?(Hash) && files.any?
+        files.each do |name, checksum|
+          file = File.join(snapshot_path, name.to_s)
+          unless File.file?(file) && Digest::SHA256.file(file).hexdigest == checksum
+            raise SnapshotError, "Snapshot checksum mismatch: #{name}"
+          end
+        end
+      end
+
+      def calculate_snapshot_size(snapshot_path)
+        Dir.glob(File.join(snapshot_path, "**/*")).sum do |f|
+          File.size(f) if File.file?(f)
+        end || 0
+      end
+
+      def clean_old_snapshots
+        snapshots = list_snapshots
+
+        # Remove by retention days
+        cutoff = Time.now - @retention_days * 24 * 60 * 60
+        snapshots.select { |s| Time.parse(s[:created_at]) < cutoff }.each do |s|
+          delete_snapshot(s[:name])
+        end
+
+        # Remove by count
+        snapshots = list_snapshots
+        if snapshots.size > @max_snapshots
+          snapshots.last(snapshots.size - @max_snapshots).each do |s|
+            delete_snapshot(s[:name])
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rubydb/backup/verification.rb

@@ -0,0 +1,279 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "time"
+require "json"
+require "digest"
+require "monitor"
+require "tmpdir"
+require_relative "restore"
+require_relative "../storage/engine"
+
+module RubyDB
+  module Backup
+    # Verification - Verifies backup integrity
+    class Verification
+      attr_reader :stats
+
+      def initialize(config = {})
+        @config = config
+        @verification_dir = config[:verification_dir] || "verifications"
+        @stats = {
+          verifications: 0,
+          successful: 0,
+          failed: 0,
+          total_time_ms: 0,
+          avg_time_ms: 0,
+          last_verification: nil,
+          errors: 0
+        }
+        @lock = Monitor.new
+        @cache = {}
+
+        FileUtils.mkdir_p(@verification_dir)
+      end
+
+      def verify_backup(backup_path, options = {})
+        @lock.synchronize do
+          start_time = Time.now
+          @stats[:verifications] += 1
+
+          unless Dir.exist?(backup_path)
+            return { success: false, error: "Backup path does not exist" }
+          end
+
+          manifest_path = File.join(backup_path, "manifest.json")
+          unless File.exist?(manifest_path)
+            return { success: false, error: "Manifest not found" }
+          end
+
+          begin
+            metadata = JSON.parse(File.read(manifest_path), symbolize_names: true)
+
+            results = {
+              manifest_valid: true,
+              files_valid: true,
+              checksums_valid: true,
+              database_valid: true,
+              details: {}
+            }
+
+            # Verify manifest
+            results[:details][:manifest] = verify_manifest(metadata)
+
+            # Verify files
+            files_result = verify_files(backup_path, metadata[:files] || [])
+            results[:files_valid] = files_result[:valid]
+            results[:details][:files] = files_result
+
+            # Verify checksums
+            if metadata[:checksum]
+              checksum_result = verify_checksums(backup_path, metadata[:checksum])
+              results[:checksums_valid] = checksum_result[:valid]
+              results[:details][:checksums] = checksum_result
+            end
+
+            # Verify database consistency (restore to temp and verify)
+            if options[:verify_database] != false
+              db_result = verify_database(backup_path, metadata)
+              results[:database_valid] = db_result[:valid]
+              results[:details][:database] = db_result
+            end
+
+            success = results[:manifest_valid] && results[:files_valid] &&
+                      results[:checksums_valid] && results[:database_valid]
+
+            # Save verification result
+            save_verification_result(backup_path, results)
+
+            elapsed_ms = (Time.now - start_time) * 1000
+            @stats[:total_time_ms] += elapsed_ms
+            @stats[:avg_time_ms] = @stats[:total_time_ms] / @stats[:verifications]
+
+            if success
+              @stats[:successful] += 1
+            else
+              @stats[:failed] += 1
+            end
+
+            @stats[:last_verification] = Time.now
+
+            { success: success, results: results, elapsed_ms: elapsed_ms }
+
+          rescue => e
+            @stats[:errors] += 1
+            @stats[:failed] += 1
+            { success: false, error: e.message }
+          end
+        end
+      end
+
+      def verify_restore(backup_path, options = {})
+        @lock.synchronize do
+          # Create temporary restore destination
+          temp_dir = File.join(@verification_dir, "temp_restore_#{Time.now.to_i}")
+          FileUtils.mkdir_p(temp_dir)
+
+          begin
+            manifest_path = File.join(backup_path, "manifest.json")
+            metadata = JSON.parse(File.read(manifest_path), symbolize_names: true)
+            # Restore to temporary location
+            restore = Restore.new(nil, @config)
+            restore_result = restore.restore(backup_path, destination: temp_dir)
+
+            unless restore_result[:success]
+              return { success: false, error: "Restore failed: #{restore_result[:error]}" }
+            end
+
+            database_path = Dir.glob(File.join(temp_dir, "*.rdb")).first
+            return { success: false, error: "Restored database file not found" } unless database_path
+
+            # Verify that the extracted files can be opened by the real engine,
+            # not merely that their checksums and names look correct.
+            engine = Storage::Engine.new(database_path, auto_cleanup: false, auto_vacuum: false)
+            actual_tables = engine.list_tables.map(&:to_s).sort
+            expected_tables = Array(metadata[:tables]).map(&:to_s).sort
+            unless actual_tables == expected_tables
+              return { success: false, error: "Restored catalog mismatch", expected_tables: expected_tables, actual_tables: actual_tables }
+            end
+            row_counts = actual_tables.to_h do |table|
+              [table, engine.table_row_count(table)]
+            end
+            engine.close
+            {
+              success: true,
+              restored_path: temp_dir,
+              tables: actual_tables,
+              row_counts: row_counts,
+              message: "Restore verification successful"
+            }
+
+          rescue => e
+            { success: false, error: e.message }
+
+          ensure
+            engine&.close if defined?(engine) && engine&.open?
+            # Clean up temp directory
+            FileUtils.rm_rf(temp_dir) if Dir.exist?(temp_dir)
+          end
+        end
+      end
+
+      def verify_chain(base_backup, incrementals, options = {})
+        @lock.synchronize do
+          start_time = Time.now
+          results = []
+
+          # Verify base backup
+          base_result = verify_backup(base_backup, options)
+          results << { backup: base_backup, result: base_result }
+
+          # Verify each incremental
+          incrementals.each do |inc|
+            inc_result = verify_backup(inc, options)
+            results << { backup: inc, result: inc_result }
+          end
+
+          # Verify chain integrity
+          chain_valid = results.all? { |r| r[:result][:success] }
+
+          {
+            success: chain_valid,
+            results: results,
+            elapsed_ms: (Time.now - start_time) * 1000
+          }
+        end
+      end
+
+      def stats
+        @lock.synchronize do
+          @stats.merge({
+            verification_dir: @verification_dir,
+            cache_size: @cache.size
+          })
+        end
+      end
+
+      private
+
+      def verify_manifest(metadata)
+        required_fields = [:name, :type, :created_at, :files]
+        missing = required_fields - metadata.keys
+
+        if missing.any?
+          { valid: false, error: "Missing required fields: #{missing.join(', ')}" }
+        else
+          { valid: true }
+        end
+      end
+
+      def verify_files(backup_path, files)
+        missing = []
+        corrupted = []
+
+        files.each do |file|
+          file_path = File.join(backup_path, file)
+          unless File.exist?(file_path)
+            missing << file
+            next
+          end
+
+          if File.size(file_path) == 0
+            corrupted << file
+          end
+        end
+
+        if missing.any?
+          { valid: false, missing: missing, message: "Missing files: #{missing.join(', ')}" }
+        elsif corrupted.any?
+          { valid: false, corrupted: corrupted, message: "Corrupted files: #{corrupted.join(', ')}" }
+        else
+          { valid: true, file_count: files.size }
+        end
+      end
+
+      def verify_checksums(backup_path, expected_checksum)
+        digest = Digest::SHA256.new
+        Dir.glob(File.join(backup_path, "**/*")).select { |path| File.file?(path) }.sort.each do |path|
+          relative = path.delete_prefix("#{backup_path}#{File::SEPARATOR}")
+          next if relative == "manifest.json"
+          digest.update(relative)
+          digest.update(File.binread(path))
+        end
+        actual = digest.hexdigest
+        { valid: actual == expected_checksum, expected: expected_checksum, actual: actual }
+      end
+
+      def verify_database(backup_path, metadata)
+        temp_dir = Dir.mktmpdir("rubydb_verify_")
+        begin
+          restore_result = Restore.new(nil, @config).restore(backup_path, destination: temp_dir)
+          return { valid: false, error: restore_result[:error] } unless restore_result[:success]
+
+          data_path = Dir.glob(File.join(temp_dir, "*.rdb")).first
+          return { valid: false, error: "Restored database file not found" } unless data_path
+
+          engine = Storage::Engine.new(data_path, auto_vacuum: false)
+          actual_tables = engine.list_tables.map(&:to_s).sort
+          expected_tables = Array(metadata[:tables]).map(&:to_s).sort
+          valid = actual_tables == expected_tables
+          { valid: valid, expected_tables: expected_tables, actual_tables: actual_tables }
+        rescue StandardError => e
+          { valid: false, error: e.message }
+        ensure
+          engine&.close
+          FileUtils.rm_rf(temp_dir) if Dir.exist?(temp_dir)
+        end
+      end
+
+      def save_verification_result(backup_path, results)
+        result_path = File.join(@verification_dir, "#{File.basename(backup_path)}_verify.json")
+        File.write(result_path, JSON.generate({
+          backup: File.basename(backup_path),
+          timestamp: Time.now.iso8601,
+          results: results
+        }))
+      end
+    end
+  end
+end