rubydb 0.1.0

data/lib/rubydb/mvcc/version_store.rb

@@ -0,0 +1,398 @@
+# frozen_string_literal: true
+
+require "set"
+require "json"
+require "fileutils"
+require "time"
+require "monitor"
+require_relative "version"
+
+module RubyDB
+  module MVCC
+    # VersionStore - Stores and manages all versions of rows
+    class VersionStore
+      attr_reader :stats, :version_count, :row_count
+
+      def initialize(config = {})
+        @versions = {}  # row_id => [versions]
+        @version_map = {}  # version_id => version
+        @version_keys = {} # version_id => storage key
+        @active_versions = Set.new
+        @stats = {
+          versions_created: 0,
+          versions_committed: 0,
+          versions_aborted: 0,
+          versions_deleted: 0,
+          versions_pruned: 0,
+          cache_hits: 0,
+          cache_misses: 0
+        }
+        @version_count = 0
+        @row_count = 0
+        @max_versions_per_row = config[:max_versions_per_row] || 100
+        @lock = Monitor.new
+        @persistence_path = config[:persistence_path]
+        @dirty = false
+        load if @persistence_path && File.exist?(@persistence_path)
+      end
+
+      def create_version(row_id, data, transaction_id, key: row_id)
+        @lock.synchronize do
+          # Get current latest version
+          versions = @versions[key] || []
+          prev_version_id = versions.last&.version_id
+
+          # Create new version
+          version = Version.new(row_id, data, transaction_id, prev_version_id)
+
+          # Store version
+          @versions[key] ||= []
+          @versions[key] << version
+          @version_map[version.version_id] = version
+          @version_keys[version.version_id] = key
+
+          # Update active versions
+          @active_versions.add(version.version_id)
+
+          @version_count += 1
+          @row_count = @versions.size
+          @stats[:versions_created] += 1
+          @dirty = true
+
+          version
+        end
+      end
+
+      def commit_version(version, commit_id = nil)
+        @lock.synchronize do
+          version.commit(commit_id)
+          @active_versions.delete(version.version_id)
+          @stats[:versions_committed] += 1
+          @dirty = true
+        end
+      end
+
+      def abort_version(version)
+        @lock.synchronize do
+          version.abort
+          @active_versions.delete(version.version_id)
+
+          # Remove from version chain if it was the latest
+          key = @version_keys[version.version_id] || version.row_id
+          versions = @versions[key]
+          if versions && versions.last&.version_id == version.version_id
+            versions.pop
+            @version_map.delete(version.version_id)
+            @version_keys.delete(version.version_id)
+          end
+
+          @stats[:versions_aborted] += 1
+          @dirty = true
+        end
+      end
+
+      def get_latest_version(row_id, transaction_id = nil, snapshot = nil, key: row_id)
+        @lock.synchronize do
+          # Check cache for snapshot
+          cache_key = "#{row_id}:#{transaction_id}:#{snapshot&.id}"
+
+          versions = @versions[key]
+          return nil unless versions
+
+          # Get the latest committed version visible to transaction
+          versions.reverse_each do |version|
+            if snapshot
+              if version.visible_to?(transaction_id, snapshot)
+                return nil if version.is_deleted
+                @stats[:cache_hits] += 1
+                return version
+              end
+            elsif version.is_committed
+              # A nil transaction ID means "latest committed". Otherwise a
+              # version is visible only when its commit ID is not newer than
+              # the reader's transaction ID.
+              if transaction_id.nil? || version.commit_id.nil? || version.commit_id <= transaction_id
+                @stats[:cache_hits] += 1
+                return version.is_deleted ? nil : version
+              end
+            end
+          end
+
+          @stats[:cache_misses] += 1
+          nil
+        end
+      end
+
+      def get_version_at_time(row_id, time)
+        @lock.synchronize do
+          versions = @versions[row_id]
+          return nil unless versions
+
+          # Find version that was active at the given time
+          versions.reverse_each do |version|
+            if version.created_at <= time
+              return version
+            end
+          end
+
+          nil
+        end
+      end
+
+      def get_all_versions(row_id)
+        @lock.synchronize do
+          @versions[row_id] || []
+        end
+      end
+
+      def keys(prefix = nil)
+        @lock.synchronize do
+          values = @versions.keys
+          prefix ? values.select { |key| key.to_s.start_with?(prefix.to_s) } : values
+        end
+      end
+
+      def validate_serializable!(snapshot, read_keys, write_keys, read_predicates: [])
+        @lock.synchronize do
+          dependencies = Set.new(read_keys.to_a + write_keys.to_a)
+          @versions.each do |key, versions|
+            predicate_conflict = read_predicates.any? { |predicate| key.to_s.start_with?(predicate.to_s) }
+            next unless dependencies.include?(key) || predicate_conflict
+
+            versions.each do |version|
+              next unless version.is_committed
+              next unless version.commit_id && version.commit_id.to_i > snapshot.transaction_id.to_i
+              next if snapshot.committed_transactions.include?(version.commit_id)
+
+              raise RubyDB::DatabaseError,
+                "serialization failure: concurrent commit conflicts with #{key.inspect}"
+            end
+          end
+          true
+        end
+      end
+
+      def get_version(version_id)
+        @lock.synchronize do
+          @version_map[version_id]
+        end
+      end
+
+      def delete_version(version)
+        @lock.synchronize do
+          version.mark_deleted
+          @active_versions.delete(version.version_id)
+          @stats[:versions_deleted] += 1
+          @dirty = true
+        end
+      end
+
+      def prune_versions(row_id, keep_count = nil)
+        @lock.synchronize do
+          versions = @versions[row_id]
+          return 0 unless versions
+
+          keep_count ||= @max_versions_per_row
+          pruned = 0
+
+          # Remove old versions (keeping the most recent ones)
+          while versions.size > keep_count
+            old_version = versions.shift
+            next unless old_version
+
+            # Only prune committed versions that are not active
+            if old_version.is_committed && !@active_versions.include?(old_version.version_id)
+              @version_map.delete(old_version.version_id)
+              @version_keys.delete(old_version.version_id)
+              @stats[:versions_pruned] += 1
+              pruned += 1
+            else
+              # Can't prune active versions, move to end
+              versions << old_version
+              break
+            end
+          end
+
+          @dirty = true if pruned > 0
+          pruned
+        end
+      end
+
+      def active_version_count
+        @active_versions.size
+      end
+
+      def prune_all(max_versions = nil)
+        @lock.synchronize do
+          total_pruned = 0
+          @versions.keys.each do |row_id|
+            versions = @versions[row_id]
+            keep_count = max_versions || @max_versions_per_row
+            while versions && versions.size > keep_count
+              version = versions.first
+              break unless version.is_committed && !@active_versions.include?(version.version_id)
+
+              versions.shift
+              @version_map.delete(version.version_id)
+              @version_keys.delete(version.version_id)
+              @stats[:versions_pruned] += 1
+              total_pruned += 1
+            end
+          end
+          @dirty = true if total_pruned > 0
+          total_pruned
+        end
+      end
+
+      def vacuum(min_active_transaction_id: nil)
+        @lock.synchronize do
+          # Remove versions that are no longer needed
+          removed = 0
+          @versions.each do |row_id, versions|
+            # Keep the newest version as the base visible version. Older
+            # committed history may be removed only before the oldest active
+            # transaction; active and uncommitted versions are never removed.
+            newest = versions.last
+            protected_base = versions.reverse.find(&:is_committed)
+            versions.delete_if do |version|
+              removable = version.is_aborted ||
+                (version != newest && version != protected_base && version.is_committed &&
+                 min_active_transaction_id && version.commit_id &&
+                 version.commit_id < min_active_transaction_id)
+              if removable && !@active_versions.include?(version.version_id)
+                @version_map.delete(version.version_id)
+                removed += 1
+                true
+              else
+                false
+              end
+            end
+
+            # Remove empty arrays
+            if versions.empty?
+              @versions.delete(row_id)
+            end
+          end
+
+          @row_count = @versions.size
+          @stats[:versions_pruned] += removed
+          @dirty = true if removed > 0
+          removed
+        end
+      end
+
+      def serialize
+        @lock.synchronize do
+          data = {
+            versions: {},
+            version_count: @version_count,
+            row_count: @row_count,
+            active_versions: @active_versions.to_a,
+            timestamp: Time.now.iso8601
+          }
+
+          @versions.each do |row_key, versions|
+            data[:versions][row_key.to_s] = versions.map do |v|
+              {
+                row_id: v.row_id,
+                version_id: v.version_id,
+                created_at: v.created_at.iso8601,
+                transaction_id: v.transaction_id,
+                prev_version_id: v.prev_version_id,
+                is_committed: v.is_committed,
+                is_aborted: v.is_aborted,
+                is_deleted: v.is_deleted,
+                commit_id: v.commit_id,
+                visibility: v.visibility,
+                data: v.data
+              }
+            end
+          end
+
+          JSON.generate(data)
+        end
+      end
+
+      def deserialize(json_data)
+        @lock.synchronize do
+          data = JSON.parse(json_data, symbolize_names: true)
+
+          @versions.clear
+          @version_map.clear
+          @version_keys.clear
+          @active_versions.clear
+
+          data[:versions].each do |row_key, versions_data|
+            row_key = row_key.to_s
+            row_key = row_key.to_i if row_key.match?(/\A\d+\z/)
+            @versions[row_key] = versions_data.map do |v_data|
+              version = Version.new(
+                v_data[:row_id],
+                v_data[:data],
+                v_data[:transaction_id],
+                v_data[:prev_version_id]
+              )
+              version.instance_variable_set(:@version_id, v_data[:version_id])
+              version.instance_variable_set(:@created_at, Time.parse(v_data[:created_at]))
+              version.instance_variable_set(:@is_committed, v_data[:is_committed])
+              version.instance_variable_set(:@is_aborted, v_data[:is_aborted])
+              version.instance_variable_set(:@is_deleted, v_data[:is_deleted])
+              version.instance_variable_set(:@commit_id, v_data[:commit_id])
+              version.instance_variable_set(:@visibility, v_data[:visibility].to_sym)
+
+              @version_map[version.version_id] = version
+              @version_keys[version.version_id] = row_key
+              version
+            end
+          end
+
+          @version_count = data[:version_count] || 0
+          @row_count = data[:row_count] || 0
+          @active_versions = Set.new(data[:active_versions] || [])
+          @dirty = false
+        end
+      end
+
+      def persist
+        return unless @persistence_path && @dirty
+
+        @lock.synchronize do
+          FileUtils.mkdir_p(File.dirname(@persistence_path))
+          temp_path = "#{@persistence_path}.tmp"
+          File.open(temp_path, "w") do |file|
+            file.write(serialize)
+            file.flush
+            file.fsync
+          end
+          File.rename(temp_path, @persistence_path)
+          @dirty = false
+        end
+      end
+
+      def load
+        return unless @persistence_path && File.exist?(@persistence_path)
+
+        @lock.synchronize do
+          json_data = File.read(@persistence_path)
+          deserialize(json_data)
+        end
+      end
+
+      def stats
+        @lock.synchronize do
+          @stats.merge({
+            version_count: @version_count,
+            row_count: @row_count,
+            active_versions: @active_versions.size,
+            rows_with_versions: @versions.size,
+            dirty: @dirty
+          })
+        end
+      end
+
+      def inspect
+        "#<VersionStore versions=#{@version_count} rows=#{@row_count} active=#{@active_versions.size}>"
+      end
+    end
+  end
+end

data/lib/rubydb/mvcc/visibility.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+require "set"
+
+module RubyDB
+  module MVCC
+    # Visibility - Determines visibility of versions to transactions
+    class Visibility
+      attr_reader :stats
+
+      def initialize
+        @visibility_cache = {}
+        @stats = {
+          cache_hits: 0,
+          cache_misses: 0,
+          visibility_checks: 0,
+          cache_evictions: 0
+        }
+        @cache_size = 10000
+        @lock = Mutex.new
+      end
+
+      def visible?(version, transaction_id, snapshot = nil)
+        @lock.synchronize do
+          @stats[:visibility_checks] += 1
+
+          # Check cache
+          cache_key = "#{version.version_id}:#{transaction_id}"
+          if @visibility_cache.key?(cache_key)
+            @stats[:cache_hits] += 1
+            return @visibility_cache[cache_key]
+          end
+
+          result = check_visibility(version, transaction_id, snapshot)
+
+          # Cache result
+          if @visibility_cache.size >= @cache_size
+            evict_oldest_cache_entry
+          end
+
+          @visibility_cache[cache_key] = result
+          @stats[:cache_misses] += 1
+
+          result
+        end
+      end
+
+      def check_visibility(version, transaction_id, snapshot = nil)
+        # Check if version is from the same transaction
+        return true if version.transaction_id == transaction_id
+
+        # Check snapshot
+        if snapshot
+          return snapshot.visible?(version, transaction_id)
+        end
+
+        # If not committed, invisible to others
+        return false unless version.is_committed
+
+        # Check if deleted
+        return false if version.is_deleted && version.visibility == :deleted
+
+        # Check commit visibility
+        if version.commit_id
+          return version.commit_id <= transaction_id
+        end
+
+        true
+      end
+
+      def clear_cache
+        @lock.synchronize do
+          @visibility_cache.clear
+          @stats[:cache_evictions] += 1
+        end
+      end
+
+      def cache_size
+        @visibility_cache.size
+      end
+
+      def stats
+        @lock.synchronize do
+          @stats.merge({
+            cache_size: @visibility_cache.size,
+            hit_rate: hit_rate
+          })
+        end
+      end
+
+      private
+
+      def hit_rate
+        total = @stats[:cache_hits] + @stats[:cache_misses]
+        return 0.0 if total == 0
+        (@stats[:cache_hits].to_f / total * 100).round(2)
+      end
+
+      def evict_oldest_cache_entry
+        # Remove first entry (oldest)
+        if @visibility_cache.any?
+          key = @visibility_cache.keys.first
+          @visibility_cache.delete(key)
+          @stats[:cache_evictions] += 1
+        end
+      end
+    end
+  end
+end

data/lib/rubydb/protocol/capabilities.rb

@@ -0,0 +1,125 @@
+# frozen_string_literal: true
+
+module RubyDB
+  module Protocol
+    class Capabilities
+      COMPRESSION = 0x0001
+      ENCRYPTION = 0x0002
+      PIPELINING = 0x0004
+      BATCHING = 0x0008
+      PREPARED_STATEMENTS = 0x0010
+      CURSORS = 0x0020
+      NOTIFICATIONS = 0x0040
+      LISTEN_NOTIFY = 0x0080
+      LARGE_OBJECTS = 0x0100
+      COPY = 0x0200
+      TRANSACTIONS = 0x0400
+      SAVEPOINTS = 0x0800
+      TWO_PHASE_COMMIT = 0x1000
+      SECURE_CONNECTION = 0x2000
+      LOAD_BALANCING = 0x4000
+      AUTO_RECONNECT = 0x8000
+
+      attr_reader :supported, :enabled, :negotiated, :features, :limits
+
+      def initialize
+        @supported = 0
+        @enabled = 0
+        @negotiated = 0
+        @limits = {}
+        @features = {}
+      end
+
+      def set_supported(flags)
+        @supported |= flags
+      end
+
+      def enable(flags)
+        return unless (@supported & flags) == flags
+        @enabled |= flags
+      end
+
+      def disable(flags)
+        @enabled &= ~flags
+      end
+
+      def has?(flags)
+        (@enabled & flags) == flags
+      end
+
+      def supported?(flags)
+        (@supported & flags) == flags
+      end
+
+      def negotiate(client_capabilities)
+        negotiated = client_capabilities & @supported
+        @negotiated = negotiated
+        @enabled = negotiated
+
+        @features[:compression] = has?(COMPRESSION)
+        @features[:encryption] = has?(ENCRYPTION)
+        @features[:pipelining] = has?(PIPELINING)
+        @features[:batching] = has?(BATCHING)
+        @features[:prepared_statements] = has?(PREPARED_STATEMENTS)
+        @features[:cursors] = has?(CURSORS)
+        @features[:notifications] = has?(NOTIFICATIONS)
+        @features[:listen_notify] = has?(LISTEN_NOTIFY)
+        @features[:large_objects] = has?(LARGE_OBJECTS)
+        @features[:copy] = has?(COPY)
+        @features[:transactions] = has?(TRANSACTIONS)
+        @features[:savepoints] = has?(SAVEPOINTS)
+        @features[:two_phase_commit] = has?(TWO_PHASE_COMMIT)
+        @features[:secure_connection] = has?(SECURE_CONNECTION)
+        @features[:load_balancing] = has?(LOAD_BALANCING)
+        @features[:auto_reconnect] = has?(AUTO_RECONNECT)
+
+        @negotiated
+      end
+
+      def set_limit(name, value)
+        @limits[name] = value
+      end
+
+      def get_limit(name)
+        @limits[name]
+      end
+
+      def to_hash
+        {
+          supported: @supported,
+          enabled: @enabled,
+          negotiated: @negotiated,
+          features: @features,
+          limits: @limits
+        }
+      end
+
+      def self.default_server
+        caps = new
+        caps.set_supported(
+          COMPRESSION | ENCRYPTION | PIPELINING | BATCHING |
+          PREPARED_STATEMENTS | CURSORS | NOTIFICATIONS |
+          LISTEN_NOTIFY | TRANSACTIONS | SAVEPOINTS |
+          TWO_PHASE_COMMIT | LOAD_BALANCING
+        )
+        caps.set_limit(:max_batch_size, 1000)
+        caps.set_limit(:max_rows, 100000)
+        caps.set_limit(:max_timeout, 300)
+        caps
+      end
+
+      def self.default_client
+        caps = new
+        caps.set_supported(
+          COMPRESSION | ENCRYPTION | PIPELINING | BATCHING |
+          PREPARED_STATEMENTS | CURSORS | TRANSACTIONS |
+          SAVEPOINTS | AUTO_RECONNECT
+        )
+        caps.set_limit(:batch_size, 100)
+        caps.set_limit(:max_rows, 10000)
+        caps.set_limit(:timeout, 30)
+        caps
+      end
+    end
+  end
+end