rubydb 0.1.0

data/lib/rubydb/storage/buffer_pool.rb

@@ -0,0 +1,155 @@
+# frozen_string_literal: true
+
+module RubyDB
+  module Storage
+    # BufferPool - Caches pages in memory
+    class BufferPool
+      attr_reader :size, :hit_count, :miss_count
+
+      def initialize(size = Constants::DEFAULT_BUFFER_POOL_SIZE)
+        @size = size
+        @frames = {}
+        @lru_list = []
+        @hit_count = 0
+        @miss_count = 0
+        @lock = Mutex.new
+        @page_manager = nil
+      end
+
+      def set_page_manager(page_manager)
+        @page_manager = page_manager
+      end
+
+      def get_page(page_number)
+        @lock.synchronize do
+          if @frames.key?(page_number)
+            @hit_count += 1
+            touch_page(page_number)
+            return @frames[page_number]
+          end
+
+          @miss_count += 1
+
+          # Evict if necessary
+          if @frames.size >= @size
+            evict_page
+          end
+
+          # Load from disk
+          page = @page_manager.get_page(page_number)
+          frame = BufferFrame.new(page)
+
+          @frames[page_number] = frame
+          @lru_list.unshift(page_number)
+
+          frame
+        end
+      end
+
+      def write_page(page)
+        @lock.synchronize do
+          if @frames.key?(page.page_number)
+            frame = @frames[page.page_number]
+            frame.dirty = true
+            frame.page = page
+            touch_page(page.page_number)
+          else
+            # Write directly to disk
+            @page_manager.write_page(page)
+          end
+        end
+      end
+
+      def flush_all
+        @lock.synchronize do
+          @frames.each do |page_number, frame|
+            if frame.dirty
+              @page_manager.write_page(frame.page)
+              frame.dirty = false
+            end
+          end
+        end
+      end
+
+      def flush_page(page_number)
+        @lock.synchronize do
+          if @frames.key?(page_number) && @frames[page_number].dirty
+            @page_manager.write_page(@frames[page_number].page)
+            @frames[page_number].dirty = false
+          end
+        end
+      end
+
+      def remove_page(page_number)
+        @lock.synchronize do
+          @frames.delete(page_number)
+          @lru_list.delete(page_number)
+        end
+      end
+
+      def clear
+        @lock.synchronize do
+          flush_all
+          @frames.clear
+          @lru_list.clear
+        end
+      end
+
+      def size
+        @frames.size
+      end
+
+      def hit_rate
+        total = @hit_count + @miss_count
+        return 0.0 if total == 0
+        @hit_count.to_f / total
+      end
+
+      def stats
+        {
+          size: @frames.size,
+          max_size: @size,
+          hit_count: @hit_count,
+          miss_count: @miss_count,
+          hit_rate: hit_rate
+        }
+      end
+
+      private
+
+      def touch_page(page_number)
+        @lru_list.delete(page_number)
+        @lru_list.unshift(page_number)
+      end
+
+      def evict_page
+        # Find a clean page to evict (LRU)
+        evict_candidates = @lru_list.reverse
+
+        evict_candidates.each do |page_number|
+          frame = @frames[page_number]
+          next if frame.dirty
+
+          # Evict this page
+          @frames.delete(page_number)
+          @lru_list.delete(page_number)
+          return true
+        end
+
+        # If all pages are dirty, force flush the least recently used
+        if @lru_list.any?
+          page_number = @lru_list.last
+          frame = @frames[page_number]
+          @page_manager.write_page(frame.page)
+          frame.dirty = false
+
+          @frames.delete(page_number)
+          @lru_list.delete(page_number)
+          return true
+        end
+
+        false
+      end
+    end
+  end
+end

data/lib/rubydb/storage/database_lock.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+require "monitor"
+require "fileutils"
+
+module RubyDB
+  module Storage
+    # Owns an embedded database path for exactly one engine process. RubyDB's
+    # page cache, metadata sidecars, WAL and MVCC files are coordinated inside
+    # one engine instance; opening them from independent processes is unsafe.
+    class DatabaseLock
+      @registry_lock = Monitor.new
+      @open_paths = {}
+
+      class << self
+        attr_reader :registry_lock, :open_paths
+      end
+
+      attr_reader :path
+
+      def initialize(database_path)
+        expanded = File.expand_path(database_path)
+        FileUtils.mkdir_p(File.dirname(expanded))
+        @database_path = if File.exist?(expanded)
+          File.realpath(expanded)
+        else
+          File.join(File.realpath(File.dirname(expanded)), File.basename(expanded))
+        end
+        @path = "#{@database_path}.lock"
+        @file = nil
+      end
+
+      def acquire!
+        self.class.registry_lock.synchronize do
+          raise DatabaseError, "Database '#{@database_path}' is already open in this process" if self.class.open_paths.key?(@database_path)
+
+          FileUtils.mkdir_p(File.dirname(@path))
+          @file = File.open(@path, File::RDWR | File::CREAT, 0o600)
+          unless @file.flock(File::LOCK_EX | File::LOCK_NB)
+            close_file
+            raise DatabaseError, "Database '#{@database_path}' is already open by another process"
+          end
+
+          self.class.open_paths[@database_path] = self
+          true
+        end
+      rescue Errno::EACCES, Errno::EAGAIN
+        close_file
+        raise DatabaseError, "Database '#{@database_path}' is already open by another process"
+      rescue SystemCallError => error
+        close_file
+        raise DatabaseError, "Unable to lock database '#{@database_path}': #{error.message}"
+      end
+
+      def release
+        self.class.registry_lock.synchronize do
+          return false unless self.class.open_paths[@database_path].equal?(self)
+
+          self.class.open_paths.delete(@database_path)
+          @file&.flock(File::LOCK_UN)
+          close_file
+          true
+        end
+      end
+
+      private
+
+      def close_file
+        @file&.close unless @file&.closed?
+        @file = nil
+      end
+    end
+  end
+end

data/lib/rubydb/storage/deserializer.rb

@@ -0,0 +1,342 @@
+# frozen_string_literal: true
+
+require "json"
+require "bigdecimal"
+require "date"
+require "time"
+
+module RubyDB
+  module Storage
+    # Deserializer - Converts binary format to Ruby objects with full type support
+    class Deserializer
+      # Deserialize a single value based on its type
+      def self.deserialize(data, type, options = {})
+        return nil if data.nil? || data.empty?
+        
+        type_obj = Types::TypeRegistry.lookup(type)
+        type_obj.deserialize(data)
+      rescue => e
+        raise CorruptionError, "Failed to deserialize value of type #{type}: #{e.message}"
+      end
+
+      # Deserialize a full row from binary data
+      def self.deserialize_row(data, columns, options = {})
+        return {} if data.nil? || data.empty?
+        
+        row = {}
+        null_bitmap = options[:null_bitmap]
+        bitmap_size = null_bitmap ? (columns.size + 7) / 8 : 0
+        bitmap = null_bitmap ? data.byteslice(0, bitmap_size).bytes : []
+        offset = bitmap_size
+        fixed_sizes = { integer: 4, bigint: 8, smallint: 2, float: 8, boolean: 1, date: 8, time: 8, timestamp: 8 }
+        variable_length_prefixes = options[:variable_length_prefixes]
+        
+        columns.each_with_index do |col, idx|
+          begin
+            col_type = col.type_class
+            col_name = col.name
+            has_default = col.has_default?
+            
+            # Check if fixed-size type
+            if fixed_sizes.key?(col_type)
+              length = fixed_sizes[col_type]
+              if offset + length <= data.bytesize
+                value_data = data[offset, length]
+                offset += length
+                row[col_name] = bitmap[col_index = idx / 8] && (bitmap[col_index] & (1 << (idx % 8))) != 0 ? nil : deserialize(value_data, col_type)
+              else
+                row[col_name] = has_default ? col.default : nil
+              end
+            else
+              if variable_length_prefixes
+                raise CorruptionError, "Missing length prefix for column '#{col_name}'" if offset + 4 > data.bytesize
+
+                length = data.byteslice(offset, 4).unpack1("N")
+                offset += 4
+                raise CorruptionError, "Invalid length for column '#{col_name}'" if offset + length > data.bytesize
+
+                value_data = data.byteslice(offset, length)
+                offset += length
+                row[col_name] = if bitmap[idx / 8] && (bitmap[idx / 8] & (1 << (idx % 8))) != 0
+                                  nil
+                                else
+                                  deserialize(value_data, col_type)
+                                end
+                row[col_name] = col.default if row[col_name].nil? && has_default
+              # Legacy records did not store variable-length field sizes.
+              # Only the final variable-width column can be recovered safely.
+              elsif idx == columns.size - 1
+                value_data = data[offset..-1]
+                row[col_name] = if bitmap[idx / 8] && (bitmap[idx / 8] & (1 << (idx % 8))) != 0
+                                  nil
+                                elsif value_data.bytesize > 0
+                                  deserialize(value_data, col_type)
+                                end
+                row[col_name] = col.default if row[col_name].nil? && has_default
+              else
+                row[col_name] = has_default ? col.default : nil
+              end
+            end
+          rescue => e
+            row[col_name] = col.default if col.has_default?
+            row[col_name] = nil if col.nullable?
+            if row[col_name].nil? && !col.nullable? && !col.has_default?
+              raise CorruptionError, "Failed to deserialize column '#{col_name}': #{e.message}"
+            end
+          end
+        end
+        
+        row
+      end
+
+      def self.variable_length_type?(type)
+        !%i[integer bigint smallint float boolean date time timestamp].include?(type.to_sym)
+      end
+
+      # Deserialize a row with a header
+      def self.deserialize_row_with_header(data, columns)
+        return nil if data.nil? || data.empty?
+        
+        # Parse header
+        header_data = deserialize_header(data)
+        
+        # Extract row data (after header)
+        row_data = data[header_data[:header_size]..-1]
+        
+        # Deserialize row
+        row = deserialize_row(row_data, columns, has_length_prefix: true)
+        
+        # Add header info to row
+        row[:_row_id] = header_data[:row_id]
+        row[:_timestamp] = header_data[:timestamp]
+        row[:_version] = header_data[:version]
+        
+        row
+      end
+
+      # Deserialize a record header
+      def self.deserialize_header(data)
+        return { header_size: 0 } if data.nil? || data.bytesize < 16
+        
+        header_size = 0
+        
+        # Parse row header
+        row_id, version, timestamp, flags, column_count = data.unpack("Q>Q>Q>C>S")
+        header_size = 8 + 8 + 8 + 1 + 2  # row_id(8) + version(8) + timestamp(8) + flags(1) + column_count(2)
+        
+        {
+          row_id: row_id,
+          version: version,
+          timestamp: timestamp,
+          flags: flags,
+          column_count: column_count,
+          header_size: header_size
+        }
+      rescue => e
+        { header_size: 0, error: e.message }
+      end
+
+      # Deserialize table metadata
+      def self.deserialize_metadata(data)
+        return {} if data.nil? || data.empty?
+        
+        begin
+          JSON.parse(data.force_encoding("UTF-8"), symbolize_names: true)
+        rescue JSON::ParserError => e
+          # Try to handle different encodings
+          begin
+            JSON.parse(data.force_encoding("ASCII-8BIT"), symbolize_names: true)
+          rescue JSON::ParserError
+            {}
+          end
+        end
+      end
+
+      # Deserialize page header
+      def self.deserialize_page_header(data)
+        return nil if data.nil? || data.bytesize < PageHeader::SIZE
+        
+        PageHeader.deserialize(data)
+      end
+
+      # Deserialize a record from a page
+      def self.deserialize_record(page, offset, columns)
+        return nil if page.nil? || offset.nil?
+        
+        # Read record header
+        record_header = page.read(offset, 16)
+        return nil if record_header.nil? || record_header.bytesize < 16
+        
+        # Parse record header
+        record_id, record_size, flags, column_count = record_header.unpack("Q>L>C>S")
+        
+        # Validate record size
+        if record_size <= 0 || record_size > 65535
+          raise CorruptionError, "Invalid record size: #{record_size}"
+        end
+        
+        # Read record data
+        record_data = page.read(offset + 16, record_size)
+        return nil if record_data.nil?
+        
+        # Deserialize row
+        row = deserialize_row(record_data, columns, has_length_prefix: true)
+        
+        # Add record metadata
+        row[:_record_id] = record_id
+        row[:_record_size] = record_size
+        row[:_flags] = flags
+        
+        row
+      end
+
+      # Deserialize multiple records from a page
+      def self.deserialize_records(page, columns, max_records = nil)
+        records = []
+        offset = PageHeader::SIZE
+        count = 0
+        
+        while offset < page.header.data_end
+          break if max_records && count >= max_records
+          
+          record = deserialize_record(page, offset, columns)
+          break if record.nil?
+          
+          records << record
+          offset += 16 + record[:_record_size]
+          count += 1
+        end
+        
+        records
+      end
+
+      # Deserialize a tuple from binary data
+      def self.deserialize_tuple(data, columns)
+        return nil if data.nil? || data.empty?
+        
+        tuple = Tuple.new(columns)
+        tuple.deserialize(data)
+        tuple
+      end
+
+      # Deserialize a value with type and length prefix
+      def self.deserialize_value_with_length(data, offset, type)
+        return [nil, offset] if data.nil? || offset >= data.bytesize
+        
+        # Read length prefix (2 bytes)
+        if offset + 2 <= data.bytesize
+          length = data[offset, 2].unpack("S").first
+          offset += 2
+          
+          if length == 0xFFFF  # NULL marker
+            return [nil, offset]
+          elsif length > 0 && offset + length <= data.bytesize
+            value_data = data[offset, length]
+            offset += length
+            return [deserialize(value_data, type), offset]
+          end
+        end
+        
+        [nil, offset]
+      end
+
+      # Deserialize an array of values
+      def self.deserialize_array(data, type, count)
+        return [] if data.nil? || data.empty? || count <= 0
+        
+        values = []
+        offset = 0
+        
+        count.times do
+          value, new_offset = deserialize_value_with_length(data, offset, type)
+          break if new_offset == offset  # No progress
+          values << value
+          offset = new_offset
+        end
+        
+        values
+      end
+
+      # Deserialize a JSON object from binary
+      def self.deserialize_json(data)
+        return nil if data.nil? || data.empty?
+        
+        begin
+          JSON.parse(data.force_encoding("UTF-8"))
+        rescue JSON::ParserError
+          nil
+        end
+      end
+
+      # Deserialize a bitmap
+      def self.deserialize_bitmap(data, bit_count)
+        return [] if data.nil? || data.empty? || bit_count <= 0
+        
+        bitmap = []
+        data.bytes.each_with_index do |byte, byte_idx|
+          (0..7).each do |bit_idx|
+            bit_pos = byte_idx * 8 + bit_idx
+            break if bit_pos >= bit_count
+            bitmap << ((byte >> bit_idx) & 1) == 1
+          end
+        end
+        
+        bitmap
+      end
+
+      # Deserialize variable-length data with length prefix
+      def self.deserialize_variable(data, offset)
+        return [nil, offset] if data.nil? || offset >= data.bytesize
+        
+        if offset + 2 <= data.bytesize
+          length = data[offset, 2].unpack("S").first
+          offset += 2
+          
+          if length == 0xFFFF
+            return [nil, offset]
+          elsif length > 0 && offset + length <= data.bytesize
+            value_data = data[offset, length]
+            offset += length
+            return [value_data, offset]
+          end
+        end
+        
+        [nil, offset]
+      end
+
+      private
+
+      # Get column length from data
+      def self.get_column_length(data, offset, col, columns, idx)
+        col_type = col.type_class
+        
+        # Fixed length types - return their serialized size
+        fixed_sizes = {
+          integer: 8,
+          bigint: 8,
+          smallint: 2,
+          float: 8,
+          boolean: 1,
+          date: 8,
+          time: 8,
+          timestamp: 8
+        }
+        
+        if fixed_sizes.key?(col_type)
+          return fixed_sizes[col_type]
+        else
+          # Variable length (text, json, etc.) - read until end of data or next field boundary
+          # A final variable-width column consumes the remaining record bytes
+          # for compatibility with records that omit a trailing length prefix.
+          if idx == columns.size - 1
+            return data.bytesize - offset
+          else
+            # For middle columns, we need a length prefix or a delimiter
+            # For simplicity, use remaining data if we don't have more info
+            return nil
+          end
+        end
+      end
+    end
+  end
+end