rubydb 0.1.0

data/lib/rubydb/storage/page_header.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module RubyDB
+  module Storage
+    # PageHeader - Metadata for a page
+    class PageHeader
+      SIZE = 64  # Fixed header size
+
+      attr_accessor :page_number, :page_size, :header_size, :data_end,
+                   :flags, :checksum, :version, :page_type, :next_page,
+                   :prev_page, :free_start, :free_end
+
+      def initialize
+        @page_number = 0
+        @page_size = Constants::DEFAULT_PAGE_SIZE
+        @header_size = SIZE
+        @data_end = SIZE
+        @flags = 0
+        @checksum = 0
+        @version = 1
+        @page_type = 0  # 0=normal, 1=index, 2=overflow, 3=free
+        @next_page = 0
+        @prev_page = 0
+        @free_start = SIZE
+        @free_end = @page_size
+      end
+
+      def flags=(value)
+        @flags = value
+      end
+
+      def page_type=(value)
+        @page_type = value
+      end
+
+      def serialize
+        [
+          @page_number, @page_size, @header_size, @data_end,
+          @flags, @checksum, @version, @page_type,
+          @next_page, @prev_page, @free_start, @free_end
+        ].pack("Q>Q>I>I>I>I>I>I>Q>Q>I>I")
+      end
+
+      def self.deserialize(data)
+        header = new
+        values = data.unpack("Q>Q>I>I>I>I>I>I>Q>Q>I>I")
+        header.page_number = values[0]
+        header.page_size = values[1]
+        header.header_size = values[2]
+        header.data_end = values[3]
+        header.flags = values[4]
+        header.checksum = values[5]
+        header.version = values[6]
+        header.page_type = values[7]
+        header.next_page = values[8]
+        header.prev_page = values[9]
+        header.free_start = values[10]
+        header.free_end = values[11]
+        header
+      end
+
+      def to_s
+        "PageHeader(page=#{@page_number}, type=#{@page_type}, data_end=#{@data_end})"
+      end
+    end
+  end
+end

data/lib/rubydb/storage/page_manager.rb

@@ -0,0 +1,131 @@
+# frozen_string_literal: true
+
+module RubyDB
+  module Storage
+    # PageManager - Manages page allocation and deallocation
+    class PageManager
+      attr_reader :file_manager
+
+      def initialize(file_manager)
+        @file_manager = file_manager
+        @free_list = []
+        @page_type_map = {}
+        @lock = Mutex.new
+        load_free_list
+      end
+
+      def allocate_page(page_type = 0)
+        @lock.synchronize do
+          page_number = nil
+
+          # Try to get from free list
+          if @free_list.any?
+            page_number = @free_list.pop
+            @page_type_map[page_number] = page_type
+            return page_number
+          end
+
+          # Allocate new page
+          page_number = @file_manager.num_pages
+          @file_manager.extend_file(1)
+
+          # Initialize page
+          page = Page.new(page_number, @file_manager.page_size)
+          page.header.page_type = page_type
+          page.write_header
+          @file_manager.write_page(page_number, page.data)
+
+          @page_type_map[page_number] = page_type
+          page_number
+        end
+      end
+
+      def free_page(page_number)
+        @lock.synchronize do
+          return unless page_exists?(page_number)
+
+          # Clear the page
+          page = Page.new(page_number, @file_manager.page_size)
+          page.header.page_type = 3  # Free page type
+          page.write_header
+          @file_manager.write_page(page_number, page.data)
+
+          @free_list << page_number
+          @page_type_map.delete(page_number)
+          true
+        end
+      end
+
+      def page_exists?(page_number)
+        page_number < @file_manager.num_pages
+      end
+
+      def page_type(page_number)
+        @page_type_map[page_number] || 0
+      end
+
+      def get_page(page_number)
+        @lock.synchronize do
+          raise StorageError, "Page #{page_number} does not exist" unless page_exists?(page_number)
+
+          data = @file_manager.read_page(page_number)
+          page = Page.new(page_number, @file_manager.page_size)
+          page.instance_variable_set(:@data, data)
+          page.read_header
+          page
+        end
+      end
+
+      def write_page(page)
+        @lock.synchronize do
+          raise StorageError, "Page #{page.page_number} does not exist" unless page_exists?(page.page_number)
+
+          page.write_header
+          @file_manager.write_page(page.page_number, page.data)
+          page.instance_variable_set(:@dirty, false)
+        end
+      end
+
+      def total_pages
+        @file_manager.num_pages
+      end
+
+      def free_pages
+        @free_list.size
+      end
+
+      def used_pages
+        total_pages - free_pages
+      end
+
+      def load_free_list
+        # Scan all pages to find free ones
+        @free_list = []
+        @page_type_map = {}
+
+        (0...@file_manager.num_pages).each do |page_number|
+          begin
+            data = @file_manager.read_page(page_number)
+            header = PageHeader.deserialize(data[0, PageHeader::SIZE])
+
+            if header.page_type == 3  # Free page
+              @free_list << page_number
+            else
+              @page_type_map[page_number] = header.page_type
+            end
+          rescue => e
+            # Ignore errors during free list loading
+          end
+        end
+
+        # Don't include page 0 (superblock)
+        @free_list.delete(0)
+        @free_list.sort!
+      end
+
+      def flush
+        @file_manager.sync
+      end
+    end
+  end
+end

data/lib/rubydb/storage/record.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module RubyDB
+  module Storage
+    # Record - A single record stored on a page
+    class Record
+      attr_reader :page_number, :offset, :length, :data
+      attr_accessor :row_id
+
+      def initialize(page_number, offset, length, data = nil)
+        @page_number = page_number
+        @offset = offset
+        @length = length
+        @data = data || "\x00".b * length
+        @row_id = nil
+        @dirty = false
+        @deleted = false
+      end
+
+      def read(storage_manager)
+        return @data unless @data.nil?
+
+        @data = storage_manager.read_record(@page_number, @offset, @length)
+        @data
+      end
+
+      def write(storage_manager)
+        return unless @dirty
+
+        storage_manager.write_record(@page_number, @offset, @data)
+        @dirty = false
+      end
+
+      def mark_deleted
+        @deleted = true
+        @dirty = true
+      end
+
+      def deleted?
+        @deleted
+      end
+
+      def update(new_data)
+        @data = new_data
+        @length = new_data.bytesize
+        @dirty = true
+      end
+
+      def to_s
+        "Record(row_id=#{@row_id}, page=#{@page_number}, offset=#{@offset}, length=#{@length}, deleted=#{@deleted})"
+      end
+
+      def inspect
+        to_s
+      end
+    end
+  end
+end

data/lib/rubydb/storage/row.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module RubyDB
+  module Storage
+    # Row - Represents a database row with typed values
+    class Row
+      attr_reader :row_id, :columns, :values
+      attr_accessor :record
+
+      def initialize(row_id, columns, values = {})
+        @row_id = row_id
+        @columns = columns
+        @values = {}
+        @record = nil
+        @dirty = false
+
+        normalized_values = if values.is_a?(Array)
+          columns.each_with_index.each_with_object({}) do |(col, index), result|
+            result[col.name] = values[index]
+          end
+        else
+          values
+        end
+
+        # Initialize values
+        columns.each do |col|
+          value = if normalized_values.key?(col.name)
+                    normalized_values[col.name]
+                  elsif normalized_values.key?(col.name.to_sym)
+                    normalized_values[col.name.to_sym]
+                  else
+                    col.default
+                  end
+          @values[col.name] = value
+        end
+      end
+
+      def [](column_name)
+        @values[column_name]
+      end
+
+      def []=(column_name, value)
+        @values[column_name] = value
+        @dirty = true
+      end
+
+      def column_value(column_index)
+        column_name = @columns[column_index]&.name
+        @values[column_name]
+      end
+
+      def to_hash
+        @values.dup
+      end
+
+      def to_ary
+        @columns.map { |col| @values[col.name] }
+      end
+
+      def to_sql
+        values = @columns.map { |col| @values[col.name] }
+        "ROW(#{values.map { |v| v.nil? ? 'NULL' : v.inspect }.join(', ')})"
+      end
+
+      def dirty?
+        @dirty
+      end
+
+      def mark_clean
+        @dirty = false
+      end
+
+      def clone
+        Row.new(@row_id, @columns, @values.dup)
+      end
+    end
+  end
+end

data/lib/rubydb/storage/serializer.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module RubyDB
+  module Storage
+    # Serializer - Converts Ruby objects to binary format
+    class Serializer
+      def self.serialize(value, type)
+        type_obj = Types::TypeRegistry.lookup(type)
+        type_obj.serialize(value)
+      end
+
+      def self.serialize_row(row, columns, null_bitmap: false)
+        data = +""
+        if null_bitmap
+          bitmap_size = (columns.size + 7) / 8
+          bitmap = Array.new(bitmap_size, 0)
+          columns.each_with_index do |col, index|
+            bitmap[index / 8] |= (1 << (index % 8)) if row[col.name].nil?
+          end
+          data << bitmap.pack("C*")
+        end
+        columns.each do |col|
+          value = row[col.name]
+          serialized = serialize(value, col.type_class)
+          if variable_length_type?(col.type_class)
+            data << [serialized.bytesize].pack("N")
+          end
+          data << serialized
+        end
+        data
+      end
+
+      def self.variable_length_type?(type)
+        !%i[integer bigint smallint float boolean date time timestamp].include?(type.to_sym)
+      end
+
+      def self.serialize_header(row_id, column_count)
+        [row_id, column_count, Time.now.to_i].pack("Q>I>Q>")
+      end
+
+      def self.serialize_metadata(table_name, column_names, column_types)
+        {
+          table_name: table_name,
+          column_names: column_names,
+          column_types: column_types,
+          created_at: Time.now.iso8601
+        }.to_json
+      end
+    end
+  end
+end

data/lib/rubydb/storage/storage_layout.rb

@@ -0,0 +1,151 @@
+# frozen_string_literal: true
+
+module RubyDB
+  module Storage
+    # StorageLayout - Defines the on-disk storage format
+    module StorageLayout
+      # Magic number for file identification
+      MAGIC_NUMBER = "RUBYDB".freeze
+
+      # Current format version
+      FORMAT_VERSION = 1
+
+      # Page types
+      PAGE_TYPE_SUPERBLOCK = 0
+      PAGE_TYPE_FREE = 1
+      PAGE_TYPE_TABLE = 2
+      PAGE_TYPE_INDEX = 3
+      PAGE_TYPE_OVERFLOW = 4
+      PAGE_TYPE_INDEX_LEAF = 5
+      PAGE_TYPE_INDEX_INTERNAL = 6
+
+      # Superblock layout
+      class Superblock
+        attr_accessor :magic, :version, :page_size, :num_pages,
+                      :root_page, :created_at, :modified_at
+
+        def initialize
+          @magic = MAGIC_NUMBER
+          @version = FORMAT_VERSION
+          @page_size = Constants::DEFAULT_PAGE_SIZE
+          @num_pages = 1
+          @root_page = 0
+          @created_at = Time.now.to_i
+          @modified_at = Time.now.to_i
+        end
+
+        def serialize
+          data = ""
+          data << @magic.ljust(16)
+          data << [@version, @page_size, @num_pages, @root_page].pack("I>I>Q>Q>")
+          data << [@created_at, @modified_at].pack("Q>Q>")
+          data
+        end
+
+        def self.deserialize(data)
+          sb = new
+          sb.magic = data[0, 16].strip
+          sb.version, sb.page_size, sb.num_pages, sb.root_page = data[16, 16].unpack("I>I>Q>Q>")
+          sb.created_at, sb.modified_at = data[32, 16].unpack("Q>Q>")
+          sb
+        end
+      end
+
+      # Table metadata layout
+      class TableMetadata
+        attr_accessor :table_id, :table_name, :column_count,
+                      :row_count, :first_page, :last_page,
+                      :created_at, :updated_at
+
+        def initialize
+          @table_id = 0
+          @table_name = ""
+          @column_count = 0
+          @row_count = 0
+          @first_page = 0
+          @last_page = 0
+          @created_at = Time.now.to_i
+          @updated_at = Time.now.to_i
+        end
+
+        def serialize
+          data = +""
+          data << [@table_id].pack("Q>")
+          data << @table_name.to_s.ljust(64)
+          data << [@column_count, @row_count].pack("I>Q>")
+          data << [@first_page, @last_page].pack("Q>Q>")
+          data << [@created_at, @updated_at].pack("Q>Q>")
+          data
+        end
+
+        def self.deserialize(data)
+          tm = new
+          tm.table_id = data[0, 8].unpack("Q>").first
+          tm.table_name = data[8, 64].strip
+          tm.column_count, tm.row_count = data[72, 12].unpack("I>Q>")
+          tm.first_page, tm.last_page = data[84, 16].unpack("Q>Q>")
+          tm.created_at, tm.updated_at = data[100, 16].unpack("Q>Q>")
+          tm
+        end
+      end
+
+      # Column metadata layout
+      class ColumnMetadata
+        attr_accessor :column_id, :column_name, :data_type,
+                      :is_nullable, :is_primary_key, :position,
+                      :default, :created_at
+
+        def initialize
+          @column_id = 0
+          @column_name = ""
+          @data_type = :text
+          @is_nullable = true
+          @is_primary_key = false
+          @position = 0
+          @default = nil
+          @created_at = Time.now.to_i
+        end
+
+        def serialize
+          data = +""
+          data << [@column_id, @position].pack("I>I>")
+          data << @column_name.to_s.ljust(64)
+          data << [@data_type.to_s].pack("Z*")
+          flags = 0
+          flags |= 1 if @is_nullable
+          flags |= 2 if @is_primary_key
+          data << [flags].pack("C")
+          default_payload = @default.nil? ? "" : @default.to_s
+          data << default_payload.ljust(64)
+          data << [@created_at].pack("Q>")
+          data
+        end
+
+        def self.deserialize(data)
+          cm = new
+          cm.column_id, cm.position = data[0, 8].unpack("I>I>")
+          cm.column_name = data[8, 64].strip
+          cm.data_type = data[72].unpack("Z*").first.to_sym
+          flags = data[72 + cm.data_type.to_s.length + 1].unpack("C").first
+          cm.is_nullable = (flags & 1) != 0
+          cm.is_primary_key = (flags & 2) != 0
+          cm.default = data[73 + cm.data_type.to_s.length, 64].strip unless data[73 + cm.data_type.to_s.length, 64].strip.empty?
+          cm.created_at = data[137, 8].unpack("Q>").first
+          cm
+        end
+      end
+
+      def self.page_header_size
+        PageHeader::SIZE
+      end
+
+      def self.record_header_size
+        8  # [row_id(4)] + [length(4)]
+      end
+
+      def self.max_record_size(page_size)
+        page_size - PageHeader::SIZE - 4  # 4 bytes for record pointer
+      end
+    end
+  end
+end

data/lib/rubydb/storage/storage_manager.rb

@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+module RubyDB
+  module Storage
+    # StorageManager - Central storage management
+    class StorageManager
+      attr_reader :file_manager, :page_manager, :page_allocator,
+                  :buffer_pool, :free_space_map, :visibility_map, :page_size
+
+      def initialize(path, config = {})
+        @path = path
+        @page_size = config[:page_size] || Constants::DEFAULT_PAGE_SIZE
+        @buffer_size = config[:buffer_size] || Constants::DEFAULT_BUFFER_POOL_SIZE
+
+        # Initialize components
+        @file_manager = FileManager.new(path, @page_size, config)
+        @page_manager = PageManager.new(@file_manager)
+        @buffer_pool = BufferPool.new(@buffer_size)
+        @free_space_map = FreeSpaceMap.new(@page_manager)
+        visibility_path = config[:visibility_path] || "#{path}.visibility"
+        @visibility_map = VisibilityMap.new(@page_manager, config.merge(visibility_path: visibility_path))
+        @page_allocator = PageAllocator.new(@page_manager, buffer_pool: @buffer_pool)
+
+        # Connect components
+        @buffer_pool.set_page_manager(@page_manager)
+
+        @is_open = false
+        @stats = {
+          reads: 0,
+          writes: 0,
+          allocations: 0,
+          frees: 0
+        }
+      end
+
+      def open
+        @file_manager.create_or_open
+        @page_manager.load_free_list
+        @is_open = true
+        self
+      rescue => e
+        raise StorageError, "Failed to open storage: #{e.message}"
+      end
+
+      def close
+        return unless @is_open
+
+        flush
+        @file_manager.close
+        @is_open = false
+        true
+      end
+
+      def flush
+        @buffer_pool.flush_all
+        @file_manager.sync
+      end
+
+      def read_page(page_number)
+        @stats[:reads] += 1
+        frame = @buffer_pool.get_page(page_number)
+        frame.access
+        frame.page
+      end
+
+      def write_page(page)
+        @stats[:writes] += 1
+        @buffer_pool.write_page(page)
+      end
+
+      def allocate_page(page_type = 0)
+        @stats[:allocations] += 1
+        @page_manager.allocate_page(page_type)
+      end
+
+      def free_page(page_number)
+        @stats[:frees] += 1
+        @page_manager.free_page(page_number)
+        @buffer_pool.remove_page(page_number)
+      end
+
+      def read_record(page_number, offset, length)
+        page = read_page(page_number)
+        page.read(offset, length)
+      end
+
+      def write_record(page_number, offset, data)
+        page = read_page(page_number)
+        page.write(offset, data)
+        write_page(page)
+      end
+
+      def stats
+        {
+          page_size: @page_size,
+          total_pages: @page_manager.total_pages,
+          free_pages: @page_manager.free_pages,
+          used_pages: @page_manager.used_pages,
+          buffer_hits: @buffer_pool.hit_count,
+          buffer_misses: @buffer_pool.miss_count,
+          buffer_hit_rate: @buffer_pool.hit_rate,
+          reads: @stats[:reads],
+          writes: @stats[:writes],
+          allocations: @stats[:allocations],
+          frees: @stats[:frees]
+        }
+      end
+
+      def open?
+        @is_open
+      end
+    end
+  end
+end