rubydb 0.1.0

data/lib/rubydb/wal/checkpoint.rb

@@ -0,0 +1,183 @@
+# frozen_string_literal: true
+
+require "time"
+
+module RubyDB
+  module WAL
+    # Checkpoint - Manages checkpoints for recovery
+    class Checkpoint
+      attr_reader :last_checkpoint_lsn, :stats
+
+      def initialize(wal_writer, config = {})
+        @wal_writer = wal_writer
+        @config = config
+        @checkpoint_interval = config[:interval] || 300  # 5 minutes
+        @min_checkpoint_age = config[:min_age] || 60    # 1 minute
+        @last_checkpoint_time = Time.now
+        @last_checkpoint_lsn = nil
+        @last_checkpoint_data = nil
+        @stats = {
+          checkpoints_created: 0,
+          checkpoints_restored: 0,
+          checkpoint_failures: 0,
+          last_error: nil,
+          total_checkpoint_time_ms: 0,
+          avg_checkpoint_time_ms: 0,
+          last_checkpoint_size: 0
+        }
+        @lock = Mutex.new
+        @running = false
+        @checkpoint_thread = nil
+      end
+
+      def start
+        @lock.synchronize do
+          return if @running
+
+          @running = true
+          @checkpoint_thread = Thread.new do
+            checkpoint_loop
+          end
+        end
+      end
+
+      def stop
+        @lock.synchronize do
+          @running = false
+          @checkpoint_thread&.kill
+          @checkpoint_thread = nil
+        end
+      end
+
+      def create_checkpoint(force = false)
+        @lock.synchronize do
+          start_time = Time.now
+
+          # Check if enough time has passed
+          if !force && (Time.now - @last_checkpoint_time) < @min_checkpoint_age
+            return false
+          end
+
+          begin
+            # Flush WAL
+            @wal_writer.flush
+
+            # Get current LSN
+            current_lsn = @wal_writer.current_lsn
+
+            # Create checkpoint record
+            checkpoint_data = {
+              lsn: current_lsn.to_s,
+              timestamp: Time.now.iso8601,
+              wal_segment: @wal_writer.current_segment.segment_id,
+              wal_offset: current_lsn.offset
+            }
+
+            # Write checkpoint record to WAL
+            record = Record.new(:checkpoint, checkpoint_data)
+            @wal_writer.write_record(record)
+            # A checkpoint is a durability boundary even when ordinary WAL
+            # writes use asynchronous buffering. Do not publish its state or
+            # return success until the checkpoint frame is flushed and synced.
+            @wal_writer.flush
+            @wal_writer.sync
+
+            # Update checkpoint info
+            @last_checkpoint_lsn = current_lsn
+            @last_checkpoint_data = checkpoint_data
+            @last_checkpoint_time = Time.now
+
+            elapsed_ms = (Time.now - start_time) * 1000
+            @stats[:checkpoints_created] += 1
+            @stats[:total_checkpoint_time_ms] += elapsed_ms
+            @stats[:avg_checkpoint_time_ms] = @stats[:total_checkpoint_time_ms] / @stats[:checkpoints_created]
+            @stats[:last_checkpoint_size] = estimate_checkpoint_size
+
+            true
+
+          rescue => e
+            @stats[:checkpoint_failures] += 1
+            @stats[:last_error] = "#{e.class}: #{e.message}"
+            raise
+          end
+        end
+      end
+
+      def restore_checkpoint(reader)
+        @lock.synchronize do
+          start_time = Time.now
+
+          begin
+            # Find the latest checkpoint in the WAL
+            all_records = reader.read_all
+            checkpoint_records = all_records.select { |r| r.type == :checkpoint }
+
+            if checkpoint_records.empty?
+              return nil
+            end
+
+            # Get the latest checkpoint
+            latest = checkpoint_records.last
+            # `latest.data[:lsn]` is the last record covered by the
+            # checkpoint and is an inclusive replay boundary. Return the
+            # marker's own LSN so recovery starts after the covered records.
+            @last_checkpoint_lsn = latest.lsn
+            @last_checkpoint_data = latest.data
+            @last_checkpoint_time = Time.parse(latest.data[:timestamp])
+
+            @stats[:checkpoints_restored] += 1
+            @last_checkpoint_lsn
+
+          rescue => e
+            @stats[:checkpoint_failures] += 1
+            raise
+          end
+        end
+      end
+
+      def checkpoint_exists?
+        !@last_checkpoint_lsn.nil?
+      end
+
+      def checkpoint_age
+        return nil unless @last_checkpoint_lsn
+        Time.now - @last_checkpoint_time
+      end
+
+      def stats
+        @lock.synchronize do
+          @stats.merge({
+            running: @running,
+            last_checkpoint: @last_checkpoint_time.iso8601,
+            last_checkpoint_lsn: @last_checkpoint_lsn&.to_s,
+            checkpoint_age: checkpoint_age,
+            interval: @checkpoint_interval,
+            min_age: @min_checkpoint_age
+          })
+        end
+      end
+
+      private
+
+      def checkpoint_loop
+        while @running
+          sleep(@checkpoint_interval)
+          begin
+            create_checkpoint
+          rescue => e
+            # Log error but continue
+          end
+        end
+      end
+
+      def estimate_checkpoint_size
+        segment = @wal_writer.current_segment
+        return 0 unless segment
+
+        # The checkpoint covers the flushed WAL through the current segment.
+        # Report the actual segment bytes instead of a fixed placeholder.
+        segment.size.to_i + @wal_writer.stats.fetch(:buffer_bytes, 0).to_i
+      end
+    end
+  end
+end

data/lib/rubydb/wal/lsn.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+require "time"
+
+module RubyDB
+  module WAL
+    # LSN - Log Sequence Number for identifying WAL records
+    class LSN
+      attr_reader :segment_id, :offset, :timestamp
+
+      def initialize(segment_id = 0, offset = 0, timestamp = nil)
+        @segment_id = segment_id
+        @offset = offset
+        @timestamp = timestamp || Time.now
+      end
+
+      def to_i
+        (@segment_id << 32) | @offset
+      end
+
+      def to_s
+        "LSN(#{@segment_id}, #{@offset})"
+      end
+
+      def inspect
+        to_s
+      end
+
+      def ==(other)
+        other.is_a?(LSN) && @segment_id == other.segment_id && @offset == other.offset
+      end
+
+      def <(other)
+        return false unless other.is_a?(LSN)
+        @segment_id < other.segment_id || (@segment_id == other.segment_id && @offset < other.offset)
+      end
+
+      def <=(other)
+        self < other || self == other
+      end
+
+      def >(other)
+        return false unless other.is_a?(LSN)
+        @segment_id > other.segment_id || (@segment_id == other.segment_id && @offset > other.offset)
+      end
+
+      def >=(other)
+        self > other || self == other
+      end
+
+      def to_i64
+        [@segment_id, @offset].pack("Q>Q>")
+      end
+
+      def self.from_i64(data)
+        segment_id, offset = data.unpack("Q>Q>")
+        new(segment_id, offset)
+      end
+
+      def self.from_i(value)
+        segment_id = value >> 32
+        offset = value & 0xFFFFFFFF
+        new(segment_id, offset)
+      end
+
+      def self.from_s(value)
+        match = value.to_s.match(/\ALSN\((\d+),\s*(\d+)\)\z/)
+        raise ArgumentError, "Invalid LSN: #{value.inspect}" unless match
+
+        new(match[1].to_i, match[2].to_i)
+      end
+
+      def self.null
+        new(0, 0)
+      end
+
+      def null?
+        @segment_id == 0 && @offset == 0
+      end
+
+      def serialize
+        {
+          segment_id: @segment_id,
+          offset: @offset,
+          timestamp: @timestamp.iso8601
+        }
+      end
+
+      def self.deserialize(data)
+        new(data[:segment_id], data[:offset], Time.parse(data[:timestamp]))
+      end
+    end
+  end
+end

data/lib/rubydb/wal/reader.rb

@@ -0,0 +1,260 @@
+# frozen_string_literal: true
+
+module RubyDB
+  module WAL
+    # Reader - Reads records from the WAL
+    class Reader
+      attr_reader :stats
+
+      def initialize(wal_dir, config = {})
+        @wal_dir = wal_dir
+        @config = config
+        @segments = []
+        @current_segment_index = 0
+        @stats = {
+          records_read: 0,
+          bytes_read: 0,
+          segments_read: 0,
+          segment_skips: 0,
+          corrupted_records: 0
+        }
+        @lock = Mutex.new
+        @batch_size = config[:batch_size] || 1000
+
+        load_segments
+      end
+
+      def read_all
+        @lock.synchronize do
+          records = []
+          @segments.each do |segment|
+            records.concat(_read_segment(segment))
+          end
+          records
+        end
+      end
+
+      def read_from_lsn(start_lsn)
+        @lock.synchronize do
+          records = []
+          found_start = false
+
+          @segments.each do |segment|
+            if segment.segment_id < start_lsn.segment_id
+              next
+            end
+
+            if segment.segment_id == start_lsn.segment_id
+              # Read from offset
+              records.concat(_read_segment_from_offset(segment, start_lsn.offset))
+            else
+              records.concat(_read_segment(segment))
+            end
+
+            break if @config[:limit] && records.size >= @config[:limit]
+          end
+
+          records
+        end
+      end
+
+      def read_to_lsn(end_lsn)
+        @lock.synchronize do
+          records = []
+
+          @segments.each do |segment|
+            break if segment.segment_id > end_lsn.segment_id
+
+            if segment.segment_id == end_lsn.segment_id
+              records.concat(_read_segment_to_offset(segment, end_lsn.offset))
+            else
+              records.concat(_read_segment(segment))
+            end
+          end
+
+          records
+        end
+      end
+
+      def read_range(start_lsn, end_lsn)
+        @lock.synchronize do
+          records = []
+
+          @segments.each do |segment|
+            break if segment.segment_id > end_lsn.segment_id
+            next if segment.segment_id < start_lsn.segment_id
+
+            if segment.segment_id == start_lsn.segment_id
+              records.concat(_read_segment_range(segment, start_lsn.offset, end_lsn.offset))
+            elsif segment.segment_id == end_lsn.segment_id
+              records.concat(_read_segment_to_offset(segment, end_lsn.offset))
+            else
+              records.concat(_read_segment(segment))
+            end
+          end
+
+          records
+        end
+      end
+
+      def read_segment(segment)
+        @lock.synchronize do
+          _read_segment(segment)
+        end
+      end
+
+      private def _read_segment(segment)
+        records = []
+        offset = 0
+
+        puts "WAL DEBUG: _read_segment starting for segment #{segment.segment_id}" if ENV['DEBUG_RECOVERY']
+        while true
+          record_data = segment.read_record_at(offset)
+          break if record_data.nil? || record_data.empty?
+          puts "WAL DEBUG: read record at offset #{offset}, size=#{record_data.bytesize}" if ENV['DEBUG_RECOVERY']
+
+          begin
+            lsn = LSN.new(segment.segment_id, offset)
+            record = Record.deserialize(record_data, lsn)
+            records << record
+            @stats[:records_read] += 1
+            @stats[:bytes_read] += record_data.bytesize
+            offset += Segment::FRAME_HEADER_SIZE + record_data.bytesize
+            puts "WAL DEBUG: deserialized record #{record.type}" if ENV['DEBUG_RECOVERY']
+          rescue => e
+            @stats[:corrupted_records] += 1
+            puts "WAL DEBUG: failed to deserialize at offset #{offset}: #{e.message}" if ENV['DEBUG_RECOVERY']
+            break
+          end
+        end
+
+        @stats[:segments_read] += 1
+        puts "WAL DEBUG: finished reading segment #{segment.segment_id}, found #{records.count} records" if ENV['DEBUG_RECOVERY']
+        records
+      end
+
+      private def _read_segment_from_offset(segment, offset)
+        records = []
+        current_offset = offset
+
+        while true
+          record_data = segment.read_record_at(current_offset)
+          break if record_data.nil? || record_data.empty?
+
+          begin
+            lsn = LSN.new(segment.segment_id, current_offset)
+            record = Record.deserialize(record_data, lsn)
+            records << record
+            @stats[:records_read] += 1
+            @stats[:bytes_read] += record_data.bytesize
+            current_offset += Segment::FRAME_HEADER_SIZE + record_data.bytesize
+          rescue => e
+            @stats[:corrupted_records] += 1
+            break
+          end
+        end
+
+        records
+      end
+
+      private def _read_segment_to_offset(segment, offset)
+        records = []
+        current_offset = 0
+
+        while current_offset < offset
+          record_data = segment.read_record_at(current_offset)
+          break if record_data.nil? || record_data.empty?
+
+          begin
+            lsn = LSN.new(segment.segment_id, current_offset)
+            record = Record.deserialize(record_data, lsn)
+            records << record
+            @stats[:records_read] += 1
+            @stats[:bytes_read] += record_data.bytesize
+            current_offset += Segment::FRAME_HEADER_SIZE + record_data.bytesize
+          rescue => e
+            @stats[:corrupted_records] += 1
+            break
+          end
+        end
+
+        records
+      end
+
+      private def _read_segment_range(segment, start_offset, end_offset)
+        records = []
+        current_offset = start_offset
+
+        while current_offset < end_offset
+          record_data = segment.read_record_at(current_offset)
+          break if record_data.nil? || record_data.empty?
+
+          begin
+            lsn = LSN.new(segment.segment_id, current_offset)
+            record = Record.deserialize(record_data, lsn)
+            records << record
+            @stats[:records_read] += 1
+            @stats[:bytes_read] += record_data.bytesize
+            current_offset += Segment::FRAME_HEADER_SIZE + record_data.bytesize
+          rescue => e
+            @stats[:corrupted_records] += 1
+            break
+          end
+        end
+
+        records
+      end
+
+      def reload
+        @lock.synchronize do
+          @segments.each(&:close)
+          @segments.clear
+          @current_segment_index = 0
+          load_segments
+          true
+        end
+      end
+
+      def close
+        @lock.synchronize do
+          @segments.each(&:close)
+          @segments.clear
+          true
+        end
+      end
+
+      def stats
+        @lock.synchronize do
+          @stats.merge({
+            segments_loaded: @segments.size,
+            current_segment: @current_segment_index,
+            batch_size: @batch_size
+          })
+        end
+      end
+
+      private
+
+      def load_segments
+        @segments = []
+
+        # Find all WAL segment files
+        wal_files = Dir.glob(File.join(@wal_dir, "wal_*.log"))
+        puts "WAL DEBUG: found #{wal_files.count} WAL files: #{wal_files.map { |f| File.basename(f) }.inspect}" if ENV['DEBUG_RECOVERY']
+        wal_files.sort.each do |file_path|
+          # Extract segment ID from filename
+          if file_path =~ /wal_(\d+)\.log$/
+            segment_id = $1.to_i
+            segment = Segment.new(segment_id, @wal_dir)
+            if segment.exists?
+              segment.open
+              @segments << segment
+              puts "WAL DEBUG: loaded segment #{segment_id}" if ENV['DEBUG_RECOVERY']
+            end
+          end
+        end
+        puts "WAL DEBUG: total segments loaded: #{@segments.count}" if ENV['DEBUG_RECOVERY']
+      end
+    end
+  end
+end

data/lib/rubydb/wal/record.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+require "json"
+require "digest"
+require "time"
+require_relative "../errors/corruption_error"
+
+module RubyDB
+  module WAL
+    # Record - A single WAL record
+    class Record
+      # Record types
+      TYPE_BEGIN = :begin
+      TYPE_COMMIT = :commit
+      TYPE_ROLLBACK = :rollback
+      TYPE_INSERT = :insert
+      TYPE_UPDATE = :update
+      TYPE_DELETE = :delete
+      TYPE_CREATE_TABLE = :create_table
+      TYPE_DROP_TABLE = :drop_table
+      TYPE_CREATE_INDEX = :create_index
+      TYPE_DROP_INDEX = :drop_index
+      TYPE_CHECKPOINT = :checkpoint
+      TYPE_SCHEMA_CHANGE = :schema_change
+      TYPE_PREPARE = :prepare
+      TYPE_SAVEPOINT = :savepoint
+
+      attr_reader :lsn, :type, :data, :transaction_id, :timestamp
+      attr_reader :checksum, :prev_lsn, :size
+
+      def initialize(type, data = {}, transaction_id: nil, prev_lsn: nil)
+        @type = type
+        @data = data
+        @transaction_id = transaction_id
+        @prev_lsn = prev_lsn
+        @timestamp = Time.now
+        @lsn = nil
+        @checksum = nil
+        @size = 0
+      end
+
+      def serialize
+        # Build record data
+        record_data = {
+          type: @type,
+          transaction_id: @transaction_id,
+          prev_lsn: @prev_lsn&.to_s,
+          timestamp: @timestamp.iso8601,
+          data: @data
+        }
+
+        json_data = JSON.generate(record_data)
+        # The serialized payload is checksum (16 ASCII bytes) followed by JSON.
+        # Segment framing is owned by Segment, so this is the payload size.
+        @size = json_data.bytesize + 16
+
+        # Calculate checksum
+        @checksum = Digest::SHA256.hexdigest(json_data)[0...16]
+
+        # Return serialized record
+        {
+          checksum: @checksum,
+          data: json_data,
+          size: @size
+        }
+      end
+
+      def self.deserialize(raw_data, lsn)
+        checksum = raw_data[0...16]
+        json_data = raw_data[16..-1]
+
+        # Verify checksum
+        calculated = Digest::SHA256.hexdigest(json_data)[0...16]
+        raise CorruptionError, "WAL record checksum mismatch" if calculated != checksum
+
+        parsed = JSON.parse(json_data, symbolize_names: true)
+
+        record = new(
+          parsed[:type].to_sym,
+          parsed[:data] || {},
+          transaction_id: parsed[:transaction_id],
+          prev_lsn: parsed[:prev_lsn] ? LSN.from_s(parsed[:prev_lsn]) : nil
+        )
+        record.instance_variable_set(:@timestamp, Time.parse(parsed[:timestamp]))
+        record.instance_variable_set(:@lsn, lsn)
+        record.instance_variable_set(:@checksum, checksum)
+        record.instance_variable_set(:@size, raw_data.bytesize)
+
+        record
+      end
+
+      def to_s
+        "WALRecord(lsn=#{@lsn}, type=#{@type}, txn=#{@transaction_id})"
+      end
+
+      def inspect
+        to_s
+      end
+
+      def size
+        @size || 0
+      end
+    end
+  end
+end