rubydb 0.1.0

data/lib/rubydb/wal/segment.rb

@@ -0,0 +1,193 @@
+# frozen_string_literal: true
+
+require "fileutils"
+
+module RubyDB
+  module WAL
+    # Segment - A WAL segment file
+    class Segment
+      attr_reader :segment_id, :path, :size, :max_size
+      attr_reader :first_lsn, :last_lsn, :record_count
+
+      DEFAULT_MAX_SIZE = 16 * 1024 * 1024  # 16MB
+      FRAME_HEADER_SIZE = 4
+
+      def initialize(segment_id, wal_dir, max_size = DEFAULT_MAX_SIZE)
+        @segment_id = segment_id
+        @wal_dir = wal_dir
+        @max_size = max_size
+        @path = File.join(wal_dir, "wal_#{segment_id.to_s.rjust(8, '0')}.log")
+        @size = 0
+        @record_count = 0
+        @first_lsn = nil
+        @last_lsn = nil
+        @file = nil
+        @is_open = false
+        @is_full = false
+        @lock = Mutex.new
+      end
+
+      def open
+        @lock.synchronize do
+          return if @is_open
+
+          FileUtils.mkdir_p(@wal_dir) unless Dir.exist?(@wal_dir)
+
+          if File.exist?(@path)
+            @file = File.open(@path, "a+b")
+            @size = File.size(@path)
+            @record_count = count_records
+          else
+            @file = File.open(@path, "w+b")
+            @size = 0
+            @record_count = 0
+          end
+
+          @is_open = true
+          self
+        end
+      end
+
+      def close
+        @lock.synchronize do
+          return unless @is_open
+
+          @file.close if @file
+          @is_open = false
+          true
+        end
+      end
+
+      def append(record_data, lsn)
+        @lock.synchronize do
+          raise "Segment is full" if @is_full
+          raise "Segment not open" unless @is_open
+
+          # Each record is framed as a 4-byte big-endian payload length followed
+          # by the checksum+JSON payload. This makes records independently
+          # readable and lets recovery safely stop at a torn tail record.
+          frame = [record_data.bytesize].pack("N") + record_data
+
+          # Write record
+          @file.seek(0, IO::SEEK_END)
+          @file.write(frame)
+          @file.flush
+
+          @size += frame.bytesize
+          @record_count += 1
+
+          @first_lsn ||= lsn
+          @last_lsn = lsn
+
+          # Check if segment is full
+          if @size >= @max_size
+            @is_full = true
+            return :full
+          end
+
+          :ok
+        end
+      end
+
+      def read_at(offset, length)
+        @lock.synchronize do
+          raise "Segment not open" unless @is_open
+
+          @file.seek(offset)
+          @file.read(length)
+        end
+      end
+
+      def read_record_at(offset)
+        @lock.synchronize do
+          raise "Segment not open" unless @is_open
+
+          @file.seek(offset)
+          header = @file.read(FRAME_HEADER_SIZE)
+          return nil if header.nil? || header.empty?
+          return nil if header.bytesize < FRAME_HEADER_SIZE
+
+          payload_size = header.unpack1("N")
+          return nil if payload_size < 16
+
+          payload = @file.read(payload_size)
+          return nil if payload.nil? || payload.bytesize < payload_size
+
+          payload
+        end
+      end
+
+      def truncate
+        @lock.synchronize do
+          return unless @is_open
+
+          @file.truncate(0)
+          @size = 0
+          @record_count = 0
+          @first_lsn = nil
+          @last_lsn = nil
+          @is_full = false
+          true
+        end
+      end
+
+      def full?
+        @is_full
+      end
+
+      def open?
+        @is_open
+      end
+
+      def exists?
+        File.exist?(@path)
+      end
+
+      def delete
+        @lock.synchronize do
+          close if @is_open
+          File.delete(@path) if exists?
+          true
+        end
+      end
+
+      def archive
+        # Move segment to archive
+        archived_path = File.join(@wal_dir, "archive", "wal_#{@segment_id.to_s.rjust(8, '0')}.log")
+        FileUtils.mkdir_p(File.dirname(archived_path))
+        FileUtils.mv(@path, archived_path)
+        archived_path
+      end
+
+      def to_s
+        "Segment(#{@segment_id}, size=#{@size}, records=#{@record_count}, full=#{@is_full})"
+      end
+
+      def inspect
+        to_s
+      end
+
+      private
+
+      def count_records
+        count = 0
+        offset = 0
+        loop do
+          @file.seek(offset)
+          header = @file.read(FRAME_HEADER_SIZE)
+          break if header.nil? || header.bytesize < FRAME_HEADER_SIZE
+
+          payload_size = header.unpack1("N")
+          break if payload_size < 16
+
+          payload = @file.read(payload_size)
+          break if payload.nil? || payload.bytesize < payload_size
+
+          count += 1
+          offset += FRAME_HEADER_SIZE + payload_size
+        end
+        count
+      end
+    end
+  end
+end

data/lib/rubydb/wal/wal.rb

@@ -0,0 +1,452 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "time"
+
+# Import all WAL components
+require_relative "lsn"
+require_relative "record"
+require_relative "segment"
+require_relative "writer"
+require_relative "reader"
+require_relative "checkpoint"
+require_relative "archive"
+require_relative "../errors/recovery_error"
+
+module RubyDB
+  module WAL
+    # WAL - Main Write-Ahead Log interface
+    class WAL
+      attr_reader :writer, :reader, :checkpoint, :archive, :stats
+      attr_reader :wal_dir, :current_lsn
+
+      def initialize(wal_dir, config = {})
+        @wal_dir = wal_dir
+        @config = config
+        @engine = config[:engine]
+        @stats = {
+          writes: 0,
+          reads: 0,
+          checkpoints: 0,
+          archives: 0,
+          restores: 0,
+          errors: 0,
+          total_write_time_ms: 0,
+          total_read_time_ms: 0,
+          avg_write_time_ms: 0,
+          avg_read_time_ms: 0
+        }
+        @lock = Mutex.new
+        @running = false
+        @shutdown = false
+        @current_lsn = LSN.null
+
+        # Create WAL directory
+        FileUtils.mkdir_p(wal_dir) unless Dir.exist?(wal_dir)
+
+        begin
+          # Initialize components
+          initialize_components(config)
+
+          # Start checkpoint if configured
+          if config[:auto_checkpoint] != false
+            @checkpoint.start
+          end
+
+          # Start recovery if needed
+          if config[:recovery] != false
+            recover
+          end
+
+          @running = true
+        rescue StandardError
+          # Recovery can fail after the writer has opened a segment. Close all
+          # partially initialized components before re-raising so callers can
+          # replace/remove a failed WAL directory on every supported platform.
+          @checkpoint&.stop rescue nil
+          @writer&.shutdown rescue nil
+          @reader&.close rescue nil
+          raise
+        end
+      end
+
+      def attach_engine(engine)
+        @lock.synchronize { @engine = engine }
+        self
+      end
+
+      def write(record)
+        @lock.synchronize do
+          start_time = Time.now
+          @stats[:writes] += 1
+
+          begin
+            lsn = @writer.write_record(record)
+            @current_lsn = lsn
+
+            elapsed_ms = (Time.now - start_time) * 1000
+            @stats[:total_write_time_ms] += elapsed_ms
+            @stats[:avg_write_time_ms] = @stats[:total_write_time_ms] / @stats[:writes]
+
+            lsn
+          rescue => e
+            @stats[:errors] += 1
+            raise
+          end
+        end
+      end
+
+      def write_batch(records)
+        @lock.synchronize do
+          start_time = Time.now
+          @stats[:writes] += records.size
+
+          begin
+            result = @writer.write_batch(records)
+            @current_lsn = result[:end_lsn] if result[:end_lsn]
+
+            elapsed_ms = (Time.now - start_time) * 1000
+            @stats[:total_write_time_ms] += elapsed_ms
+            @stats[:avg_write_time_ms] = @stats[:total_write_time_ms] / @stats[:writes]
+
+            result
+          rescue => e
+            @stats[:errors] += 1
+            raise
+          end
+        end
+      end
+
+      def read_all
+        @lock.synchronize do
+          start_time = Time.now
+          @stats[:reads] += 1
+
+          begin
+            records = @reader.read_all
+
+            elapsed_ms = (Time.now - start_time) * 1000
+            @stats[:total_read_time_ms] += elapsed_ms
+            @stats[:avg_read_time_ms] = @stats[:total_read_time_ms] / @stats[:reads]
+
+            records
+          rescue => e
+            @stats[:errors] += 1
+            raise
+          end
+        end
+      end
+
+      def read_from(lsn)
+        @lock.synchronize do
+          start_time = Time.now
+          @stats[:reads] += 1
+
+          begin
+            records = @reader.read_from_lsn(lsn)
+
+            elapsed_ms = (Time.now - start_time) * 1000
+            @stats[:total_read_time_ms] += elapsed_ms
+            @stats[:avg_read_time_ms] = @stats[:total_read_time_ms] / @stats[:reads]
+
+            records
+          rescue => e
+            @stats[:errors] += 1
+            raise
+          end
+        end
+      end
+
+      def read_range(start_lsn, end_lsn)
+        @lock.synchronize do
+          start_time = Time.now
+          @stats[:reads] += 1
+
+          begin
+            records = @reader.read_range(start_lsn, end_lsn)
+
+            elapsed_ms = (Time.now - start_time) * 1000
+            @stats[:total_read_time_ms] += elapsed_ms
+            @stats[:avg_read_time_ms] = @stats[:total_read_time_ms] / @stats[:reads]
+
+            records
+          rescue => e
+            @stats[:errors] += 1
+            raise
+          end
+        end
+      end
+
+      def create_checkpoint(force = false)
+        @lock.synchronize do
+          start_time = Time.now
+          @stats[:checkpoints] += 1
+
+          begin
+            result = @checkpoint.create_checkpoint(force)
+            @stats[:checkpoints] += 1 if result
+            result
+          rescue => e
+            @stats[:errors] += 1
+            raise
+          end
+        end
+      end
+
+      def archive_segment(segment_id)
+        @lock.synchronize do
+          @stats[:archives] += 1
+
+          begin
+            segment = Segment.new(segment_id, @wal_dir)
+            if segment.exists?
+              segment.open
+              result = @archive.archive_segment(segment)
+              segment.close
+              result
+            else
+              false
+            end
+          rescue => e
+            @stats[:errors] += 1
+            raise
+          end
+        end
+      end
+
+      def archive_all_segments
+        @lock.synchronize do
+          archived = 0
+
+          Dir.glob(File.join(@wal_dir, "wal_*.log")).each do |file_path|
+            if file_path =~ /wal_(\d+)\.log$/
+              segment_id = $1.to_i
+              if archive_segment(segment_id)
+                archived += 1
+              end
+            end
+          end
+
+          archived
+        end
+      end
+
+      def restore_from_archive(segment_id)
+        @lock.synchronize do
+          @stats[:restores] += 1
+
+          begin
+            result = @archive.restore_archive(segment_id)
+            # Reload reader after restore
+            @reader.reload if result
+            result
+          rescue => e
+            @stats[:errors] += 1
+            raise
+          end
+        end
+      end
+
+      def restore_latest_archive
+        @lock.synchronize do
+          archives = @archive.list_archives
+          return false if archives.empty?
+
+          latest = archives.max_by { |a| a[:segment_id] }
+          restore_from_archive(latest[:segment_id])
+        end
+      end
+
+      def recover
+        @lock.synchronize do
+          # Find the latest checkpoint
+          checkpoint_lsn = @checkpoint.restore_checkpoint(@reader)
+
+          if checkpoint_lsn
+            # Read records from the checkpoint marker. The reader includes
+            # the starting frame; the marker is ignored by replay and all
+            # subsequent WAL records are applied.
+            records = @reader.read_from_lsn(checkpoint_lsn)
+
+            # Replay committed transactions
+            committed_transactions = Set.new
+            records.each do |record|
+              case record.type
+              when :commit
+                committed_transactions.add(record.transaction_id)
+              when :prepare
+                # Handle prepared transactions
+                if can_commit_prepared?(record)
+                  committed_transactions.add(record.transaction_id)
+                end
+              end
+            end
+
+            # REDO committed transactions
+            redo_records = records.select do |r|
+              committed_transactions.include?(r.transaction_id) &&
+              [:insert, :update, :delete, :create_table, :drop_table].include?(r.type)
+            end
+
+            redo_records.each do |record|
+              replay_record(record)
+            end
+
+            # UNDO uncommitted transactions
+            uncommitted = records.select do |r|
+              !committed_transactions.include?(r.transaction_id) &&
+              [:insert, :update, :delete].include?(r.type)
+            end
+
+            uncommitted.reverse_each do |record|
+              undo_record(record)
+            end
+          end
+
+          true
+        end
+      end
+
+      def flush
+        @lock.synchronize do
+          @writer.flush
+        end
+      end
+
+      def sync
+        @lock.synchronize do
+          @writer.sync
+        end
+      end
+
+      def shutdown(wait = true)
+        @lock.synchronize do
+          @shutdown = true
+          @running = false
+
+          @checkpoint.stop if @checkpoint
+          @writer.shutdown(wait)
+          @reader.close if @reader
+        end
+      end
+
+      def running?
+        @running
+      end
+
+      def stats
+        @lock.synchronize do
+          writer_stats = @writer.stats rescue {}
+          reader_stats = @reader.stats rescue {}
+          checkpoint_stats = @checkpoint.stats rescue {}
+          archive_stats = @archive.stats rescue {}
+
+          @stats.merge({
+            writer: writer_stats,
+            reader: reader_stats,
+            checkpoint: checkpoint_stats,
+            archive: archive_stats,
+            current_lsn: @current_lsn.to_s,
+            running: @running,
+            wal_dir: @wal_dir
+          })
+        end
+      end
+
+      private
+
+      def initialize_components(config)
+        @writer = Writer.new(@wal_dir, config)
+        @reader = Reader.new(@wal_dir, config)
+        @checkpoint = Checkpoint.new(@writer, config)
+        @archive = Archive.new(@wal_dir, config[:archive_dir], config)
+      end
+
+      def can_commit_prepared?(record)
+        data = record&.data
+        resources = data && data[:resources]
+        return true unless resources
+
+        Array(resources).all? do |resource|
+          case resource
+          when String then File.exist?(resource)
+          when Hash
+            path = resource[:path] || resource["path"] || resource[:file] || resource["file"]
+            path && File.exist?(path)
+          else
+            resource.respond_to?(:available?) && resource.available?
+          end
+        end
+      end
+
+      def replay_record(record)
+        engine = @engine
+        raise RubyDB::RecoveryError, "WAL replay requires an attached engine" unless engine
+        data = record.data
+
+        case record.type
+        when :insert
+          # Insert row
+          engine.insert_row(
+            data[:table],
+            data[:columns],
+            data[:values]
+          )
+        when :update
+          # Update row
+          engine.update_row(
+            data[:table],
+            data[:row_id],
+            data[:values]
+          )
+        when :delete
+          # Delete row
+          engine.delete_row(
+            data[:table],
+            data[:row_id]
+          )
+        when :create_table
+          # Create table
+          engine.create_table(
+            data[:table_name],
+            data[:columns]
+          )
+        when :drop_table
+          # Drop table
+          engine.drop_table(
+            data[:table_name]
+          )
+        end
+      end
+
+      def undo_record(record)
+        engine = @engine
+        raise RubyDB::RecoveryError, "WAL undo requires an attached engine" unless engine
+        data = record.data
+
+        case record.type
+        when :insert
+          # Delete the inserted row
+          engine.delete_row(
+            data[:table],
+            data[:row_id]
+          )
+        when :update
+          # Restore old values
+          engine.update_row(
+            data[:table],
+            data[:row_id],
+            data[:old_values]
+          )
+        when :delete
+          # Reinsert the deleted row
+          engine.insert_row(
+            data[:table],
+            data[:columns],
+            data[:row_data]
+          )
+        end
+      end
+    end
+  end
+end