rubydb 0.1.0

data/lib/rubydb/replication/replica.rb

@@ -0,0 +1,384 @@
+# frozen_string_literal: true
+
+require "socket"
+require "time"
+require "monitor"
+require "json"
+require "fileutils"
+
+module RubyDB
+  module Replication
+    # Replica - Replica database server
+    class Replica
+      attr_reader :config, :engine, :primary_host, :primary_port
+      attr_reader :stats
+
+      # Replica states
+      STATE_INIT = :init
+      STATE_CONNECTING = :connecting
+      STATE_STREAMING = :streaming
+      STATE_REPLAYING = :replaying
+      STATE_CATCHING_UP = :catching_up
+      STATE_SYNCED = :synced
+      STATE_DISCONNECTED = :disconnected
+      STATE_FAILED = :failed
+
+      def initialize(engine, config = {})
+        @engine = engine
+        @config = config
+        @primary_host = config[:primary_host] || "localhost"
+        @primary_port = config[:primary_port] || 7433
+        @replication_port = config[:replication_port] || 7434
+        @state = STATE_INIT
+        @connection = nil
+        @replication_stream = nil
+        @receive_buffer = +""
+        @max_frame_size = config[:max_replication_frame_bytes] || 1_048_576
+        @last_received_lsn = nil
+        @last_replayed_lsn = nil
+        @catchup_start_time = nil
+        @stats = {
+          bytes_received: 0,
+          transactions_replayed: 0,
+          replication_lag_ms: 0,
+          reconnect_attempts: 0,
+          last_connect_time: nil,
+          last_disconnect_time: nil,
+          total_uptime_ms: 0,
+          last_error: nil
+        }
+        @lock = Monitor.new
+        @running = false
+        @replication_thread = nil
+        @recovery = nil
+        @retry_interval = config[:retry_interval] || 5
+        @max_retry_attempts = config[:max_retry_attempts] || 10
+        @heartbeat_interval = config[:heartbeat_interval] || 5
+        @retry_count = 0
+        @state_path = config[:state_path] || "#{@engine.path}.replica_state.json"
+        @state_write_lock = Mutex.new
+        @state_write_sequence = 0
+        load_state
+        @engine.set_replication_read_only(true) if @engine.respond_to?(:set_replication_read_only)
+      end
+
+      def start
+        @lock.synchronize do
+          return if @running
+
+          @running = true
+          @state = STATE_CONNECTING
+          @replication_thread = Thread.new { replication_loop }
+
+          puts "Replica started, connecting to primary at #{@primary_host}:#{@replication_port}"
+          true
+        end
+      end
+
+      def stop
+        thread, connection = @lock.synchronize do
+          return unless @running
+
+          @running = false
+          @state = STATE_DISCONNECTED
+          [@replication_thread, @connection]
+        end
+
+        # Closing the socket wakes a blocked read. Let an in-flight frame finish
+        # its durable state publication before joining the worker; killing it can
+        # interrupt File.open/rename and leave replica state ambiguous.
+        connection&.close
+        thread&.join(5)
+        if thread&.alive?
+          thread.kill
+          thread.join
+        end
+
+        @lock.synchronize do
+          @connection = nil
+          @replication_thread = nil
+          @stats[:last_disconnect_time] = Time.now
+        end
+        true
+      end
+
+      def running?
+        @lock.synchronize { @running }
+      end
+
+      def promote_to_primary(recovery_point: nil)
+        @lock.synchronize do
+          validate_promotion!(recovery_point)
+        end
+
+        # Stop outside the monitor so the worker can finish without waiting on
+        # the promotion caller's lock.
+        stop
+
+        # Promotion is an explicit operator action. Re-enable ordinary
+        # mutations only after the replication stream has been stopped.
+        @engine.set_replication_read_only(false) if @engine.respond_to?(:set_replication_read_only)
+        @lock.synchronize { @state = STATE_SYNCED }
+        true
+      end
+
+      def replay_transaction(transaction_data, lsn = nil)
+        @lock.synchronize do
+          # Apply transaction to engine
+          @engine.apply_transaction(transaction_data)
+          @stats[:transactions_replayed] += 1
+          @last_replayed_lsn = lsn || transaction_data[:lsn]
+          persist_state
+
+          true
+        end
+      end
+
+      def replication_status
+        @lock.synchronize do
+          {
+            state: @state,
+            primary_host: @primary_host,
+            primary_port: @primary_port,
+            last_received_lsn: @last_received_lsn,
+            last_replayed_lsn: @last_replayed_lsn,
+            lag_ms: @stats[:replication_lag_ms],
+            bytes_received: @stats[:bytes_received],
+            transactions_replayed: @stats[:transactions_replayed],
+            connected: @state == STATE_STREAMING || @state == STATE_SYNCED
+          }
+        end
+      end
+
+      private
+
+      def replication_loop
+        while @running
+          begin
+            case @state
+            when STATE_INIT, STATE_CONNECTING
+              connect_to_primary
+            when STATE_STREAMING
+              stream_replication
+            when STATE_CATCHING_UP
+              catch_up
+            when STATE_SYNCED
+              sleep(1)
+            when STATE_DISCONNECTED, STATE_FAILED
+              reconnect
+            end
+          rescue => e
+            @stats[:last_error] = "#{e.class}: #{e.message}"
+            @state = STATE_FAILED
+            @stats[:reconnect_attempts] += 1
+            sleep(@retry_interval)
+          end
+        end
+      end
+
+      def connect_to_primary
+        @lock.synchronize do
+          begin
+            @connection = TCPSocket.new(@primary_host, @replication_port)
+            @state = STATE_CONNECTING
+            @stats[:last_connect_time] = Time.now
+
+            # Send replica handshake
+            handshake = {
+              type: "replica_handshake",
+              replica_id: @config[:replica_id] || "replica_#{Process.pid}",
+              protocol_version: 1,
+              wal_position: @last_replayed_lsn || 0,
+              auth_token: @config[:replication_auth_token] || @config[:auth_token]
+            }
+
+            @connection.write(JSON.generate(handshake) + "\n")
+            response = JSON.parse(@connection.readline)
+
+            if response["success"]
+              @state = STATE_STREAMING
+              @stats[:reconnect_attempts] = 0
+              @retry_count = 0
+              puts "Connected to primary as replica"
+            else
+              @state = STATE_FAILED
+            end
+
+          rescue => e
+            @stats[:last_error] = "#{e.class}: #{e.message}"
+            @state = STATE_FAILED
+            @stats[:reconnect_attempts] += 1
+            raise
+          end
+        end
+      end
+
+      def stream_replication
+        while @running && @state == STATE_STREAMING
+          begin
+            # Read replication data
+            if @connection
+              ready = IO.select([@connection], nil, nil, @heartbeat_interval)
+              unless ready
+                @connection.write(
+                  JSON.generate(
+                    type: "heartbeat",
+                    wal_position: @last_replayed_lsn || 0
+                  ) + "\n"
+                )
+                @connection.flush
+                next
+              end
+
+              data = @connection.readpartial(4096)
+              @stats[:bytes_received] += data.bytesize
+
+              # Process replication data
+              process_replication_data(data)
+            else
+              @state = STATE_DISCONNECTED
+              break
+            end
+          rescue EOFError, Errno::ECONNRESET
+            @state = STATE_DISCONNECTED
+            break
+          rescue => e
+            @state = STATE_FAILED
+            break
+          end
+        end
+      end
+
+      def process_replication_data(data)
+        @receive_buffer << data.to_s
+        if @receive_buffer.bytesize > @max_frame_size
+          raise RubyDB::ReplicationError, "Replication frame exceeds #{@max_frame_size} bytes"
+        end
+
+        while (newline = @receive_buffer.index("\n"))
+          line = @receive_buffer.slice!(0, newline + 1).strip
+          next if line.empty?
+
+          message = JSON.parse(line, symbolize_names: true)
+          if message[:type].to_s == "replication_bootstrap"
+            apply_bootstrap_schema(message[:schema].to_s)
+            next
+          end
+          next unless message[:type].to_s == "replication_data"
+
+          Array(message[:data]).each do |entry|
+            next if entry[:lsn] && @last_replayed_lsn && entry[:lsn] <= @last_replayed_lsn
+            replay_transaction(entry[:data] || entry, entry[:lsn])
+            @last_received_lsn = entry[:lsn]
+            persist_state
+            @connection&.write(JSON.generate(type: "ack", lsn: @last_received_lsn) + "\n")
+          end
+        end
+      end
+
+      def catch_up
+        @state = if @last_received_lsn && @last_received_lsn == @last_replayed_lsn
+                   STATE_SYNCED
+                 else
+                   STATE_STREAMING
+                 end
+      end
+
+      def apply_bootstrap_schema(schema)
+        return if schema.empty? || !@engine.respond_to?(:with_replication_apply)
+
+        @engine.with_replication_apply do
+          statements = RubyDB::SQL::Parser.new(RubyDB::SQL::Lexer.new(schema).tokenize).parse
+          statements.each do |statement|
+            plan = RubyDB::Execution::Planner.new(@engine).plan(statement)
+            RubyDB::Execution::Executor.new(@engine).execute(plan)
+          end
+        end
+      rescue StandardError => error
+        raise RubyDB::ReplicationError, "Replica bootstrap failed: #{error.message}"
+      end
+
+      def load_state
+        return unless File.file?(@state_path)
+        state = JSON.parse(File.read(@state_path), symbolize_names: true)
+        @last_received_lsn = state[:last_received_lsn]
+        @last_replayed_lsn = state[:last_replayed_lsn]
+      rescue JSON::ParserError => error
+        raise RubyDB::ReplicationError, "Invalid replica state #{@state_path}: #{error.message}"
+      end
+
+      def persist_state
+        @state_write_lock.synchronize do
+          FileUtils.mkdir_p(File.dirname(@state_path))
+          @state_write_sequence += 1
+          temporary = "#{@state_path}.tmp-#{Process.pid}-#{Thread.current.object_id}-#{@state_write_sequence}"
+          payload = JSON.generate(
+            last_received_lsn: @last_received_lsn,
+            last_replayed_lsn: @last_replayed_lsn,
+            updated_at: Time.now.iso8601
+          )
+          File.open(temporary, "wb") do |file|
+            file.write(payload)
+            file.flush
+            file.fsync
+          end
+          begin
+            File.rename(temporary, @state_path)
+          rescue Errno::EACCES, Errno::EPERM
+            # Windows does not replace an existing destination with rename(2).
+            # FileUtils performs the platform-appropriate forced move while
+            # retaining the fsynced temporary payload as the source.
+            raise unless Gem.win_platform? && File.file?(@state_path)
+
+            FileUtils.mv(temporary, @state_path, force: true)
+          end
+        ensure
+          File.delete(temporary) if defined?(temporary) && File.file?(temporary)
+        end
+      end
+
+      def reconnect
+        @retry_count += 1
+
+        if @retry_count > @max_retry_attempts
+          @state = STATE_FAILED
+          return
+        end
+
+        sleep(@retry_interval * @retry_count)
+        @state = STATE_CONNECTING
+      end
+
+      def disconnect
+        @connection&.close
+        @connection = nil
+        @state = STATE_DISCONNECTED
+        @stats[:last_disconnect_time] = Time.now
+      end
+
+      def validate_promotion!(recovery_point)
+        promotable_state = [STATE_STREAMING, STATE_SYNCED, STATE_DISCONNECTED, STATE_CONNECTING].include?(@state)
+        caught_up_after_disconnect = [STATE_DISCONNECTED, STATE_CONNECTING].include?(@state) &&
+                                     @last_replayed_lsn &&
+                                     @last_received_lsn == @last_replayed_lsn
+        unless promotable_state && ([STATE_STREAMING, STATE_SYNCED].include?(@state) || caught_up_after_disconnect)
+          raise ReplicationError, "Replica is not synchronized enough for promotion"
+        end
+
+        if @last_received_lsn != @last_replayed_lsn
+          raise ReplicationError,
+                "Replica has unapplied replication data (received=#{@last_received_lsn.inspect}, " \
+                "replayed=#{@last_replayed_lsn.inspect})"
+        end
+
+        return true if recovery_point.nil?
+
+        target = recovery_point.to_i
+        replayed = @last_replayed_lsn.to_i
+        raise ReplicationError, "Replica has not reached recovery point #{target}" if replayed < target
+
+        true
+      end
+    end
+  end
+end

data/lib/rubydb/replication/replication_log.rb

@@ -0,0 +1,200 @@
+# frozen_string_literal: true
+
+require "json"
+require "fileutils"
+require "time"
+require "monitor"
+
+module RubyDB
+  module Replication
+    # ReplicationLog - Logs replication transactions
+    class ReplicationLog
+      attr_reader :stats
+
+      def initialize(engine, config = {})
+        @engine = engine
+        @config = config
+        # A replication log belongs to one database. A process-wide relative
+        # directory can mix records from unrelated clusters after restart.
+        @log_dir = config[:log_dir] || "#{@engine.path}.replication_log"
+        @max_segment_size = config[:max_segment_size] || 16 * 1024 * 1024
+        @keep_segments = config[:keep_segments] || 100
+        @current_segment = nil
+        @current_segment_size = 0
+        @segment_counter = 0
+        @stats = {
+          transactions_logged: 0,
+          bytes_logged: 0,
+          segments_created: 0,
+          segments_archived: 0,
+          segments_deleted: 0
+        }
+        @lock = Monitor.new
+
+        create_log_directory
+        open_current_segment
+      end
+
+      def log_transaction(transaction_data, lsn)
+        @lock.synchronize do
+          entry = {
+            lsn: lsn,
+            timestamp: Time.now.iso8601,
+            transaction_id: transaction_data[:id],
+            data: transaction_data
+          }
+
+          json = JSON.generate(entry) + "\n"
+          @current_segment.write(json)
+          @current_segment.flush
+
+          @stats[:transactions_logged] += 1
+          @stats[:bytes_logged] += json.bytesize
+          @current_segment_size += json.bytesize
+
+          # Rotate if segment is full
+          if @current_segment_size >= @max_segment_size
+            rotate_segment
+          end
+
+          true
+        end
+      end
+
+      def read_transactions(from_lsn = nil, to_lsn = nil)
+        @lock.synchronize do
+          transactions = []
+          segments = list_segments
+
+          segments.each do |segment_path|
+            File.open(segment_path, "r") do |file|
+              file.each_line do |line|
+                begin
+                  entry = JSON.parse(line, symbolize_names: true)
+
+                  # Filter by LSN range
+                  if from_lsn && entry[:lsn] < from_lsn
+                    next
+                  end
+                  if to_lsn && entry[:lsn] > to_lsn
+                    next
+                  end
+
+                  transactions << entry
+                rescue JSON::ParserError
+                  # Skip malformed entries
+                end
+              end
+            end
+          end
+
+          transactions
+        end
+      end
+
+      def get_last_lsn
+        @lock.synchronize do
+          list_segments.reverse_each do |segment_path|
+            begin
+              File.readlines(segment_path).reverse_each do |line|
+                begin
+                  lsn = JSON.parse(line, symbolize_names: true)[:lsn]
+                  return lsn unless lsn.nil?
+                rescue JSON::ParserError
+                  next
+                end
+              end
+            rescue Errno::ENOENT
+              # A concurrently rotated segment may disappear between the
+              # directory listing and the read. Continue with the remaining
+              # durable segments for LSN recovery.
+            end
+          end
+          nil
+        end
+      end
+
+      def archive_segment(segment_name)
+        @lock.synchronize do
+          archive_dir = File.join(@log_dir, "archive")
+          FileUtils.mkdir_p(archive_dir)
+
+          src = File.join(@log_dir, segment_name)
+          dst = File.join(archive_dir, segment_name)
+
+          if File.exist?(src)
+            FileUtils.mv(src, dst)
+            @stats[:segments_archived] += 1
+            return true
+          end
+
+          false
+        end
+      end
+
+      def delete_old_segments
+        @lock.synchronize do
+          segments = list_segments
+          return if segments.size <= @keep_segments
+
+          to_delete = segments.first(segments.size - @keep_segments)
+          to_delete.each do |segment_path|
+            File.delete(segment_path)
+            @stats[:segments_deleted] += 1
+          end
+        end
+      end
+
+      def stats
+        @lock.synchronize do
+          stats_hash = @stats.dup
+          stats_hash[:current_segment] = File.basename(@current_segment.path) if @current_segment
+          stats_hash[:current_segment_size] = @current_segment_size
+          stats_hash[:segments_count] = list_segments.size
+          stats_hash[:log_dir] = @log_dir
+          stats_hash
+        end
+      end
+
+      def close
+        @lock.synchronize do
+          @current_segment&.close unless @current_segment&.closed?
+          @current_segment = nil
+          true
+        end
+      end
+
+      def closed?
+        @current_segment.nil? || @current_segment.closed?
+      end
+
+      private
+
+      def create_log_directory
+        FileUtils.mkdir_p(@log_dir)
+        FileUtils.mkdir_p(File.join(@log_dir, "archive"))
+      end
+
+      def open_current_segment
+        @segment_counter += 1
+        segment_name = "replication_#{Time.now.strftime('%Y%m%d_%H%M%S')}_#{@segment_counter}.log"
+        segment_path = File.join(@log_dir, segment_name)
+
+        @current_segment = File.open(segment_path, "a+")
+        @current_segment.sync = false
+        @current_segment_size = File.size(segment_path)
+        @stats[:segments_created] += 1
+      end
+
+      def rotate_segment
+        @current_segment.close
+        delete_old_segments
+        open_current_segment
+      end
+
+      def list_segments
+        Dir.glob(File.join(@log_dir, "replication_*.log")).sort
+      end
+    end
+  end
+end