mysql_replicator 0.1.0

data/lib/mysql_replicator/binlogs/json_parser.rb

@@ -0,0 +1,335 @@
+# frozen_string_literal: true
+# rbs_inline: enabled
+
+require 'json'
+
+module MysqlReplicator
+  module Binlogs
+    class JsonParser
+      # @rbs!
+      #   type jsonValue = String | Integer | Float | bool | nil
+
+      # @rbs!
+      #   type json = Hash[Symbol, json] | Array[json] | jsonValue
+
+      JSONB_TYPE_SMALL_OBJECT = 0x00 #: Integer
+      JSONB_TYPE_LARGE_OBJECT = 0x01 #: Integer
+      JSONB_TYPE_SMALL_ARRAY  = 0x02 #: Integer
+      JSONB_TYPE_LARGE_ARRAY  = 0x03 #: Integer
+      JSONB_TYPE_LITERAL      = 0x04 #: Integer
+      JSONB_TYPE_INT16        = 0x05 #: Integer
+      JSONB_TYPE_UINT16       = 0x06 #: Integer
+      JSONB_TYPE_INT32        = 0x07 #: Integer
+      JSONB_TYPE_UINT32       = 0x08 #: Integer
+      JSONB_TYPE_INT64        = 0x09 #: Integer
+      JSONB_TYPE_UINT64       = 0x0A #: Integer
+      JSONB_TYPE_DOUBLE       = 0x0B #: Integer
+      JSONB_TYPE_STRING       = 0x0C #: Integer
+      JSONB_TYPE_OPAQUE       = 0x0F #: Integer
+
+      JSONB_NULL  = 0x00 #: Integer
+      JSONB_TRUE  = 0x01 #: Integer
+      JSONB_FALSE = 0x02 #: Integer
+
+      # @rbs payload: String | nil
+      # @rbs return: json
+      def self.parse(payload)
+        return nil if payload.nil? || payload.empty?
+
+        data = payload.dup.force_encoding(Encoding::BINARY)
+
+        type = MysqlReplicator::StringUtil.read_uint8(data[0])
+        parse_value(type, data, 1)
+      end
+
+      # @rbs type: Integer
+      # @rbs data: String
+      # @rbs pos: Integer
+      # @rbs return: json
+      def self.parse_value(type, data, pos)
+        case type
+        when JSONB_TYPE_SMALL_OBJECT
+          parse_object(data, pos, small: true)
+        when JSONB_TYPE_LARGE_OBJECT
+          parse_object(data, pos, small: false)
+        when JSONB_TYPE_SMALL_ARRAY
+          parse_array(data, pos, small: true)
+        when JSONB_TYPE_LARGE_ARRAY
+          parse_array(data, pos, small: false)
+        when JSONB_TYPE_LITERAL
+          parse_literal(MysqlReplicator::StringUtil.read_uint8(data[pos]))
+        when JSONB_TYPE_INT16
+          MysqlReplicator::StringUtil.read_int16(data[pos, 2])
+        when JSONB_TYPE_UINT16
+          MysqlReplicator::StringUtil.read_uint16(data[pos, 2])
+        when JSONB_TYPE_INT32
+          MysqlReplicator::StringUtil.read_int32(data[pos, 4])
+        when JSONB_TYPE_UINT32
+          MysqlReplicator::StringUtil.read_uint32(data[pos, 4])
+        when JSONB_TYPE_INT64
+          MysqlReplicator::StringUtil.read_int64(data[pos, 8])
+        when JSONB_TYPE_UINT64
+          MysqlReplicator::StringUtil.read_uint64(data[pos, 8])
+        when JSONB_TYPE_DOUBLE
+          MysqlReplicator::StringUtil.read_double64(data[pos, 8])
+        when JSONB_TYPE_STRING
+          parse_string(data, pos)
+        when JSONB_TYPE_OPAQUE
+          code = MysqlReplicator::StringUtil.read_uint8(data[pos])
+          raise "OPAQUE type is not supported (type code: 0x#{code.to_s(16)})"
+        else
+          raise "Unknown JSON type: 0x#{type.to_s(16)} at position #{pos}"
+        end
+      end
+
+      # @rbs type: Integer
+      # @rbs data: String
+      # @rbs offset: Integer
+      # @rbs base_offset: Integer
+      # @rbs return: json
+      def self.parse_value_at_offset(type, data, offset, base_offset)
+        abs_pos = base_offset + 1 + offset
+
+        case type
+        when JSONB_TYPE_SMALL_OBJECT
+          parse_object(data, abs_pos, small: true)
+        when JSONB_TYPE_LARGE_OBJECT
+          parse_object(data, abs_pos, small: false)
+        when JSONB_TYPE_SMALL_ARRAY
+          parse_array(data, abs_pos, small: true)
+        when JSONB_TYPE_LARGE_ARRAY
+          parse_array(data, abs_pos, small: false)
+        when JSONB_TYPE_STRING
+          parse_string(data, abs_pos)
+        when JSONB_TYPE_INT64
+          MysqlReplicator::StringUtil.read_int64(data[abs_pos, 8])
+        when JSONB_TYPE_UINT64
+          MysqlReplicator::StringUtil.read_uint64(data[abs_pos, 8])
+        when JSONB_TYPE_DOUBLE
+          MysqlReplicator::StringUtil.read_double64(data[abs_pos, 8])
+        when JSONB_TYPE_OPAQUE
+          code = MysqlReplicator::StringUtil.read_uint8(data[abs_pos])
+          raise "OPAQUE type is not supported (type code: 0x#{code.to_s(16)})"
+        else
+          raise "Unexpected type at offset: 0x#{type.to_s(16)}"
+        end
+      end
+
+      # SMALL: under 65535 elements, and 2byte offsets
+      # LARGE: 65535 elements or more, and 4byte offsets
+      #
+      # @rbs data: String
+      # @rbs pos: Integer
+      # @rbs small: bool
+      # @rbs return: Hash[Symbol, json]
+      def self.parse_object(data, pos, small:)
+        offset_size = small ? 2 : 4
+
+        # base offset of this object
+        base_offset = pos - 1
+
+        # Read header
+        # element count is json key count
+        element_count = if small
+                          MysqlReplicator::StringUtil.read_uint16(data[pos, 2])
+                        else
+                          MysqlReplicator::StringUtil.read_uint32(data[pos, 4])
+                        end
+        pos += offset_size
+        _byte_size = if small
+                       MysqlReplicator::StringUtil.read_uint16(data[pos, 2])
+                     else
+                       MysqlReplicator::StringUtil.read_uint32(data[pos, 4])
+                     end
+        pos += offset_size
+
+        # Read key entries
+        key_entries = []
+        element_count.times do
+          key_offset = if small
+                         MysqlReplicator::StringUtil.read_uint16(data[pos, 2])
+                       else
+                         MysqlReplicator::StringUtil.read_uint32(data[pos, 4])
+                       end
+          pos += offset_size
+          key_length = MysqlReplicator::StringUtil.read_uint16(data[pos, 2])
+          pos += 2
+          key_entries << { offset: key_offset, length: key_length }
+        end
+
+        # Read value entries
+        value_entries = []
+        element_count.times do
+          value_type = MysqlReplicator::StringUtil.read_uint8(data[pos])
+          pos += 1
+
+          if inlined_type?(value_type, small)
+            # Small values ​​(LITERAL, INT16, etc.) are embedded directly
+            inline_value = read_inlined_value(data, pos, value_type)
+            pos += offset_size
+            value_entries << { type: value_type, inline: true, value: inline_value }
+          else
+            # Large values ​​are stored as offsets
+            value_offset = if small
+                             MysqlReplicator::StringUtil.read_uint16(data[pos, 2])
+                           else
+                             MysqlReplicator::StringUtil.read_uint32(data[pos, 4])
+                           end
+            pos += offset_size
+            value_entries << { type: value_type, inline: false, offset: value_offset }
+          end
+        end
+
+        # Get real key string
+        keys = key_entries.map do |entry|
+          abs_pos = base_offset + 1 + entry[:offset]
+          str = MysqlReplicator::StringUtil.read_str(data[abs_pos, entry[:length]])
+          str.force_encoding(Encoding::UTF_8)
+        end
+
+        # Get real value
+        values = value_entries.map do |entry|
+          if entry[:inline]
+            entry[:value]
+          else
+            parse_value_at_offset(entry[:type], data, entry[:offset], base_offset)
+          end
+        end
+
+        # Return key-value hash
+        keys.zip(values).to_h
+      end
+
+      # @rbs data: String
+      # @rbs pos: Integer
+      # @rbs small: bool
+      # @rbs return: Array[json]
+      def self.parse_array(data, pos, small:)
+        offset_size = small ? 2 : 4
+
+        # base offset of this object
+        base_offset = pos - 1
+
+        # Read header
+        # element count is json key count
+        element_count = if small
+                          MysqlReplicator::StringUtil.read_uint16(data[pos, 2])
+                        else
+                          MysqlReplicator::StringUtil.read_uint32(data[pos, 4])
+                        end
+        pos += offset_size
+        _byte_size = if small
+                       MysqlReplicator::StringUtil.read_uint16(data[pos, 2])
+                     else
+                       MysqlReplicator::StringUtil.read_uint32(data[pos, 4])
+                     end
+        pos += offset_size
+
+        # Read value entries
+        value_entries = []
+        element_count.times do
+          value_type = MysqlReplicator::StringUtil.read_uint8(data[pos])
+          pos += 1
+
+          if inlined_type?(value_type, small)
+            # Small values ​​(LITERAL, INT16, etc.) are embedded directly
+            inline_value = read_inlined_value(data, pos, value_type)
+            pos += offset_size
+            value_entries << { type: value_type, inline: true, value: inline_value }
+          else
+            # Large values ​​are stored as offsets
+            value_offset = if small
+                             MysqlReplicator::StringUtil.read_uint16(data[pos, 2])
+                           else
+                             MysqlReplicator::StringUtil.read_uint32(data[pos, 4])
+                           end
+            pos += offset_size
+            value_entries << { type: value_type, inline: false, offset: value_offset }
+          end
+        end
+
+        # Return real value array
+        value_entries.map do |entry|
+          if entry[:inline]
+            entry[:value]
+          else
+            parse_value_at_offset(entry[:type], data, entry[:offset], base_offset)
+          end
+        end
+      end
+
+      # @rbs literal_type: Integer
+      # @rbs return: bool | nil
+      def self.parse_literal(literal_type)
+        case literal_type
+        when JSONB_NULL  then nil
+        when JSONB_TRUE  then true
+        when JSONB_FALSE then false
+        else
+          raise "Unknown literal type: #{literal_type}"
+        end
+      end
+
+      # @rbs data: String
+      # @rbs pos: Integer
+      # @rbs return: String
+      def self.parse_string(data, pos)
+        length, bytes_read = read_variable_length(data, pos)
+        value = MysqlReplicator::StringUtil.read_str(data[pos + bytes_read, length])
+        value.force_encoding(Encoding::UTF_8)
+      end
+
+      # @rbs type: Integer
+      # @rbs small: bool
+      # @rbs return: bool
+      def self.inlined_type?(type, small)
+        case type
+        when JSONB_TYPE_LITERAL, JSONB_TYPE_INT16, JSONB_TYPE_UINT16
+          true
+        when JSONB_TYPE_INT32, JSONB_TYPE_UINT32
+          !small
+        else
+          false
+        end
+      end
+
+      # @rbs data: String
+      # @rbs pos: Integer
+      # @rbs return: Integer | bool | nil
+      def self.read_inlined_value(data, pos, type)
+        case type
+        when JSONB_TYPE_LITERAL
+          parse_literal(MysqlReplicator::StringUtil.read_uint8(data[pos]))
+        when JSONB_TYPE_INT16
+          MysqlReplicator::StringUtil.read_int16(data[pos, 2])
+        when JSONB_TYPE_UINT16
+          MysqlReplicator::StringUtil.read_uint16(data[pos, 2])
+        when JSONB_TYPE_INT32
+          MysqlReplicator::StringUtil.read_int32(data[pos, 4])
+        when JSONB_TYPE_UINT32
+          MysqlReplicator::StringUtil.read_uint32(data[pos, 4])
+        end
+      end
+
+      # @rbs data: String
+      # @rbs pos: Integer
+      # @rbs return: [Integer, Integer]
+      def self.read_variable_length(data, pos)
+        length = 0
+        shift = 0
+        bytes_read = 0
+
+        loop do
+          byte = MysqlReplicator::StringUtil.read_uint8(data[pos + bytes_read])
+          bytes_read += 1
+          length |= (byte & 0x7F) << shift
+          break if (byte & 0x80).zero?
+
+          shift += 7
+        end
+
+        [length, bytes_read]
+      end
+    end
+  end
+end

data/lib/mysql_replicator/binlogs/query_event_parser.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+# rbs_inline: enabled
+
+module MysqlReplicator
+  module Binlogs
+    class QueryEventParser
+      # @rbs!
+      #   type execution = {
+      #     thread_id: Integer,
+      #     exec_time: Integer,
+      #     error_code: Integer,
+      #     database: String | nil,
+      #     sql: String | nil
+      #   }
+
+      # @rbs payload: String
+      # @rbs checksum_enabled: bool
+      # @rbs return: execution
+      def self.parse(payload, checksum_enabled)
+        offset = 0
+
+        thread_id = MysqlReplicator::StringUtil.read_uint32(payload[offset, 4])
+        offset += 4
+
+        exec_time = MysqlReplicator::StringUtil.read_uint32(payload[offset, 4])
+        offset += 4
+
+        db_len = MysqlReplicator::StringUtil.read_uint8(payload[offset])
+        offset += 1
+
+        error_code = MysqlReplicator::StringUtil.read_uint16(payload[offset, 2])
+        offset += 2
+
+        # Skip status variables
+        status_vars_len = MysqlReplicator::StringUtil.read_uint16(payload[offset, 2])
+        offset += 2
+        if status_vars_len > 0
+          offset += status_vars_len
+        end
+
+        # Database name (null-terminated)
+        if db_len > 0
+          database = payload[offset, db_len]
+          offset += db_len
+        end
+
+        # Skip null terminator for database name
+        if offset < payload.length && payload[offset] == "\x00"
+          offset += 1
+        end
+
+        # The rest is the SQL query
+        if offset < payload.length
+          # Remove checksum if present (last 4 bytes)
+          sql_end = checksum_enabled ? -5 : -1
+          sql = payload[offset..sql_end]
+        end
+
+        {
+          thread_id: thread_id,
+          exec_time: exec_time,
+          error_code: error_code,
+          database: database,
+          sql: sql
+        }
+      end
+    end
+  end
+end

data/lib/mysql_replicator/binlogs/rotate_event_parser.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+# rbs_inline: enabled
+
+module MysqlReplicator
+  module Binlogs
+    class RotateEventParser
+      # @rbs!
+      #   type execution = {
+      #     position: Integer,
+      #     filename: String
+      #   }
+
+      # @rbs payload: String
+      # @rbs checksum_enabled: bool
+      # @rbs return: execution
+      def self.parse(payload, checksum_enabled = false)
+        # Position (8 bytes, where the next binlog starts, Little Endian 64-bit)
+        position = MysqlReplicator::StringUtil.read_uint64(payload[0, 8])
+        # Filename (remaining bytes, new binlog filename)
+        filename = MysqlReplicator::StringUtil.read_str(payload[8..])
+
+        # Remove checksum if present (last 4 bytes)
+        if checksum_enabled && filename.length >= 4
+          filename = MysqlReplicator::StringUtil.read_str(filename[0...-4])
+        end
+
+        # Find null terminator or use entire remaining data
+        null_pos = filename.index("\x00")
+        filename = MysqlReplicator::StringUtil.read_str(filename[0...null_pos]) if null_pos
+        # Clean up any non-printable characters
+        filename = filename.force_encoding(Encoding::UTF_8).scrub
+
+        { position: position, filename: filename }
+      end
+    end
+  end
+end

data/lib/mysql_replicator/binlogs/rows_event_parser.rb

@@ -0,0 +1,161 @@
+# frozen_string_literal: true
+# rbs_inline: enabled
+
+require 'stringio'
+
+module MysqlReplicator
+  module Binlogs
+    class RowsEventParser
+      # @rbs!
+      #   type rowData = {
+      #     ordinal_position: Integer,
+      #     data_type: String,
+      #     column_name: String,
+      #     value: String | Integer | bool | nil,
+      #     primary_key: bool
+      #   }
+
+      # @rbs!
+      #   type execution = {
+      #     table_id: Integer,
+      #     flags: Integer,
+      #     extra_data_length: Integer,
+      #     column_count: Integer,
+      #     rows: Array[rowData]
+      #   }
+
+      # @rbs event_type: :WRITE_ROWS | :UPDATE_ROWS | :DELETE_ROWS
+      # @rbs payload: String
+      # @rbs checksum_enabled bool
+      # @rbs table_map: Hash[Integer, MysqlReplicator::Binlogs::TableMapEventParser::execution]
+      # @rbs return: execution
+      def self.parse(event_type, payload, checksum_enabled, table_map)
+        io = StringIO.new(payload)
+        io.set_encoding(Encoding::BINARY)
+
+        # 4bytes checksum at the end if CRC32 checksum is enabled
+        payload_size = checksum_enabled ? payload.bytesize - 4 : payload.bytesize
+
+        # Table ID (6 bytes)
+        table_id = StringIOUtil.read_uint48(io)
+        # Flags (2 bytes)
+        flags = StringIOUtil.read_uint16(io)
+        # Extra data length (2 bytes)
+        extra_data_length = StringIOUtil.read_uint16(io)
+        # Skip extra data if present
+        io.read(extra_data_length - 2) if extra_data_length > 2
+
+        # Column count (variable length encoded)
+        column_count = read_packed_integer(io)
+
+        # Columns present bitmap
+        bitmap_size = (column_count + 7) / 8
+        # A bitmap indicating which columns are present in the row data
+        columns_present_bitmap = MysqlReplicator::StringIOUtil.read_array_from_int8(io, bitmap_size)
+        # if UPDATE_ROWS event, after this bitmap, there is another bitmap for "columns present in the after image"
+        columns_after_bitmap = event_type == :UPDATE_ROWS ? MysqlReplicator::StringIOUtil.read_array_from_int8(io, bitmap_size) : []
+
+        # Parse row data
+        table_def = table_map[table_id]
+        rows = []
+        while io.pos < payload_size
+          if event_type == :UPDATE_ROWS
+            before_row = parse_row(io, column_count, columns_present_bitmap, table_def)
+            after_row = parse_row(io, column_count, columns_after_bitmap, table_def)
+            rows << { before: before_row, after: after_row }
+          else
+            row = parse_row(io, column_count, columns_present_bitmap, table_def)
+            rows << row
+          end
+        end
+
+        {
+          table_id: table_id,
+          flags: flags,
+          extra_data_length: extra_data_length,
+          column_count: column_count,
+          rows: rows
+        }
+      end
+
+      # @rbs io: StringIO
+      # @rbs column_count: Integer
+      # @rbs columns_present_bitmap: Array[Integer]
+      # @rbs table_def: MysqlReplicator::Binlogs::TableMapEventParser::execution
+      # @rbs return: Array[rowData]
+      def self.parse_row(io, column_count, columns_present_bitmap, table_def)
+        # Null bitmap
+        # A bitmap indicating which columns are NULL
+        present_count = count_bits(columns_present_bitmap, column_count)
+        null_bitmap_size = (present_count + 7) / 8
+        null_bitmap = MysqlReplicator::StringIOUtil.read_array_from_int8(io, null_bitmap_size)
+        null_bit_index = 0
+
+        row = []
+        table_def[:columns].each_with_index do |column_def, column_index|
+          unless bit_set?(columns_present_bitmap, column_index)
+            next
+          end
+
+          column_name = column_def[:column_name]
+
+          # Check if the column is NULL
+          value = if bit_set?(null_bitmap, null_bit_index)
+                    nil
+                  else
+                    MysqlReplicator::Binlogs::ColumnParser.parse(io, column_def)
+                  end
+          null_bit_index += 1
+
+          row << {
+            ordinal_position: column_def[:ordinal_position].to_i,
+            data_type: column_def[:data_type],
+            column_name: column_name,
+            value: value,
+            primary_key: column_def[:primary_key]
+          }
+        end
+
+        row
+      end
+
+      # @rbs io: StringIO
+      # @rbs return: Integer
+      def self.read_packed_integer(io)
+        first = StringIOUtil.read_uint8(io)
+        case first
+        when 0..250
+          first
+        when 252
+          StringIOUtil.read_uint16(io)
+        when 253
+          StringIOUtil.read_uint24(io)
+        when 254
+          StringIOUtil.read_uint64(io)
+        else
+          raise "Invalid packed integer: #{first}"
+        end
+      end
+
+      # @rbs bitmap: Array[Integer]
+      # @rbs index: Integer
+      # @rbs return: bool
+      def self.bit_set?(bitmap, index)
+        byte_index = index / 8
+        bit_index = index % 8
+        (bitmap[byte_index] & (1 << bit_index)) != 0
+      end
+
+      # @rbs bitmap: Array[Integer]
+      # @rbs max_bits: Integer
+      # @rbs return: Integer
+      def self.count_bits(bitmap, max_bits)
+        count = 0
+        max_bits.times do |i|
+          count += 1 if bit_set?(bitmap, i)
+        end
+        count
+      end
+    end
+  end
+end