mysql_replicator 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.rspec +3 -0
- data/.rubocop.yml +79 -0
- data/CHANGELOG.md +5 -0
- data/CODE_OF_CONDUCT.md +132 -0
- data/LICENSE.txt +21 -0
- data/README.md +115 -0
- data/Rakefile +12 -0
- data/Steepfile +22 -0
- data/docker-compose.yml +13 -0
- data/lib/mysql_replicator/binlog_client.rb +201 -0
- data/lib/mysql_replicator/binlogs/column_parser.rb +425 -0
- data/lib/mysql_replicator/binlogs/constants.rb +74 -0
- data/lib/mysql_replicator/binlogs/event_parser.rb +134 -0
- data/lib/mysql_replicator/binlogs/format_description_event_parser.rb +24 -0
- data/lib/mysql_replicator/binlogs/json_parser.rb +335 -0
- data/lib/mysql_replicator/binlogs/query_event_parser.rb +69 -0
- data/lib/mysql_replicator/binlogs/rotate_event_parser.rb +37 -0
- data/lib/mysql_replicator/binlogs/rows_event_parser.rb +161 -0
- data/lib/mysql_replicator/binlogs/table_map_event_parser.rb +155 -0
- data/lib/mysql_replicator/binlogs/xid_event_parser.rb +25 -0
- data/lib/mysql_replicator/connection.rb +226 -0
- data/lib/mysql_replicator/connections/auth.rb +303 -0
- data/lib/mysql_replicator/connections/handshake.rb +132 -0
- data/lib/mysql_replicator/connections/query.rb +322 -0
- data/lib/mysql_replicator/error.rb +6 -0
- data/lib/mysql_replicator/logger.rb +43 -0
- data/lib/mysql_replicator/string_io_util.rb +199 -0
- data/lib/mysql_replicator/string_util.rb +106 -0
- data/lib/mysql_replicator/version.rb +6 -0
- data/lib/mysql_replicator.rb +51 -0
- data/sig/generated/mysql_replicator/binlog_client.rbs +52 -0
- data/sig/generated/mysql_replicator/binlogs/column_parser.rbs +134 -0
- data/sig/generated/mysql_replicator/binlogs/constants.rbs +69 -0
- data/sig/generated/mysql_replicator/binlogs/event_parser.rbs +35 -0
- data/sig/generated/mysql_replicator/binlogs/format_description_event_parser.rbs +13 -0
- data/sig/generated/mysql_replicator/binlogs/json_parser.rbs +101 -0
- data/sig/generated/mysql_replicator/binlogs/query_event_parser.rbs +14 -0
- data/sig/generated/mysql_replicator/binlogs/rotate_event_parser.rbs +14 -0
- data/sig/generated/mysql_replicator/binlogs/rows_event_parser.rbs +39 -0
- data/sig/generated/mysql_replicator/binlogs/table_map_event_parser.rbs +31 -0
- data/sig/generated/mysql_replicator/binlogs/xid_event_parser.rbs +13 -0
- data/sig/generated/mysql_replicator/connection.rbs +103 -0
- data/sig/generated/mysql_replicator/connections/auth.rbs +76 -0
- data/sig/generated/mysql_replicator/connections/handshake.rbs +21 -0
- data/sig/generated/mysql_replicator/connections/query.rbs +62 -0
- data/sig/generated/mysql_replicator/error.rbs +6 -0
- data/sig/generated/mysql_replicator/logger.rbs +26 -0
- data/sig/generated/mysql_replicator/string_io_util.rbs +75 -0
- data/sig/generated/mysql_replicator/string_util.rbs +45 -0
- data/sig/generated/mysql_replicator/types.rbs +19 -0
- data/sig/generated/mysql_replicator/version.rbs +5 -0
- data/sig/generated/mysql_replicator.rbs +16 -0
- metadata +124 -0
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
# rbs_inline: enabled
|
|
3
|
+
|
|
4
|
+
module MysqlReplicator
|
|
5
|
+
module Binlogs
|
|
6
|
+
class TableMapEventParser
|
|
7
|
+
# @rbs!
|
|
8
|
+
# type columnData = {
|
|
9
|
+
# ordinal_position: Integer,
|
|
10
|
+
# data_type: String,
|
|
11
|
+
# column_name: String,
|
|
12
|
+
# column_type: String,
|
|
13
|
+
# enum_values: Array[String] | nil,
|
|
14
|
+
# nullable: bool,
|
|
15
|
+
# column_default: String | nil,
|
|
16
|
+
# numeric_precision: Integer,
|
|
17
|
+
# numeric_scale: Integer,
|
|
18
|
+
# character_maximum_length: Integer,
|
|
19
|
+
# character_set_name: String,
|
|
20
|
+
# collation_name: String,
|
|
21
|
+
# primary_key: bool
|
|
22
|
+
# }
|
|
23
|
+
|
|
24
|
+
# @rbs!
|
|
25
|
+
# type execution = {
|
|
26
|
+
# database: String | nil,
|
|
27
|
+
# table: String | nil,
|
|
28
|
+
# table_id: Integer,
|
|
29
|
+
# columns: Array[columnData],
|
|
30
|
+
# flags: Integer
|
|
31
|
+
# }
|
|
32
|
+
|
|
33
|
+
# @rbs payload: String
|
|
34
|
+
# @rbs connection: MysqlReplicator::Connection
|
|
35
|
+
# @rbs return: execution
|
|
36
|
+
def self.parse(payload, connection)
|
|
37
|
+
offset = 0
|
|
38
|
+
|
|
39
|
+
# Table ID (6 bytes)
|
|
40
|
+
table_id = to_little_endian(MysqlReplicator::StringUtil.read_array_from_int8(payload[0, 6]))
|
|
41
|
+
offset += 6
|
|
42
|
+
|
|
43
|
+
# Flags (2 bytes)
|
|
44
|
+
flags = MysqlReplicator::StringUtil.read_uint16(payload[offset, 2])
|
|
45
|
+
offset += 2
|
|
46
|
+
|
|
47
|
+
# Database name length (1 byte) + database name + null terminator
|
|
48
|
+
db_name_len = MysqlReplicator::StringUtil.read_uint8(payload[offset, 1])
|
|
49
|
+
offset += 1
|
|
50
|
+
database_name = MysqlReplicator::StringUtil.read_str(payload[offset, db_name_len])
|
|
51
|
+
offset += db_name_len + 1 # +1 for null terminator
|
|
52
|
+
|
|
53
|
+
# Table name length (1 byte) + table name + null terminator
|
|
54
|
+
table_name_len = MysqlReplicator::StringUtil.read_uint8(payload[offset, 1])
|
|
55
|
+
offset += 1
|
|
56
|
+
table_name = MysqlReplicator::StringUtil.read_str(payload[offset, table_name_len])
|
|
57
|
+
|
|
58
|
+
# Get actual column names from table schema
|
|
59
|
+
columns = get_table_columns(connection, database_name, table_name)
|
|
60
|
+
|
|
61
|
+
{
|
|
62
|
+
database: database_name,
|
|
63
|
+
table: table_name,
|
|
64
|
+
table_id: table_id,
|
|
65
|
+
columns: columns,
|
|
66
|
+
flags: flags
|
|
67
|
+
}
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
# @rbs bytes: Array[Integer]
|
|
71
|
+
# @rbs return: Integer
|
|
72
|
+
def self.to_little_endian(bytes)
|
|
73
|
+
result = 0
|
|
74
|
+
bytes.each_with_index do |byte, i|
|
|
75
|
+
result |= (byte << (i * 8))
|
|
76
|
+
end
|
|
77
|
+
result
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
# @rbs connection: MysqlReplicator::Connection
|
|
81
|
+
# @rbs database: String
|
|
82
|
+
# @rbs table: String
|
|
83
|
+
# @rbs return: Array[columnData]
|
|
84
|
+
def self.get_table_columns(connection, database, table)
|
|
85
|
+
# Create a separate connection to query table structure
|
|
86
|
+
query_connection = connection.dup
|
|
87
|
+
|
|
88
|
+
# Query table structure
|
|
89
|
+
# IMPORTANT: Column data is stored in ascending order of ORDINAL_POSITION
|
|
90
|
+
query = <<~SQL
|
|
91
|
+
SELECT
|
|
92
|
+
ORDINAL_POSITION,
|
|
93
|
+
DATA_TYPE,
|
|
94
|
+
COLUMN_NAME,
|
|
95
|
+
COLUMN_TYPE,
|
|
96
|
+
IS_NULLABLE,
|
|
97
|
+
COLUMN_DEFAULT,
|
|
98
|
+
NUMERIC_PRECISION,
|
|
99
|
+
NUMERIC_SCALE,
|
|
100
|
+
CHARACTER_MAXIMUM_LENGTH,
|
|
101
|
+
CHARACTER_SET_NAME,
|
|
102
|
+
COLLATION_NAME,
|
|
103
|
+
COLUMN_KEY
|
|
104
|
+
FROM INFORMATION_SCHEMA.COLUMNS
|
|
105
|
+
WHERE TABLE_SCHEMA = '#{database}'
|
|
106
|
+
AND TABLE_NAME = '#{table}'
|
|
107
|
+
ORDER BY ORDINAL_POSITION
|
|
108
|
+
SQL
|
|
109
|
+
result = query_connection.query(query)
|
|
110
|
+
|
|
111
|
+
# Close the separated connection
|
|
112
|
+
query_connection.close
|
|
113
|
+
|
|
114
|
+
if result.nil? || !result[:rows].is_a?(Array)
|
|
115
|
+
return []
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
result[:rows].map do |row|
|
|
119
|
+
{
|
|
120
|
+
ordinal_position: row[:ordinal_position].to_i,
|
|
121
|
+
data_type: row[:data_type].to_s,
|
|
122
|
+
column_name: row[:column_name].to_s,
|
|
123
|
+
column_type: row[:column_type].to_s,
|
|
124
|
+
enum_values: extract_enum_from_column_type(row[:data_type].to_s, row[:column_type].to_s),
|
|
125
|
+
nullable: row[:is_nullable] == 'YES',
|
|
126
|
+
column_default: row[:column_default],
|
|
127
|
+
numeric_precision: row[:numeric_precision].to_i,
|
|
128
|
+
numeric_scale: row[:numeric_scale].to_i,
|
|
129
|
+
character_maximum_length: row[:character_maximum_length].to_i,
|
|
130
|
+
character_set_name: row[:character_set_name].to_s,
|
|
131
|
+
collation_name: row[:collation_name].to_s,
|
|
132
|
+
primary_key: row[:column_key] == 'PRI'
|
|
133
|
+
}
|
|
134
|
+
end
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
# @rbs data_type: String
|
|
138
|
+
# @rbs column_type: String
|
|
139
|
+
# @rbs return: Array[String] | nil
|
|
140
|
+
def self.extract_enum_from_column_type(data_type, column_type)
|
|
141
|
+
return nil unless data_type.downcase == 'enum'
|
|
142
|
+
|
|
143
|
+
# Extract values from ENUM('value1','value2','value3')
|
|
144
|
+
if column_type =~ /enum\((.*)\)/i
|
|
145
|
+
enum_string = ::Regexp.last_match(1) || ''
|
|
146
|
+
# Extract value by arround single quote
|
|
147
|
+
values = enum_string.scan(/'([^']*)'/).flatten
|
|
148
|
+
return values
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
nil
|
|
152
|
+
end
|
|
153
|
+
end
|
|
154
|
+
end
|
|
155
|
+
end
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
# rbs_inline: enabled
|
|
3
|
+
|
|
4
|
+
require 'stringio'
|
|
5
|
+
|
|
6
|
+
module MysqlReplicator
|
|
7
|
+
module Binlogs
|
|
8
|
+
class XidEventParser
|
|
9
|
+
# @rbs!
|
|
10
|
+
# type execution = { xid: Integer }
|
|
11
|
+
|
|
12
|
+
# @rbs payload: String
|
|
13
|
+
# @rbs return: execution
|
|
14
|
+
def self.parse(payload)
|
|
15
|
+
io = StringIO.new(payload)
|
|
16
|
+
io.set_encoding(Encoding::BINARY)
|
|
17
|
+
|
|
18
|
+
# XID (8 bytes)
|
|
19
|
+
xid = StringIOUtil.read_uint64(io)
|
|
20
|
+
|
|
21
|
+
{ xid: xid }
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
end
|
|
@@ -0,0 +1,226 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
# rbs_inline: enabled
|
|
3
|
+
|
|
4
|
+
require 'socket'
|
|
5
|
+
|
|
6
|
+
module MysqlReplicator
|
|
7
|
+
class Connection
|
|
8
|
+
# @rbs!
|
|
9
|
+
# type packet = {
|
|
10
|
+
# length: Integer,
|
|
11
|
+
# sequence_id: Integer,
|
|
12
|
+
# payload: String
|
|
13
|
+
# }
|
|
14
|
+
|
|
15
|
+
# @rbs @host: String
|
|
16
|
+
# @rbs @port: Integer
|
|
17
|
+
# @rbs @user: String
|
|
18
|
+
# @rbs @password: String
|
|
19
|
+
# @rbs @database: String
|
|
20
|
+
# @rbs @sequence_id: Integer
|
|
21
|
+
# @rbs @connected: bool
|
|
22
|
+
# @rbs @socket: TCPSocket
|
|
23
|
+
# @rbs @handshake_info: MysqlReplicator::Connections::Handshake::handshake
|
|
24
|
+
|
|
25
|
+
# @rbs! attr_reader host: String
|
|
26
|
+
# @rbs! attr_reader port: Integer
|
|
27
|
+
# @rbs! attr_reader user: String
|
|
28
|
+
# @rbs! attr_reader password: String
|
|
29
|
+
# @rbs! attr_reader database: String
|
|
30
|
+
attr_reader :host, :port, :user, :password, :database
|
|
31
|
+
|
|
32
|
+
# @rbs host: String
|
|
33
|
+
# @rbs port: Integer
|
|
34
|
+
# @rbs user: String
|
|
35
|
+
# @rbs password: String
|
|
36
|
+
# @rbs database: String
|
|
37
|
+
# @rbs return: void
|
|
38
|
+
def initialize(host: 'localhost', port: 3306, user: 'root', password: '', database: '')
|
|
39
|
+
@host = host
|
|
40
|
+
@port = port
|
|
41
|
+
@user = user
|
|
42
|
+
@password = password
|
|
43
|
+
@database = database
|
|
44
|
+
|
|
45
|
+
@sequence_id = 0
|
|
46
|
+
@connected = false
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
# @rbs return: void
|
|
50
|
+
def reset_sequence_id
|
|
51
|
+
@sequence_id = 0
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
# @rbs return: -> bool
|
|
55
|
+
def connected?
|
|
56
|
+
@connected
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
# @rbs return: -> void
|
|
60
|
+
def connect
|
|
61
|
+
if @connected
|
|
62
|
+
MysqlReplicator::Logger.warn 'Connection is already connected'
|
|
63
|
+
return
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
@socket = TCPSocket.new(@host, @port)
|
|
67
|
+
@socket.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1)
|
|
68
|
+
|
|
69
|
+
@handshake_info = MysqlReplicator::Connections::Handshake.execute(self)
|
|
70
|
+
MysqlReplicator::Connections::Auth.execute(self, @user, @password, @database, @handshake_info)
|
|
71
|
+
|
|
72
|
+
@connected = true
|
|
73
|
+
MysqlReplicator::Logger.info "Connected to MySQL server at #{@host}:#{@port}"
|
|
74
|
+
rescue => e
|
|
75
|
+
close
|
|
76
|
+
raise e
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
# @rbs sql: String
|
|
80
|
+
# @rbs return: MysqlReplicator::Connections::Query::queryResult | nil
|
|
81
|
+
def query(sql)
|
|
82
|
+
unless @connected
|
|
83
|
+
MysqlReplicator::Logger.warn 'Connection is not connected'
|
|
84
|
+
return
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
reset_sequence_id
|
|
88
|
+
flush_socket_buffer
|
|
89
|
+
|
|
90
|
+
MysqlReplicator::Connections::Query.execute(self, sql)
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
# @rbs return: 'PONG' | 'ERROR'
|
|
94
|
+
def ping
|
|
95
|
+
unless @connected
|
|
96
|
+
MysqlReplicator::Logger.warn 'Connection is not connected'
|
|
97
|
+
return 'ERROR'
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
reset_sequence_id
|
|
101
|
+
|
|
102
|
+
ping_payload = [0x0E].pack('C')
|
|
103
|
+
send_packet(ping_payload)
|
|
104
|
+
|
|
105
|
+
response = read_packet
|
|
106
|
+
success = MysqlReplicator::StringUtil.read_uint8(response[:payload][0]) == 0x00
|
|
107
|
+
success ? 'PONG' : 'ERROR'
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
# @rbs return: void
|
|
111
|
+
def close
|
|
112
|
+
if !@connected && (@socket.nil? || @socket.closed?)
|
|
113
|
+
MysqlReplicator::Logger.warn 'Connection is not connected'
|
|
114
|
+
return
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
reset_sequence_id
|
|
118
|
+
|
|
119
|
+
if @connected
|
|
120
|
+
quit_payload = [0x01].pack('C')
|
|
121
|
+
send_packet(quit_payload)
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
@socket.close unless @socket.closed?
|
|
125
|
+
|
|
126
|
+
@connected = false
|
|
127
|
+
MysqlReplicator::Logger.info "Disconnected to MySQL server at #{@host}:#{@port}"
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
# @rbs return: packet
|
|
131
|
+
def read_packet
|
|
132
|
+
if @socket.nil?
|
|
133
|
+
raise MysqlReplicator::Error, 'TCPSocket is nil'
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
header = @socket.read(4)
|
|
137
|
+
if header.nil? || header.length != 4
|
|
138
|
+
raise MysqlReplicator::Error, 'Failed to read packet header'
|
|
139
|
+
end
|
|
140
|
+
|
|
141
|
+
# Little-endian 24-bit
|
|
142
|
+
packet_length = MysqlReplicator::StringUtil.read_uint8(header[0]) |
|
|
143
|
+
(MysqlReplicator::StringUtil.read_uint8(header[1]) << 8) |
|
|
144
|
+
(MysqlReplicator::StringUtil.read_uint8(header[2]) << 16)
|
|
145
|
+
sequence_id = MysqlReplicator::StringUtil.read_uint8(header[3])
|
|
146
|
+
|
|
147
|
+
payload = @socket.read(packet_length)
|
|
148
|
+
if payload.nil? || payload.length != packet_length
|
|
149
|
+
raise MysqlReplicator::Error,
|
|
150
|
+
"Failed to read packet payload: expected #{packet_length} bytes, got #{payload&.length || 0}"
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
packet = { length: packet_length, sequence_id: sequence_id, payload: payload }
|
|
154
|
+
MysqlReplicator::Logger.debug "Received packet: #{packet.inspect}"
|
|
155
|
+
|
|
156
|
+
# Update to next expected sequence ID
|
|
157
|
+
@sequence_id = (sequence_id + 1) % 256
|
|
158
|
+
|
|
159
|
+
{
|
|
160
|
+
length: packet_length,
|
|
161
|
+
sequence_id: sequence_id,
|
|
162
|
+
payload: payload
|
|
163
|
+
}
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
# @rbs payload: String
|
|
167
|
+
# @rbs return: void
|
|
168
|
+
def send_packet(payload)
|
|
169
|
+
if @socket.nil?
|
|
170
|
+
raise MysqlReplicator::Error, 'TCPSocket is nil'
|
|
171
|
+
end
|
|
172
|
+
|
|
173
|
+
packet_length = payload.length
|
|
174
|
+
header = ([packet_length].pack('V')[0..2] || '') + [@sequence_id].pack('C').to_s
|
|
175
|
+
@socket.write(header + payload)
|
|
176
|
+
|
|
177
|
+
packet = { length: packet_length, sequence_id: @sequence_id, payload: payload }
|
|
178
|
+
MysqlReplicator::Logger.debug "Sent packet: #{packet.inspect}"
|
|
179
|
+
end
|
|
180
|
+
|
|
181
|
+
# @rbs return: void
|
|
182
|
+
def flush_socket_buffer
|
|
183
|
+
return if @socket.nil?
|
|
184
|
+
|
|
185
|
+
flushed_data = ''
|
|
186
|
+
|
|
187
|
+
begin
|
|
188
|
+
# Read all unread data in non-blocking mode
|
|
189
|
+
while @socket.ready?
|
|
190
|
+
data = @socket.read_nonblock(1024)
|
|
191
|
+
flushed_data += data
|
|
192
|
+
MysqlReplicator::Logger.debug \
|
|
193
|
+
"Found unread data: #{MysqlReplicator::StringUtil.read_array_from_int8(data).map { |b| format('%02X', b) }.join(' ')}"
|
|
194
|
+
end
|
|
195
|
+
|
|
196
|
+
sleep 0.1
|
|
197
|
+
rescue IO::WaitReadable
|
|
198
|
+
# Not at all if no data
|
|
199
|
+
rescue => e
|
|
200
|
+
MysqlReplicator::Logger.error "Buffer clear error: #{e.message}"
|
|
201
|
+
end
|
|
202
|
+
|
|
203
|
+
return if flushed_data.empty?
|
|
204
|
+
|
|
205
|
+
MysqlReplicator::Logger.debug "#{flushed_data.length} bytes of unread data cleared"
|
|
206
|
+
end
|
|
207
|
+
|
|
208
|
+
# @rbs return: Integer
|
|
209
|
+
def connection_id
|
|
210
|
+
@handshake_info[:connection_id]
|
|
211
|
+
end
|
|
212
|
+
|
|
213
|
+
# @rbs return: MysqlReplicator::Connection
|
|
214
|
+
def dup
|
|
215
|
+
new_connection = self.class.new(
|
|
216
|
+
host: @host,
|
|
217
|
+
port: @port,
|
|
218
|
+
user: @user,
|
|
219
|
+
password: @password,
|
|
220
|
+
database: @database
|
|
221
|
+
)
|
|
222
|
+
new_connection.connect
|
|
223
|
+
new_connection
|
|
224
|
+
end
|
|
225
|
+
end
|
|
226
|
+
end
|
|
@@ -0,0 +1,303 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
# rbs_inline: enabled
|
|
3
|
+
|
|
4
|
+
require 'digest'
|
|
5
|
+
require 'openssl'
|
|
6
|
+
|
|
7
|
+
module MysqlReplicator
|
|
8
|
+
module Connections
|
|
9
|
+
class Auth
|
|
10
|
+
CLIENT_PLUGIN_AUTH = 0x00080000 #: Integer
|
|
11
|
+
CLIENT_SECURE_CONNECTION = 0x00008000 #: Integer
|
|
12
|
+
CLIENT_PROTOCOL_41 = 0x00000200 #: Integer
|
|
13
|
+
CLIENT_CONNECT_WITH_DB = 0x00000008 #: Integer
|
|
14
|
+
CLIENT_MULTI_STATEMENTS = 0x00010000 #: Integer
|
|
15
|
+
CLIENT_MULTI_RESULTS = 0x00020000 #: Integer
|
|
16
|
+
|
|
17
|
+
# @rbs connection: MysqlReplicator::Connection
|
|
18
|
+
# @rbs user: String
|
|
19
|
+
# @rbs password: String
|
|
20
|
+
# @rbs database: String
|
|
21
|
+
# @rbs handshake_info: MysqlReplicator::Connections::Handshake::handshake
|
|
22
|
+
# @rbs return: void
|
|
23
|
+
def self.execute(connection, user, password, database, handshake_info)
|
|
24
|
+
auth_plugin_name = handshake_info[:auth_plugin_name]
|
|
25
|
+
|
|
26
|
+
case auth_plugin_name
|
|
27
|
+
when 'caching_sha2_password'
|
|
28
|
+
caching_sha2_password_auth(connection, user, password, database, handshake_info)
|
|
29
|
+
MysqlReplicator::Logger.debug 'Authentication by caching_sha2_password is successful!'
|
|
30
|
+
when 'mysql_native_password'
|
|
31
|
+
mysql_native_password_auth(connection, password, handshake_info)
|
|
32
|
+
MysqlReplicator::Logger.debug 'Authentication by mysql_native_password is successful!'
|
|
33
|
+
else
|
|
34
|
+
raise MysqlReplicator::Error, "Unsupported auth plugin name: #{auth_plugin_name}"
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
# Clear EOF packet (at finish)
|
|
38
|
+
connection.flush_socket_buffer
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
# @rbs connection: MysqlReplicator::Connection
|
|
42
|
+
# @rbs user: String
|
|
43
|
+
# @rbs password: String
|
|
44
|
+
# @rbs database: String
|
|
45
|
+
# @rbs handshake_info: MysqlReplicator::Connections::Handshake::handshake
|
|
46
|
+
# @rbs return: void
|
|
47
|
+
def self.caching_sha2_password_auth(connection, user, password, database, handshake_info)
|
|
48
|
+
auth_payload = build_caching_sha2_password_payload(user, password, database, handshake_info)
|
|
49
|
+
debug_caching_sha2_password_payload(auth_payload, !database.empty?)
|
|
50
|
+
connection.send_packet(auth_payload)
|
|
51
|
+
|
|
52
|
+
auth_response_packet = connection.read_packet
|
|
53
|
+
if handle_caching_sha2_password_response(auth_response_packet) == :success
|
|
54
|
+
return
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
MysqlReplicator::Logger.debug 'Trying RSA encryption authentication...'
|
|
58
|
+
|
|
59
|
+
# Request public key for RSA encryption
|
|
60
|
+
public_key_payload = [0x02].pack('C')
|
|
61
|
+
connection.send_packet(public_key_payload)
|
|
62
|
+
|
|
63
|
+
public_key_response_packet = connection.read_packet
|
|
64
|
+
if MysqlReplicator::StringUtil.read_uint8(public_key_response_packet[:payload][0]) != 0x01
|
|
65
|
+
raise MysqlReplicator::Error, 'Failed to retrieve public key'
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
# Auth with RSA encryption
|
|
69
|
+
public_key = MysqlReplicator::StringUtil.read_str(public_key_response_packet[:payload][1..])
|
|
70
|
+
encrypted_password_payload = build_rsa_encrypt_password_payload(password, public_key, handshake_info[:auth_plugin_data])
|
|
71
|
+
connection.send_packet(encrypted_password_payload)
|
|
72
|
+
|
|
73
|
+
final_auth_response_packet = connection.read_packet
|
|
74
|
+
return unless MysqlReplicator::StringUtil.read_uint8(final_auth_response_packet[:payload][0]) != 0x00
|
|
75
|
+
|
|
76
|
+
raise MysqlReplicator::Error, 'RSA encryption authentication failed'
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
# @rbs packet: MysqlReplicator::Connection::packet
|
|
80
|
+
# @rbs return: :success | :challenge
|
|
81
|
+
def self.handle_caching_sha2_password_response(packet)
|
|
82
|
+
payload = packet[:payload]
|
|
83
|
+
|
|
84
|
+
# First byte of the response is result type
|
|
85
|
+
first_byte = MysqlReplicator::StringUtil.read_uint8(payload[0])
|
|
86
|
+
case first_byte
|
|
87
|
+
when 0x00
|
|
88
|
+
:success
|
|
89
|
+
when 0x01
|
|
90
|
+
more_data = payload[1..] || ['']
|
|
91
|
+
command = MysqlReplicator::StringUtil.read_uint8(more_data[0])
|
|
92
|
+
case command
|
|
93
|
+
when 0x03
|
|
94
|
+
:success
|
|
95
|
+
when 0x04
|
|
96
|
+
:challenge
|
|
97
|
+
else
|
|
98
|
+
raise MysqlReplicator::Error, "Unexpected command: #{format('%02X', command)}"
|
|
99
|
+
end
|
|
100
|
+
else
|
|
101
|
+
raise MysqlReplicator::Error,
|
|
102
|
+
'Authentication Error: ' \
|
|
103
|
+
"first_byte = #{first_byte}, " \
|
|
104
|
+
"code = #{(payload[1..2] || '').unpack('v')[0]}, " \
|
|
105
|
+
"sql_state_marker = #{(payload[3] || '').chr}, " \
|
|
106
|
+
"sql_state = #{payload[4..8]}, " \
|
|
107
|
+
"message = #{payload[9..]}"
|
|
108
|
+
end
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
# @rbs user: String
|
|
112
|
+
# @rbs password: String
|
|
113
|
+
# @rbs database: String
|
|
114
|
+
# @rbs handshake_info: MysqlReplicator::Connections::Handshake::handshake
|
|
115
|
+
# @rbs return: String
|
|
116
|
+
def self.build_caching_sha2_password_payload(user, password, database, handshake_info)
|
|
117
|
+
# Client feature flag
|
|
118
|
+
client_flags = CLIENT_PROTOCOL_41 |
|
|
119
|
+
CLIENT_SECURE_CONNECTION |
|
|
120
|
+
CLIENT_PLUGIN_AUTH |
|
|
121
|
+
CLIENT_MULTI_STATEMENTS |
|
|
122
|
+
CLIENT_MULTI_RESULTS
|
|
123
|
+
client_flags |= CLIENT_CONNECT_WITH_DB if database && !database.empty?
|
|
124
|
+
|
|
125
|
+
# Max packet size (4 bytes)
|
|
126
|
+
max_packet_size = 0x01000000
|
|
127
|
+
|
|
128
|
+
# Character set
|
|
129
|
+
# MySQL 8.0 uses utf8mb4_0900_ai_ci as the default character set
|
|
130
|
+
charset = handshake_info[:charset]
|
|
131
|
+
|
|
132
|
+
# Reserved (23 bytes)
|
|
133
|
+
reserved_data = "\x00" * 23
|
|
134
|
+
|
|
135
|
+
# Username
|
|
136
|
+
username_data = user + "\x00"
|
|
137
|
+
|
|
138
|
+
# Hash for caching_sha2_password
|
|
139
|
+
challenge_hash = build_caching_sha2_password_hash(password, handshake_info[:auth_plugin_data])
|
|
140
|
+
|
|
141
|
+
# Database (optional)
|
|
142
|
+
database_data = database && !database.empty? ? database + "\x00" : ''
|
|
143
|
+
|
|
144
|
+
# Authentication plugin name
|
|
145
|
+
auth_plugin_name_data = handshake_info[:auth_plugin_name].to_s + "\x00"
|
|
146
|
+
|
|
147
|
+
# Payload of packet
|
|
148
|
+
[client_flags].pack('V') +
|
|
149
|
+
[max_packet_size].pack('V') +
|
|
150
|
+
[charset].pack('C') +
|
|
151
|
+
reserved_data +
|
|
152
|
+
username_data +
|
|
153
|
+
[challenge_hash.length].pack('C') + challenge_hash +
|
|
154
|
+
database_data +
|
|
155
|
+
auth_plugin_name_data
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
# Hash value for caching_sha2_password
|
|
159
|
+
# SHA256(password) XOR SHA256(SHA256(SHA256(password)) + salt)
|
|
160
|
+
#
|
|
161
|
+
# @rbs password: String
|
|
162
|
+
# @rbs salt: String
|
|
163
|
+
# @rbs return: String
|
|
164
|
+
def self.build_caching_sha2_password_hash(password, salt)
|
|
165
|
+
return '' if password.empty?
|
|
166
|
+
|
|
167
|
+
# SHA256(password)
|
|
168
|
+
hash1 = Digest::SHA256.digest(password.encode('utf-8'))
|
|
169
|
+
# SHA256(SHA256(password))
|
|
170
|
+
hash2 = Digest::SHA256.digest(hash1)
|
|
171
|
+
# SHA256(SHA256(SHA256(password)), salt)
|
|
172
|
+
hash3 = Digest::SHA256.digest(hash2 + salt)
|
|
173
|
+
|
|
174
|
+
# XOR hash1 and hash3
|
|
175
|
+
payload = ''
|
|
176
|
+
hash1.each_byte.with_index do |byte, i|
|
|
177
|
+
payload += (byte ^ hash3[i].to_s.ord).chr
|
|
178
|
+
end
|
|
179
|
+
payload
|
|
180
|
+
end
|
|
181
|
+
|
|
182
|
+
# @rbs password: String
|
|
183
|
+
# @rbs public_key: String
|
|
184
|
+
# @rbs scramble: String
|
|
185
|
+
# @rbs return: String
|
|
186
|
+
def self.build_rsa_encrypt_password_payload(password, public_key, scramble)
|
|
187
|
+
rsa_public_key = OpenSSL::PKey::RSA.new(public_key)
|
|
188
|
+
|
|
189
|
+
# Password is null-terminated string
|
|
190
|
+
password_with_null = password + "\x00"
|
|
191
|
+
|
|
192
|
+
password_bytes = password_with_null.encode(Encoding::UTF_8).bytes
|
|
193
|
+
scramble_bytes = scramble.bytes
|
|
194
|
+
|
|
195
|
+
xor_result = []
|
|
196
|
+
password_bytes.each_with_index do |byte, index|
|
|
197
|
+
scramble_byte = scramble_bytes[index % scramble_bytes.length]
|
|
198
|
+
xor_result << (byte ^ scramble_byte)
|
|
199
|
+
end
|
|
200
|
+
data_to_encrypt = xor_result.pack('C*')
|
|
201
|
+
|
|
202
|
+
begin
|
|
203
|
+
# First, try OAEP padding (MySQL 8.0.5+)
|
|
204
|
+
rsa_public_key.public_encrypt(data_to_encrypt, OpenSSL::PKey::RSA::PKCS1_OAEP_PADDING)
|
|
205
|
+
rescue OpenSSL::PKey::RSAError
|
|
206
|
+
# If OAEP fails, use PKCS#1 (MySQL 8.0.4 and earlier)
|
|
207
|
+
rsa_public_key.public_encrypt(data_to_encrypt, OpenSSL::PKey::RSA::PKCS1_PADDING)
|
|
208
|
+
end
|
|
209
|
+
end
|
|
210
|
+
|
|
211
|
+
# @rbs payload: String
|
|
212
|
+
# @rbs with_database: bool
|
|
213
|
+
# @rbs return: void
|
|
214
|
+
def self.debug_caching_sha2_password_payload(payload, with_database)
|
|
215
|
+
offset = 0
|
|
216
|
+
|
|
217
|
+
client_flags = MysqlReplicator::StringUtil.read_uint32(payload[offset..(offset + 3)])
|
|
218
|
+
offset += 4
|
|
219
|
+
|
|
220
|
+
max_packet_size = MysqlReplicator::StringUtil.read_uint32(payload[offset..(offset + 3)])
|
|
221
|
+
offset += 4
|
|
222
|
+
|
|
223
|
+
character_set = MysqlReplicator::StringUtil.read_uint8(payload[offset])
|
|
224
|
+
offset += 1
|
|
225
|
+
|
|
226
|
+
reserved = MysqlReplicator::StringUtil.read_str(payload[offset..(offset + 22)])
|
|
227
|
+
offset += 23
|
|
228
|
+
|
|
229
|
+
null_pos = payload.index("\x00", offset).to_i
|
|
230
|
+
user = MysqlReplicator::StringUtil.read_str(payload[offset...null_pos])
|
|
231
|
+
offset = null_pos + 1
|
|
232
|
+
|
|
233
|
+
challenge_hash_length = MysqlReplicator::StringUtil.read_uint8(payload[offset])
|
|
234
|
+
offset += 1
|
|
235
|
+
if challenge_hash_length > 0
|
|
236
|
+
challenge_hash_data = MysqlReplicator::StringUtil.read_str(payload[offset..(offset + challenge_hash_length - 1)])
|
|
237
|
+
offset += challenge_hash_length
|
|
238
|
+
end
|
|
239
|
+
|
|
240
|
+
if with_database
|
|
241
|
+
db_null_pos = payload.index("\x00", offset).to_i
|
|
242
|
+
database = MysqlReplicator::StringUtil.read_str(payload[offset...db_null_pos])
|
|
243
|
+
offset = db_null_pos + 1
|
|
244
|
+
end
|
|
245
|
+
|
|
246
|
+
plugin_null_pos = payload.index("\x00", offset).to_i
|
|
247
|
+
plugin_name = MysqlReplicator::StringUtil.read_str(payload[offset...plugin_null_pos]) if plugin_null_pos
|
|
248
|
+
|
|
249
|
+
MysqlReplicator::Logger.debug \
|
|
250
|
+
"===== Start Auth Payload =====\n" \
|
|
251
|
+
"Client flags: #{client_flags.to_i.to_s(16)}\n" \
|
|
252
|
+
"Max packet size: #{max_packet_size}\n" \
|
|
253
|
+
"Character set: #{character_set}\n" \
|
|
254
|
+
"Reserved: #{MysqlReplicator::StringUtil.read_array_from_int8(reserved).all?(&:zero?) ? 'All zero' : 'None zero'}\n" \
|
|
255
|
+
"User: #{user}\n" \
|
|
256
|
+
"Challenge hash length: #{challenge_hash_length}\n" \
|
|
257
|
+
"Challenge hash data: #{MysqlReplicator::StringUtil.read_array_from_int8(challenge_hash_data).map { |b| format('%02X', b) }.join(' ')}\n" \
|
|
258
|
+
"Database name: #{database}\n" \
|
|
259
|
+
"Auth plugin name: #{plugin_name}\n" \
|
|
260
|
+
'===== End Auth Payload ====='
|
|
261
|
+
end
|
|
262
|
+
|
|
263
|
+
# @rbs connection: MysqlReplicator::Connection
|
|
264
|
+
# @rbs password: String
|
|
265
|
+
# @rbs handshake_info: MysqlReplicator::Connections::Handshake::handshake
|
|
266
|
+
# @return: void
|
|
267
|
+
def self.mysql_native_password_auth(connection, password, handshake_info)
|
|
268
|
+
auth_payload = build_mysql_native_password_payload(password, handshake_info[:auth_plugin_data])
|
|
269
|
+
connection.send_packet(auth_payload)
|
|
270
|
+
|
|
271
|
+
auth_response_packet = connection.read_packet
|
|
272
|
+
|
|
273
|
+
return unless MysqlReplicator::StringUtil.read_uint8(auth_response_packet[:payload][0]) != 0x00
|
|
274
|
+
|
|
275
|
+
raise MysqlReplicator::Error,
|
|
276
|
+
'mysql_native_password authentication failed: ' \
|
|
277
|
+
"Payload = #{MysqlReplicator::StringUtil.read_array_from_int8(auth_response_packet[:payload]).map { |b| format('%02X', b) }.join(' ')}"
|
|
278
|
+
end
|
|
279
|
+
|
|
280
|
+
# @rbs password: String
|
|
281
|
+
# @rbs salt: String
|
|
282
|
+
# @rbs return: String
|
|
283
|
+
def self.build_mysql_native_password_payload(password, salt)
|
|
284
|
+
return '' if password.empty?
|
|
285
|
+
|
|
286
|
+
# SHA1(password)
|
|
287
|
+
hash1 = Digest::SHA1.digest(password.encode('utf-8'))
|
|
288
|
+
# SHA1(SHA1(password))
|
|
289
|
+
hash2 = Digest::SHA1.digest(hash1)
|
|
290
|
+
# SHA1(salt + SHA1(SHA1(password)))
|
|
291
|
+
hash3 = Digest::SHA1.digest(salt + hash2)
|
|
292
|
+
|
|
293
|
+
# XOR hash1 and hash3
|
|
294
|
+
payload = ''
|
|
295
|
+
hash1.each_byte.with_index do |byte, i|
|
|
296
|
+
payload += (byte ^ (hash3[i] || '').ord).chr
|
|
297
|
+
end
|
|
298
|
+
|
|
299
|
+
payload
|
|
300
|
+
end
|
|
301
|
+
end
|
|
302
|
+
end
|
|
303
|
+
end
|