groonga-delta 1.0.0 → 1.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/doc/text/news.md +42 -0
- data/lib/groonga-delta/command.rb +10 -0
- data/lib/groonga-delta/config.rb +1 -1
- data/lib/groonga-delta/error.rb +20 -0
- data/lib/groonga-delta/import-command.rb +7 -2
- data/lib/groonga-delta/import-config.rb +43 -4
- data/lib/groonga-delta/import-status.rb +12 -4
- data/lib/groonga-delta/local-delta.rb +12 -282
- data/lib/groonga-delta/local-reader.rb +353 -0
- data/lib/groonga-delta/local-source.rb +3 -3
- data/lib/groonga-delta/local-vacuumer.rb +39 -0
- data/lib/groonga-delta/{writer.rb → local-writer.rb} +17 -5
- data/lib/groonga-delta/mapping.rb +16 -3
- data/lib/groonga-delta/mysql-source.rb +84 -131
- data/lib/groonga-delta/status.rb +3 -1
- data/lib/groonga-delta/version.rb +1 -1
- metadata +5 -3
@@ -15,106 +15,25 @@
|
|
15
15
|
|
16
16
|
require "arrow"
|
17
17
|
require "mysql2"
|
18
|
+
require "mysql2-replication"
|
18
19
|
|
19
20
|
require_relative "error"
|
20
|
-
require_relative "writer"
|
21
|
+
require_relative "local-writer"
|
21
22
|
|
22
23
|
module GroongaDelta
|
23
24
|
class MySQLSource
|
24
|
-
def initialize(config, status)
|
25
|
+
def initialize(config, status, writer)
|
25
26
|
@logger = config.logger
|
26
|
-
@writer = Writer.new(@logger, config.delta_dir)
|
27
27
|
@config = config.mysql
|
28
28
|
@binlog_dir = @config.binlog_dir
|
29
29
|
@mapping = config.mapping
|
30
30
|
@status = status.mysql
|
31
|
+
@writer = writer
|
31
32
|
@tables = {}
|
32
33
|
end
|
33
34
|
|
34
35
|
def import
|
35
|
-
|
36
|
-
when "mysqlbinlog"
|
37
|
-
require "mysql_binlog"
|
38
|
-
import_mysqlbinlog
|
39
|
-
when "mysql2-replication"
|
40
|
-
require "mysql2-replication"
|
41
|
-
import_mysql2_replication
|
42
|
-
else
|
43
|
-
begin
|
44
|
-
require "mysql2-replication"
|
45
|
-
rescue LoadError
|
46
|
-
require "mysql_binlog"
|
47
|
-
import_mysqlbinlog
|
48
|
-
else
|
49
|
-
import_mysql2_replication
|
50
|
-
end
|
51
|
-
end
|
52
|
-
end
|
53
|
-
|
54
|
-
private
|
55
|
-
def import_mysqlbinlog
|
56
|
-
file, position = read_current_status
|
57
|
-
FileUtils.mkdir_p(@binlog_dir)
|
58
|
-
local_file = File.join(@binlog_dir, file)
|
59
|
-
unless File.exist?(local_file.succ)
|
60
|
-
command_line = [@config.mysqlbinlog].flatten
|
61
|
-
command_line << "--host=#{@config.host}" if @config.host
|
62
|
-
command_line << "--port=#{@config.port}" if @config.port
|
63
|
-
command_line << "--socket=#{@config.socket}" if @config.socket
|
64
|
-
if @config.replication_slave_user
|
65
|
-
command_line << "--user=#{@config.replication_slave_user}"
|
66
|
-
end
|
67
|
-
if @config.replication_slave_password
|
68
|
-
command_line << "--password=#{@config.replication_slave_password}"
|
69
|
-
end
|
70
|
-
command_line << "--read-from-remote-server"
|
71
|
-
command_line << "--raw"
|
72
|
-
command_line << "--result-file=#{@binlog_dir}/"
|
73
|
-
command_line << file
|
74
|
-
spawn_process(*command_line) do |pid, output_read, error_read|
|
75
|
-
end
|
76
|
-
end
|
77
|
-
reader = MysqlBinlog::BinlogFileReader.new(local_file)
|
78
|
-
binlog = MysqlBinlog::Binlog.new(reader)
|
79
|
-
binlog.checksum = @config.checksum
|
80
|
-
binlog.ignore_rotate = true
|
81
|
-
binlog.each_event do |event|
|
82
|
-
next if event[:position] < position
|
83
|
-
case event[:type]
|
84
|
-
when :rotate_event
|
85
|
-
@status.update("file" => event[:event][:name],
|
86
|
-
"position" => event[:event][:pos])
|
87
|
-
when :write_rows_event_v1,
|
88
|
-
:write_rows_event_v2,
|
89
|
-
:update_rows_event_v1,
|
90
|
-
:update_rows_event_v2,
|
91
|
-
:delete_rows_event_v1,
|
92
|
-
:delete_rows_event_v2
|
93
|
-
normalized_type = event[:type].to_s.gsub(/_v\d\z/, "").to_sym
|
94
|
-
import_rows_event(normalized_type,
|
95
|
-
event[:event][:table][:db],
|
96
|
-
event[:event][:table][:table],
|
97
|
-
file,
|
98
|
-
event[:header][:next_position]) do
|
99
|
-
case normalized_type
|
100
|
-
when :write_rows_event,
|
101
|
-
:update_rows_event
|
102
|
-
event[:event][:row_image].collect do |row_image|
|
103
|
-
build_row(row_image[:after][:image])
|
104
|
-
end
|
105
|
-
when :delete_rows_event
|
106
|
-
event[:event][:row_image].collect do |row_image|
|
107
|
-
build_row(row_image[:before][:image])
|
108
|
-
end
|
109
|
-
end
|
110
|
-
end
|
111
|
-
position = event[:header][:next_position]
|
112
|
-
end
|
113
|
-
end
|
114
|
-
end
|
115
|
-
|
116
|
-
def import_mysql2_replication
|
117
|
-
file, position = read_current_status
|
36
|
+
current_status = read_current_status
|
118
37
|
is_mysql_56_or_later = mysql(@config.select_user,
|
119
38
|
@config.select_password) do |select_client|
|
120
39
|
mysql_version(select_client) >= Gem::Version.new("5.6")
|
@@ -127,69 +46,85 @@ module GroongaDelta
|
|
127
46
|
replication_client = Mysql2Replication::Client.new(client,
|
128
47
|
checksum: "NONE")
|
129
48
|
end
|
130
|
-
|
131
|
-
replication_client.
|
49
|
+
current_file = current_status.last_table_map_file
|
50
|
+
replication_client.file_name = current_file
|
51
|
+
current_event_position = current_status.last_table_map_position
|
52
|
+
if current_event_position
|
53
|
+
replication_client.start_position = current_event_position
|
54
|
+
end
|
132
55
|
replication_client.open do
|
133
56
|
replication_client.each do |event|
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
when Mysql2Replication::RowsEvent
|
138
|
-
event_name = event.class.name.split("::").last
|
139
|
-
normalized_type =
|
140
|
-
event_name.scan(/[A-Z][a-z]+/).
|
141
|
-
collect(&:downcase).
|
142
|
-
join("_").
|
143
|
-
to_sym
|
144
|
-
import_rows_event(normalized_type,
|
145
|
-
event.table_map.database,
|
146
|
-
event.table_map.table,
|
147
|
-
file,
|
148
|
-
event.next_position) do
|
149
|
-
case normalized_type
|
150
|
-
when :update_rows_event
|
151
|
-
event.updated_rows
|
152
|
-
else
|
153
|
-
event.rows
|
154
|
-
end
|
57
|
+
begin
|
58
|
+
@logger.debug do
|
59
|
+
event.inspect
|
155
60
|
end
|
61
|
+
case event
|
62
|
+
when Mysql2Replication::RotateEvent
|
63
|
+
current_file = event.file_name
|
64
|
+
current_status.last_file = current_file
|
65
|
+
current_status.last_position = event.position
|
66
|
+
when Mysql2Replication::TableMapEvent
|
67
|
+
current_status.last_table_map_file = current_file
|
68
|
+
current_status.last_table_map_position = current_event_position
|
69
|
+
when Mysql2Replication::RowsEvent
|
70
|
+
next if current_file != current_status.last_file
|
71
|
+
next if current_event_position < current_status.last_position
|
72
|
+
import_rows_event(event, current_status)
|
73
|
+
end
|
74
|
+
ensure
|
75
|
+
# next_position may be 0. For example,
|
76
|
+
# Mysql2Replication::FormatDescriptionEvent#next_position
|
77
|
+
# returns 0.
|
78
|
+
next_position = event.next_position
|
79
|
+
current_event_position = next_position unless next_position.zero?
|
156
80
|
end
|
157
81
|
end
|
158
82
|
end
|
159
83
|
end
|
160
84
|
end
|
161
85
|
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
&block)
|
86
|
+
private
|
87
|
+
def import_rows_event(event, current_status)
|
88
|
+
database_name = event.table_map.database
|
89
|
+
table_name = event.table_map.table
|
90
|
+
|
168
91
|
source_table = @mapping[database_name, table_name]
|
169
92
|
return if source_table.nil?
|
170
93
|
|
171
94
|
table = find_table(database_name, table_name)
|
172
95
|
groonga_table = source_table.groonga_table
|
173
|
-
|
96
|
+
case event
|
97
|
+
when Mysql2Replication::UpdateRowsEvent
|
98
|
+
target_rows = event.updated_rows
|
99
|
+
else
|
100
|
+
target_rows = event.rows
|
101
|
+
end
|
174
102
|
groonga_records = target_rows.collect do |row|
|
175
103
|
record = build_record(table, row)
|
176
104
|
groonga_table.generate_record(record)
|
177
105
|
end
|
178
106
|
return if groonga_records.empty?
|
179
107
|
|
180
|
-
case
|
181
|
-
when
|
182
|
-
|
108
|
+
case event
|
109
|
+
when Mysql2Replication::WriteRowsEvent,
|
110
|
+
Mysql2Replication::UpdateRowsEvent
|
183
111
|
@writer.write_upserts(groonga_table.name, groonga_records)
|
184
|
-
when
|
112
|
+
when Mysql2Replication::DeleteRowsEvent
|
185
113
|
groonga_record_keys = groonga_records.collect do |record|
|
186
114
|
record[:_key]
|
187
115
|
end
|
188
116
|
@writer.write_deletes(groonga_table.name,
|
189
117
|
groonga_record_keys)
|
190
118
|
end
|
191
|
-
|
192
|
-
|
119
|
+
current_status.last_position = event.next_position
|
120
|
+
new_status = {
|
121
|
+
"last_file" => current_status.last_file,
|
122
|
+
"last_position" => current_status.last_position,
|
123
|
+
"last_table_map_file" => current_status.last_table_map_file,
|
124
|
+
"last_table_map_position" => current_status.last_table_map_position,
|
125
|
+
}
|
126
|
+
@logger.info("Update status: #{new_status.inspect}")
|
127
|
+
@status.update(new_status)
|
193
128
|
end
|
194
129
|
|
195
130
|
def wait_process(command_line, pid, output_read, error_read)
|
@@ -263,17 +198,20 @@ module GroongaDelta
|
|
263
198
|
end
|
264
199
|
|
265
200
|
def read_current_status
|
266
|
-
if @status.
|
267
|
-
|
201
|
+
if @status.last_file
|
202
|
+
CurrentStatus.new(@status.last_file,
|
203
|
+
@status.last_position,
|
204
|
+
@status.last_table_map_file,
|
205
|
+
@status.last_table_map_position)
|
268
206
|
else
|
269
|
-
|
270
|
-
|
207
|
+
last_file = nil
|
208
|
+
last_position = 0
|
271
209
|
mysql(@config.replication_client_user,
|
272
210
|
@config.replication_client_password) do |replication_client|
|
273
211
|
replication_client.query("FLUSH TABLES WITH READ LOCK")
|
274
212
|
result = replication_client.query("SHOW MASTER STATUS").first
|
275
|
-
|
276
|
-
|
213
|
+
last_file = result["File"]
|
214
|
+
last_position = result["Position"]
|
277
215
|
mysql(@config.select_user,
|
278
216
|
@config.select_password) do |select_client|
|
279
217
|
start_transaction = "START TRANSACTION " +
|
@@ -287,9 +225,19 @@ module GroongaDelta
|
|
287
225
|
select_client.query("ROLLBACK")
|
288
226
|
end
|
289
227
|
end
|
290
|
-
|
291
|
-
|
292
|
-
|
228
|
+
current_status = CurrentStatus.new(last_file,
|
229
|
+
last_position,
|
230
|
+
last_file,
|
231
|
+
last_position)
|
232
|
+
new_status = {
|
233
|
+
"last_file" => current_status.last_file,
|
234
|
+
"last_position" => current_status.last_position,
|
235
|
+
"last_table_map_file" => current_status.last_file,
|
236
|
+
"last_table_map_position" => current_status.last_position,
|
237
|
+
}
|
238
|
+
@logger.info("Update status: #{new_status.inspect}")
|
239
|
+
@status.update(new_status)
|
240
|
+
current_status
|
293
241
|
end
|
294
242
|
end
|
295
243
|
|
@@ -387,5 +335,10 @@ module GroongaDelta
|
|
387
335
|
end
|
388
336
|
record
|
389
337
|
end
|
338
|
+
|
339
|
+
CurrentStatus = Struct.new(:last_file,
|
340
|
+
:last_position,
|
341
|
+
:last_table_map_file,
|
342
|
+
:last_table_map_position)
|
390
343
|
end
|
391
344
|
end
|
data/lib/groonga-delta/status.rb
CHANGED
@@ -35,9 +35,11 @@ module GroongaDelta
|
|
35
35
|
def update(data)
|
36
36
|
@data.update(data)
|
37
37
|
FileUtils.mkdir_p(@dir)
|
38
|
-
|
38
|
+
path_writing = "#{@path}.writing"
|
39
|
+
File.open(path_writing, "w") do |output|
|
39
40
|
output.puts(YAML.dump(@data))
|
40
41
|
end
|
42
|
+
FileUtils.mv(path_writing, @path)
|
41
43
|
end
|
42
44
|
end
|
43
45
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: groonga-delta
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.
|
4
|
+
version: 1.0.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sutou Kouhei
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2022-
|
11
|
+
date: 2022-07-20 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: groonga-client
|
@@ -80,13 +80,15 @@ files:
|
|
80
80
|
- lib/groonga-delta/import-config.rb
|
81
81
|
- lib/groonga-delta/import-status.rb
|
82
82
|
- lib/groonga-delta/local-delta.rb
|
83
|
+
- lib/groonga-delta/local-reader.rb
|
83
84
|
- lib/groonga-delta/local-source.rb
|
85
|
+
- lib/groonga-delta/local-vacuumer.rb
|
86
|
+
- lib/groonga-delta/local-writer.rb
|
84
87
|
- lib/groonga-delta/ltsv-log-formatter.rb
|
85
88
|
- lib/groonga-delta/mapping.rb
|
86
89
|
- lib/groonga-delta/mysql-source.rb
|
87
90
|
- lib/groonga-delta/status.rb
|
88
91
|
- lib/groonga-delta/version.rb
|
89
|
-
- lib/groonga-delta/writer.rb
|
90
92
|
homepage: https://github.com/groonga/groonga-delta
|
91
93
|
licenses:
|
92
94
|
- GPL-3.0+
|