groonga-delta 1.0.0 → 1.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -15,106 +15,25 @@
15
15
 
16
16
  require "arrow"
17
17
  require "mysql2"
18
+ require "mysql2-replication"
18
19
 
19
20
  require_relative "error"
20
- require_relative "writer"
21
+ require_relative "local-writer"
21
22
 
22
23
  module GroongaDelta
23
24
  class MySQLSource
24
- def initialize(config, status)
25
+ def initialize(config, status, writer)
25
26
  @logger = config.logger
26
- @writer = Writer.new(@logger, config.delta_dir)
27
27
  @config = config.mysql
28
28
  @binlog_dir = @config.binlog_dir
29
29
  @mapping = config.mapping
30
30
  @status = status.mysql
31
+ @writer = writer
31
32
  @tables = {}
32
33
  end
33
34
 
34
35
  def import
35
- case ENV["GROONGA_DELTA_IMPORT_MYSQL_SOURCE_BACKEND"]
36
- when "mysqlbinlog"
37
- require "mysql_binlog"
38
- import_mysqlbinlog
39
- when "mysql2-replication"
40
- require "mysql2-replication"
41
- import_mysql2_replication
42
- else
43
- begin
44
- require "mysql2-replication"
45
- rescue LoadError
46
- require "mysql_binlog"
47
- import_mysqlbinlog
48
- else
49
- import_mysql2_replication
50
- end
51
- end
52
- end
53
-
54
- private
55
- def import_mysqlbinlog
56
- file, position = read_current_status
57
- FileUtils.mkdir_p(@binlog_dir)
58
- local_file = File.join(@binlog_dir, file)
59
- unless File.exist?(local_file.succ)
60
- command_line = [@config.mysqlbinlog].flatten
61
- command_line << "--host=#{@config.host}" if @config.host
62
- command_line << "--port=#{@config.port}" if @config.port
63
- command_line << "--socket=#{@config.socket}" if @config.socket
64
- if @config.replication_slave_user
65
- command_line << "--user=#{@config.replication_slave_user}"
66
- end
67
- if @config.replication_slave_password
68
- command_line << "--password=#{@config.replication_slave_password}"
69
- end
70
- command_line << "--read-from-remote-server"
71
- command_line << "--raw"
72
- command_line << "--result-file=#{@binlog_dir}/"
73
- command_line << file
74
- spawn_process(*command_line) do |pid, output_read, error_read|
75
- end
76
- end
77
- reader = MysqlBinlog::BinlogFileReader.new(local_file)
78
- binlog = MysqlBinlog::Binlog.new(reader)
79
- binlog.checksum = @config.checksum
80
- binlog.ignore_rotate = true
81
- binlog.each_event do |event|
82
- next if event[:position] < position
83
- case event[:type]
84
- when :rotate_event
85
- @status.update("file" => event[:event][:name],
86
- "position" => event[:event][:pos])
87
- when :write_rows_event_v1,
88
- :write_rows_event_v2,
89
- :update_rows_event_v1,
90
- :update_rows_event_v2,
91
- :delete_rows_event_v1,
92
- :delete_rows_event_v2
93
- normalized_type = event[:type].to_s.gsub(/_v\d\z/, "").to_sym
94
- import_rows_event(normalized_type,
95
- event[:event][:table][:db],
96
- event[:event][:table][:table],
97
- file,
98
- event[:header][:next_position]) do
99
- case normalized_type
100
- when :write_rows_event,
101
- :update_rows_event
102
- event[:event][:row_image].collect do |row_image|
103
- build_row(row_image[:after][:image])
104
- end
105
- when :delete_rows_event
106
- event[:event][:row_image].collect do |row_image|
107
- build_row(row_image[:before][:image])
108
- end
109
- end
110
- end
111
- position = event[:header][:next_position]
112
- end
113
- end
114
- end
115
-
116
- def import_mysql2_replication
117
- file, position = read_current_status
36
+ current_status = read_current_status
118
37
  is_mysql_56_or_later = mysql(@config.select_user,
119
38
  @config.select_password) do |select_client|
120
39
  mysql_version(select_client) >= Gem::Version.new("5.6")
@@ -127,69 +46,85 @@ module GroongaDelta
127
46
  replication_client = Mysql2Replication::Client.new(client,
128
47
  checksum: "NONE")
129
48
  end
130
- replication_client.file_name = file
131
- replication_client.start_position = position
49
+ current_file = current_status.last_table_map_file
50
+ replication_client.file_name = current_file
51
+ current_event_position = current_status.last_table_map_position
52
+ if current_event_position
53
+ replication_client.start_position = current_event_position
54
+ end
132
55
  replication_client.open do
133
56
  replication_client.each do |event|
134
- case event
135
- when Mysql2Replication::RotateEvent
136
- file = event.file_name
137
- when Mysql2Replication::RowsEvent
138
- event_name = event.class.name.split("::").last
139
- normalized_type =
140
- event_name.scan(/[A-Z][a-z]+/).
141
- collect(&:downcase).
142
- join("_").
143
- to_sym
144
- import_rows_event(normalized_type,
145
- event.table_map.database,
146
- event.table_map.table,
147
- file,
148
- event.next_position) do
149
- case normalized_type
150
- when :update_rows_event
151
- event.updated_rows
152
- else
153
- event.rows
154
- end
57
+ begin
58
+ @logger.debug do
59
+ event.inspect
155
60
  end
61
+ case event
62
+ when Mysql2Replication::RotateEvent
63
+ current_file = event.file_name
64
+ current_status.last_file = current_file
65
+ current_status.last_position = event.position
66
+ when Mysql2Replication::TableMapEvent
67
+ current_status.last_table_map_file = current_file
68
+ current_status.last_table_map_position = current_event_position
69
+ when Mysql2Replication::RowsEvent
70
+ next if current_file != current_status.last_file
71
+ next if current_event_position < current_status.last_position
72
+ import_rows_event(event, current_status)
73
+ end
74
+ ensure
75
+ # next_position may be 0. For example,
76
+ # Mysql2Replication::FormatDescriptionEvent#next_position
77
+ # returns 0.
78
+ next_position = event.next_position
79
+ current_event_position = next_position unless next_position.zero?
156
80
  end
157
81
  end
158
82
  end
159
83
  end
160
84
  end
161
85
 
162
- def import_rows_event(type,
163
- database_name,
164
- table_name,
165
- file,
166
- next_position,
167
- &block)
86
+ private
87
+ def import_rows_event(event, current_status)
88
+ database_name = event.table_map.database
89
+ table_name = event.table_map.table
90
+
168
91
  source_table = @mapping[database_name, table_name]
169
92
  return if source_table.nil?
170
93
 
171
94
  table = find_table(database_name, table_name)
172
95
  groonga_table = source_table.groonga_table
173
- target_rows = block.call
96
+ case event
97
+ when Mysql2Replication::UpdateRowsEvent
98
+ target_rows = event.updated_rows
99
+ else
100
+ target_rows = event.rows
101
+ end
174
102
  groonga_records = target_rows.collect do |row|
175
103
  record = build_record(table, row)
176
104
  groonga_table.generate_record(record)
177
105
  end
178
106
  return if groonga_records.empty?
179
107
 
180
- case type
181
- when :write_rows_event,
182
- :update_rows_event
108
+ case event
109
+ when Mysql2Replication::WriteRowsEvent,
110
+ Mysql2Replication::UpdateRowsEvent
183
111
  @writer.write_upserts(groonga_table.name, groonga_records)
184
- when :delete_rows_event
112
+ when Mysql2Replication::DeleteRowsEvent
185
113
  groonga_record_keys = groonga_records.collect do |record|
186
114
  record[:_key]
187
115
  end
188
116
  @writer.write_deletes(groonga_table.name,
189
117
  groonga_record_keys)
190
118
  end
191
- @status.update("file" => file,
192
- "position" => next_position)
119
+ current_status.last_position = event.next_position
120
+ new_status = {
121
+ "last_file" => current_status.last_file,
122
+ "last_position" => current_status.last_position,
123
+ "last_table_map_file" => current_status.last_table_map_file,
124
+ "last_table_map_position" => current_status.last_table_map_position,
125
+ }
126
+ @logger.info("Update status: #{new_status.inspect}")
127
+ @status.update(new_status)
193
128
  end
194
129
 
195
130
  def wait_process(command_line, pid, output_read, error_read)
@@ -263,17 +198,20 @@ module GroongaDelta
263
198
  end
264
199
 
265
200
  def read_current_status
266
- if @status.file
267
- [@status.file, @status.position]
201
+ if @status.last_file
202
+ CurrentStatus.new(@status.last_file,
203
+ @status.last_position,
204
+ @status.last_table_map_file,
205
+ @status.last_table_map_position)
268
206
  else
269
- file = nil
270
- position = 0
207
+ last_file = nil
208
+ last_position = 0
271
209
  mysql(@config.replication_client_user,
272
210
  @config.replication_client_password) do |replication_client|
273
211
  replication_client.query("FLUSH TABLES WITH READ LOCK")
274
212
  result = replication_client.query("SHOW MASTER STATUS").first
275
- file = result["File"]
276
- position = result["Position"]
213
+ last_file = result["File"]
214
+ last_position = result["Position"]
277
215
  mysql(@config.select_user,
278
216
  @config.select_password) do |select_client|
279
217
  start_transaction = "START TRANSACTION " +
@@ -287,9 +225,19 @@ module GroongaDelta
287
225
  select_client.query("ROLLBACK")
288
226
  end
289
227
  end
290
- @status.update("file" => file,
291
- "position" => position)
292
- [file, position]
228
+ current_status = CurrentStatus.new(last_file,
229
+ last_position,
230
+ last_file,
231
+ last_position)
232
+ new_status = {
233
+ "last_file" => current_status.last_file,
234
+ "last_position" => current_status.last_position,
235
+ "last_table_map_file" => current_status.last_file,
236
+ "last_table_map_position" => current_status.last_position,
237
+ }
238
+ @logger.info("Update status: #{new_status.inspect}")
239
+ @status.update(new_status)
240
+ current_status
293
241
  end
294
242
  end
295
243
 
@@ -387,5 +335,10 @@ module GroongaDelta
387
335
  end
388
336
  record
389
337
  end
338
+
339
+ CurrentStatus = Struct.new(:last_file,
340
+ :last_position,
341
+ :last_table_map_file,
342
+ :last_table_map_position)
390
343
  end
391
344
  end
@@ -35,9 +35,11 @@ module GroongaDelta
35
35
  def update(data)
36
36
  @data.update(data)
37
37
  FileUtils.mkdir_p(@dir)
38
- File.open(@path, "w") do |output|
38
+ path_writing = "#{@path}.writing"
39
+ File.open(path_writing, "w") do |output|
39
40
  output.puts(YAML.dump(@data))
40
41
  end
42
+ FileUtils.mv(path_writing, @path)
41
43
  end
42
44
  end
43
45
  end
@@ -14,5 +14,5 @@
14
14
  # along with this program. If not, see <http://www.gnu.org/licenses/>.
15
15
 
16
16
  module GroongaDelta
17
- VERSION = "1.0.0"
17
+ VERSION = "1.0.3"
18
18
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: groonga-delta
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.0
4
+ version: 1.0.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Sutou Kouhei
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2022-03-07 00:00:00.000000000 Z
11
+ date: 2022-07-20 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: groonga-client
@@ -80,13 +80,15 @@ files:
80
80
  - lib/groonga-delta/import-config.rb
81
81
  - lib/groonga-delta/import-status.rb
82
82
  - lib/groonga-delta/local-delta.rb
83
+ - lib/groonga-delta/local-reader.rb
83
84
  - lib/groonga-delta/local-source.rb
85
+ - lib/groonga-delta/local-vacuumer.rb
86
+ - lib/groonga-delta/local-writer.rb
84
87
  - lib/groonga-delta/ltsv-log-formatter.rb
85
88
  - lib/groonga-delta/mapping.rb
86
89
  - lib/groonga-delta/mysql-source.rb
87
90
  - lib/groonga-delta/status.rb
88
91
  - lib/groonga-delta/version.rb
89
- - lib/groonga-delta/writer.rb
90
92
  homepage: https://github.com/groonga/groonga-delta
91
93
  licenses:
92
94
  - GPL-3.0+