flydata 0.1.7 → 0.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/VERSION +1 -1
- data/flydata.gemspec +15 -3
- data/lib/flydata/cli.rb +1 -1
- data/lib/flydata/command/sender.rb +35 -10
- data/lib/flydata/command/sync.rb +188 -24
- data/lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb +101 -218
- data/lib/flydata/fluent-plugins/mysql/alter_table_query_handler.rb +34 -0
- data/lib/flydata/fluent-plugins/mysql/binlog_position.rb +20 -0
- data/lib/flydata/fluent-plugins/mysql/binlog_query_dispatcher.rb +39 -0
- data/lib/flydata/fluent-plugins/mysql/binlog_query_handler.rb +11 -0
- data/lib/flydata/fluent-plugins/mysql/binlog_record_dispatcher.rb +50 -0
- data/lib/flydata/fluent-plugins/mysql/binlog_record_handler.rb +100 -0
- data/lib/flydata/fluent-plugins/mysql/context.rb +25 -0
- data/lib/flydata/fluent-plugins/mysql/dml_record_handler.rb +82 -0
- data/lib/flydata/fluent-plugins/mysql/query_parser.rb +69 -0
- data/lib/flydata/sync_file_manager.rb +119 -12
- data/lib/flydata/table_def/mysql_table_def.rb +52 -22
- data/lib/flydata/table_def/redshift_table_def.rb +17 -9
- data/spec/flydata/command/sync_spec.rb +5 -5
- data/spec/flydata/fluent-plugins/in_mysql_binlog_flydata_spec.rb +86 -15
- data/spec/flydata/fluent-plugins/mysql/binlog_position_spec.rb +33 -0
- data/spec/flydata/fluent-plugins/mysql/query_parser_spec.rb +54 -0
- data/spec/flydata/table_def/mysql_table_def_spec.rb +2 -2
- data/spec/flydata/table_def/redshift_table_def_spec.rb +40 -0
- metadata +16 -4
@@ -19,6 +19,7 @@ class MysqlTableDef
|
|
19
19
|
'fixed' => 'numeric',
|
20
20
|
'float' => 'float4',
|
21
21
|
'int' => 'int4',
|
22
|
+
'integer' => 'int4',
|
22
23
|
'longblob' => 'varbinary',
|
23
24
|
'longtext' => 'text',
|
24
25
|
'mediumblob' => 'varbinary',
|
@@ -36,6 +37,31 @@ class MysqlTableDef
|
|
36
37
|
'varchar' => 'varchar',
|
37
38
|
}
|
38
39
|
|
40
|
+
def self.convert_to_flydata_type(type)
|
41
|
+
TYPE_MAP_M2F.each do |mysql_type, flydata_type|
|
42
|
+
if /^#{mysql_type}\(|^#{mysql_type}$/.match(type)
|
43
|
+
ret_type = type.gsub(/^#{mysql_type}/, flydata_type)
|
44
|
+
ret_type = check_and_set_varchar_length(ret_type, mysql_type, flydata_type)
|
45
|
+
return ret_type
|
46
|
+
end
|
47
|
+
end
|
48
|
+
nil
|
49
|
+
end
|
50
|
+
|
51
|
+
# Check and set the varchar(char) size which is converted from
|
52
|
+
# length to byte size.
|
53
|
+
# On Mysql the record size of varchar(char) is a length of characters.
|
54
|
+
# ex) varchar(6) on mysql -> varchar(18) on flydata
|
55
|
+
def self.check_and_set_varchar_length(type, mysql_type, flydata_type)
|
56
|
+
return type unless %w(char varchar).include?(mysql_type)
|
57
|
+
if type =~ /\((\d+)\)/
|
58
|
+
# expect 3 byte UTF-8 character
|
59
|
+
"#{flydata_type}(#{$1.to_i * 3})"
|
60
|
+
else
|
61
|
+
raise "Invalid varchar type. It must be a bug... type:#{type}"
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
39
65
|
def self.create(io)
|
40
66
|
params = _create(io)
|
41
67
|
params ? self.new(*params) : nil
|
@@ -74,7 +100,7 @@ class MysqlTableDef
|
|
74
100
|
stripped_line = line.strip
|
75
101
|
# `col_smallint` smallint(6) DEFAULT NULL,
|
76
102
|
if stripped_line.start_with?('`')
|
77
|
-
columns <<
|
103
|
+
columns << parse_one_column_def(line)
|
78
104
|
# PRIMARY KEY (`id`)
|
79
105
|
elsif stripped_line.start_with?("PRIMARY KEY")
|
80
106
|
parse_primary_key(line, columns)
|
@@ -112,49 +138,56 @@ class MysqlTableDef
|
|
112
138
|
tabledef
|
113
139
|
end
|
114
140
|
|
115
|
-
|
116
|
-
|
117
|
-
def self.parse_column_line(line)
|
118
|
-
line = line.strip
|
141
|
+
def self.parse_one_column_def(query)
|
142
|
+
line = query.strip
|
119
143
|
line = line[0..-2] if line.end_with?(',')
|
120
144
|
pos = 0
|
121
145
|
cond = :column_name
|
122
146
|
column = {}
|
123
147
|
|
124
148
|
while pos < line.length
|
125
|
-
#cur_char = line[pos]
|
126
149
|
case cond
|
127
150
|
when :column_name #`column_name` ...
|
128
|
-
pos = line.index('
|
129
|
-
column[:name] = line[
|
151
|
+
pos = line.index(' ', 1)
|
152
|
+
column[:name] = if line[0] == '`'
|
153
|
+
line[1..pos-2]
|
154
|
+
else
|
155
|
+
line[0..pos-1]
|
156
|
+
end
|
130
157
|
cond = :column_type
|
131
158
|
pos += 1
|
132
159
|
when :column_type #... formattype(,,,) ...
|
133
160
|
pos += 1 until line[pos] != ' '
|
134
161
|
start_pos = pos
|
135
162
|
pos += 1 until line[pos].nil? || line[pos] =~ /\s|\(/
|
163
|
+
|
164
|
+
# meta
|
136
165
|
if line[pos] == '('
|
137
|
-
|
166
|
+
#TODO: implement better parser
|
167
|
+
pos = line.index(')', pos)
|
138
168
|
pos += 1
|
139
169
|
end
|
170
|
+
|
171
|
+
# type
|
140
172
|
type = line[start_pos..pos-1]
|
141
|
-
|
142
|
-
|
143
|
-
end
|
144
|
-
column[:type] = type
|
173
|
+
column[:type] = convert_to_flydata_type(type)
|
174
|
+
|
145
175
|
cond = :options
|
146
176
|
when :options
|
147
|
-
column[:auto_increment] = true if line =~ /AUTO_INCREMENT/
|
148
|
-
column[:not_null] = true if line =~ /NOT NULL/
|
149
|
-
if /DEFAULT\s+((?:[^'\s]+\b)|(?:'(?:\\'|[^'])*'))
|
177
|
+
column[:auto_increment] = true if line =~ /AUTO_INCREMENT/i
|
178
|
+
column[:not_null] = true if line =~ /NOT NULL/i
|
179
|
+
if /DEFAULT\s+((?:[^'\s]+\b)|(?:'(?:\\'|[^'])*'))/i.match(line)
|
150
180
|
val = $1
|
151
181
|
val = val.slice(1..-1) if val.start_with?("'")
|
152
182
|
val = val.slice(0..-2) if val.end_with?("'")
|
153
183
|
column[:default] = val == "NULL" ? nil : val
|
154
184
|
end
|
155
|
-
if /COMMENT\s+'(((?:\\'|[^'])*))'
|
185
|
+
if /COMMENT\s+'(((?:\\'|[^'])*))'/i.match(line)
|
156
186
|
column[:comment] = $1
|
157
187
|
end
|
188
|
+
if block_given?
|
189
|
+
column = yield(column, query, pos)
|
190
|
+
end
|
158
191
|
break
|
159
192
|
else
|
160
193
|
raise "Invalid condition. It must be a bug..."
|
@@ -163,6 +196,8 @@ class MysqlTableDef
|
|
163
196
|
column
|
164
197
|
end
|
165
198
|
|
199
|
+
private
|
200
|
+
|
166
201
|
def self.parse_primary_key(line, columns)
|
167
202
|
primary_keys = line.scan(/`(.*?)`/).collect{|item| item[0]}
|
168
203
|
primary_keys.each do |primary_key|
|
@@ -173,11 +208,6 @@ class MysqlTableDef
|
|
173
208
|
column[:primary_key] = true
|
174
209
|
end
|
175
210
|
end
|
176
|
-
|
177
|
-
def self.parse_column_type_values(line, pos)
|
178
|
-
#TODO: more better parse
|
179
|
-
line.index(')', pos)
|
180
|
-
end
|
181
211
|
end
|
182
212
|
|
183
213
|
end
|
@@ -27,7 +27,7 @@ class RedshiftTableDef
|
|
27
27
|
'text' => {type: 'varchar(max)'},
|
28
28
|
'time' => {type: 'timestamp'},
|
29
29
|
'varbinary' => {type: 'varchar', use_params: true},
|
30
|
-
'varchar' => {type: 'varchar', use_params: true},
|
30
|
+
'varchar' => {type: 'varchar', use_params: true, max_size: 65535},
|
31
31
|
}
|
32
32
|
def self.from_flydata_tabledef(flydata_tabledef, options = {})
|
33
33
|
options[:flydata_ctl_table] = true unless options.has_key?(:flydata_ctl_table)
|
@@ -40,11 +40,14 @@ class RedshiftTableDef
|
|
40
40
|
end
|
41
41
|
|
42
42
|
CREATE_FLYDATA_CTL_TABLE_SQL = <<EOS
|
43
|
+
DROP TABLE flydata_ctl_columns;
|
43
44
|
CREATE TABLE flydata_ctl_columns (
|
44
45
|
id integer NOT NULL IDENTITY(1,1),
|
45
46
|
table_name varchar(128) NOT NULL,
|
46
47
|
column_name varchar(128) NOT NULL,
|
47
48
|
src_data_type varchar(1024) NOT NULL,
|
49
|
+
revision int NOT NULL DEFAULT 1,
|
50
|
+
ordinal_position int NOT NULL,
|
48
51
|
PRIMARY KEY(id)
|
49
52
|
) DISTKEY(table_name) SORTKEY(table_name);
|
50
53
|
EOS
|
@@ -73,17 +76,22 @@ EOS
|
|
73
76
|
|
74
77
|
def self.column_def_sql(column)
|
75
78
|
type = column[:type]
|
76
|
-
if type =~ /\(
|
79
|
+
if type =~ /\((.*?)\)/
|
77
80
|
type = $` + $'
|
78
|
-
params =
|
81
|
+
params = $1
|
79
82
|
end
|
80
83
|
|
81
84
|
type_info = TYPE_MAP_F2R[type]
|
82
85
|
raise "Unsupported type '#{column[:type]}'" if type_info.nil?
|
83
86
|
|
84
|
-
rs_type =
|
85
|
-
|
86
|
-
|
87
|
+
rs_type = if type_info[:use_params] && params && !params.nil?
|
88
|
+
if type_info[:max_size] && /\d+/.match(params) && params.to_i > type_info[:max_size]
|
89
|
+
params = type_info[:max_size]
|
90
|
+
end
|
91
|
+
type_info[:type] + "(#{params})"
|
92
|
+
else
|
93
|
+
type_info[:type]
|
94
|
+
end
|
87
95
|
line = %Q| "#{column[:name]}" #{rs_type}|
|
88
96
|
line += " NOT NULL" if column[:not_null]
|
89
97
|
if (column.has_key?(:default))
|
@@ -127,13 +135,13 @@ EOS
|
|
127
135
|
|
128
136
|
FLYDATA_CTL_COLUMNS_SQL = <<EOS
|
129
137
|
DELETE FROM flydata_ctl_columns WHERE table_name = '%s';
|
130
|
-
INSERT INTO flydata_ctl_columns (table_name, column_name, src_data_type) VALUES
|
138
|
+
INSERT INTO flydata_ctl_columns (table_name, column_name, src_data_type, ordinal_position) VALUES
|
131
139
|
EOS
|
132
140
|
def self.flydata_ctl_sql(flydata_tabledef)
|
133
141
|
sql = FLYDATA_CTL_COLUMNS_SQL % [ flydata_tabledef[:table_name] ]
|
134
142
|
values = []
|
135
|
-
flydata_tabledef[:columns].each do |col|
|
136
|
-
values << "('#{flydata_tabledef[:table_name]}', '#{col[:name]}', '#{escape(col[:type])}')"
|
143
|
+
flydata_tabledef[:columns].each.with_index(1) do |col, i|
|
144
|
+
values << "('#{flydata_tabledef[:table_name]}', '#{col[:name]}', '#{escape(col[:type])}', #{i})"
|
137
145
|
end
|
138
146
|
sql += values.join(",\n") + ';'
|
139
147
|
sql
|
@@ -384,7 +384,7 @@ module Flydata
|
|
384
384
|
end
|
385
385
|
|
386
386
|
module Mysql
|
387
|
-
describe
|
387
|
+
describe MysqlDumpGeneratorMasterData do
|
388
388
|
let(:stdout) { double(:stdout) }
|
389
389
|
let(:stderr) { double(:stderr) }
|
390
390
|
let(:status) { double(:status) }
|
@@ -393,21 +393,21 @@ module Flydata
|
|
393
393
|
'host' => 'localhost', 'port' => 3306, 'username' => 'admin',
|
394
394
|
'password' => 'pass', 'database' => 'dev', 'tables' => 'users,groups',
|
395
395
|
} }
|
396
|
-
let(:default_dump_generator) {
|
396
|
+
let(:default_dump_generator) { MysqlDumpGeneratorMasterData.new(default_conf) }
|
397
397
|
|
398
398
|
describe '#initialize' do
|
399
399
|
context 'with password' do
|
400
400
|
subject { default_dump_generator.instance_variable_get(:@dump_cmd) }
|
401
401
|
it { should eq('mysqldump --protocol=tcp -h localhost -P 3306 -uadmin -ppass --skip-lock-tables ' +
|
402
|
-
'--single-transaction --
|
402
|
+
'--single-transaction --hex-blob --flush-logs --master-data=2 dev users groups') }
|
403
403
|
end
|
404
404
|
context 'without password' do
|
405
405
|
let (:dump_generator) do
|
406
|
-
|
406
|
+
MysqlDumpGeneratorMasterData.new(default_conf.merge({'password' => ''}))
|
407
407
|
end
|
408
408
|
subject { dump_generator.instance_variable_get(:@dump_cmd) }
|
409
409
|
it { should eq('mysqldump --protocol=tcp -h localhost -P 3306 -uadmin --skip-lock-tables ' +
|
410
|
-
'--single-transaction --
|
410
|
+
'--single-transaction --hex-blob --flush-logs --master-data=2 dev users groups') }
|
411
411
|
end
|
412
412
|
end
|
413
413
|
|
@@ -12,6 +12,7 @@ module Fluent
|
|
12
12
|
TEST_SEQUENCE_FILE = /positions\/#{TEST_TABLE}.pos$/
|
13
13
|
TEST_TABLES = "#{TEST_TABLE},test_table_1,test_table_2,test_table_3"
|
14
14
|
TEST_POSITION_FILE = "test_position.log"
|
15
|
+
TEST_REVISION_FILE = "#{ENV['HOME']}/.flydata/positions/#{TEST_TABLE}.rev"
|
15
16
|
TEST_TIMESTAMP = 1389214083
|
16
17
|
TEST_CONFIG = <<EOT
|
17
18
|
tag #{TEST_TAG}
|
@@ -66,27 +67,27 @@ EOT
|
|
66
67
|
EOT
|
67
68
|
# QUERY: create table
|
68
69
|
TEST_EVENT_QUERY_CREATE_TABLE = <<EOT
|
69
|
-
{"marker"=>0, "timestamp"
|
70
|
+
{"marker"=>0, "timestamp"=>#{TEST_TIMESTAMP}, "type_code"=>2, "server_id"=>1, "event_length"=>121, "next_position"=>317, "flags"=>0, "event_type"=>"Query", "thread_id"=>42, "exec_time"=>0, "error_code"=>0, "variables"=>[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3, 115, 116, 100, 4, 33, 0, 33, 0, 8, 0], "db_name"=>"test_db", "query"=>"create table test_table(id int primary key, value text)"}
|
70
71
|
EOT
|
71
72
|
# QUERY: drop table
|
72
73
|
TEST_EVENT_QUERY_DROP_TABLE = <<EOT
|
73
|
-
{"marker"=>0, "timestamp"
|
74
|
+
{"marker"=>0, "timestamp"=>#{TEST_TIMESTAMP}, "type_code"=>2, "server_id"=>1, "event_length"=>115, "next_position"=>568, "flags"=>0, "event_type"=>"Query", "thread_id"=>42, "exec_time"=>0, "error_code"=>0, "variables"=>[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3, 115, 116, 100, 4, 33, 0, 33, 0, 8, 0], "db_name"=>"test_db", "query"=>"DROP TABLE `test_table` /* generated by server */"}
|
74
75
|
EOT
|
75
76
|
# QUERY: alter table add column
|
76
77
|
TEST_EVENT_QUERY_ALTER_TABLE_ADD_COLUMN = <<EOT
|
77
|
-
{"marker"=>0, "timestamp"
|
78
|
+
{"marker"=>0, "timestamp"=>#{TEST_TIMESTAMP}, "type_code"=>2, "server_id"=>1, "event_length"=>111, "next_position"=>800, "flags"=>0, "event_type"=>"Query", "thread_id"=>42, "exec_time"=>0, "error_code"=>0, "variables"=>[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3, 115, 116, 100, 4, 33, 0, 33, 0, 8, 0], "db_name"=>"test_db", "query"=>"alter table test_table add column sum integer"}
|
78
79
|
EOT
|
79
80
|
# QUERY: alter table drop column
|
80
81
|
TEST_EVENT_QUERY_ALTER_TABLE_DROP_COLUMN = <<EOT
|
81
|
-
{"marker"=>0, "timestamp"
|
82
|
+
{"marker"=>0, "timestamp"=>#{TEST_TIMESTAMP}, "type_code"=>2, "server_id"=>1, "event_length"=>104, "next_position"=>904, "flags"=>0, "event_type"=>"Query", "thread_id"=>42, "exec_time"=>0, "error_code"=>0, "variables"=>[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3, 115, 116, 100, 4, 33, 0, 33, 0, 8, 0], "db_name"=>"test_db", "query"=>"alter table test_table drop column sum"}
|
82
83
|
EOT
|
83
84
|
# QUERY: alter table modify column
|
84
85
|
TEST_EVENT_QUERY_ALTER_TABLE_MODIFY_COLUMN = <<EOT
|
85
|
-
{"marker"=>0, "timestamp"
|
86
|
+
{"marker"=>0, "timestamp"=>#{TEST_TIMESTAMP}, "type_code"=>2, "server_id"=>1, "event_length"=>112, "next_position"=>1352, "flags"=>0, "event_type"=>"Query", "thread_id"=>48, "exec_time"=>0, "error_code"=>0, "variables"=>[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3, 115, 116, 100, 4, 33, 0, 33, 0, 8, 0], "db_name"=>"test_db", "query"=>"alter table test_table modify column sum float"}
|
86
87
|
EOT
|
87
88
|
# - 03 STOP_EVENT
|
88
89
|
TEST_EVENT_STOP = <<EOT
|
89
|
-
{"marker"=>0, "timestamp"
|
90
|
+
{"marker"=>0, "timestamp"=>#{TEST_TIMESTAMP}, "type_code"=>3, "server_id"=>1, "event_length"=>19, "next_position"=>2929, "flags"=>0, "event_type"=>"Stop"}
|
90
91
|
EOT
|
91
92
|
# - 04 ROTATE_EVENT
|
92
93
|
TEST_EVENT_ROTATE = <<EOT
|
@@ -137,7 +138,8 @@ EOT
|
|
137
138
|
def expect_emitted_records_with_rows(event, type, table, position, binlog_file, rows)
|
138
139
|
rows = [rows] unless rows.kind_of?(Array)
|
139
140
|
records = rows.collect do |row|
|
140
|
-
{
|
141
|
+
{ :type=>type, :table_name=>table, :respect_order=>true, :seq=>2,
|
142
|
+
:src_pos=>"#{binlog_file}\t#{position}", :table_rev=>1, :row=>row }
|
141
143
|
end
|
142
144
|
expect_emitted_records(event, records)
|
143
145
|
end
|
@@ -152,6 +154,7 @@ EOT
|
|
152
154
|
let(:update_event) { create_event(TEST_EVENT_UPDATE) }
|
153
155
|
let(:update_two_byte_event) { create_event(TEST_EVENT_TWO_BYTE_UPDATE) }
|
154
156
|
let(:update_three_byte_event) { create_event(TEST_EVENT_THREE_BYTE_UPDATE) }
|
157
|
+
let(:alter_table_add_column_event) { create_event(TEST_EVENT_QUERY_ALTER_TABLE_ADD_COLUMN) }
|
155
158
|
|
156
159
|
let(:query_event) { create_event(TEST_EVENT_QUERY_CREATE_DATABSE) }
|
157
160
|
let(:table_map_event) { create_event(TEST_EVENT_TABLE_MAP) }
|
@@ -173,6 +176,7 @@ EOT
|
|
173
176
|
allow(MysqlBinlogInput::BinlogUtil).to receive(:to_hash) {|e| e}
|
174
177
|
allow(File).to receive(:open).with(TEST_POSITION_FILE).and_return('test_position')
|
175
178
|
allow(File).to receive(:exists?).with(TEST_POSITION_FILE).and_return(true)
|
179
|
+
allow(File).to receive(:exists?).with(TEST_REVISION_FILE).and_return(false)
|
176
180
|
allow(File).to receive(:open).with(TEST_SEQUENCE_FILE, "r+").and_yield(table_seq_file)
|
177
181
|
Timecop.freeze(now)
|
178
182
|
end
|
@@ -185,7 +189,9 @@ EOT
|
|
185
189
|
|
186
190
|
context 'when received rotate event' do
|
187
191
|
before do
|
188
|
-
|
192
|
+
allow(File).to receive(:exists?).with(/binlog.pos/).and_return(false)
|
193
|
+
allow(File).to receive(:exists?).with(/\.rev$/).and_return(false)
|
194
|
+
Test.configure_plugin(plugin, TEST_CONFIG)
|
189
195
|
plugin.event_listener(rotate_event)
|
190
196
|
end
|
191
197
|
|
@@ -261,6 +267,38 @@ EOT
|
|
261
267
|
end
|
262
268
|
end
|
263
269
|
|
270
|
+
context 'when received alter table event' do
|
271
|
+
before do
|
272
|
+
allow(File).to receive(:open).with(/test_table\.rev/, 'w')
|
273
|
+
end
|
274
|
+
it do
|
275
|
+
expect(table_seq_file).to receive(:write).with(2).never
|
276
|
+
expect_no_emitted_record(alter_table_add_column_event)
|
277
|
+
end
|
278
|
+
end
|
279
|
+
|
280
|
+
#TODO: Uncomment the following test and delete the above one after supporting alter table
|
281
|
+
=begin
|
282
|
+
context 'when received alter table event' do
|
283
|
+
before do
|
284
|
+
allow(File).to receive(:open).with(/test_table\.rev/, 'w')
|
285
|
+
end
|
286
|
+
it do
|
287
|
+
expect(table_seq_file).to receive(:write).with(2).once
|
288
|
+
expect_emitted_records(alter_table_add_column_event, {
|
289
|
+
subtype: :add_column,
|
290
|
+
table_name: "test_table",
|
291
|
+
type: :alter_table,
|
292
|
+
respect_order: true,
|
293
|
+
src_pos: "mysql-bin.000048\t689",
|
294
|
+
table_rev: 2, # increment revision
|
295
|
+
seq: 2,
|
296
|
+
column: {:name=>"sum", :type=>'int4'},
|
297
|
+
})
|
298
|
+
end
|
299
|
+
end
|
300
|
+
=end
|
301
|
+
|
264
302
|
context 'when received event with another database name' do
|
265
303
|
it do
|
266
304
|
event = insert_event
|
@@ -285,9 +323,10 @@ EOT
|
|
285
323
|
end
|
286
324
|
end
|
287
325
|
end
|
288
|
-
|
326
|
+
|
289
327
|
context 'when rotate event is not received' do
|
290
328
|
before do
|
329
|
+
allow(File).to receive(:exists?).with(/binlog.pos/).and_return(false)
|
291
330
|
Test.configure_plugin(plugin, TEST_CONFIG)
|
292
331
|
end
|
293
332
|
|
@@ -311,7 +350,7 @@ EOT
|
|
311
350
|
expect_emitted_records_with_rows(delete_event, :delete, TEST_TABLE, 5324, "",
|
312
351
|
[{"1"=>"0SL00000002", "2"=>"var"}, {"1"=>"0SL00000003", "2"=>"hoge"}])
|
313
352
|
end
|
314
|
-
|
353
|
+
|
315
354
|
it 'logs warning once when it receives consecutive events' do
|
316
355
|
table_seq_file.should_receive(:write).exactly(10).with(2)
|
317
356
|
expect($log).to receive(:warn).with("Binlog file name is empty. Rotate event not received!").once
|
@@ -322,24 +361,56 @@ EOT
|
|
322
361
|
expect_emitted_records_with_rows(insert_event, :insert, TEST_TABLE, 628, "",
|
323
362
|
[{"1"=>"0SL00000001", "2"=>"foo"}, {"1"=>"0SL00000002", "2"=>"var"}, {"1"=>"0SL00000003", "2"=>"hoge"}])
|
324
363
|
expect_emitted_records_with_rows(delete_event, :delete, TEST_TABLE, 5324, "",
|
325
|
-
[{"1"=>"0SL00000002", "2"=>"var"}, {"1"=>"0SL00000003", "2"=>"hoge"}])
|
326
|
-
end
|
364
|
+
[{"1"=>"0SL00000002", "2"=>"var"}, {"1"=>"0SL00000003", "2"=>"hoge"}])
|
365
|
+
end
|
327
366
|
end
|
328
367
|
|
329
368
|
context 'when received rotate event with missing binlog file' do
|
330
369
|
before do
|
370
|
+
allow(File).to receive(:exists?).with(/binlog.pos/).and_return(false)
|
331
371
|
Test.configure_plugin(plugin, TEST_CONFIG)
|
332
372
|
plugin.event_listener(rotate_event_corrupt)
|
333
373
|
end
|
334
|
-
|
374
|
+
|
335
375
|
it 'logs a warning and emits FET with a blank binlog file name, when it receives an insert event' do
|
336
376
|
table_seq_file.should_receive(:write).exactly(3).with(2)
|
337
377
|
expect($log).to receive(:warn).with("Binlog file name is empty. Rotate event not received!").once
|
338
378
|
expect_emitted_records_with_rows(insert_event, :insert, TEST_TABLE, 628, "",
|
339
379
|
[{"1"=>"0SL00000001", "2"=>"foo"}, {"1"=>"0SL00000002", "2"=>"var"}, {"1"=>"0SL00000003", "2"=>"hoge"}])
|
340
|
-
end
|
380
|
+
end
|
381
|
+
end
|
382
|
+
|
383
|
+
context 'when per-table binlog file is present' do
|
384
|
+
before do
|
385
|
+
f = double('per-table-binlog-file')
|
386
|
+
allow(f).to receive(:readline).and_return("mysql-bin.000048\t1010")
|
387
|
+
allow(File).to receive(:exists?).with(/binlog.pos/).and_return(false)
|
388
|
+
allow(File).to receive(:exists?).with(/test_table.binlog.pos/).and_return(true)
|
389
|
+
allow(File).to receive(:open).with(/test_table.binlog.pos/, "r").and_return(f)
|
390
|
+
Test.configure_plugin(plugin, TEST_CONFIG)
|
391
|
+
plugin.event_listener(rotate_event)
|
392
|
+
end
|
393
|
+
|
394
|
+
it 'emits no record when it receives an event whose binlog position is less than per-table binlog position' do
|
395
|
+
expect_no_emitted_record(insert_event)
|
396
|
+
end
|
397
|
+
|
398
|
+
it 'deletes the per-table-binlog file (if there is one) as soon as a record with bigger binlog position is received' do
|
399
|
+
event = insert_event
|
400
|
+
event['next_position'] = 1250
|
401
|
+
table_seq_file.should_receive(:write).exactly(3).with(2)
|
402
|
+
expect(FileUtils).to receive(:rm).with(/test_table.binlog.pos/, :force => true)
|
403
|
+
expect_emitted_records_with_rows(event, :insert, TEST_TABLE, 1211, "mysql-bin.000048",
|
404
|
+
[{"1"=>"0SL00000001", "2"=>"foo"}, {"1"=>"0SL00000002", "2"=>"var"}, {"1"=>"0SL00000003", "2"=>"hoge"}])
|
405
|
+
end
|
406
|
+
|
407
|
+
it 'emits no record when it receives an event whose binlog position is equal to per-table binlog position' do
|
408
|
+
event = insert_event
|
409
|
+
event['next_position'] = 1049
|
410
|
+
expect_no_emitted_record(event)
|
411
|
+
end
|
341
412
|
end
|
342
|
-
|
343
413
|
end
|
344
414
|
end
|
415
|
+
|
345
416
|
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
require_relative '../../../spec_helper'
|
2
|
+
|
3
|
+
module Mysql
|
4
|
+
describe BinLogPosition do
|
5
|
+
let(:pos1) { BinLogPosition.new('mysql-bin.000064 107') }
|
6
|
+
let(:pos2) { BinLogPosition.new('mysql-bin.000064 978') }
|
7
|
+
let(:pos3) { BinLogPosition.new('mysql-bin.000065 107') }
|
8
|
+
let(:pos4) { BinLogPosition.new('mysql-bin.000064 1107') }
|
9
|
+
let(:pos5) { BinLogPosition.new('mysql-bin.000064 1107') }
|
10
|
+
|
11
|
+
it 'should respond to greater than or equal to operator' do
|
12
|
+
pos1.respond_to?('>=').should be_true
|
13
|
+
end
|
14
|
+
|
15
|
+
context 'when testing greater than or equal to operator' do
|
16
|
+
it 'should return true when it is compared with another object with smaller position value' do
|
17
|
+
(pos2 >= pos1).should be_true
|
18
|
+
end
|
19
|
+
|
20
|
+
it 'should return false when it is compared with another object with bigger binlogfile value' do
|
21
|
+
(pos2 >= pos3).should be_false
|
22
|
+
end
|
23
|
+
|
24
|
+
it 'should compare the position as an integer (and not string)' do
|
25
|
+
(pos2 >= pos4).should be_false
|
26
|
+
end
|
27
|
+
|
28
|
+
it 'should return true when it is compared with another object with equal value' do
|
29
|
+
(pos4 >= pos5).should be_true
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|