flydata 0.3.16 → 0.3.17
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/VERSION +1 -1
- data/flydata-core/lib/flydata-core/record/record.rb +13 -0
- data/flydata-core/lib/flydata-core/table_def/mysql_table_def.rb +107 -5
- data/flydata-core/lib/flydata-core/table_def/redshift_table_def.rb +62 -11
- data/flydata-core/spec/table_def/mysql_table_def_spec.rb +37 -1
- data/flydata-core/spec/table_def/mysql_to_redshift_table_def_spec.rb +120 -0
- data/flydata-core/spec/table_def/mysqldump_test_column_charset.dump +45 -0
- data/flydata-core/spec/table_def/redshift_table_def_spec.rb +70 -88
- data/flydata.gemspec +13 -8
- data/lib/flydata/command/setup.rb +4 -4
- data/lib/flydata/command/sync.rb +18 -29
- data/lib/flydata/compatibility_check.rb +2 -2
- data/lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb +15 -10
- data/lib/flydata/fluent-plugins/mysql/binlog_record_handler.rb +6 -3
- data/lib/flydata/fluent-plugins/mysql/dml_record_handler.rb +15 -8
- data/lib/flydata/fluent-plugins/mysql/table_meta.rb +6 -34
- data/lib/flydata/{fluent-plugins/mysql → mysql}/binlog_position.rb +2 -0
- data/lib/flydata/{util → mysql}/mysql_util.rb +34 -1
- data/lib/flydata/mysql/table_ddl.rb +118 -0
- data/lib/flydata/parser/mysql/dump_parser.rb +5 -5
- data/lib/flydata/parser/mysql/mysql_alter_table.treetop +29 -5
- data/lib/flydata/sync_file_manager.rb +15 -2
- data/spec/flydata/command/sync_spec.rb +22 -1
- data/spec/flydata/fluent-plugins/in_mysql_binlog_flydata_spec.rb +7 -2
- data/spec/flydata/fluent-plugins/mysql/dml_record_handler_spec.rb +130 -0
- data/spec/flydata/fluent-plugins/mysql/shared_query_handler_context.rb +1 -0
- data/spec/flydata/fluent-plugins/mysql/table_meta_spec.rb +14 -8
- data/spec/flydata/fluent-plugins/mysql/truncate_query_handler_spec.rb +2 -1
- data/spec/flydata/{fluent-plugins/mysql → mysql}/binlog_position_spec.rb +3 -2
- data/spec/flydata/{util → mysql}/mysql_util_spec.rb +2 -2
- data/spec/flydata/mysql/table_ddl_spec.rb +193 -0
- data/spec/flydata/parser/mysql/alter_table_parser_spec.rb +37 -9
- data/spec/flydata/sync_file_manager_spec.rb +102 -27
- metadata +12 -7
@@ -215,7 +215,9 @@ grammar MysqlAlterTable
|
|
215
215
|
}
|
216
216
|
/ convert_sym sp to_sym sp charset sp charset_name_or_default opt_collate {
|
217
217
|
def action
|
218
|
-
{ action: :convert_charset
|
218
|
+
{ action: :convert_charset,
|
219
|
+
cs: charset_name_or_default.charset
|
220
|
+
}
|
219
221
|
end
|
220
222
|
}
|
221
223
|
/ ( create_table_options_space_separated ) 1..1 {
|
@@ -795,6 +797,8 @@ grammar MysqlAlterTable
|
|
795
797
|
/ key_opt { def option; key_opt_option; end }
|
796
798
|
/ comment_opt { def option; comment_opt_option; end }
|
797
799
|
/ column_format_opt { def option; column_format_opt_option; end }
|
800
|
+
/ character_set_opt { def option; character_set_opt_option; end }
|
801
|
+
/ collate_opt { def option; collate_opt_option; end }
|
798
802
|
/ storage_opt { def option; storage_opt_option; end }
|
799
803
|
#TODO: / reference_definition
|
800
804
|
end
|
@@ -889,6 +893,20 @@ grammar MysqlAlterTable
|
|
889
893
|
end
|
890
894
|
}
|
891
895
|
end
|
896
|
+
rule character_set_opt
|
897
|
+
'character set'i sp value {
|
898
|
+
def character_set_opt_option
|
899
|
+
{ cs: FlydataCore::TableDef::MysqlTableDef.flydata_charset(value.raw_value) }
|
900
|
+
end
|
901
|
+
}
|
902
|
+
end
|
903
|
+
rule collate_opt
|
904
|
+
'collate'i sp value {
|
905
|
+
def collate_opt_option
|
906
|
+
{} # Not supported
|
907
|
+
end
|
908
|
+
}
|
909
|
+
end
|
892
910
|
rule storage_opt
|
893
911
|
'storage'i sp ( 'disk'i / 'memory'i / 'default'i ) {
|
894
912
|
def storage_opt_option
|
@@ -1012,8 +1030,8 @@ grammar MysqlAlterTable
|
|
1012
1030
|
end
|
1013
1031
|
|
1014
1032
|
rule charset_name_or_default
|
1015
|
-
default_sym
|
1016
|
-
/ charset_name
|
1033
|
+
default_sym { def charset; 'DEFAULT'; end }
|
1034
|
+
/ charset_name { def charset; FlydataCore::TableDef::MysqlTableDef.flydata_charset(text_value); end }
|
1017
1035
|
end
|
1018
1036
|
|
1019
1037
|
rule collation_name
|
@@ -1047,7 +1065,9 @@ grammar MysqlAlterTable
|
|
1047
1065
|
rule create_table_option
|
1048
1066
|
default_charset {
|
1049
1067
|
def action
|
1050
|
-
{ action: :default_charset
|
1068
|
+
{ action: :default_charset,
|
1069
|
+
cs: charset
|
1070
|
+
}
|
1051
1071
|
end
|
1052
1072
|
}
|
1053
1073
|
/ default_collation {
|
@@ -1073,7 +1093,11 @@ grammar MysqlAlterTable
|
|
1073
1093
|
end
|
1074
1094
|
|
1075
1095
|
rule default_charset
|
1076
|
-
opt_default nsp charset opt_equal nsp charset_name_or_default
|
1096
|
+
opt_default nsp charset opt_equal nsp charset_name_or_default {
|
1097
|
+
def charset
|
1098
|
+
charset_name_or_default.charset
|
1099
|
+
end
|
1100
|
+
}
|
1077
1101
|
end
|
1078
1102
|
|
1079
1103
|
rule opt_default
|
@@ -41,14 +41,27 @@ module Flydata
|
|
41
41
|
state: items[5], substate: items[6], mysql_table: mysql_table}
|
42
42
|
end
|
43
43
|
|
44
|
-
def
|
44
|
+
def load_generated_ddl(tables)
|
45
|
+
tables = [ tables ] unless tables.kind_of?(Array)
|
46
|
+
paths = table_ddl_file_paths(*tables)
|
47
|
+
paths.collect{|path|
|
48
|
+
begin
|
49
|
+
File.open(path) {|f| f.read }
|
50
|
+
rescue Errno::ENOENT
|
51
|
+
nil
|
52
|
+
end
|
53
|
+
}
|
54
|
+
end
|
55
|
+
|
56
|
+
def save_generated_ddl(tables, contents = "1")
|
57
|
+
tables = [ tables ] unless tables.kind_of?(Array)
|
45
58
|
table_positions_dir_path = ENV['FLYDATA_TABLE_POSITIONS'] || File.join(FLYDATA_HOME, 'positions')
|
46
59
|
#Create positions if dir does not exist
|
47
60
|
unless File.directory?(table_positions_dir_path)
|
48
61
|
FileUtils.mkdir_p(table_positions_dir_path)
|
49
62
|
end
|
50
63
|
tables.each do |tab|
|
51
|
-
File.open(File.join(table_positions_dir_path, "#{tab}.generated_ddl"), 'w') {|f| f.write(
|
64
|
+
File.open(File.join(table_positions_dir_path, "#{tab}.generated_ddl"), 'w') {|f| f.write(contents) }
|
52
65
|
end
|
53
66
|
end
|
54
67
|
|
@@ -5,6 +5,9 @@ require 'flydata/command/sync'
|
|
5
5
|
module Flydata
|
6
6
|
module Command
|
7
7
|
describe Sync do
|
8
|
+
let(:subject_object){ described_class.new }
|
9
|
+
subject { subject_object }
|
10
|
+
|
8
11
|
let(:default_mysqldump_dir) do
|
9
12
|
File.join('/tmp', "sync_dump_#{Time.now.to_i}")
|
10
13
|
end
|
@@ -35,6 +38,14 @@ module Flydata
|
|
35
38
|
"data_servers"=>"localhost:9905" }
|
36
39
|
}
|
37
40
|
end
|
41
|
+
let(:mysql_table_columns) {
|
42
|
+
{"id"=>{:column_name=>"id", :format_type_str=>"int(11)", :format_type=>"int", :format_size=>11}, "name"=>{:column_name=>"name", :format_type_str=>"varchar(40)", :format_type=>"varchar", :format_size=>40, :default=>nil}, "created_at"=>{:column_name=>"created_at", :format_type_str=>"timestamp", :format_type=>"timestamp", :default=>"CURRENT_TIMESTAMP"}, "bin"=>{:column_name=>"bin", :format_type_str=>"binary(34)", :format_type=>"binary", :format_size=>34, :default=>nil}, "bin2"=>{:column_name=>"bin2", :format_type_str=>"blob", :format_type=>"blob"}, "varbin"=>{:column_name=>"varbin", :format_type_str=>"varchar(34)", :format_type=>"varchar", :format_size=>34, :default=>nil}}
|
43
|
+
}
|
44
|
+
let(:mysql_table) do
|
45
|
+
mt = double('mysql_table')
|
46
|
+
allow(mt).to receive(:columns).and_return(mysql_table_columns)
|
47
|
+
mt
|
48
|
+
end
|
38
49
|
|
39
50
|
after :each do
|
40
51
|
if Dir.exists?(default_mysqldump_dir)
|
@@ -45,7 +56,6 @@ module Flydata
|
|
45
56
|
end
|
46
57
|
end
|
47
58
|
|
48
|
-
subject { Sync.new }
|
49
59
|
describe '#cleanup_sync_server' do
|
50
60
|
let(:rest_client) { double('rest_client') }
|
51
61
|
|
@@ -232,6 +242,17 @@ module Flydata
|
|
232
242
|
include_examples 'throws an error'
|
233
243
|
end
|
234
244
|
end
|
245
|
+
describe '#convert_to_flydata_values' do
|
246
|
+
subject { subject_object.send(:convert_to_flydata_values, mysql_table, values) }
|
247
|
+
let(:values) { [4, 'John', nil, '0xC0448200', nil, nil] }
|
248
|
+
|
249
|
+
it 'calls convert_to_valydata_value for each value' do
|
250
|
+
mysql_table.columns.values.each_with_index do |col_info, i|
|
251
|
+
expect(FlydataCore::TableDef::MysqlTableDef).to receive(:convert_to_flydata_value).with(values[i], col_info[:format_type]).and_return(values[i])
|
252
|
+
end
|
253
|
+
is_expected.to eq values
|
254
|
+
end
|
255
|
+
end
|
235
256
|
end
|
236
257
|
end
|
237
258
|
end
|
@@ -167,7 +167,8 @@ EOT
|
|
167
167
|
record = row.kind_of?(Hash) && row.keys.include?(:row) ? row : { row: row }
|
168
168
|
@seq += 1
|
169
169
|
record.merge({ :type=>type, :table_name=>table, :respect_order=>true,
|
170
|
-
:seq=>@seq, :src_pos=>"#{binlog_file}\t#{position}", :table_rev=>1
|
170
|
+
:seq=>@seq, :src_pos=>"#{binlog_file}\t#{position}", :table_rev=>1,
|
171
|
+
:v=>2 })
|
171
172
|
end
|
172
173
|
expect_emitted_records(event, records)
|
173
174
|
end
|
@@ -193,6 +194,7 @@ EOT
|
|
193
194
|
let(:rotate_event_corrupt){ create_event(TEST_EVENT_ROTATE_MISSING_BINLOG_FILE) }
|
194
195
|
|
195
196
|
let(:now) { Time.now }
|
197
|
+
let(:flydata_record_version) { 2 }
|
196
198
|
|
197
199
|
let(:binlog_position_file) do
|
198
200
|
f = double('binlog_position_file')
|
@@ -226,7 +228,7 @@ EOT
|
|
226
228
|
setup_initial_flydata_files
|
227
229
|
allow(MysqlBinlogInput::BinlogUtil).to receive(:to_hash) {|e| e}
|
228
230
|
allow(Mysql::BinLogPositionFile).to receive(:new).with(TEST_POSITION_FILE).and_return(binlog_position_file)
|
229
|
-
allow(Mysql::TableMeta).to receive(:update)
|
231
|
+
allow(Flydata::Mysql::TableMeta).to receive(:update)
|
230
232
|
allow(plugin).to receive(:`).with("mysql -V").and_return("mysql Ver 14.14 Distrib 5.5.40")
|
231
233
|
allow(plugin).to receive(:`).with(/^echo 'select version\(\);'/).and_return("version()\n5.5.40-0ubuntu0.12.04.1-log")
|
232
234
|
Timecop.freeze(now)
|
@@ -326,6 +328,7 @@ EOT
|
|
326
328
|
src_pos: "mysql-bin.000048\t689",
|
327
329
|
table_rev: 2, # increment revision
|
328
330
|
seq: 2,
|
331
|
+
v: flydata_record_version,
|
329
332
|
actions: [{
|
330
333
|
action: :add_column, column: "sum", :type=>'int4(11)', :query=>'add column sum integer'}],
|
331
334
|
})
|
@@ -341,6 +344,7 @@ EOT
|
|
341
344
|
src_pos: "mysql-bin.000048\t800",
|
342
345
|
table_rev: 2, # increment revision
|
343
346
|
seq: 2,
|
347
|
+
v: flydata_record_version,
|
344
348
|
actions: [{
|
345
349
|
action: :drop_column, column: "sum", :query=>'drop column sum'}],
|
346
350
|
})
|
@@ -357,6 +361,7 @@ EOT
|
|
357
361
|
src_pos: "mysql-bin.000048\t#{337 - 217}",
|
358
362
|
table_rev: 1,
|
359
363
|
seq: 2,
|
364
|
+
v: flydata_record_version,
|
360
365
|
actions: [{
|
361
366
|
action: :add_index,
|
362
367
|
support_level: :nonbreaking,
|
@@ -0,0 +1,130 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
require 'fluent_plugins_spec_helper'
|
4
|
+
require 'flydata/fluent-plugins/mysql/dml_record_handler'
|
5
|
+
|
6
|
+
module Mysql
|
7
|
+
|
8
|
+
describe DmlRecordHandler do
|
9
|
+
let(:subject_object) { described_class.new(context) }
|
10
|
+
|
11
|
+
let(:table_meta) { double('table_meta') }
|
12
|
+
let(:tables) { ['items', 'orders'] }
|
13
|
+
let(:sync_fm) {
|
14
|
+
sfm = double('sync_fm')
|
15
|
+
allow(sfm).to receive(:get_table_binlog_pos).and_return nil
|
16
|
+
sfm
|
17
|
+
}
|
18
|
+
let(:context) {
|
19
|
+
c = double('context')
|
20
|
+
allow(c).to receive(:table_meta).and_return(table_meta)
|
21
|
+
allow(c).to receive(:tables).and_return(tables)
|
22
|
+
allow(c).to receive(:sync_fm).and_return(sync_fm)
|
23
|
+
c
|
24
|
+
}
|
25
|
+
|
26
|
+
describe '#encode_row_value' do
|
27
|
+
subject { subject_object.send(:encode_row_value, value) }
|
28
|
+
|
29
|
+
let(:normal_utf8_string) { "ABC新年Ã" }
|
30
|
+
let(:valid_utf8_string) { "\x00abc" } # Yes, this is a valid UTF-8 string
|
31
|
+
let(:invalid_utf8_string) { "新年".encode('shift_jis') }
|
32
|
+
let(:latin1_string_happened_to_be_a_valid_utf8_string) { "ã".encode("iso-8859-1") }
|
33
|
+
|
34
|
+
shared_examples 'encoding values differently based on their contents not based on encoding' do
|
35
|
+
context 'with a normal utf-8 string' do
|
36
|
+
let(:value) { normal_utf8_string }
|
37
|
+
it 'returns a utf8 string' do
|
38
|
+
result = subject
|
39
|
+
expect(result).to eq normal_utf8_string
|
40
|
+
expect(result.encoding).to eq Encoding.find('utf-8')
|
41
|
+
end
|
42
|
+
end
|
43
|
+
context 'with a valid utf-8 string' do
|
44
|
+
let(:value) { valid_utf8_string }
|
45
|
+
it 'returns a utf8 string' do
|
46
|
+
result = subject
|
47
|
+
expect(result).to eq valid_utf8_string
|
48
|
+
expect(result.encoding).to eq Encoding.find('utf-8')
|
49
|
+
end
|
50
|
+
end
|
51
|
+
context 'with an invalid utf-8 string' do
|
52
|
+
let(:value) { invalid_utf8_string }
|
53
|
+
it 'returns a FlyData binary format string as BINARY encoding' do
|
54
|
+
result = subject
|
55
|
+
expect(result).to eq "0x9056944E"
|
56
|
+
expect(result.encoding).to eq Encoding::BINARY
|
57
|
+
end
|
58
|
+
end
|
59
|
+
context 'with a latin1 string which happens to be a valid utf8 character' do
|
60
|
+
let(:value) { latin1_string_happened_to_be_a_valid_utf8_string }
|
61
|
+
it 'returns a utf8 string' do
|
62
|
+
result = subject
|
63
|
+
expect(result).to eq latin1_string_happened_to_be_a_valid_utf8_string
|
64
|
+
expect(result).to eq "ã" # #C3A3
|
65
|
+
expect(result.encoding).to eq Encoding.find('utf-8')
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
context 'with strings whose encoding is utf-8' do
|
71
|
+
before do
|
72
|
+
normal_utf8_string.force_encoding 'utf-8'
|
73
|
+
valid_utf8_string.force_encoding 'utf-8'
|
74
|
+
invalid_utf8_string.force_encoding 'utf-8'
|
75
|
+
latin1_string_happened_to_be_a_valid_utf8_string.force_encoding 'utf-8'
|
76
|
+
end
|
77
|
+
it_behaves_like 'encoding values differently based on their contents not based on encoding'
|
78
|
+
end
|
79
|
+
|
80
|
+
context 'with strings whose encoding is binary' do
|
81
|
+
before do
|
82
|
+
normal_utf8_string.force_encoding 'binary'
|
83
|
+
valid_utf8_string.force_encoding 'binary'
|
84
|
+
invalid_utf8_string.force_encoding 'binary'
|
85
|
+
latin1_string_happened_to_be_a_valid_utf8_string.force_encoding 'binary'
|
86
|
+
end
|
87
|
+
it_behaves_like 'encoding values differently based on their contents not based on encoding'
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
describe '#convert_to_flydata_row_format' do
|
92
|
+
subject { subject_object.send(:convert_to_flydata_row_format, row) }
|
93
|
+
|
94
|
+
let(:row) { [ value ] }
|
95
|
+
before do
|
96
|
+
end
|
97
|
+
|
98
|
+
shared_examples "returning a json hash with no attrs" do
|
99
|
+
it do
|
100
|
+
is_expected.to eq({"1" => value })
|
101
|
+
end
|
102
|
+
end
|
103
|
+
|
104
|
+
context 'with a row containing a string value' do
|
105
|
+
let(:value) { "a normal string" }
|
106
|
+
|
107
|
+
before do
|
108
|
+
expect(subject_object).to receive(:encode_row_value).with(value).and_return(value)
|
109
|
+
end
|
110
|
+
|
111
|
+
it_behaves_like "returning a json hash with no attrs"
|
112
|
+
end
|
113
|
+
context 'with a row containing a binary value' do
|
114
|
+
let(:value) { "\x00abc".force_encoding('binary') }
|
115
|
+
before do
|
116
|
+
expect(subject_object).to receive(:encode_row_value).with(value).and_return(value)
|
117
|
+
end
|
118
|
+
it "returns a json hash with attrs" do
|
119
|
+
is_expected.to eq({"1" => value, "attrs" => {"1" => {"enc" => "b" }}})
|
120
|
+
end
|
121
|
+
end
|
122
|
+
context 'with a row containing an integer value' do
|
123
|
+
let(:value) { 4 }
|
124
|
+
|
125
|
+
it_behaves_like "returning a json hash with no attrs"
|
126
|
+
end
|
127
|
+
end
|
128
|
+
end
|
129
|
+
|
130
|
+
end
|
@@ -1,6 +1,7 @@
|
|
1
1
|
require 'fluent_plugins_spec_helper'
|
2
2
|
require 'flydata/fluent-plugins/mysql/table_meta'
|
3
3
|
|
4
|
+
module Flydata
|
4
5
|
module Mysql
|
5
6
|
describe TableMeta do
|
6
7
|
let(:db_opts) { { host: 'test-host', port: 3306, username: 'test-user', password: 'test-pswd', database: 'test-db' } }
|
@@ -9,6 +10,9 @@ module Mysql
|
|
9
10
|
|
10
11
|
let(:conn) { double('conn') }
|
11
12
|
let(:table_meta) { TableMeta.new(db_opts.merge(database: database, tables: tables)) }
|
13
|
+
let(:utf8) { 'utf8' }
|
14
|
+
let(:latin1) { 'latin1' }
|
15
|
+
let(:cp932) { 'cp932' }
|
12
16
|
|
13
17
|
before do
|
14
18
|
allow(conn).to receive(:close)
|
@@ -24,23 +28,26 @@ module Mysql
|
|
24
28
|
context 'when character_set_name is supported' do
|
25
29
|
context 'when encoding is not necessary' do
|
26
30
|
let(:query_result) {[
|
27
|
-
{'table_name' => 'a_table', 'character_set_name' =>
|
31
|
+
{'table_name' => 'a_table', 'character_set_name' => utf8 }
|
28
32
|
]}
|
29
33
|
it 'set nil for encoding' do
|
30
34
|
table_meta.update
|
31
|
-
expect(table_meta['a_table'][:encoding]).to
|
35
|
+
expect(table_meta['a_table'][:encoding]).to eq Encoding::UTF_8
|
36
|
+
expect(table_meta['a_table'][:mysql_charset]).to eq utf8
|
32
37
|
end
|
33
38
|
end
|
34
39
|
|
35
40
|
context 'when encoding is needed' do
|
36
41
|
let(:query_result) {[
|
37
|
-
{'table_name' => 'a_table', 'character_set_name' =>
|
38
|
-
{'table_name' => 'b_table', 'character_set_name' =>
|
42
|
+
{'table_name' => 'a_table', 'character_set_name' => latin1 },
|
43
|
+
{'table_name' => 'b_table', 'character_set_name' => cp932 }
|
39
44
|
]}
|
40
45
|
it 'set ruby encoding encoding' do
|
41
46
|
table_meta.update
|
42
|
-
expect(table_meta['a_table'][:encoding]).to eq
|
43
|
-
expect(table_meta['
|
47
|
+
expect(table_meta['a_table'][:encoding]).to eq Encoding::ISO_8859_1
|
48
|
+
expect(table_meta['a_table'][:mysql_charset]).to eq latin1
|
49
|
+
expect(table_meta['b_table'][:encoding]).to eq Encoding::CP932
|
50
|
+
expect(table_meta['b_table'][:mysql_charset]).to eq cp932
|
44
51
|
end
|
45
52
|
end
|
46
53
|
end
|
@@ -56,5 +63,4 @@ module Mysql
|
|
56
63
|
end
|
57
64
|
end
|
58
65
|
end
|
59
|
-
|
60
|
-
|
66
|
+
end
|
@@ -1,6 +1,6 @@
|
|
1
1
|
require 'fluent_plugins_spec_helper'
|
2
|
-
require 'mysql/binlog_position'
|
3
|
-
|
2
|
+
require 'flydata/mysql/binlog_position'
|
3
|
+
module Flydata
|
4
4
|
module Mysql
|
5
5
|
describe BinLogPosition do
|
6
6
|
let(:pos1) { BinLogPosition.new('mysql-bin.000064 107') }
|
@@ -32,3 +32,4 @@ module Mysql
|
|
32
32
|
end
|
33
33
|
end
|
34
34
|
end
|
35
|
+
end
|
@@ -0,0 +1,193 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'flydata/mysql/table_ddl'
|
3
|
+
|
4
|
+
module Flydata
|
5
|
+
module Mysql
|
6
|
+
|
7
|
+
describe TableDdl do
|
8
|
+
let(:table_name1) { "items" }
|
9
|
+
let(:table_name2) { "orders" }
|
10
|
+
let(:table_names) { [table_name1, table_name2] }
|
11
|
+
let(:table1_charset) { 'utf8' }
|
12
|
+
let(:table2_charset) { 'sjis' }
|
13
|
+
let(:at_charset_event_table1) { double('at_charset_event_table1') }
|
14
|
+
let(:at_charset_event_table2) { double('at_charset_event_table2') }
|
15
|
+
let(:at_column_charset_event_table1) { double('at_column_charset_event_table1') }
|
16
|
+
let(:expected_alter_table_charset_events) { [ at_charset_event_table1, at_charset_event_table2] }
|
17
|
+
let(:at_charset_query) { "ALTER TABLE `#{database_name}`.`#{at_table}` CHARACTER SET = #{charset};" }
|
18
|
+
let(:query) { "ALTER TABLE `#{database_name}`.`#{event_table_name}` CHARACTER SET = #{charset};" }
|
19
|
+
let(:column_query1) { "ALTER TABLE `#{database_name}`.`#{event_table_name}` CHANGE COLUMN `#{table1_varchar_column}` #{table1_varchar_column_def};" }
|
20
|
+
let(:charset) { 'utf8' }
|
21
|
+
let(:event_type) { 'Query' }
|
22
|
+
let(:database_name) { "a_database" }
|
23
|
+
let(:sync_fm) { double('sync_fm') }
|
24
|
+
let(:master_binlog_position) { "#{binlog_file}\t#{binlog_position}" }
|
25
|
+
let(:binlog_file) { "mysql-binlog.00123" }
|
26
|
+
let(:binlog_position) { 28393 }
|
27
|
+
let(:original_current_binlog_file) { double('original_current_binlog_file') }
|
28
|
+
let(:mysql_opts) { { host: double('host'), port: double('port'),
|
29
|
+
username: double('username'),
|
30
|
+
password: double('password'),
|
31
|
+
database: database_name,
|
32
|
+
} }
|
33
|
+
let(:position_file) { double('position_file') }
|
34
|
+
let(:context) { double('context') }
|
35
|
+
let(:mysql_tabledef) { { table_name1 => mysql_tabledef1,
|
36
|
+
table_name2 => mysql_tabledef2,
|
37
|
+
} }
|
38
|
+
let(:mysql_tabledef1) { double('mysql_tabledef1') }
|
39
|
+
let(:mysql_tabledef2) { double('mysql_tabledef2') }
|
40
|
+
let(:column_def1) { { "id" => "`id` integer NOT NULL",
|
41
|
+
table1_varchar_column => table1_varchar_column_def,
|
42
|
+
} }
|
43
|
+
let(:table1_varchar_column) { 'name' }
|
44
|
+
let(:table1_varchar_column_def) { "`#{table1_varchar_column}` varchar(45)#{charset_option1} NULL" }
|
45
|
+
let(:column_def2) { { "id" => "`id` integer NOT NULL",
|
46
|
+
"address" => "`address` varchar(45) NULL"
|
47
|
+
} }
|
48
|
+
let(:charset_option1) { "" }
|
49
|
+
let(:save) { {} }
|
50
|
+
|
51
|
+
before do
|
52
|
+
save[:log] = $log
|
53
|
+
$log = double('$log')
|
54
|
+
allow($log).to receive(:info)
|
55
|
+
allow($log).to receive(:debug)
|
56
|
+
allow($log).to receive(:error)
|
57
|
+
allow($log).to receive(:trace)
|
58
|
+
|
59
|
+
allow(mysql_tabledef1).to receive(:default_charset_mysql).and_return(table1_charset)
|
60
|
+
allow(mysql_tabledef1).to receive(:table_name).and_return(table_name1)
|
61
|
+
allow(mysql_tabledef1).to receive(:column_def).and_return(column_def1)
|
62
|
+
allow(mysql_tabledef2).to receive(:default_charset_mysql).and_return(table2_charset)
|
63
|
+
allow(mysql_tabledef2).to receive(:table_name).and_return(table_name2)
|
64
|
+
allow(mysql_tabledef2).to receive(:column_def).and_return(column_def2)
|
65
|
+
end
|
66
|
+
|
67
|
+
after do
|
68
|
+
$log = save[:log]
|
69
|
+
end
|
70
|
+
|
71
|
+
describe '.migrate_to_v2' do
|
72
|
+
let(:subject_object) { described_class }
|
73
|
+
let(:subject_params) { [:migrate_to_v2, table_names, mysql_opts, sync_fm, position_file, context] }
|
74
|
+
|
75
|
+
let(:target_version) { "2" }
|
76
|
+
|
77
|
+
shared_examples "create and send charset records and increment version" do
|
78
|
+
let(:next_position) { binlog_position }
|
79
|
+
|
80
|
+
it do
|
81
|
+
matcher = receive(:each_mysql_tabledef).with(table_names, mysql_opts)
|
82
|
+
matcher = table_names.inject(matcher) {|m, tn| m.and_yield(mysql_tabledef[tn], nil) }
|
83
|
+
expect(MysqlUtil).to matcher
|
84
|
+
expect(File).to receive(:open).with(position_file).and_return(master_binlog_position)
|
85
|
+
expect(context).to receive(:current_binlog_file).and_return(original_current_binlog_file)
|
86
|
+
expect(context).to receive(:current_binlog_file=).with(binlog_file)
|
87
|
+
expect(context).to receive(:current_binlog_file=).with(original_current_binlog_file)
|
88
|
+
expect(sync_fm).to receive(:save_generated_ddl).with(table_names, target_version)
|
89
|
+
|
90
|
+
expect{|b| subject_object.send(*subject_params, &b) }.to yield_successive_args(*expected_alter_table_charset_events)
|
91
|
+
end
|
92
|
+
end
|
93
|
+
shared_examples "does nothing" do
|
94
|
+
it do
|
95
|
+
expect(sync_fm).not_to receive(:save_generated_ddl)
|
96
|
+
|
97
|
+
expect{|b| subject_object.send(*subject_params, &b) }.not_to yield_control
|
98
|
+
end
|
99
|
+
end
|
100
|
+
|
101
|
+
let(:table_versions) { table_names.size.times.collect{ current_version } }
|
102
|
+
before do
|
103
|
+
expect(sync_fm).to receive(:load_generated_ddl).with(table_names).and_return table_versions
|
104
|
+
end
|
105
|
+
|
106
|
+
context "with a single table" do
|
107
|
+
let(:table_names) { [table_name1] }
|
108
|
+
let(:event_table_name) { table_name1 }
|
109
|
+
let(:charset) { table1_charset }
|
110
|
+
let(:expected_alter_table_charset_events) { [ at_charset_event_table1 ] }
|
111
|
+
|
112
|
+
context "with a table whose version is 0" do
|
113
|
+
let(:current_version) { nil } # generate_ddl file doesn't exist in case of version "0"
|
114
|
+
before do
|
115
|
+
expect(QueryEvent).to receive(:new).with(event_type, database_name, event_table_name, next_position, 0, query, Integer).and_return(at_charset_event_table1)
|
116
|
+
end
|
117
|
+
it_behaves_like "create and send charset records and increment version"
|
118
|
+
end
|
119
|
+
context "with a table whose version is 1" do
|
120
|
+
let(:current_version) { "1" }
|
121
|
+
before do
|
122
|
+
expect(QueryEvent).to receive(:new).with(event_type, database_name, event_table_name, next_position, 0, query, Integer).and_return(at_charset_event_table1)
|
123
|
+
end
|
124
|
+
it_behaves_like "create and send charset records and increment version"
|
125
|
+
end
|
126
|
+
context "with a table whose version is 2" do
|
127
|
+
let(:current_version) { "2" }
|
128
|
+
it_behaves_like "does nothing"
|
129
|
+
end
|
130
|
+
context "with a table whose version is 3" do
|
131
|
+
let(:current_version) { "3" }
|
132
|
+
it_behaves_like "does nothing"
|
133
|
+
end
|
134
|
+
context 'with a single table who has a column with charset' do
|
135
|
+
let(:charset_option1) { " CHARACTER SET 'latin5'" }
|
136
|
+
let(:expected_alter_table_charset_events) { [ at_column_charset_event_table1, at_charset_event_table1 ] }
|
137
|
+
|
138
|
+
context "with a table whose version is 0" do
|
139
|
+
let(:current_version) { nil } # generate_ddl file doesn't exist in case of version "0"
|
140
|
+
before do
|
141
|
+
expect(QueryEvent).to receive(:new).with(event_type, database_name, event_table_name, next_position, 0, column_query1, Integer).and_return(at_column_charset_event_table1)
|
142
|
+
expect(QueryEvent).to receive(:new).with(event_type, database_name, event_table_name, next_position, 0, query, Integer).and_return(at_charset_event_table1)
|
143
|
+
end
|
144
|
+
it_behaves_like "create and send charset records and increment version"
|
145
|
+
end
|
146
|
+
end
|
147
|
+
end
|
148
|
+
end
|
149
|
+
end
|
150
|
+
|
151
|
+
describe QueryEvent do
|
152
|
+
let(:subject_object) { described_class.new(event_type, database_name, event_table_name, next_position, event_length, query, timestamp) }
|
153
|
+
|
154
|
+
let(:event_type) { 'Query' }
|
155
|
+
let(:database_name) { 'mydb' }
|
156
|
+
let(:event_table_name) { 'mytable' }
|
157
|
+
let(:next_position) { 38293 }
|
158
|
+
let(:event_length) { 2093 }
|
159
|
+
let(:query) { 'ALTER TABLE CHARACTER SET sjis' }
|
160
|
+
let(:timestamp) { Time.now.to_i }
|
161
|
+
|
162
|
+
describe "#event_type" do
|
163
|
+
subject { subject_object.event_type }
|
164
|
+
it { is_expected.to eq event_type }
|
165
|
+
end
|
166
|
+
describe "#database_name" do
|
167
|
+
subject { subject_object.database_name }
|
168
|
+
it { is_expected.to eq database_name}
|
169
|
+
end
|
170
|
+
describe "#event_table_name" do
|
171
|
+
subject { subject_object.event_table_name }
|
172
|
+
it { is_expected.to eq event_table_name}
|
173
|
+
end
|
174
|
+
describe "#next_position" do
|
175
|
+
subject { subject_object.next_position }
|
176
|
+
it { is_expected.to eq next_position}
|
177
|
+
end
|
178
|
+
describe "#event_length" do
|
179
|
+
subject { subject_object.event_length }
|
180
|
+
it { is_expected.to eq event_length}
|
181
|
+
end
|
182
|
+
describe "#query" do
|
183
|
+
subject { subject_object.query }
|
184
|
+
it { is_expected.to eq query}
|
185
|
+
end
|
186
|
+
describe "#timestamp" do
|
187
|
+
subject { subject_object.timestamp }
|
188
|
+
it { is_expected.to eq timestamp}
|
189
|
+
end
|
190
|
+
end
|
191
|
+
|
192
|
+
end
|
193
|
+
end
|