flydata 0.3.16 → 0.3.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/VERSION +1 -1
- data/flydata-core/lib/flydata-core/record/record.rb +13 -0
- data/flydata-core/lib/flydata-core/table_def/mysql_table_def.rb +107 -5
- data/flydata-core/lib/flydata-core/table_def/redshift_table_def.rb +62 -11
- data/flydata-core/spec/table_def/mysql_table_def_spec.rb +37 -1
- data/flydata-core/spec/table_def/mysql_to_redshift_table_def_spec.rb +120 -0
- data/flydata-core/spec/table_def/mysqldump_test_column_charset.dump +45 -0
- data/flydata-core/spec/table_def/redshift_table_def_spec.rb +70 -88
- data/flydata.gemspec +13 -8
- data/lib/flydata/command/setup.rb +4 -4
- data/lib/flydata/command/sync.rb +18 -29
- data/lib/flydata/compatibility_check.rb +2 -2
- data/lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb +15 -10
- data/lib/flydata/fluent-plugins/mysql/binlog_record_handler.rb +6 -3
- data/lib/flydata/fluent-plugins/mysql/dml_record_handler.rb +15 -8
- data/lib/flydata/fluent-plugins/mysql/table_meta.rb +6 -34
- data/lib/flydata/{fluent-plugins/mysql → mysql}/binlog_position.rb +2 -0
- data/lib/flydata/{util → mysql}/mysql_util.rb +34 -1
- data/lib/flydata/mysql/table_ddl.rb +118 -0
- data/lib/flydata/parser/mysql/dump_parser.rb +5 -5
- data/lib/flydata/parser/mysql/mysql_alter_table.treetop +29 -5
- data/lib/flydata/sync_file_manager.rb +15 -2
- data/spec/flydata/command/sync_spec.rb +22 -1
- data/spec/flydata/fluent-plugins/in_mysql_binlog_flydata_spec.rb +7 -2
- data/spec/flydata/fluent-plugins/mysql/dml_record_handler_spec.rb +130 -0
- data/spec/flydata/fluent-plugins/mysql/shared_query_handler_context.rb +1 -0
- data/spec/flydata/fluent-plugins/mysql/table_meta_spec.rb +14 -8
- data/spec/flydata/fluent-plugins/mysql/truncate_query_handler_spec.rb +2 -1
- data/spec/flydata/{fluent-plugins/mysql → mysql}/binlog_position_spec.rb +3 -2
- data/spec/flydata/{util → mysql}/mysql_util_spec.rb +2 -2
- data/spec/flydata/mysql/table_ddl_spec.rb +193 -0
- data/spec/flydata/parser/mysql/alter_table_parser_spec.rb +37 -9
- data/spec/flydata/sync_file_manager_spec.rb +102 -27
- metadata +12 -7
@@ -215,7 +215,9 @@ grammar MysqlAlterTable
|
|
215
215
|
}
|
216
216
|
/ convert_sym sp to_sym sp charset sp charset_name_or_default opt_collate {
|
217
217
|
def action
|
218
|
-
{ action: :convert_charset
|
218
|
+
{ action: :convert_charset,
|
219
|
+
cs: charset_name_or_default.charset
|
220
|
+
}
|
219
221
|
end
|
220
222
|
}
|
221
223
|
/ ( create_table_options_space_separated ) 1..1 {
|
@@ -795,6 +797,8 @@ grammar MysqlAlterTable
|
|
795
797
|
/ key_opt { def option; key_opt_option; end }
|
796
798
|
/ comment_opt { def option; comment_opt_option; end }
|
797
799
|
/ column_format_opt { def option; column_format_opt_option; end }
|
800
|
+
/ character_set_opt { def option; character_set_opt_option; end }
|
801
|
+
/ collate_opt { def option; collate_opt_option; end }
|
798
802
|
/ storage_opt { def option; storage_opt_option; end }
|
799
803
|
#TODO: / reference_definition
|
800
804
|
end
|
@@ -889,6 +893,20 @@ grammar MysqlAlterTable
|
|
889
893
|
end
|
890
894
|
}
|
891
895
|
end
|
896
|
+
rule character_set_opt
|
897
|
+
'character set'i sp value {
|
898
|
+
def character_set_opt_option
|
899
|
+
{ cs: FlydataCore::TableDef::MysqlTableDef.flydata_charset(value.raw_value) }
|
900
|
+
end
|
901
|
+
}
|
902
|
+
end
|
903
|
+
rule collate_opt
|
904
|
+
'collate'i sp value {
|
905
|
+
def collate_opt_option
|
906
|
+
{} # Not supported
|
907
|
+
end
|
908
|
+
}
|
909
|
+
end
|
892
910
|
rule storage_opt
|
893
911
|
'storage'i sp ( 'disk'i / 'memory'i / 'default'i ) {
|
894
912
|
def storage_opt_option
|
@@ -1012,8 +1030,8 @@ grammar MysqlAlterTable
|
|
1012
1030
|
end
|
1013
1031
|
|
1014
1032
|
rule charset_name_or_default
|
1015
|
-
default_sym
|
1016
|
-
/ charset_name
|
1033
|
+
default_sym { def charset; 'DEFAULT'; end }
|
1034
|
+
/ charset_name { def charset; FlydataCore::TableDef::MysqlTableDef.flydata_charset(text_value); end }
|
1017
1035
|
end
|
1018
1036
|
|
1019
1037
|
rule collation_name
|
@@ -1047,7 +1065,9 @@ grammar MysqlAlterTable
|
|
1047
1065
|
rule create_table_option
|
1048
1066
|
default_charset {
|
1049
1067
|
def action
|
1050
|
-
{ action: :default_charset
|
1068
|
+
{ action: :default_charset,
|
1069
|
+
cs: charset
|
1070
|
+
}
|
1051
1071
|
end
|
1052
1072
|
}
|
1053
1073
|
/ default_collation {
|
@@ -1073,7 +1093,11 @@ grammar MysqlAlterTable
|
|
1073
1093
|
end
|
1074
1094
|
|
1075
1095
|
rule default_charset
|
1076
|
-
opt_default nsp charset opt_equal nsp charset_name_or_default
|
1096
|
+
opt_default nsp charset opt_equal nsp charset_name_or_default {
|
1097
|
+
def charset
|
1098
|
+
charset_name_or_default.charset
|
1099
|
+
end
|
1100
|
+
}
|
1077
1101
|
end
|
1078
1102
|
|
1079
1103
|
rule opt_default
|
@@ -41,14 +41,27 @@ module Flydata
|
|
41
41
|
state: items[5], substate: items[6], mysql_table: mysql_table}
|
42
42
|
end
|
43
43
|
|
44
|
-
def
|
44
|
+
def load_generated_ddl(tables)
|
45
|
+
tables = [ tables ] unless tables.kind_of?(Array)
|
46
|
+
paths = table_ddl_file_paths(*tables)
|
47
|
+
paths.collect{|path|
|
48
|
+
begin
|
49
|
+
File.open(path) {|f| f.read }
|
50
|
+
rescue Errno::ENOENT
|
51
|
+
nil
|
52
|
+
end
|
53
|
+
}
|
54
|
+
end
|
55
|
+
|
56
|
+
def save_generated_ddl(tables, contents = "1")
|
57
|
+
tables = [ tables ] unless tables.kind_of?(Array)
|
45
58
|
table_positions_dir_path = ENV['FLYDATA_TABLE_POSITIONS'] || File.join(FLYDATA_HOME, 'positions')
|
46
59
|
#Create positions if dir does not exist
|
47
60
|
unless File.directory?(table_positions_dir_path)
|
48
61
|
FileUtils.mkdir_p(table_positions_dir_path)
|
49
62
|
end
|
50
63
|
tables.each do |tab|
|
51
|
-
File.open(File.join(table_positions_dir_path, "#{tab}.generated_ddl"), 'w') {|f| f.write(
|
64
|
+
File.open(File.join(table_positions_dir_path, "#{tab}.generated_ddl"), 'w') {|f| f.write(contents) }
|
52
65
|
end
|
53
66
|
end
|
54
67
|
|
@@ -5,6 +5,9 @@ require 'flydata/command/sync'
|
|
5
5
|
module Flydata
|
6
6
|
module Command
|
7
7
|
describe Sync do
|
8
|
+
let(:subject_object){ described_class.new }
|
9
|
+
subject { subject_object }
|
10
|
+
|
8
11
|
let(:default_mysqldump_dir) do
|
9
12
|
File.join('/tmp', "sync_dump_#{Time.now.to_i}")
|
10
13
|
end
|
@@ -35,6 +38,14 @@ module Flydata
|
|
35
38
|
"data_servers"=>"localhost:9905" }
|
36
39
|
}
|
37
40
|
end
|
41
|
+
let(:mysql_table_columns) {
|
42
|
+
{"id"=>{:column_name=>"id", :format_type_str=>"int(11)", :format_type=>"int", :format_size=>11}, "name"=>{:column_name=>"name", :format_type_str=>"varchar(40)", :format_type=>"varchar", :format_size=>40, :default=>nil}, "created_at"=>{:column_name=>"created_at", :format_type_str=>"timestamp", :format_type=>"timestamp", :default=>"CURRENT_TIMESTAMP"}, "bin"=>{:column_name=>"bin", :format_type_str=>"binary(34)", :format_type=>"binary", :format_size=>34, :default=>nil}, "bin2"=>{:column_name=>"bin2", :format_type_str=>"blob", :format_type=>"blob"}, "varbin"=>{:column_name=>"varbin", :format_type_str=>"varchar(34)", :format_type=>"varchar", :format_size=>34, :default=>nil}}
|
43
|
+
}
|
44
|
+
let(:mysql_table) do
|
45
|
+
mt = double('mysql_table')
|
46
|
+
allow(mt).to receive(:columns).and_return(mysql_table_columns)
|
47
|
+
mt
|
48
|
+
end
|
38
49
|
|
39
50
|
after :each do
|
40
51
|
if Dir.exists?(default_mysqldump_dir)
|
@@ -45,7 +56,6 @@ module Flydata
|
|
45
56
|
end
|
46
57
|
end
|
47
58
|
|
48
|
-
subject { Sync.new }
|
49
59
|
describe '#cleanup_sync_server' do
|
50
60
|
let(:rest_client) { double('rest_client') }
|
51
61
|
|
@@ -232,6 +242,17 @@ module Flydata
|
|
232
242
|
include_examples 'throws an error'
|
233
243
|
end
|
234
244
|
end
|
245
|
+
describe '#convert_to_flydata_values' do
|
246
|
+
subject { subject_object.send(:convert_to_flydata_values, mysql_table, values) }
|
247
|
+
let(:values) { [4, 'John', nil, '0xC0448200', nil, nil] }
|
248
|
+
|
249
|
+
it 'calls convert_to_valydata_value for each value' do
|
250
|
+
mysql_table.columns.values.each_with_index do |col_info, i|
|
251
|
+
expect(FlydataCore::TableDef::MysqlTableDef).to receive(:convert_to_flydata_value).with(values[i], col_info[:format_type]).and_return(values[i])
|
252
|
+
end
|
253
|
+
is_expected.to eq values
|
254
|
+
end
|
255
|
+
end
|
235
256
|
end
|
236
257
|
end
|
237
258
|
end
|
@@ -167,7 +167,8 @@ EOT
|
|
167
167
|
record = row.kind_of?(Hash) && row.keys.include?(:row) ? row : { row: row }
|
168
168
|
@seq += 1
|
169
169
|
record.merge({ :type=>type, :table_name=>table, :respect_order=>true,
|
170
|
-
:seq=>@seq, :src_pos=>"#{binlog_file}\t#{position}", :table_rev=>1
|
170
|
+
:seq=>@seq, :src_pos=>"#{binlog_file}\t#{position}", :table_rev=>1,
|
171
|
+
:v=>2 })
|
171
172
|
end
|
172
173
|
expect_emitted_records(event, records)
|
173
174
|
end
|
@@ -193,6 +194,7 @@ EOT
|
|
193
194
|
let(:rotate_event_corrupt){ create_event(TEST_EVENT_ROTATE_MISSING_BINLOG_FILE) }
|
194
195
|
|
195
196
|
let(:now) { Time.now }
|
197
|
+
let(:flydata_record_version) { 2 }
|
196
198
|
|
197
199
|
let(:binlog_position_file) do
|
198
200
|
f = double('binlog_position_file')
|
@@ -226,7 +228,7 @@ EOT
|
|
226
228
|
setup_initial_flydata_files
|
227
229
|
allow(MysqlBinlogInput::BinlogUtil).to receive(:to_hash) {|e| e}
|
228
230
|
allow(Mysql::BinLogPositionFile).to receive(:new).with(TEST_POSITION_FILE).and_return(binlog_position_file)
|
229
|
-
allow(Mysql::TableMeta).to receive(:update)
|
231
|
+
allow(Flydata::Mysql::TableMeta).to receive(:update)
|
230
232
|
allow(plugin).to receive(:`).with("mysql -V").and_return("mysql Ver 14.14 Distrib 5.5.40")
|
231
233
|
allow(plugin).to receive(:`).with(/^echo 'select version\(\);'/).and_return("version()\n5.5.40-0ubuntu0.12.04.1-log")
|
232
234
|
Timecop.freeze(now)
|
@@ -326,6 +328,7 @@ EOT
|
|
326
328
|
src_pos: "mysql-bin.000048\t689",
|
327
329
|
table_rev: 2, # increment revision
|
328
330
|
seq: 2,
|
331
|
+
v: flydata_record_version,
|
329
332
|
actions: [{
|
330
333
|
action: :add_column, column: "sum", :type=>'int4(11)', :query=>'add column sum integer'}],
|
331
334
|
})
|
@@ -341,6 +344,7 @@ EOT
|
|
341
344
|
src_pos: "mysql-bin.000048\t800",
|
342
345
|
table_rev: 2, # increment revision
|
343
346
|
seq: 2,
|
347
|
+
v: flydata_record_version,
|
344
348
|
actions: [{
|
345
349
|
action: :drop_column, column: "sum", :query=>'drop column sum'}],
|
346
350
|
})
|
@@ -357,6 +361,7 @@ EOT
|
|
357
361
|
src_pos: "mysql-bin.000048\t#{337 - 217}",
|
358
362
|
table_rev: 1,
|
359
363
|
seq: 2,
|
364
|
+
v: flydata_record_version,
|
360
365
|
actions: [{
|
361
366
|
action: :add_index,
|
362
367
|
support_level: :nonbreaking,
|
@@ -0,0 +1,130 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
require 'fluent_plugins_spec_helper'
|
4
|
+
require 'flydata/fluent-plugins/mysql/dml_record_handler'
|
5
|
+
|
6
|
+
module Mysql
|
7
|
+
|
8
|
+
describe DmlRecordHandler do
|
9
|
+
let(:subject_object) { described_class.new(context) }
|
10
|
+
|
11
|
+
let(:table_meta) { double('table_meta') }
|
12
|
+
let(:tables) { ['items', 'orders'] }
|
13
|
+
let(:sync_fm) {
|
14
|
+
sfm = double('sync_fm')
|
15
|
+
allow(sfm).to receive(:get_table_binlog_pos).and_return nil
|
16
|
+
sfm
|
17
|
+
}
|
18
|
+
let(:context) {
|
19
|
+
c = double('context')
|
20
|
+
allow(c).to receive(:table_meta).and_return(table_meta)
|
21
|
+
allow(c).to receive(:tables).and_return(tables)
|
22
|
+
allow(c).to receive(:sync_fm).and_return(sync_fm)
|
23
|
+
c
|
24
|
+
}
|
25
|
+
|
26
|
+
describe '#encode_row_value' do
|
27
|
+
subject { subject_object.send(:encode_row_value, value) }
|
28
|
+
|
29
|
+
let(:normal_utf8_string) { "ABC新年Ã" }
|
30
|
+
let(:valid_utf8_string) { "\x00abc" } # Yes, this is a valid UTF-8 string
|
31
|
+
let(:invalid_utf8_string) { "新年".encode('shift_jis') }
|
32
|
+
let(:latin1_string_happened_to_be_a_valid_utf8_string) { "ã".encode("iso-8859-1") }
|
33
|
+
|
34
|
+
shared_examples 'encoding values differently based on their contents not based on encoding' do
|
35
|
+
context 'with a normal utf-8 string' do
|
36
|
+
let(:value) { normal_utf8_string }
|
37
|
+
it 'returns a utf8 string' do
|
38
|
+
result = subject
|
39
|
+
expect(result).to eq normal_utf8_string
|
40
|
+
expect(result.encoding).to eq Encoding.find('utf-8')
|
41
|
+
end
|
42
|
+
end
|
43
|
+
context 'with a valid utf-8 string' do
|
44
|
+
let(:value) { valid_utf8_string }
|
45
|
+
it 'returns a utf8 string' do
|
46
|
+
result = subject
|
47
|
+
expect(result).to eq valid_utf8_string
|
48
|
+
expect(result.encoding).to eq Encoding.find('utf-8')
|
49
|
+
end
|
50
|
+
end
|
51
|
+
context 'with an invalid utf-8 string' do
|
52
|
+
let(:value) { invalid_utf8_string }
|
53
|
+
it 'returns a FlyData binary format string as BINARY encoding' do
|
54
|
+
result = subject
|
55
|
+
expect(result).to eq "0x9056944E"
|
56
|
+
expect(result.encoding).to eq Encoding::BINARY
|
57
|
+
end
|
58
|
+
end
|
59
|
+
context 'with a latin1 string which happens to be a valid utf8 character' do
|
60
|
+
let(:value) { latin1_string_happened_to_be_a_valid_utf8_string }
|
61
|
+
it 'returns a utf8 string' do
|
62
|
+
result = subject
|
63
|
+
expect(result).to eq latin1_string_happened_to_be_a_valid_utf8_string
|
64
|
+
expect(result).to eq "ã" # #C3A3
|
65
|
+
expect(result.encoding).to eq Encoding.find('utf-8')
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
context 'with strings whose encoding is utf-8' do
|
71
|
+
before do
|
72
|
+
normal_utf8_string.force_encoding 'utf-8'
|
73
|
+
valid_utf8_string.force_encoding 'utf-8'
|
74
|
+
invalid_utf8_string.force_encoding 'utf-8'
|
75
|
+
latin1_string_happened_to_be_a_valid_utf8_string.force_encoding 'utf-8'
|
76
|
+
end
|
77
|
+
it_behaves_like 'encoding values differently based on their contents not based on encoding'
|
78
|
+
end
|
79
|
+
|
80
|
+
context 'with strings whose encoding is binary' do
|
81
|
+
before do
|
82
|
+
normal_utf8_string.force_encoding 'binary'
|
83
|
+
valid_utf8_string.force_encoding 'binary'
|
84
|
+
invalid_utf8_string.force_encoding 'binary'
|
85
|
+
latin1_string_happened_to_be_a_valid_utf8_string.force_encoding 'binary'
|
86
|
+
end
|
87
|
+
it_behaves_like 'encoding values differently based on their contents not based on encoding'
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
describe '#convert_to_flydata_row_format' do
|
92
|
+
subject { subject_object.send(:convert_to_flydata_row_format, row) }
|
93
|
+
|
94
|
+
let(:row) { [ value ] }
|
95
|
+
before do
|
96
|
+
end
|
97
|
+
|
98
|
+
shared_examples "returning a json hash with no attrs" do
|
99
|
+
it do
|
100
|
+
is_expected.to eq({"1" => value })
|
101
|
+
end
|
102
|
+
end
|
103
|
+
|
104
|
+
context 'with a row containing a string value' do
|
105
|
+
let(:value) { "a normal string" }
|
106
|
+
|
107
|
+
before do
|
108
|
+
expect(subject_object).to receive(:encode_row_value).with(value).and_return(value)
|
109
|
+
end
|
110
|
+
|
111
|
+
it_behaves_like "returning a json hash with no attrs"
|
112
|
+
end
|
113
|
+
context 'with a row containing a binary value' do
|
114
|
+
let(:value) { "\x00abc".force_encoding('binary') }
|
115
|
+
before do
|
116
|
+
expect(subject_object).to receive(:encode_row_value).with(value).and_return(value)
|
117
|
+
end
|
118
|
+
it "returns a json hash with attrs" do
|
119
|
+
is_expected.to eq({"1" => value, "attrs" => {"1" => {"enc" => "b" }}})
|
120
|
+
end
|
121
|
+
end
|
122
|
+
context 'with a row containing an integer value' do
|
123
|
+
let(:value) { 4 }
|
124
|
+
|
125
|
+
it_behaves_like "returning a json hash with no attrs"
|
126
|
+
end
|
127
|
+
end
|
128
|
+
end
|
129
|
+
|
130
|
+
end
|
@@ -1,6 +1,7 @@
|
|
1
1
|
require 'fluent_plugins_spec_helper'
|
2
2
|
require 'flydata/fluent-plugins/mysql/table_meta'
|
3
3
|
|
4
|
+
module Flydata
|
4
5
|
module Mysql
|
5
6
|
describe TableMeta do
|
6
7
|
let(:db_opts) { { host: 'test-host', port: 3306, username: 'test-user', password: 'test-pswd', database: 'test-db' } }
|
@@ -9,6 +10,9 @@ module Mysql
|
|
9
10
|
|
10
11
|
let(:conn) { double('conn') }
|
11
12
|
let(:table_meta) { TableMeta.new(db_opts.merge(database: database, tables: tables)) }
|
13
|
+
let(:utf8) { 'utf8' }
|
14
|
+
let(:latin1) { 'latin1' }
|
15
|
+
let(:cp932) { 'cp932' }
|
12
16
|
|
13
17
|
before do
|
14
18
|
allow(conn).to receive(:close)
|
@@ -24,23 +28,26 @@ module Mysql
|
|
24
28
|
context 'when character_set_name is supported' do
|
25
29
|
context 'when encoding is not necessary' do
|
26
30
|
let(:query_result) {[
|
27
|
-
{'table_name' => 'a_table', 'character_set_name' =>
|
31
|
+
{'table_name' => 'a_table', 'character_set_name' => utf8 }
|
28
32
|
]}
|
29
33
|
it 'set nil for encoding' do
|
30
34
|
table_meta.update
|
31
|
-
expect(table_meta['a_table'][:encoding]).to
|
35
|
+
expect(table_meta['a_table'][:encoding]).to eq Encoding::UTF_8
|
36
|
+
expect(table_meta['a_table'][:mysql_charset]).to eq utf8
|
32
37
|
end
|
33
38
|
end
|
34
39
|
|
35
40
|
context 'when encoding is needed' do
|
36
41
|
let(:query_result) {[
|
37
|
-
{'table_name' => 'a_table', 'character_set_name' =>
|
38
|
-
{'table_name' => 'b_table', 'character_set_name' =>
|
42
|
+
{'table_name' => 'a_table', 'character_set_name' => latin1 },
|
43
|
+
{'table_name' => 'b_table', 'character_set_name' => cp932 }
|
39
44
|
]}
|
40
45
|
it 'set ruby encoding encoding' do
|
41
46
|
table_meta.update
|
42
|
-
expect(table_meta['a_table'][:encoding]).to eq
|
43
|
-
expect(table_meta['
|
47
|
+
expect(table_meta['a_table'][:encoding]).to eq Encoding::ISO_8859_1
|
48
|
+
expect(table_meta['a_table'][:mysql_charset]).to eq latin1
|
49
|
+
expect(table_meta['b_table'][:encoding]).to eq Encoding::CP932
|
50
|
+
expect(table_meta['b_table'][:mysql_charset]).to eq cp932
|
44
51
|
end
|
45
52
|
end
|
46
53
|
end
|
@@ -56,5 +63,4 @@ module Mysql
|
|
56
63
|
end
|
57
64
|
end
|
58
65
|
end
|
59
|
-
|
60
|
-
|
66
|
+
end
|
@@ -1,6 +1,6 @@
|
|
1
1
|
require 'fluent_plugins_spec_helper'
|
2
|
-
require 'mysql/binlog_position'
|
3
|
-
|
2
|
+
require 'flydata/mysql/binlog_position'
|
3
|
+
module Flydata
|
4
4
|
module Mysql
|
5
5
|
describe BinLogPosition do
|
6
6
|
let(:pos1) { BinLogPosition.new('mysql-bin.000064 107') }
|
@@ -32,3 +32,4 @@ module Mysql
|
|
32
32
|
end
|
33
33
|
end
|
34
34
|
end
|
35
|
+
end
|
@@ -0,0 +1,193 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'flydata/mysql/table_ddl'
|
3
|
+
|
4
|
+
module Flydata
|
5
|
+
module Mysql
|
6
|
+
|
7
|
+
describe TableDdl do
|
8
|
+
let(:table_name1) { "items" }
|
9
|
+
let(:table_name2) { "orders" }
|
10
|
+
let(:table_names) { [table_name1, table_name2] }
|
11
|
+
let(:table1_charset) { 'utf8' }
|
12
|
+
let(:table2_charset) { 'sjis' }
|
13
|
+
let(:at_charset_event_table1) { double('at_charset_event_table1') }
|
14
|
+
let(:at_charset_event_table2) { double('at_charset_event_table2') }
|
15
|
+
let(:at_column_charset_event_table1) { double('at_column_charset_event_table1') }
|
16
|
+
let(:expected_alter_table_charset_events) { [ at_charset_event_table1, at_charset_event_table2] }
|
17
|
+
let(:at_charset_query) { "ALTER TABLE `#{database_name}`.`#{at_table}` CHARACTER SET = #{charset};" }
|
18
|
+
let(:query) { "ALTER TABLE `#{database_name}`.`#{event_table_name}` CHARACTER SET = #{charset};" }
|
19
|
+
let(:column_query1) { "ALTER TABLE `#{database_name}`.`#{event_table_name}` CHANGE COLUMN `#{table1_varchar_column}` #{table1_varchar_column_def};" }
|
20
|
+
let(:charset) { 'utf8' }
|
21
|
+
let(:event_type) { 'Query' }
|
22
|
+
let(:database_name) { "a_database" }
|
23
|
+
let(:sync_fm) { double('sync_fm') }
|
24
|
+
let(:master_binlog_position) { "#{binlog_file}\t#{binlog_position}" }
|
25
|
+
let(:binlog_file) { "mysql-binlog.00123" }
|
26
|
+
let(:binlog_position) { 28393 }
|
27
|
+
let(:original_current_binlog_file) { double('original_current_binlog_file') }
|
28
|
+
let(:mysql_opts) { { host: double('host'), port: double('port'),
|
29
|
+
username: double('username'),
|
30
|
+
password: double('password'),
|
31
|
+
database: database_name,
|
32
|
+
} }
|
33
|
+
let(:position_file) { double('position_file') }
|
34
|
+
let(:context) { double('context') }
|
35
|
+
let(:mysql_tabledef) { { table_name1 => mysql_tabledef1,
|
36
|
+
table_name2 => mysql_tabledef2,
|
37
|
+
} }
|
38
|
+
let(:mysql_tabledef1) { double('mysql_tabledef1') }
|
39
|
+
let(:mysql_tabledef2) { double('mysql_tabledef2') }
|
40
|
+
let(:column_def1) { { "id" => "`id` integer NOT NULL",
|
41
|
+
table1_varchar_column => table1_varchar_column_def,
|
42
|
+
} }
|
43
|
+
let(:table1_varchar_column) { 'name' }
|
44
|
+
let(:table1_varchar_column_def) { "`#{table1_varchar_column}` varchar(45)#{charset_option1} NULL" }
|
45
|
+
let(:column_def2) { { "id" => "`id` integer NOT NULL",
|
46
|
+
"address" => "`address` varchar(45) NULL"
|
47
|
+
} }
|
48
|
+
let(:charset_option1) { "" }
|
49
|
+
let(:save) { {} }
|
50
|
+
|
51
|
+
before do
|
52
|
+
save[:log] = $log
|
53
|
+
$log = double('$log')
|
54
|
+
allow($log).to receive(:info)
|
55
|
+
allow($log).to receive(:debug)
|
56
|
+
allow($log).to receive(:error)
|
57
|
+
allow($log).to receive(:trace)
|
58
|
+
|
59
|
+
allow(mysql_tabledef1).to receive(:default_charset_mysql).and_return(table1_charset)
|
60
|
+
allow(mysql_tabledef1).to receive(:table_name).and_return(table_name1)
|
61
|
+
allow(mysql_tabledef1).to receive(:column_def).and_return(column_def1)
|
62
|
+
allow(mysql_tabledef2).to receive(:default_charset_mysql).and_return(table2_charset)
|
63
|
+
allow(mysql_tabledef2).to receive(:table_name).and_return(table_name2)
|
64
|
+
allow(mysql_tabledef2).to receive(:column_def).and_return(column_def2)
|
65
|
+
end
|
66
|
+
|
67
|
+
after do
|
68
|
+
$log = save[:log]
|
69
|
+
end
|
70
|
+
|
71
|
+
describe '.migrate_to_v2' do
|
72
|
+
let(:subject_object) { described_class }
|
73
|
+
let(:subject_params) { [:migrate_to_v2, table_names, mysql_opts, sync_fm, position_file, context] }
|
74
|
+
|
75
|
+
let(:target_version) { "2" }
|
76
|
+
|
77
|
+
shared_examples "create and send charset records and increment version" do
|
78
|
+
let(:next_position) { binlog_position }
|
79
|
+
|
80
|
+
it do
|
81
|
+
matcher = receive(:each_mysql_tabledef).with(table_names, mysql_opts)
|
82
|
+
matcher = table_names.inject(matcher) {|m, tn| m.and_yield(mysql_tabledef[tn], nil) }
|
83
|
+
expect(MysqlUtil).to matcher
|
84
|
+
expect(File).to receive(:open).with(position_file).and_return(master_binlog_position)
|
85
|
+
expect(context).to receive(:current_binlog_file).and_return(original_current_binlog_file)
|
86
|
+
expect(context).to receive(:current_binlog_file=).with(binlog_file)
|
87
|
+
expect(context).to receive(:current_binlog_file=).with(original_current_binlog_file)
|
88
|
+
expect(sync_fm).to receive(:save_generated_ddl).with(table_names, target_version)
|
89
|
+
|
90
|
+
expect{|b| subject_object.send(*subject_params, &b) }.to yield_successive_args(*expected_alter_table_charset_events)
|
91
|
+
end
|
92
|
+
end
|
93
|
+
shared_examples "does nothing" do
|
94
|
+
it do
|
95
|
+
expect(sync_fm).not_to receive(:save_generated_ddl)
|
96
|
+
|
97
|
+
expect{|b| subject_object.send(*subject_params, &b) }.not_to yield_control
|
98
|
+
end
|
99
|
+
end
|
100
|
+
|
101
|
+
let(:table_versions) { table_names.size.times.collect{ current_version } }
|
102
|
+
before do
|
103
|
+
expect(sync_fm).to receive(:load_generated_ddl).with(table_names).and_return table_versions
|
104
|
+
end
|
105
|
+
|
106
|
+
context "with a single table" do
|
107
|
+
let(:table_names) { [table_name1] }
|
108
|
+
let(:event_table_name) { table_name1 }
|
109
|
+
let(:charset) { table1_charset }
|
110
|
+
let(:expected_alter_table_charset_events) { [ at_charset_event_table1 ] }
|
111
|
+
|
112
|
+
context "with a table whose version is 0" do
|
113
|
+
let(:current_version) { nil } # generate_ddl file doesn't exist in case of version "0"
|
114
|
+
before do
|
115
|
+
expect(QueryEvent).to receive(:new).with(event_type, database_name, event_table_name, next_position, 0, query, Integer).and_return(at_charset_event_table1)
|
116
|
+
end
|
117
|
+
it_behaves_like "create and send charset records and increment version"
|
118
|
+
end
|
119
|
+
context "with a table whose version is 1" do
|
120
|
+
let(:current_version) { "1" }
|
121
|
+
before do
|
122
|
+
expect(QueryEvent).to receive(:new).with(event_type, database_name, event_table_name, next_position, 0, query, Integer).and_return(at_charset_event_table1)
|
123
|
+
end
|
124
|
+
it_behaves_like "create and send charset records and increment version"
|
125
|
+
end
|
126
|
+
context "with a table whose version is 2" do
|
127
|
+
let(:current_version) { "2" }
|
128
|
+
it_behaves_like "does nothing"
|
129
|
+
end
|
130
|
+
context "with a table whose version is 3" do
|
131
|
+
let(:current_version) { "3" }
|
132
|
+
it_behaves_like "does nothing"
|
133
|
+
end
|
134
|
+
context 'with a single table who has a column with charset' do
|
135
|
+
let(:charset_option1) { " CHARACTER SET 'latin5'" }
|
136
|
+
let(:expected_alter_table_charset_events) { [ at_column_charset_event_table1, at_charset_event_table1 ] }
|
137
|
+
|
138
|
+
context "with a table whose version is 0" do
|
139
|
+
let(:current_version) { nil } # generate_ddl file doesn't exist in case of version "0"
|
140
|
+
before do
|
141
|
+
expect(QueryEvent).to receive(:new).with(event_type, database_name, event_table_name, next_position, 0, column_query1, Integer).and_return(at_column_charset_event_table1)
|
142
|
+
expect(QueryEvent).to receive(:new).with(event_type, database_name, event_table_name, next_position, 0, query, Integer).and_return(at_charset_event_table1)
|
143
|
+
end
|
144
|
+
it_behaves_like "create and send charset records and increment version"
|
145
|
+
end
|
146
|
+
end
|
147
|
+
end
|
148
|
+
end
|
149
|
+
end
|
150
|
+
|
151
|
+
describe QueryEvent do
|
152
|
+
let(:subject_object) { described_class.new(event_type, database_name, event_table_name, next_position, event_length, query, timestamp) }
|
153
|
+
|
154
|
+
let(:event_type) { 'Query' }
|
155
|
+
let(:database_name) { 'mydb' }
|
156
|
+
let(:event_table_name) { 'mytable' }
|
157
|
+
let(:next_position) { 38293 }
|
158
|
+
let(:event_length) { 2093 }
|
159
|
+
let(:query) { 'ALTER TABLE CHARACTER SET sjis' }
|
160
|
+
let(:timestamp) { Time.now.to_i }
|
161
|
+
|
162
|
+
describe "#event_type" do
|
163
|
+
subject { subject_object.event_type }
|
164
|
+
it { is_expected.to eq event_type }
|
165
|
+
end
|
166
|
+
describe "#database_name" do
|
167
|
+
subject { subject_object.database_name }
|
168
|
+
it { is_expected.to eq database_name}
|
169
|
+
end
|
170
|
+
describe "#event_table_name" do
|
171
|
+
subject { subject_object.event_table_name }
|
172
|
+
it { is_expected.to eq event_table_name}
|
173
|
+
end
|
174
|
+
describe "#next_position" do
|
175
|
+
subject { subject_object.next_position }
|
176
|
+
it { is_expected.to eq next_position}
|
177
|
+
end
|
178
|
+
describe "#event_length" do
|
179
|
+
subject { subject_object.event_length }
|
180
|
+
it { is_expected.to eq event_length}
|
181
|
+
end
|
182
|
+
describe "#query" do
|
183
|
+
subject { subject_object.query }
|
184
|
+
it { is_expected.to eq query}
|
185
|
+
end
|
186
|
+
describe "#timestamp" do
|
187
|
+
subject { subject_object.timestamp }
|
188
|
+
it { is_expected.to eq timestamp}
|
189
|
+
end
|
190
|
+
end
|
191
|
+
|
192
|
+
end
|
193
|
+
end
|