flydata 0.3.16 → 0.3.17

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. checksums.yaml +4 -4
  2. data/VERSION +1 -1
  3. data/flydata-core/lib/flydata-core/record/record.rb +13 -0
  4. data/flydata-core/lib/flydata-core/table_def/mysql_table_def.rb +107 -5
  5. data/flydata-core/lib/flydata-core/table_def/redshift_table_def.rb +62 -11
  6. data/flydata-core/spec/table_def/mysql_table_def_spec.rb +37 -1
  7. data/flydata-core/spec/table_def/mysql_to_redshift_table_def_spec.rb +120 -0
  8. data/flydata-core/spec/table_def/mysqldump_test_column_charset.dump +45 -0
  9. data/flydata-core/spec/table_def/redshift_table_def_spec.rb +70 -88
  10. data/flydata.gemspec +13 -8
  11. data/lib/flydata/command/setup.rb +4 -4
  12. data/lib/flydata/command/sync.rb +18 -29
  13. data/lib/flydata/compatibility_check.rb +2 -2
  14. data/lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb +15 -10
  15. data/lib/flydata/fluent-plugins/mysql/binlog_record_handler.rb +6 -3
  16. data/lib/flydata/fluent-plugins/mysql/dml_record_handler.rb +15 -8
  17. data/lib/flydata/fluent-plugins/mysql/table_meta.rb +6 -34
  18. data/lib/flydata/{fluent-plugins/mysql → mysql}/binlog_position.rb +2 -0
  19. data/lib/flydata/{util → mysql}/mysql_util.rb +34 -1
  20. data/lib/flydata/mysql/table_ddl.rb +118 -0
  21. data/lib/flydata/parser/mysql/dump_parser.rb +5 -5
  22. data/lib/flydata/parser/mysql/mysql_alter_table.treetop +29 -5
  23. data/lib/flydata/sync_file_manager.rb +15 -2
  24. data/spec/flydata/command/sync_spec.rb +22 -1
  25. data/spec/flydata/fluent-plugins/in_mysql_binlog_flydata_spec.rb +7 -2
  26. data/spec/flydata/fluent-plugins/mysql/dml_record_handler_spec.rb +130 -0
  27. data/spec/flydata/fluent-plugins/mysql/shared_query_handler_context.rb +1 -0
  28. data/spec/flydata/fluent-plugins/mysql/table_meta_spec.rb +14 -8
  29. data/spec/flydata/fluent-plugins/mysql/truncate_query_handler_spec.rb +2 -1
  30. data/spec/flydata/{fluent-plugins/mysql → mysql}/binlog_position_spec.rb +3 -2
  31. data/spec/flydata/{util → mysql}/mysql_util_spec.rb +2 -2
  32. data/spec/flydata/mysql/table_ddl_spec.rb +193 -0
  33. data/spec/flydata/parser/mysql/alter_table_parser_spec.rb +37 -9
  34. data/spec/flydata/sync_file_manager_spec.rb +102 -27
  35. metadata +12 -7
@@ -215,7 +215,9 @@ grammar MysqlAlterTable
215
215
  }
216
216
  / convert_sym sp to_sym sp charset sp charset_name_or_default opt_collate {
217
217
  def action
218
- { action: :convert_charset }
218
+ { action: :convert_charset,
219
+ cs: charset_name_or_default.charset
220
+ }
219
221
  end
220
222
  }
221
223
  / ( create_table_options_space_separated ) 1..1 {
@@ -795,6 +797,8 @@ grammar MysqlAlterTable
795
797
  / key_opt { def option; key_opt_option; end }
796
798
  / comment_opt { def option; comment_opt_option; end }
797
799
  / column_format_opt { def option; column_format_opt_option; end }
800
+ / character_set_opt { def option; character_set_opt_option; end }
801
+ / collate_opt { def option; collate_opt_option; end }
798
802
  / storage_opt { def option; storage_opt_option; end }
799
803
  #TODO: / reference_definition
800
804
  end
@@ -889,6 +893,20 @@ grammar MysqlAlterTable
889
893
  end
890
894
  }
891
895
  end
896
+ rule character_set_opt
897
+ 'character set'i sp value {
898
+ def character_set_opt_option
899
+ { cs: FlydataCore::TableDef::MysqlTableDef.flydata_charset(value.raw_value) }
900
+ end
901
+ }
902
+ end
903
+ rule collate_opt
904
+ 'collate'i sp value {
905
+ def collate_opt_option
906
+ {} # Not supported
907
+ end
908
+ }
909
+ end
892
910
  rule storage_opt
893
911
  'storage'i sp ( 'disk'i / 'memory'i / 'default'i ) {
894
912
  def storage_opt_option
@@ -1012,8 +1030,8 @@ grammar MysqlAlterTable
1012
1030
  end
1013
1031
 
1014
1032
  rule charset_name_or_default
1015
- default_sym
1016
- / charset_name
1033
+ default_sym { def charset; 'DEFAULT'; end }
1034
+ / charset_name { def charset; FlydataCore::TableDef::MysqlTableDef.flydata_charset(text_value); end }
1017
1035
  end
1018
1036
 
1019
1037
  rule collation_name
@@ -1047,7 +1065,9 @@ grammar MysqlAlterTable
1047
1065
  rule create_table_option
1048
1066
  default_charset {
1049
1067
  def action
1050
- { action: :default_charset }
1068
+ { action: :default_charset,
1069
+ cs: charset
1070
+ }
1051
1071
  end
1052
1072
  }
1053
1073
  / default_collation {
@@ -1073,7 +1093,11 @@ grammar MysqlAlterTable
1073
1093
  end
1074
1094
 
1075
1095
  rule default_charset
1076
- opt_default nsp charset opt_equal nsp charset_name_or_default
1096
+ opt_default nsp charset opt_equal nsp charset_name_or_default {
1097
+ def charset
1098
+ charset_name_or_default.charset
1099
+ end
1100
+ }
1077
1101
  end
1078
1102
 
1079
1103
  rule opt_default
@@ -41,14 +41,27 @@ module Flydata
41
41
  state: items[5], substate: items[6], mysql_table: mysql_table}
42
42
  end
43
43
 
44
- def mark_generated_tables(tables)
44
+ def load_generated_ddl(tables)
45
+ tables = [ tables ] unless tables.kind_of?(Array)
46
+ paths = table_ddl_file_paths(*tables)
47
+ paths.collect{|path|
48
+ begin
49
+ File.open(path) {|f| f.read }
50
+ rescue Errno::ENOENT
51
+ nil
52
+ end
53
+ }
54
+ end
55
+
56
+ def save_generated_ddl(tables, contents = "1")
57
+ tables = [ tables ] unless tables.kind_of?(Array)
45
58
  table_positions_dir_path = ENV['FLYDATA_TABLE_POSITIONS'] || File.join(FLYDATA_HOME, 'positions')
46
59
  #Create positions if dir does not exist
47
60
  unless File.directory?(table_positions_dir_path)
48
61
  FileUtils.mkdir_p(table_positions_dir_path)
49
62
  end
50
63
  tables.each do |tab|
51
- File.open(File.join(table_positions_dir_path, "#{tab}.generated_ddl"), 'w') {|f| f.write("1") }
64
+ File.open(File.join(table_positions_dir_path, "#{tab}.generated_ddl"), 'w') {|f| f.write(contents) }
52
65
  end
53
66
  end
54
67
 
@@ -5,6 +5,9 @@ require 'flydata/command/sync'
5
5
  module Flydata
6
6
  module Command
7
7
  describe Sync do
8
+ let(:subject_object){ described_class.new }
9
+ subject { subject_object }
10
+
8
11
  let(:default_mysqldump_dir) do
9
12
  File.join('/tmp', "sync_dump_#{Time.now.to_i}")
10
13
  end
@@ -35,6 +38,14 @@ module Flydata
35
38
  "data_servers"=>"localhost:9905" }
36
39
  }
37
40
  end
41
+ let(:mysql_table_columns) {
42
+ {"id"=>{:column_name=>"id", :format_type_str=>"int(11)", :format_type=>"int", :format_size=>11}, "name"=>{:column_name=>"name", :format_type_str=>"varchar(40)", :format_type=>"varchar", :format_size=>40, :default=>nil}, "created_at"=>{:column_name=>"created_at", :format_type_str=>"timestamp", :format_type=>"timestamp", :default=>"CURRENT_TIMESTAMP"}, "bin"=>{:column_name=>"bin", :format_type_str=>"binary(34)", :format_type=>"binary", :format_size=>34, :default=>nil}, "bin2"=>{:column_name=>"bin2", :format_type_str=>"blob", :format_type=>"blob"}, "varbin"=>{:column_name=>"varbin", :format_type_str=>"varchar(34)", :format_type=>"varchar", :format_size=>34, :default=>nil}}
43
+ }
44
+ let(:mysql_table) do
45
+ mt = double('mysql_table')
46
+ allow(mt).to receive(:columns).and_return(mysql_table_columns)
47
+ mt
48
+ end
38
49
 
39
50
  after :each do
40
51
  if Dir.exists?(default_mysqldump_dir)
@@ -45,7 +56,6 @@ module Flydata
45
56
  end
46
57
  end
47
58
 
48
- subject { Sync.new }
49
59
  describe '#cleanup_sync_server' do
50
60
  let(:rest_client) { double('rest_client') }
51
61
 
@@ -232,6 +242,17 @@ module Flydata
232
242
  include_examples 'throws an error'
233
243
  end
234
244
  end
245
+ describe '#convert_to_flydata_values' do
246
+ subject { subject_object.send(:convert_to_flydata_values, mysql_table, values) }
247
+ let(:values) { [4, 'John', nil, '0xC0448200', nil, nil] }
248
+
249
+ it 'calls convert_to_valydata_value for each value' do
250
+ mysql_table.columns.values.each_with_index do |col_info, i|
251
+ expect(FlydataCore::TableDef::MysqlTableDef).to receive(:convert_to_flydata_value).with(values[i], col_info[:format_type]).and_return(values[i])
252
+ end
253
+ is_expected.to eq values
254
+ end
255
+ end
235
256
  end
236
257
  end
237
258
  end
@@ -167,7 +167,8 @@ EOT
167
167
  record = row.kind_of?(Hash) && row.keys.include?(:row) ? row : { row: row }
168
168
  @seq += 1
169
169
  record.merge({ :type=>type, :table_name=>table, :respect_order=>true,
170
- :seq=>@seq, :src_pos=>"#{binlog_file}\t#{position}", :table_rev=>1 })
170
+ :seq=>@seq, :src_pos=>"#{binlog_file}\t#{position}", :table_rev=>1,
171
+ :v=>2 })
171
172
  end
172
173
  expect_emitted_records(event, records)
173
174
  end
@@ -193,6 +194,7 @@ EOT
193
194
  let(:rotate_event_corrupt){ create_event(TEST_EVENT_ROTATE_MISSING_BINLOG_FILE) }
194
195
 
195
196
  let(:now) { Time.now }
197
+ let(:flydata_record_version) { 2 }
196
198
 
197
199
  let(:binlog_position_file) do
198
200
  f = double('binlog_position_file')
@@ -226,7 +228,7 @@ EOT
226
228
  setup_initial_flydata_files
227
229
  allow(MysqlBinlogInput::BinlogUtil).to receive(:to_hash) {|e| e}
228
230
  allow(Mysql::BinLogPositionFile).to receive(:new).with(TEST_POSITION_FILE).and_return(binlog_position_file)
229
- allow(Mysql::TableMeta).to receive(:update)
231
+ allow(Flydata::Mysql::TableMeta).to receive(:update)
230
232
  allow(plugin).to receive(:`).with("mysql -V").and_return("mysql Ver 14.14 Distrib 5.5.40")
231
233
  allow(plugin).to receive(:`).with(/^echo 'select version\(\);'/).and_return("version()\n5.5.40-0ubuntu0.12.04.1-log")
232
234
  Timecop.freeze(now)
@@ -326,6 +328,7 @@ EOT
326
328
  src_pos: "mysql-bin.000048\t689",
327
329
  table_rev: 2, # increment revision
328
330
  seq: 2,
331
+ v: flydata_record_version,
329
332
  actions: [{
330
333
  action: :add_column, column: "sum", :type=>'int4(11)', :query=>'add column sum integer'}],
331
334
  })
@@ -341,6 +344,7 @@ EOT
341
344
  src_pos: "mysql-bin.000048\t800",
342
345
  table_rev: 2, # increment revision
343
346
  seq: 2,
347
+ v: flydata_record_version,
344
348
  actions: [{
345
349
  action: :drop_column, column: "sum", :query=>'drop column sum'}],
346
350
  })
@@ -357,6 +361,7 @@ EOT
357
361
  src_pos: "mysql-bin.000048\t#{337 - 217}",
358
362
  table_rev: 1,
359
363
  seq: 2,
364
+ v: flydata_record_version,
360
365
  actions: [{
361
366
  action: :add_index,
362
367
  support_level: :nonbreaking,
@@ -0,0 +1,130 @@
1
+ # encoding: utf-8
2
+
3
+ require 'fluent_plugins_spec_helper'
4
+ require 'flydata/fluent-plugins/mysql/dml_record_handler'
5
+
6
+ module Mysql
7
+
8
+ describe DmlRecordHandler do
9
+ let(:subject_object) { described_class.new(context) }
10
+
11
+ let(:table_meta) { double('table_meta') }
12
+ let(:tables) { ['items', 'orders'] }
13
+ let(:sync_fm) {
14
+ sfm = double('sync_fm')
15
+ allow(sfm).to receive(:get_table_binlog_pos).and_return nil
16
+ sfm
17
+ }
18
+ let(:context) {
19
+ c = double('context')
20
+ allow(c).to receive(:table_meta).and_return(table_meta)
21
+ allow(c).to receive(:tables).and_return(tables)
22
+ allow(c).to receive(:sync_fm).and_return(sync_fm)
23
+ c
24
+ }
25
+
26
+ describe '#encode_row_value' do
27
+ subject { subject_object.send(:encode_row_value, value) }
28
+
29
+ let(:normal_utf8_string) { "ABC新年Ã" }
30
+ let(:valid_utf8_string) { "\x00abc" } # Yes, this is a valid UTF-8 string
31
+ let(:invalid_utf8_string) { "新年".encode('shift_jis') }
32
+ let(:latin1_string_happened_to_be_a_valid_utf8_string) { "ã".encode("iso-8859-1") }
33
+
34
+ shared_examples 'encoding values differently based on their contents not based on encoding' do
35
+ context 'with a normal utf-8 string' do
36
+ let(:value) { normal_utf8_string }
37
+ it 'returns a utf8 string' do
38
+ result = subject
39
+ expect(result).to eq normal_utf8_string
40
+ expect(result.encoding).to eq Encoding.find('utf-8')
41
+ end
42
+ end
43
+ context 'with a valid utf-8 string' do
44
+ let(:value) { valid_utf8_string }
45
+ it 'returns a utf8 string' do
46
+ result = subject
47
+ expect(result).to eq valid_utf8_string
48
+ expect(result.encoding).to eq Encoding.find('utf-8')
49
+ end
50
+ end
51
+ context 'with an invalid utf-8 string' do
52
+ let(:value) { invalid_utf8_string }
53
+ it 'returns a FlyData binary format string as BINARY encoding' do
54
+ result = subject
55
+ expect(result).to eq "0x9056944E"
56
+ expect(result.encoding).to eq Encoding::BINARY
57
+ end
58
+ end
59
+ context 'with a latin1 string which happens to be a valid utf8 character' do
60
+ let(:value) { latin1_string_happened_to_be_a_valid_utf8_string }
61
+ it 'returns a utf8 string' do
62
+ result = subject
63
+ expect(result).to eq latin1_string_happened_to_be_a_valid_utf8_string
64
+ expect(result).to eq "ã" # #C3A3
65
+ expect(result.encoding).to eq Encoding.find('utf-8')
66
+ end
67
+ end
68
+ end
69
+
70
+ context 'with strings whose encoding is utf-8' do
71
+ before do
72
+ normal_utf8_string.force_encoding 'utf-8'
73
+ valid_utf8_string.force_encoding 'utf-8'
74
+ invalid_utf8_string.force_encoding 'utf-8'
75
+ latin1_string_happened_to_be_a_valid_utf8_string.force_encoding 'utf-8'
76
+ end
77
+ it_behaves_like 'encoding values differently based on their contents not based on encoding'
78
+ end
79
+
80
+ context 'with strings whose encoding is binary' do
81
+ before do
82
+ normal_utf8_string.force_encoding 'binary'
83
+ valid_utf8_string.force_encoding 'binary'
84
+ invalid_utf8_string.force_encoding 'binary'
85
+ latin1_string_happened_to_be_a_valid_utf8_string.force_encoding 'binary'
86
+ end
87
+ it_behaves_like 'encoding values differently based on their contents not based on encoding'
88
+ end
89
+ end
90
+
91
+ describe '#convert_to_flydata_row_format' do
92
+ subject { subject_object.send(:convert_to_flydata_row_format, row) }
93
+
94
+ let(:row) { [ value ] }
95
+ before do
96
+ end
97
+
98
+ shared_examples "returning a json hash with no attrs" do
99
+ it do
100
+ is_expected.to eq({"1" => value })
101
+ end
102
+ end
103
+
104
+ context 'with a row containing a string value' do
105
+ let(:value) { "a normal string" }
106
+
107
+ before do
108
+ expect(subject_object).to receive(:encode_row_value).with(value).and_return(value)
109
+ end
110
+
111
+ it_behaves_like "returning a json hash with no attrs"
112
+ end
113
+ context 'with a row containing a binary value' do
114
+ let(:value) { "\x00abc".force_encoding('binary') }
115
+ before do
116
+ expect(subject_object).to receive(:encode_row_value).with(value).and_return(value)
117
+ end
118
+ it "returns a json hash with attrs" do
119
+ is_expected.to eq({"1" => value, "attrs" => {"1" => {"enc" => "b" }}})
120
+ end
121
+ end
122
+ context 'with a row containing an integer value' do
123
+ let(:value) { 4 }
124
+
125
+ it_behaves_like "returning a json hash with no attrs"
126
+ end
127
+ end
128
+ end
129
+
130
+ end
@@ -14,6 +14,7 @@ module Mysql
14
14
  let(:current_binlog_file) { "mysql-bin.000066" }
15
15
  let(:tag) { "some_tag" }
16
16
  let(:table_rev) { 1 }
17
+ let(:flydata_record_version) { 2 }
17
18
  let(:context) do
18
19
  r = double('context')
19
20
  allow(r).to receive(:sync_fm).and_return(sync_fm)
@@ -1,6 +1,7 @@
1
1
  require 'fluent_plugins_spec_helper'
2
2
  require 'flydata/fluent-plugins/mysql/table_meta'
3
3
 
4
+ module Flydata
4
5
  module Mysql
5
6
  describe TableMeta do
6
7
  let(:db_opts) { { host: 'test-host', port: 3306, username: 'test-user', password: 'test-pswd', database: 'test-db' } }
@@ -9,6 +10,9 @@ module Mysql
9
10
 
10
11
  let(:conn) { double('conn') }
11
12
  let(:table_meta) { TableMeta.new(db_opts.merge(database: database, tables: tables)) }
13
+ let(:utf8) { 'utf8' }
14
+ let(:latin1) { 'latin1' }
15
+ let(:cp932) { 'cp932' }
12
16
 
13
17
  before do
14
18
  allow(conn).to receive(:close)
@@ -24,23 +28,26 @@ module Mysql
24
28
  context 'when character_set_name is supported' do
25
29
  context 'when encoding is not necessary' do
26
30
  let(:query_result) {[
27
- {'table_name' => 'a_table', 'character_set_name' => 'utf8' }
31
+ {'table_name' => 'a_table', 'character_set_name' => utf8 }
28
32
  ]}
29
33
  it 'set nil for encoding' do
30
34
  table_meta.update
31
- expect(table_meta['a_table'][:encoding]).to be_nil
35
+ expect(table_meta['a_table'][:encoding]).to eq Encoding::UTF_8
36
+ expect(table_meta['a_table'][:mysql_charset]).to eq utf8
32
37
  end
33
38
  end
34
39
 
35
40
  context 'when encoding is needed' do
36
41
  let(:query_result) {[
37
- {'table_name' => 'a_table', 'character_set_name' => 'latin1' },
38
- {'table_name' => 'b_table', 'character_set_name' => 'cp932' }
42
+ {'table_name' => 'a_table', 'character_set_name' => latin1 },
43
+ {'table_name' => 'b_table', 'character_set_name' => cp932 }
39
44
  ]}
40
45
  it 'set ruby encoding encoding' do
41
46
  table_meta.update
42
- expect(table_meta['a_table'][:encoding]).to eq('ISO-8859-1')
43
- expect(table_meta['b_table'][:encoding]).to eq('CP932')
47
+ expect(table_meta['a_table'][:encoding]).to eq Encoding::ISO_8859_1
48
+ expect(table_meta['a_table'][:mysql_charset]).to eq latin1
49
+ expect(table_meta['b_table'][:encoding]).to eq Encoding::CP932
50
+ expect(table_meta['b_table'][:mysql_charset]).to eq cp932
44
51
  end
45
52
  end
46
53
  end
@@ -56,5 +63,4 @@ module Mysql
56
63
  end
57
64
  end
58
65
  end
59
-
60
-
66
+ end
@@ -16,7 +16,8 @@ module Mysql
16
16
  respect_order: true,
17
17
  src_pos: "#{current_binlog_file}\t#{next_position - event_length}",
18
18
  table_rev: table_rev,
19
- seq: seq
19
+ seq: seq,
20
+ v: flydata_record_version
20
21
  }
21
22
  end
22
23
  before do
@@ -1,6 +1,6 @@
1
1
  require 'fluent_plugins_spec_helper'
2
- require 'mysql/binlog_position'
3
-
2
+ require 'flydata/mysql/binlog_position'
3
+ module Flydata
4
4
  module Mysql
5
5
  describe BinLogPosition do
6
6
  let(:pos1) { BinLogPosition.new('mysql-bin.000064 107') }
@@ -32,3 +32,4 @@ module Mysql
32
32
  end
33
33
  end
34
34
  end
35
+ end
@@ -1,8 +1,8 @@
1
1
  require 'spec_helper'
2
- require 'flydata/util/mysql_util'
2
+ require 'flydata/mysql/mysql_util'
3
3
 
4
4
  module Flydata
5
- module Util
5
+ module Mysql
6
6
 
7
7
  describe MysqlUtil do
8
8
  let(:default_option) { {
@@ -0,0 +1,193 @@
1
+ require 'spec_helper'
2
+ require 'flydata/mysql/table_ddl'
3
+
4
+ module Flydata
5
+ module Mysql
6
+
7
+ describe TableDdl do
8
+ let(:table_name1) { "items" }
9
+ let(:table_name2) { "orders" }
10
+ let(:table_names) { [table_name1, table_name2] }
11
+ let(:table1_charset) { 'utf8' }
12
+ let(:table2_charset) { 'sjis' }
13
+ let(:at_charset_event_table1) { double('at_charset_event_table1') }
14
+ let(:at_charset_event_table2) { double('at_charset_event_table2') }
15
+ let(:at_column_charset_event_table1) { double('at_column_charset_event_table1') }
16
+ let(:expected_alter_table_charset_events) { [ at_charset_event_table1, at_charset_event_table2] }
17
+ let(:at_charset_query) { "ALTER TABLE `#{database_name}`.`#{at_table}` CHARACTER SET = #{charset};" }
18
+ let(:query) { "ALTER TABLE `#{database_name}`.`#{event_table_name}` CHARACTER SET = #{charset};" }
19
+ let(:column_query1) { "ALTER TABLE `#{database_name}`.`#{event_table_name}` CHANGE COLUMN `#{table1_varchar_column}` #{table1_varchar_column_def};" }
20
+ let(:charset) { 'utf8' }
21
+ let(:event_type) { 'Query' }
22
+ let(:database_name) { "a_database" }
23
+ let(:sync_fm) { double('sync_fm') }
24
+ let(:master_binlog_position) { "#{binlog_file}\t#{binlog_position}" }
25
+ let(:binlog_file) { "mysql-binlog.00123" }
26
+ let(:binlog_position) { 28393 }
27
+ let(:original_current_binlog_file) { double('original_current_binlog_file') }
28
+ let(:mysql_opts) { { host: double('host'), port: double('port'),
29
+ username: double('username'),
30
+ password: double('password'),
31
+ database: database_name,
32
+ } }
33
+ let(:position_file) { double('position_file') }
34
+ let(:context) { double('context') }
35
+ let(:mysql_tabledef) { { table_name1 => mysql_tabledef1,
36
+ table_name2 => mysql_tabledef2,
37
+ } }
38
+ let(:mysql_tabledef1) { double('mysql_tabledef1') }
39
+ let(:mysql_tabledef2) { double('mysql_tabledef2') }
40
+ let(:column_def1) { { "id" => "`id` integer NOT NULL",
41
+ table1_varchar_column => table1_varchar_column_def,
42
+ } }
43
+ let(:table1_varchar_column) { 'name' }
44
+ let(:table1_varchar_column_def) { "`#{table1_varchar_column}` varchar(45)#{charset_option1} NULL" }
45
+ let(:column_def2) { { "id" => "`id` integer NOT NULL",
46
+ "address" => "`address` varchar(45) NULL"
47
+ } }
48
+ let(:charset_option1) { "" }
49
+ let(:save) { {} }
50
+
51
+ before do
52
+ save[:log] = $log
53
+ $log = double('$log')
54
+ allow($log).to receive(:info)
55
+ allow($log).to receive(:debug)
56
+ allow($log).to receive(:error)
57
+ allow($log).to receive(:trace)
58
+
59
+ allow(mysql_tabledef1).to receive(:default_charset_mysql).and_return(table1_charset)
60
+ allow(mysql_tabledef1).to receive(:table_name).and_return(table_name1)
61
+ allow(mysql_tabledef1).to receive(:column_def).and_return(column_def1)
62
+ allow(mysql_tabledef2).to receive(:default_charset_mysql).and_return(table2_charset)
63
+ allow(mysql_tabledef2).to receive(:table_name).and_return(table_name2)
64
+ allow(mysql_tabledef2).to receive(:column_def).and_return(column_def2)
65
+ end
66
+
67
+ after do
68
+ $log = save[:log]
69
+ end
70
+
71
+ describe '.migrate_to_v2' do
72
+ let(:subject_object) { described_class }
73
+ let(:subject_params) { [:migrate_to_v2, table_names, mysql_opts, sync_fm, position_file, context] }
74
+
75
+ let(:target_version) { "2" }
76
+
77
+ shared_examples "create and send charset records and increment version" do
78
+ let(:next_position) { binlog_position }
79
+
80
+ it do
81
+ matcher = receive(:each_mysql_tabledef).with(table_names, mysql_opts)
82
+ matcher = table_names.inject(matcher) {|m, tn| m.and_yield(mysql_tabledef[tn], nil) }
83
+ expect(MysqlUtil).to matcher
84
+ expect(File).to receive(:open).with(position_file).and_return(master_binlog_position)
85
+ expect(context).to receive(:current_binlog_file).and_return(original_current_binlog_file)
86
+ expect(context).to receive(:current_binlog_file=).with(binlog_file)
87
+ expect(context).to receive(:current_binlog_file=).with(original_current_binlog_file)
88
+ expect(sync_fm).to receive(:save_generated_ddl).with(table_names, target_version)
89
+
90
+ expect{|b| subject_object.send(*subject_params, &b) }.to yield_successive_args(*expected_alter_table_charset_events)
91
+ end
92
+ end
93
+ shared_examples "does nothing" do
94
+ it do
95
+ expect(sync_fm).not_to receive(:save_generated_ddl)
96
+
97
+ expect{|b| subject_object.send(*subject_params, &b) }.not_to yield_control
98
+ end
99
+ end
100
+
101
+ let(:table_versions) { table_names.size.times.collect{ current_version } }
102
+ before do
103
+ expect(sync_fm).to receive(:load_generated_ddl).with(table_names).and_return table_versions
104
+ end
105
+
106
+ context "with a single table" do
107
+ let(:table_names) { [table_name1] }
108
+ let(:event_table_name) { table_name1 }
109
+ let(:charset) { table1_charset }
110
+ let(:expected_alter_table_charset_events) { [ at_charset_event_table1 ] }
111
+
112
+ context "with a table whose version is 0" do
113
+ let(:current_version) { nil } # generate_ddl file doesn't exist in case of version "0"
114
+ before do
115
+ expect(QueryEvent).to receive(:new).with(event_type, database_name, event_table_name, next_position, 0, query, Integer).and_return(at_charset_event_table1)
116
+ end
117
+ it_behaves_like "create and send charset records and increment version"
118
+ end
119
+ context "with a table whose version is 1" do
120
+ let(:current_version) { "1" }
121
+ before do
122
+ expect(QueryEvent).to receive(:new).with(event_type, database_name, event_table_name, next_position, 0, query, Integer).and_return(at_charset_event_table1)
123
+ end
124
+ it_behaves_like "create and send charset records and increment version"
125
+ end
126
+ context "with a table whose version is 2" do
127
+ let(:current_version) { "2" }
128
+ it_behaves_like "does nothing"
129
+ end
130
+ context "with a table whose version is 3" do
131
+ let(:current_version) { "3" }
132
+ it_behaves_like "does nothing"
133
+ end
134
+ context 'with a single table who has a column with charset' do
135
+ let(:charset_option1) { " CHARACTER SET 'latin5'" }
136
+ let(:expected_alter_table_charset_events) { [ at_column_charset_event_table1, at_charset_event_table1 ] }
137
+
138
+ context "with a table whose version is 0" do
139
+ let(:current_version) { nil } # generate_ddl file doesn't exist in case of version "0"
140
+ before do
141
+ expect(QueryEvent).to receive(:new).with(event_type, database_name, event_table_name, next_position, 0, column_query1, Integer).and_return(at_column_charset_event_table1)
142
+ expect(QueryEvent).to receive(:new).with(event_type, database_name, event_table_name, next_position, 0, query, Integer).and_return(at_charset_event_table1)
143
+ end
144
+ it_behaves_like "create and send charset records and increment version"
145
+ end
146
+ end
147
+ end
148
+ end
149
+ end
150
+
151
+ describe QueryEvent do
152
+ let(:subject_object) { described_class.new(event_type, database_name, event_table_name, next_position, event_length, query, timestamp) }
153
+
154
+ let(:event_type) { 'Query' }
155
+ let(:database_name) { 'mydb' }
156
+ let(:event_table_name) { 'mytable' }
157
+ let(:next_position) { 38293 }
158
+ let(:event_length) { 2093 }
159
+ let(:query) { 'ALTER TABLE CHARACTER SET sjis' }
160
+ let(:timestamp) { Time.now.to_i }
161
+
162
+ describe "#event_type" do
163
+ subject { subject_object.event_type }
164
+ it { is_expected.to eq event_type }
165
+ end
166
+ describe "#database_name" do
167
+ subject { subject_object.database_name }
168
+ it { is_expected.to eq database_name}
169
+ end
170
+ describe "#event_table_name" do
171
+ subject { subject_object.event_table_name }
172
+ it { is_expected.to eq event_table_name}
173
+ end
174
+ describe "#next_position" do
175
+ subject { subject_object.next_position }
176
+ it { is_expected.to eq next_position}
177
+ end
178
+ describe "#event_length" do
179
+ subject { subject_object.event_length }
180
+ it { is_expected.to eq event_length}
181
+ end
182
+ describe "#query" do
183
+ subject { subject_object.query }
184
+ it { is_expected.to eq query}
185
+ end
186
+ describe "#timestamp" do
187
+ subject { subject_object.timestamp }
188
+ it { is_expected.to eq timestamp}
189
+ end
190
+ end
191
+
192
+ end
193
+ end