flydata 0.6.3 → 0.6.4

Sign up to get free protection for your applications and to get access to all the features.
Files changed (94) hide show
  1. checksums.yaml +4 -4
  2. data/Rakefile +2 -2
  3. data/VERSION +1 -1
  4. data/bin/fdredshift +78 -0
  5. data/circle.yml +1 -1
  6. data/ext/flydata/{parser/mysql → source_mysql/parser}/.gitignore +0 -0
  7. data/ext/flydata/{parser/mysql → source_mysql/parser}/dump_parser_ext.cpp +3 -3
  8. data/ext/flydata/source_mysql/parser/extconf.rb +3 -0
  9. data/ext/flydata/{parser/mysql → source_mysql/parser}/parser.txt +0 -0
  10. data/ext/flydata/{parser/mysql → source_mysql/parser}/sql_parser.cpp +0 -0
  11. data/ext/flydata/{parser/mysql → source_mysql/parser}/sql_parser.h +0 -0
  12. data/flydata-core/lib/flydata-core/mysql/binlog_pos.rb +34 -32
  13. data/flydata-core/lib/flydata-core/mysql/compatibility_checker.rb +20 -0
  14. data/flydata-core/lib/flydata-core/table_def/mysql_table_def.rb +12 -4
  15. data/flydata-core/lib/flydata-core/table_def/redshift_table_def.rb +60 -6
  16. data/flydata-core/spec/mysql/binlog_pos_spec.rb +474 -0
  17. data/flydata-core/spec/table_def/mysql_table_def_spec.rb +57 -0
  18. data/flydata-core/spec/table_def/mysql_to_redshift_table_def_spec.rb +174 -20
  19. data/flydata-core/spec/table_def/mysqldump_test_col_comment_with_AUTO_INCREMENT_keyword.dump +43 -0
  20. data/flydata-core/spec/table_def/mysqldump_test_col_comment_with_not_null_keyword.dump +43 -0
  21. data/flydata-core/spec/table_def/mysqldump_test_col_comment_with_unique_keyword.dump +43 -0
  22. data/flydata-core/spec/table_def/mysqldump_test_col_comment_with_unsigned_keyword.dump +43 -0
  23. data/flydata-core/spec/table_def/redshift_table_def_spec.rb +41 -8
  24. data/flydata.gemspec +0 -0
  25. data/lib/flydata/cli.rb +11 -5
  26. data/lib/flydata/command/base.rb +14 -1
  27. data/lib/flydata/command/exclusive_runnable.rb +42 -12
  28. data/lib/flydata/command/helper.rb +6 -6
  29. data/lib/flydata/command/sender.rb +4 -3
  30. data/lib/flydata/command/setup.rb +30 -381
  31. data/lib/flydata/command/stop.rb +1 -0
  32. data/lib/flydata/command/sync.rb +273 -301
  33. data/lib/flydata/compatibility_check.rb +24 -117
  34. data/lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb +3 -3
  35. data/lib/flydata/fluent-plugins/mysql/alter_table_query_handler.rb +2 -2
  36. data/lib/flydata/fluent-plugins/mysql/binlog_record_handler.rb +6 -6
  37. data/lib/flydata/fluent-plugins/mysql/truncate_table_query_handler.rb +0 -1
  38. data/lib/flydata/parser.rb +14 -0
  39. data/lib/flydata/{parser_provider.rb → parser/parser_provider.rb} +6 -4
  40. data/lib/flydata/parser/source_table.rb +33 -0
  41. data/lib/flydata/source.rb +105 -0
  42. data/lib/flydata/source/component.rb +21 -0
  43. data/lib/flydata/source/errors.rb +7 -0
  44. data/lib/flydata/source/generate_source_dump.rb +72 -0
  45. data/lib/flydata/source/parse_dump_and_send.rb +52 -0
  46. data/lib/flydata/source/setup.rb +31 -0
  47. data/lib/flydata/source/source_pos.rb +45 -0
  48. data/lib/flydata/source/sync.rb +56 -0
  49. data/lib/flydata/source/sync_generate_table_ddl.rb +43 -0
  50. data/lib/flydata/source_file/setup.rb +17 -0
  51. data/lib/flydata/source_file/sync.rb +14 -0
  52. data/lib/flydata/{command → source_mysql/command}/mysql.rb +2 -1
  53. data/lib/flydata/{command → source_mysql/command}/mysql_command_base.rb +2 -4
  54. data/lib/flydata/{command → source_mysql/command}/mysqlbinlog.rb +2 -1
  55. data/lib/flydata/{command → source_mysql/command}/mysqldump.rb +2 -1
  56. data/lib/flydata/source_mysql/generate_source_dump.rb +53 -0
  57. data/lib/flydata/source_mysql/mysql_compatibility_check.rb +114 -0
  58. data/lib/flydata/source_mysql/parse_dump_and_send.rb +28 -0
  59. data/lib/flydata/{parser/mysql → source_mysql/parser}/.gitignore +0 -0
  60. data/lib/flydata/{parser/mysql → source_mysql/parser}/dump_parser.rb +32 -67
  61. data/lib/flydata/{parser/mysql → source_mysql/parser}/mysql_alter_table.treetop +0 -0
  62. data/lib/flydata/source_mysql/setup.rb +24 -0
  63. data/lib/flydata/source_mysql/source_pos.rb +21 -0
  64. data/lib/flydata/source_mysql/sync.rb +45 -0
  65. data/lib/flydata/source_mysql/sync_generate_table_ddl.rb +40 -0
  66. data/lib/flydata/{mysql → source_mysql}/table_ddl.rb +6 -17
  67. data/lib/flydata/source_zendesk/sync_generate_table_ddl.rb +30 -0
  68. data/lib/flydata/source_zendesk/zendesk_flydata_tabledefs.rb +133 -0
  69. data/lib/flydata/sync_file_manager.rb +132 -73
  70. data/lib/flydata/table_ddl.rb +18 -0
  71. data/spec/flydata/cli_spec.rb +1 -0
  72. data/spec/flydata/command/exclusive_runnable_spec.rb +19 -8
  73. data/spec/flydata/command/sender_spec.rb +1 -1
  74. data/spec/flydata/command/setup_spec.rb +4 -4
  75. data/spec/flydata/command/sync_spec.rb +97 -134
  76. data/spec/flydata/compatibility_check_spec.rb +16 -289
  77. data/spec/flydata/fluent-plugins/mysql/alter_table_query_handler_spec.rb +3 -3
  78. data/spec/flydata/fluent-plugins/mysql/dml_record_handler_spec.rb +1 -1
  79. data/spec/flydata/fluent-plugins/mysql/shared_query_handler_context.rb +4 -2
  80. data/spec/flydata/fluent-plugins/mysql/truncate_query_handler_spec.rb +1 -1
  81. data/spec/flydata/source_mysql/generate_source_dump_spec.rb +69 -0
  82. data/spec/flydata/source_mysql/mysql_compatibility_check_spec.rb +280 -0
  83. data/spec/flydata/{parser/mysql → source_mysql/parser}/alter_table_parser_spec.rb +2 -2
  84. data/spec/flydata/{parser/mysql → source_mysql/parser}/dump_parser_spec.rb +75 -70
  85. data/spec/flydata/source_mysql/sync_generate_table_ddl_spec.rb +137 -0
  86. data/spec/flydata/{mysql → source_mysql}/table_ddl_spec.rb +2 -2
  87. data/spec/flydata/source_spec.rb +140 -0
  88. data/spec/flydata/source_zendesk/sync_generate_table_ddl_spec.rb +33 -0
  89. data/spec/flydata/sync_file_manager_spec.rb +157 -77
  90. data/tmpl/redshift_mysql_data_entry.conf.tmpl +1 -1
  91. metadata +56 -23
  92. data/ext/flydata/parser/mysql/extconf.rb +0 -3
  93. data/lib/flydata/mysql/binlog_position.rb +0 -22
  94. data/spec/flydata/mysql/binlog_position_spec.rb +0 -35
@@ -0,0 +1,137 @@
1
+ require 'flydata/source_mysql/sync_generate_table_ddl'
2
+
3
+ module Flydata
4
+ module SourceMysql
5
+
6
+ describe SyncGenerateTableDdl do
7
+ let(:subject_object) { described_class.new(source, dp, options) }
8
+
9
+ let(:dp) { double('dp') }
10
+ let(:de) { {
11
+ 'mysql_data_entry_preference' => {
12
+ 'host' => 'localhost',
13
+ 'port' => 3306,
14
+ 'username' => 'masashi',
15
+ 'password' => 'welcome',
16
+ 'database' => 'sync_test',
17
+ }
18
+ } }
19
+ let(:options) { {} }
20
+
21
+ let(:source) { double('source') }
22
+ before do
23
+ allow(source).to receive(:de).and_return de
24
+ end
25
+
26
+ describe '#run_compatibility_check' do
27
+ subject { subject_object.run_compatibility_check }
28
+
29
+ let(:checker) { double("checker") }
30
+ it do
31
+ expect(MysqlCompatibilityCheck).to receive(:new).and_return checker
32
+ expect(checker).to receive(:check)
33
+
34
+ subject
35
+ end
36
+ end
37
+ describe '#generate_flydata_tabledef' do
38
+ subject { subject_object.generate_flydata_tabledef(tables, options) }
39
+
40
+ let(:tables) { %w| table1 table2 table4 table3 | }
41
+
42
+ context 'with full options' do
43
+ it 'issues mysqldump command with expected parameters' do
44
+ expect(Open3).to receive(:popen3).with(
45
+ 'mysqldump -h localhost -P 3306 -umasashi -pwelcome --default-character-set=utf8 --protocol=tcp -d sync_test table1 table2 table4 table3')
46
+
47
+ subject
48
+ end
49
+ end
50
+ context 'without_host' do
51
+ before do
52
+ de['mysql_data_entry_preference'].delete('host')
53
+ end
54
+ it { expect { subject }.to raise_error }
55
+ end
56
+ context 'with empty host' do
57
+ before do
58
+ de['mysql_data_entry_preference']['host'] = ""
59
+ end
60
+ it { expect { subject }.to raise_error }
61
+ end
62
+ context 'without port' do
63
+ before do
64
+ de['mysql_data_entry_preference'].delete('port')
65
+ end
66
+ it "uses the default port" do
67
+ expect(Open3).to receive(:popen3).with(
68
+ 'mysqldump -h localhost -umasashi -pwelcome --default-character-set=utf8 --protocol=tcp -d sync_test table1 table2 table4 table3')
69
+ subject
70
+ end
71
+ end
72
+ context 'with port override' do
73
+ before do
74
+ de['mysql_data_entry_preference']['port'] = 1234
75
+ end
76
+ it "uses the specified port" do
77
+ expect(Open3).to receive(:popen3).with(
78
+ 'mysqldump -h localhost -P 1234 -umasashi -pwelcome --default-character-set=utf8 --protocol=tcp -d sync_test table1 table2 table4 table3')
79
+ subject
80
+ end
81
+ end
82
+ context 'without username' do
83
+ before do
84
+ de['mysql_data_entry_preference'].delete('username')
85
+ end
86
+ it { expect { subject }.to raise_error }
87
+ end
88
+ context 'with empty username' do
89
+ before do
90
+ de['mysql_data_entry_preference']['username'] = ""
91
+ end
92
+ it { expect { subject }.to raise_error }
93
+ end
94
+ context 'without_password' do
95
+ before do
96
+ de['mysql_data_entry_preference'].delete('password')
97
+ end
98
+ it "call mysqldump without MYSQL_PW set" do
99
+ expect(Open3).to receive(:popen3).with(
100
+ 'mysqldump -h localhost -P 3306 -umasashi --default-character-set=utf8 --protocol=tcp -d sync_test table1 table2 table4 table3')
101
+ subject
102
+ end
103
+ end
104
+ context 'with password containing symbols' do
105
+ before do
106
+ de['mysql_data_entry_preference']['password']="welcome&!@^@#^"
107
+ end
108
+ it "call mysqldump with MYSQL_PW set with correct symbols" do
109
+ expect(Open3).to receive(:popen3).with(
110
+ 'mysqldump -h localhost -P 3306 -umasashi -pwelcome\\&\\!@\\^@\\#\\^ --default-character-set=utf8 --protocol=tcp -d sync_test table1 table2 table4 table3')
111
+ subject
112
+ end
113
+ end
114
+ context 'without_database' do
115
+ before do
116
+ de['mysql_data_entry_preference'].delete('database')
117
+ end
118
+ it { expect { subject }.to raise_error }
119
+ end
120
+ context 'with empty database' do
121
+ before do
122
+ de['mysql_data_entry_preference']['database'] = ""
123
+ end
124
+ it { expect { subject }.to raise_error }
125
+ end
126
+ context 'with empty tables' do
127
+ before do
128
+ de['mysql_data_entry_preference']['tables'] = []
129
+ end
130
+ it { expect { subject }.to raise_error }
131
+ end
132
+ end
133
+ end
134
+
135
+
136
+ end
137
+ end
@@ -1,8 +1,8 @@
1
1
  require 'spec_helper'
2
- require 'flydata/mysql/table_ddl'
2
+ require 'flydata/source_mysql/table_ddl'
3
3
 
4
4
  module Flydata
5
- module Mysql
5
+ module SourceMysql
6
6
 
7
7
  describe TableDdl do
8
8
  let(:table_name1) { "items" }
@@ -0,0 +1,140 @@
1
+ require 'flydata/source'
2
+
3
+ module Flydata
4
+
5
+ describe Source do
6
+ let(:subject_object) { described_class.create(de) }
7
+
8
+ let(:target_class) { double('target_class') }
9
+ let(:type_class) { double('type_class') }
10
+
11
+ let(:expected_instance) { double('expected_instance') }
12
+ let(:de) { double('de') }
13
+ let(:opts) { double('opts') }
14
+ let(:class_name) { "replace this" }
15
+ let(:type_class_name) { "replace this" }
16
+ let(:data_entry_type) { "RedshiftMysqlDataEntry" }
17
+
18
+ before do
19
+ allow(target_class).to receive(:name).and_return(class_name)
20
+ allow(type_class).to receive(:name).and_return(type_class_name)
21
+ end
22
+ before do
23
+ allow(de).to receive(:[]).with('type').and_return(data_entry_type)
24
+ end
25
+
26
+ after do
27
+ if described_class.instance_variable_defined? :@component_classes
28
+ described_class.remove_instance_variable :@component_classes
29
+ end
30
+ end
31
+
32
+ describe '#method_missing' do
33
+ subject { subject_object.send(method_name, opts) }
34
+
35
+ before do
36
+ described_class.register(target_class, type_class)
37
+ end
38
+
39
+ context 'when the target data source exists' do
40
+ let(:data_entry_type) { "RedshiftMysqlDataEntry" }
41
+ let(:class_name) { "Flydata::SourceMysql::MySR" }
42
+
43
+ context 'when the target component exists' do
44
+ let(:method_name) { :sync_reset }
45
+ let(:type_class_name) { "Flydata::Source::SyncReset" }
46
+
47
+ it "returns an instance of the target_class" do
48
+ expect(target_class).to receive(:new).with(subject_object, opts).
49
+ and_return expected_instance
50
+
51
+ expect(subject).to eq expected_instance
52
+ end
53
+ end
54
+ context 'when the target component does not exist' do
55
+ let(:method_name) { :sync_start }
56
+ let(:type_class_name) { "Flydata::Source::SyncReset" }
57
+
58
+ it "raises an error" do
59
+ expect{ subject }.to raise_error /Component '.*?' is not defined/
60
+ end
61
+ end
62
+ end
63
+ context 'when the target data source does not exist' do
64
+ let(:data_entry_type) { "FooBarDataEntry" }
65
+ let(:class_name) { "Flydata::SourceFooBar::MySR" }
66
+
67
+ context 'when the target component exists' do
68
+ let(:method_name) { :sync_reset }
69
+ let(:type_class_name) { "Flydata::Source::SyncReset" }
70
+
71
+ it "raises an error" do
72
+ expect{ subject }.to raise_error /No source components are available/
73
+ end
74
+ end
75
+ end
76
+ end
77
+
78
+ describe '.underscored_source_module_sym' do
79
+ subject { described_class.send(:underscored_source_module_sym, target_class) }
80
+
81
+ context 'with a standard name' do
82
+ let(:class_name) { "Flydata::SourceMysql::SyncReset" }
83
+ it { is_expected.to eq :source_mysql }
84
+ end
85
+ context 'with a nested path' do
86
+ let(:class_name) { "Flydata::SourceMysql::Command::SyncReset" }
87
+ it { is_expected.to eq :source_mysql }
88
+ end
89
+ context 'with a nested path (not recommended)' do
90
+ let(:class_name) { "Flydata::Foo::Bar::SourceMysql::SyncReset" }
91
+ it { is_expected.to eq :source_mysql }
92
+ end
93
+ context 'when none of elements starts with "Source"' do
94
+ let(:class_name) { "Flydata::Foo::Bar::SyncReset" }
95
+ it { is_expected.to eq nil }
96
+ end
97
+ context 'when more than one element start with "Source"' do
98
+ let(:class_name) { "Flydata::SourceMysql::SourceFiles::SyncReset" }
99
+ it "uses the first one" do
100
+ is_expected.to eq :source_mysql
101
+ end
102
+ end
103
+ end
104
+ describe '.command_dir_paths' do
105
+ subject { described_class.command_dir_paths }
106
+
107
+ before do
108
+ allow(File).to receive(:exists?).and_return false
109
+ end
110
+ shared_examples 'returning the expected result' do
111
+ before do
112
+ expect(File).to receive(:exists?).with(/source_mysql\/command/).
113
+ and_return source_mysql_command_dir_exists
114
+ expect(File).to receive(:exists?).with(/source_file\/command/).
115
+ and_return source_file_command_dir_exists
116
+ end
117
+ it { is_expected.to eq expected_result }
118
+ end
119
+ let(:source_file_command_dir_exists) { false }
120
+
121
+ context 'with no source dir' do
122
+ let(:source_mysql_command_dir_exists) { false }
123
+ let(:expected_result) { [] }
124
+ it_behaves_like 'returning the expected result'
125
+ end
126
+ context 'with command dir' do
127
+ let(:source_mysql_command_dir_exists) { true }
128
+ let(:expected_result) { ["flydata/source_mysql/command"] }
129
+ it_behaves_like 'returning the expected result'
130
+ end
131
+ context 'with two source dirs with command dir' do
132
+ let(:source_mysql_command_dir_exists) { true }
133
+ let(:source_file_command_dir_exists) { true }
134
+ let(:expected_result) { %w(flydata/source_mysql/command flydata/source_file/command) }
135
+ it_behaves_like 'returning the expected result'
136
+ end
137
+ end
138
+ end
139
+
140
+ end
@@ -0,0 +1,33 @@
1
+ require 'flydata/source_zendesk/sync_generate_table_ddl'
2
+
3
+ module Flydata
4
+ module SourceZendesk
5
+
6
+ describe SyncGenerateTableDdl do
7
+ let(:subject_object) { described_class.new(source, dp, options) }
8
+
9
+ let(:source) { double('source') }
10
+ let(:dp) { double('dp') }
11
+ let(:options) { {} }
12
+
13
+ describe '#run_compatibility_check' do
14
+ subject { subject_object.run_compatibility_check }
15
+ it { subject }
16
+ end
17
+
18
+ describe '#generate_flydata_tabledef' do
19
+ subject { subject_object.generate_flydata_tabledef(tables, options) }
20
+
21
+ let(:supported_tables) { %w(tickets ticket_comments ticket_audits users organizations) }
22
+ let(:options) { double('options') }
23
+
24
+ context 'with all supported tables' do
25
+ let(:tables) { supported_tables }
26
+ it { subject }
27
+ end
28
+ end
29
+ end
30
+
31
+
32
+ end
33
+ end
@@ -1,15 +1,19 @@
1
1
  # coding: utf-8
2
2
  require 'spec_helper'
3
3
  require 'flydata/sync_file_manager'
4
- require 'flydata/parser/mysql/dump_parser'
4
+ require 'flydata/parser/source_table'
5
+ require 'flydata/command/base'
5
6
 
6
7
  module Flydata
7
8
  describe SyncFileManager do
8
- let(:subject_object) { described_class.new(default_data_entry) }
9
+ let(:subject_object) { described_class.new(default_data_entry, source) }
9
10
 
10
- let(:default_mysqldump_dir) do
11
+ let(:default_dump_dir) do
11
12
  File.join('/tmp', "sync_dump_#{Time.now.to_i}")
12
13
  end
14
+ let(:default_dump_dir2) do
15
+ File.join('/tmp', "sync_dump2_#{Time.now.to_i}")
16
+ end
13
17
  let(:default_data_entry) do
14
18
  {"id"=>93,
15
19
  "name"=>"flydata_sync_mysql",
@@ -33,7 +37,7 @@ module Flydata
33
37
  "mysql_data_entry_preference" =>
34
38
  { "host"=>"localhost", "port"=>3306, "username"=>"masashi",
35
39
  "password"=>"welcome", "database"=>"sync_test", "tables"=>nil,
36
- "mysqldump_dir"=>default_mysqldump_dir, "forwarder" => "tcpforwarder",
40
+ "dump_dir"=>default_dump_dir, "forwarder" => "tcpforwarder",
37
41
  "data_servers"=>"localhost:9905" }
38
42
  }
39
43
  end
@@ -42,156 +46,188 @@ module Flydata
42
46
  let (:table_name) { 'test_table' }
43
47
  let (:table_name2) { 'test_table2' }
44
48
  let (:last_pos) { 9999 }
45
- let (:binlog_pos) { {binfile: 'mysqlbin.00001', pos: 111} }
46
49
  let (:state) { 'CREATE_TABLE' }
47
50
  let (:substate) { nil }
48
51
 
52
+ let(:source_pos_str) { "a position\t\tcould include many tabs\t456" }
53
+ let(:source_pos_org) { double ('source_pos_org') }
54
+ let(:source_pos_obj) { double('source_pos_obj') }
55
+ let(:source) { double('source') }
56
+ let(:context) { double('context') }
57
+ let(:columns_hash) do
58
+ { 'id' => { column_name: 'id', format_type: 'int' },
59
+ 'value' => { column_name: 'value', format_type: 'text' } }
60
+ end
61
+ let(:dump_contents_hash) do
62
+ { status: status, table_name: table_name, last_pos: last_pos,
63
+ source_pos: source_pos_obj, state: state, substate: substate }
64
+ end
65
+
49
66
  after :each do
50
- if Dir.exists?(default_mysqldump_dir)
51
- Dir.delete(default_mysqldump_dir) rescue nil
67
+ if Dir.exists?(default_dump_dir)
68
+ Dir.delete(default_dump_dir) rescue nil
52
69
  end
53
- if File.exists?(default_mysqldump_dir)
54
- File.delete(default_mysqldump_dir) rescue nil
70
+ if File.exists?(default_dump_dir)
71
+ File.delete(default_dump_dir) rescue nil
55
72
  end
56
73
  end
57
74
 
58
75
  describe '#dump_file_path' do
59
- context 'when mysqldump_dir param is nil' do
76
+ subject { subject_object.dump_file_path }
77
+ context 'when dump_dir param is nil' do
60
78
  before do
61
- default_data_entry['mysql_data_entry_preference'].delete('mysqldump_dir')
79
+ default_data_entry['mysql_data_entry_preference'].delete('dump_dir')
62
80
  end
63
81
  it do
64
82
  stub_const('Flydata::FileUtil::SyncFileManager::DUMP_DIR', File.join(Flydata::FLYDATA_HOME, 'dump'))
65
- expect(subject_object.dump_file_path).to eq(
66
- File.join(Flydata::FLYDATA_HOME, 'dump', 'flydata_sync_mysql.dump'))
83
+ is_expected.to eq(File.join(Flydata::FLYDATA_HOME, 'dump', 'flydata_sync_mysql.dump'))
67
84
  end
68
85
  end
69
86
  context 'when file exists' do
70
- before { `touch #{default_mysqldump_dir}`}
87
+ before { `touch #{default_dump_dir}`}
71
88
  it do
72
- expect{subject_object.dump_file_path}.to raise_error
89
+ expect{ subject }.to raise_error
73
90
  end
74
91
  end
75
92
  context 'when directory exists' do
76
- before { `mkdir -p #{default_mysqldump_dir}`}
93
+ before { `mkdir -p #{default_dump_dir}`}
77
94
  it do
78
- expect(FileUtils).to receive(:mkdir_p).with(default_mysqldump_dir).never
79
- expect(subject_object.dump_file_path).to eq(
80
- File.join(default_mysqldump_dir, 'flydata_sync_mysql.dump'))
95
+ expect(FileUtils).to receive(:mkdir_p).with(default_dump_dir).never
96
+ is_expected.to eq(File.join(default_dump_dir, 'flydata_sync_mysql.dump'))
81
97
  end
82
98
  end
83
99
  context 'when directory or file does not exist' do
84
100
  it do
85
- expect(FileUtils).to receive(:mkdir_p).with(default_mysqldump_dir).once
86
- expect(subject_object.dump_file_path).to eq(
87
- File.join(default_mysqldump_dir, 'flydata_sync_mysql.dump'))
101
+ expect(FileUtils).to receive(:mkdir_p).with(default_dump_dir).once
102
+ is_expected.to eq(File.join(default_dump_dir, 'flydata_sync_mysql.dump'))
88
103
  end
89
104
  end
90
105
  context 'when file name includes "~"' do
91
- let(:default_mysqldump_dir) { "~/tmp/dump/sync_spec_#{Time.now.to_i}" }
106
+ let(:default_dump_dir) { "~/tmp/dump/sync_spec_#{Time.now.to_i}" }
92
107
  it do
93
- expected_dir = File.join(ENV['HOME'], default_mysqldump_dir[1..-1])
108
+ expected_dir = File.join(ENV['HOME'], default_dump_dir[1..-1])
94
109
  expect(FileUtils).to receive(:mkdir_p).with(expected_dir).once
95
- expect(subject_object.dump_file_path).to eq(
96
- File.join(expected_dir, 'flydata_sync_mysql.dump'))
110
+ is_expected.to eq(File.join(expected_dir, 'flydata_sync_mysql.dump'))
97
111
  end
98
112
  end
99
113
  end
100
114
 
101
115
  describe '#dump_pos_path' do
102
- it { expect(subject_object.dump_pos_path).to eq(
103
- File.join(default_mysqldump_dir, 'flydata_sync_mysql.dump.pos')) }
116
+ subject { subject_object.dump_pos_path }
117
+ it { is_expected.to eq(File.join(default_dump_dir, 'flydata_sync_mysql.dump.pos')) }
104
118
  end
105
119
 
106
120
  describe '#save_dump_pos' do
107
- context 'without mysql marshal data' do
121
+ subject { subject_object.save_dump_pos(status, table_name, last_pos, source_pos_org,
122
+ state, substate) }
123
+ context 'without source marshal data' do
108
124
  it do
109
- expect{subject_object.save_dump_pos(
110
- status, table_name, last_pos, binlog_pos, state, substate)}.not_to raise_error
111
- expect(subject_object.load_dump_pos).to eq({
112
- status: status, table_name: table_name, last_pos: last_pos,
113
- binlog_pos: binlog_pos, state: state, substate: substate,
114
- mysql_table: nil
115
- })
125
+ expect(source_pos_org).to receive(:to_s).and_return(source_pos_str)
126
+ expect{ subject }.not_to raise_error
127
+
128
+ expect(source).to receive(:source_pos).and_return(context)
129
+ expect(context).to receive(:create_source_pos).with(source_pos_str).
130
+ and_return(source_pos_obj)
131
+
132
+ expect(subject_object.load_dump_pos).to eq( dump_contents_hash.merge( { source_table: nil } ) )
116
133
  end
117
134
  end
118
135
  end
119
136
 
120
137
  describe '#load_dump_pos' do
121
- let (:mysql_table) do
122
- Flydata::Parser::Mysql::MysqlTable.new(
123
- table_name, { 'id' => { column_name: 'id', format_type: 'int' }, 'value' => { column_name: 'value', format_type: 'text' } }
124
- )
125
- end
138
+ subject { subject_object.load_dump_pos }
139
+
140
+ context 'with source marshal data' do
141
+ let(:source_table_obj) { double('source_table_obj') }
126
142
 
127
- context 'with mysql marshal data' do
143
+ let(:source_table) { Flydata::Parser::SourceTable.new( table_name, columns_hash) }
128
144
  before do
129
- subject_object.save_mysql_table_marshal_dump(mysql_table)
130
- subject_object.save_dump_pos(status, table_name, last_pos, binlog_pos, state, substate)
145
+ allow(source_pos_org).to receive(:to_s).and_return(source_pos_str)
146
+ subject_object.save_source_table_marshal_dump(source_table)
147
+ subject_object.save_dump_pos(status, table_name, last_pos, source_pos_org, state, substate)
131
148
  end
132
149
  it do
133
- ret = subject_object.load_dump_pos
134
- mt = ret.delete(:mysql_table)
135
- expect(ret).to eq({
136
- status: status, table_name: table_name, last_pos: last_pos,
137
- binlog_pos: binlog_pos, state: state, substate: substate,
138
- })
139
- expect(mt.table_name).to eq(table_name)
140
- expect(mt.columns).to eq({
141
- 'id' => { column_name: 'id', format_type: 'int' },
142
- 'value' => { column_name: 'value', format_type: 'text' },
143
- })
150
+ expect(source).to receive(:source_pos).and_return(context)
151
+ expect(context).to receive(:create_source_pos).with(source_pos_str).and_return(source_pos_obj)
152
+ expect(subject_object).to receive(:load_source_table_marshal_dump).and_return(source_table_obj)
153
+
154
+ is_expected.to eq( dump_contents_hash.merge({source_table: source_table_obj}) )
144
155
  end
145
156
  end
146
157
  end
147
158
 
148
- describe '#save_binlog' do
149
- let(:binfile) { 'mysqlbinlog.000001' }
150
- let(:pos) { 107 }
151
- let(:binlog_pos) { {binfile: binfile, pos: pos} }
159
+ describe '#load_source_table_marshal_dump' do
160
+ subject { subject_object.send(:load_source_table_marshal_dump) }
161
+
162
+ let(:source_table) { Flydata::Parser::SourceTable.new( table_name, columns_hash) }
163
+ before do
164
+ subject_object.save_source_table_marshal_dump(source_table)
165
+ end
152
166
  it do
153
- subject_object.save_binlog(binlog_pos)
154
- expect(`cat #{subject_object.binlog_path}`).to eq("#{binfile}\t#{pos}")
167
+ expect(subject.table_name).to eq(table_name)
168
+ expect(subject.columns).to eq(columns_hash)
155
169
  end
156
170
  end
157
171
 
158
- describe '#load_binlog' do
159
- let(:binfile) { 'mysqlbinlog.000001' }
160
- let(:pos) { 107 }
161
- let(:binlog_pos) { {binfile: binfile, pos: pos} }
172
+ describe '#save_source_pos' do
173
+ subject { subject_object.save_source_pos(source_pos) }
162
174
 
163
- subject { subject_object.load_binlog }
175
+ let(:source_pos) { double('source_pos') }
176
+ let(:source_pos_str) { "a position" }
177
+ it do
178
+ expect(source_pos).to receive(:to_s).and_return(source_pos_str)
179
+ subject
180
+ expect(`cat #{subject_object.source_pos_path}`).to eq(source_pos_str)
181
+ end
182
+ end
183
+
184
+ describe '#load_source_pos' do
185
+ subject { subject_object.load_source_pos }
164
186
 
165
- context 'when binlog pos does not exist' do
187
+ before do
188
+ allow(subject_object).to receive(:source).and_return(source)
189
+ end
190
+
191
+ context 'when source_pos file does not exist' do
166
192
  before do
167
- File.delete(subject_object.binlog_path) if File.exist?(subject_object.binlog_path)
193
+ File.delete(subject_object.source_pos_path) if File.exist?(subject_object.source_pos_path)
168
194
  end
169
195
  it { is_expected.to be_nil }
170
196
  end
171
197
 
172
- context 'when binlog pos exists' do
198
+ context 'when source_pos file exists' do
173
199
  before do
174
- subject_object.save_binlog(binlog_pos)
200
+ File.open(subject_object.source_pos_path, "w"){|f| f.write(source_pos_str)}
201
+ end
202
+ after do
203
+ File.delete(subject_object.source_pos_path)
204
+ end
205
+ it do
206
+ expect(source).to receive(:source_pos).and_return(context)
207
+ expect(context).to receive(:create_source_pos).with(source_pos_str).
208
+ and_return(source_pos_obj)
209
+
210
+ is_expected.to eq source_pos_obj
175
211
  end
176
- it { is_expected.to eq(binlog_pos) }
177
212
  end
178
213
  end
179
214
 
180
- describe '#binlog_path' do
181
- it { expect(subject_object.binlog_path).to eq("#{FLYDATA_HOME}/flydata_sync_mysql.binlog.pos") }
215
+ describe '#source_pos_path' do
216
+ subject { subject_object.source_pos_path }
217
+ it { is_expected.to eq("#{FLYDATA_HOME}/flydata_sync_mysql.binlog.pos") }
182
218
  end
183
219
 
184
- describe '#sent_binlog_path' do
220
+ describe '#sent_source_pos_path' do
185
221
  context 'with no args' do
186
- subject { subject_object.sent_binlog_path }
222
+ subject { subject_object.sent_source_pos_path }
187
223
  it { is_expected.to eq("#{FLYDATA_HOME}/flydata_sync_mysql.binlog.sent.pos") }
188
224
  end
189
225
  context 'with invalid args' do
190
- subject { subject_object.sent_binlog_path('/home/ec2-user/.flydata/flydata_sync.pos') }
226
+ subject { subject_object.sent_source_pos_path('/home/ec2-user/.flydata/flydata_sync.pos') }
191
227
  it { expect{subject}.to raise_error(ArgumentError) }
192
228
  end
193
229
  context 'with valid args' do
194
- subject { subject_object.sent_binlog_path('/home/ec2-user/.flydata/flydata_sync.binlog.pos') }
230
+ subject { subject_object.sent_source_pos_path('/home/ec2-user/.flydata/flydata_sync.binlog.pos') }
195
231
  it { is_expected.to eq('/home/ec2-user/.flydata/flydata_sync.binlog.sent.pos') }
196
232
  end
197
233
  end
@@ -329,5 +365,49 @@ module Flydata
329
365
  end
330
366
  end
331
367
  end
368
+ describe '#dump_dir' do
369
+ subject { subject_object.send(:dump_dir) }
370
+
371
+ before do
372
+ default_data_entry["mysql_data_entry_preference"].delete("dump_dir")
373
+ end
374
+
375
+ context 'when the new property is defined' do
376
+ before do
377
+ default_data_entry["mysql_data_entry_preference"]["dump_dir"] =
378
+ default_dump_dir
379
+ end
380
+ context 'when the old property is defined' do
381
+ before do
382
+ default_data_entry["mysql_data_entry_preference"]["mysqldump_dir"] =
383
+ default_dump_dir2
384
+ end
385
+ it 'uses the dir specified by the new property' do
386
+ is_expected.to eq default_dump_dir
387
+ end
388
+ end
389
+ context 'when the old property is not defined' do
390
+ it 'uses the dir specified by the new property' do
391
+ is_expected.to eq default_dump_dir
392
+ end
393
+ end
394
+ end
395
+ context 'when the new property is not defined' do
396
+ context 'when the old property is defined' do
397
+ before do
398
+ default_data_entry["mysql_data_entry_preference"]["mysqldump_dir"] =
399
+ default_dump_dir2
400
+ end
401
+ it 'uses the dir specified by the old property' do
402
+ is_expected.to eq default_dump_dir2
403
+ end
404
+ end
405
+ context 'when the old property is not defined' do
406
+ it 'uses the default dir' do
407
+ is_expected.to eq File.join(FLYDATA_HOME, "dump")
408
+ end
409
+ end
410
+ end
411
+ end
332
412
  end
333
413
  end