flydata 0.6.3 → 0.6.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Rakefile +2 -2
- data/VERSION +1 -1
- data/bin/fdredshift +78 -0
- data/circle.yml +1 -1
- data/ext/flydata/{parser/mysql → source_mysql/parser}/.gitignore +0 -0
- data/ext/flydata/{parser/mysql → source_mysql/parser}/dump_parser_ext.cpp +3 -3
- data/ext/flydata/source_mysql/parser/extconf.rb +3 -0
- data/ext/flydata/{parser/mysql → source_mysql/parser}/parser.txt +0 -0
- data/ext/flydata/{parser/mysql → source_mysql/parser}/sql_parser.cpp +0 -0
- data/ext/flydata/{parser/mysql → source_mysql/parser}/sql_parser.h +0 -0
- data/flydata-core/lib/flydata-core/mysql/binlog_pos.rb +34 -32
- data/flydata-core/lib/flydata-core/mysql/compatibility_checker.rb +20 -0
- data/flydata-core/lib/flydata-core/table_def/mysql_table_def.rb +12 -4
- data/flydata-core/lib/flydata-core/table_def/redshift_table_def.rb +60 -6
- data/flydata-core/spec/mysql/binlog_pos_spec.rb +474 -0
- data/flydata-core/spec/table_def/mysql_table_def_spec.rb +57 -0
- data/flydata-core/spec/table_def/mysql_to_redshift_table_def_spec.rb +174 -20
- data/flydata-core/spec/table_def/mysqldump_test_col_comment_with_AUTO_INCREMENT_keyword.dump +43 -0
- data/flydata-core/spec/table_def/mysqldump_test_col_comment_with_not_null_keyword.dump +43 -0
- data/flydata-core/spec/table_def/mysqldump_test_col_comment_with_unique_keyword.dump +43 -0
- data/flydata-core/spec/table_def/mysqldump_test_col_comment_with_unsigned_keyword.dump +43 -0
- data/flydata-core/spec/table_def/redshift_table_def_spec.rb +41 -8
- data/flydata.gemspec +0 -0
- data/lib/flydata/cli.rb +11 -5
- data/lib/flydata/command/base.rb +14 -1
- data/lib/flydata/command/exclusive_runnable.rb +42 -12
- data/lib/flydata/command/helper.rb +6 -6
- data/lib/flydata/command/sender.rb +4 -3
- data/lib/flydata/command/setup.rb +30 -381
- data/lib/flydata/command/stop.rb +1 -0
- data/lib/flydata/command/sync.rb +273 -301
- data/lib/flydata/compatibility_check.rb +24 -117
- data/lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb +3 -3
- data/lib/flydata/fluent-plugins/mysql/alter_table_query_handler.rb +2 -2
- data/lib/flydata/fluent-plugins/mysql/binlog_record_handler.rb +6 -6
- data/lib/flydata/fluent-plugins/mysql/truncate_table_query_handler.rb +0 -1
- data/lib/flydata/parser.rb +14 -0
- data/lib/flydata/{parser_provider.rb → parser/parser_provider.rb} +6 -4
- data/lib/flydata/parser/source_table.rb +33 -0
- data/lib/flydata/source.rb +105 -0
- data/lib/flydata/source/component.rb +21 -0
- data/lib/flydata/source/errors.rb +7 -0
- data/lib/flydata/source/generate_source_dump.rb +72 -0
- data/lib/flydata/source/parse_dump_and_send.rb +52 -0
- data/lib/flydata/source/setup.rb +31 -0
- data/lib/flydata/source/source_pos.rb +45 -0
- data/lib/flydata/source/sync.rb +56 -0
- data/lib/flydata/source/sync_generate_table_ddl.rb +43 -0
- data/lib/flydata/source_file/setup.rb +17 -0
- data/lib/flydata/source_file/sync.rb +14 -0
- data/lib/flydata/{command → source_mysql/command}/mysql.rb +2 -1
- data/lib/flydata/{command → source_mysql/command}/mysql_command_base.rb +2 -4
- data/lib/flydata/{command → source_mysql/command}/mysqlbinlog.rb +2 -1
- data/lib/flydata/{command → source_mysql/command}/mysqldump.rb +2 -1
- data/lib/flydata/source_mysql/generate_source_dump.rb +53 -0
- data/lib/flydata/source_mysql/mysql_compatibility_check.rb +114 -0
- data/lib/flydata/source_mysql/parse_dump_and_send.rb +28 -0
- data/lib/flydata/{parser/mysql → source_mysql/parser}/.gitignore +0 -0
- data/lib/flydata/{parser/mysql → source_mysql/parser}/dump_parser.rb +32 -67
- data/lib/flydata/{parser/mysql → source_mysql/parser}/mysql_alter_table.treetop +0 -0
- data/lib/flydata/source_mysql/setup.rb +24 -0
- data/lib/flydata/source_mysql/source_pos.rb +21 -0
- data/lib/flydata/source_mysql/sync.rb +45 -0
- data/lib/flydata/source_mysql/sync_generate_table_ddl.rb +40 -0
- data/lib/flydata/{mysql → source_mysql}/table_ddl.rb +6 -17
- data/lib/flydata/source_zendesk/sync_generate_table_ddl.rb +30 -0
- data/lib/flydata/source_zendesk/zendesk_flydata_tabledefs.rb +133 -0
- data/lib/flydata/sync_file_manager.rb +132 -73
- data/lib/flydata/table_ddl.rb +18 -0
- data/spec/flydata/cli_spec.rb +1 -0
- data/spec/flydata/command/exclusive_runnable_spec.rb +19 -8
- data/spec/flydata/command/sender_spec.rb +1 -1
- data/spec/flydata/command/setup_spec.rb +4 -4
- data/spec/flydata/command/sync_spec.rb +97 -134
- data/spec/flydata/compatibility_check_spec.rb +16 -289
- data/spec/flydata/fluent-plugins/mysql/alter_table_query_handler_spec.rb +3 -3
- data/spec/flydata/fluent-plugins/mysql/dml_record_handler_spec.rb +1 -1
- data/spec/flydata/fluent-plugins/mysql/shared_query_handler_context.rb +4 -2
- data/spec/flydata/fluent-plugins/mysql/truncate_query_handler_spec.rb +1 -1
- data/spec/flydata/source_mysql/generate_source_dump_spec.rb +69 -0
- data/spec/flydata/source_mysql/mysql_compatibility_check_spec.rb +280 -0
- data/spec/flydata/{parser/mysql → source_mysql/parser}/alter_table_parser_spec.rb +2 -2
- data/spec/flydata/{parser/mysql → source_mysql/parser}/dump_parser_spec.rb +75 -70
- data/spec/flydata/source_mysql/sync_generate_table_ddl_spec.rb +137 -0
- data/spec/flydata/{mysql → source_mysql}/table_ddl_spec.rb +2 -2
- data/spec/flydata/source_spec.rb +140 -0
- data/spec/flydata/source_zendesk/sync_generate_table_ddl_spec.rb +33 -0
- data/spec/flydata/sync_file_manager_spec.rb +157 -77
- data/tmpl/redshift_mysql_data_entry.conf.tmpl +1 -1
- metadata +56 -23
- data/ext/flydata/parser/mysql/extconf.rb +0 -3
- data/lib/flydata/mysql/binlog_position.rb +0 -22
- data/spec/flydata/mysql/binlog_position_spec.rb +0 -35
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
require 'flydata/source_mysql/sync_generate_table_ddl'
|
|
2
|
+
|
|
3
|
+
module Flydata
|
|
4
|
+
module SourceMysql
|
|
5
|
+
|
|
6
|
+
describe SyncGenerateTableDdl do
|
|
7
|
+
let(:subject_object) { described_class.new(source, dp, options) }
|
|
8
|
+
|
|
9
|
+
let(:dp) { double('dp') }
|
|
10
|
+
let(:de) { {
|
|
11
|
+
'mysql_data_entry_preference' => {
|
|
12
|
+
'host' => 'localhost',
|
|
13
|
+
'port' => 3306,
|
|
14
|
+
'username' => 'masashi',
|
|
15
|
+
'password' => 'welcome',
|
|
16
|
+
'database' => 'sync_test',
|
|
17
|
+
}
|
|
18
|
+
} }
|
|
19
|
+
let(:options) { {} }
|
|
20
|
+
|
|
21
|
+
let(:source) { double('source') }
|
|
22
|
+
before do
|
|
23
|
+
allow(source).to receive(:de).and_return de
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
describe '#run_compatibility_check' do
|
|
27
|
+
subject { subject_object.run_compatibility_check }
|
|
28
|
+
|
|
29
|
+
let(:checker) { double("checker") }
|
|
30
|
+
it do
|
|
31
|
+
expect(MysqlCompatibilityCheck).to receive(:new).and_return checker
|
|
32
|
+
expect(checker).to receive(:check)
|
|
33
|
+
|
|
34
|
+
subject
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
describe '#generate_flydata_tabledef' do
|
|
38
|
+
subject { subject_object.generate_flydata_tabledef(tables, options) }
|
|
39
|
+
|
|
40
|
+
let(:tables) { %w| table1 table2 table4 table3 | }
|
|
41
|
+
|
|
42
|
+
context 'with full options' do
|
|
43
|
+
it 'issues mysqldump command with expected parameters' do
|
|
44
|
+
expect(Open3).to receive(:popen3).with(
|
|
45
|
+
'mysqldump -h localhost -P 3306 -umasashi -pwelcome --default-character-set=utf8 --protocol=tcp -d sync_test table1 table2 table4 table3')
|
|
46
|
+
|
|
47
|
+
subject
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
context 'without_host' do
|
|
51
|
+
before do
|
|
52
|
+
de['mysql_data_entry_preference'].delete('host')
|
|
53
|
+
end
|
|
54
|
+
it { expect { subject }.to raise_error }
|
|
55
|
+
end
|
|
56
|
+
context 'with empty host' do
|
|
57
|
+
before do
|
|
58
|
+
de['mysql_data_entry_preference']['host'] = ""
|
|
59
|
+
end
|
|
60
|
+
it { expect { subject }.to raise_error }
|
|
61
|
+
end
|
|
62
|
+
context 'without port' do
|
|
63
|
+
before do
|
|
64
|
+
de['mysql_data_entry_preference'].delete('port')
|
|
65
|
+
end
|
|
66
|
+
it "uses the default port" do
|
|
67
|
+
expect(Open3).to receive(:popen3).with(
|
|
68
|
+
'mysqldump -h localhost -umasashi -pwelcome --default-character-set=utf8 --protocol=tcp -d sync_test table1 table2 table4 table3')
|
|
69
|
+
subject
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
context 'with port override' do
|
|
73
|
+
before do
|
|
74
|
+
de['mysql_data_entry_preference']['port'] = 1234
|
|
75
|
+
end
|
|
76
|
+
it "uses the specified port" do
|
|
77
|
+
expect(Open3).to receive(:popen3).with(
|
|
78
|
+
'mysqldump -h localhost -P 1234 -umasashi -pwelcome --default-character-set=utf8 --protocol=tcp -d sync_test table1 table2 table4 table3')
|
|
79
|
+
subject
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
context 'without username' do
|
|
83
|
+
before do
|
|
84
|
+
de['mysql_data_entry_preference'].delete('username')
|
|
85
|
+
end
|
|
86
|
+
it { expect { subject }.to raise_error }
|
|
87
|
+
end
|
|
88
|
+
context 'with empty username' do
|
|
89
|
+
before do
|
|
90
|
+
de['mysql_data_entry_preference']['username'] = ""
|
|
91
|
+
end
|
|
92
|
+
it { expect { subject }.to raise_error }
|
|
93
|
+
end
|
|
94
|
+
context 'without_password' do
|
|
95
|
+
before do
|
|
96
|
+
de['mysql_data_entry_preference'].delete('password')
|
|
97
|
+
end
|
|
98
|
+
it "call mysqldump without MYSQL_PW set" do
|
|
99
|
+
expect(Open3).to receive(:popen3).with(
|
|
100
|
+
'mysqldump -h localhost -P 3306 -umasashi --default-character-set=utf8 --protocol=tcp -d sync_test table1 table2 table4 table3')
|
|
101
|
+
subject
|
|
102
|
+
end
|
|
103
|
+
end
|
|
104
|
+
context 'with password containing symbols' do
|
|
105
|
+
before do
|
|
106
|
+
de['mysql_data_entry_preference']['password']="welcome&!@^@#^"
|
|
107
|
+
end
|
|
108
|
+
it "call mysqldump with MYSQL_PW set with correct symbols" do
|
|
109
|
+
expect(Open3).to receive(:popen3).with(
|
|
110
|
+
'mysqldump -h localhost -P 3306 -umasashi -pwelcome\\&\\!@\\^@\\#\\^ --default-character-set=utf8 --protocol=tcp -d sync_test table1 table2 table4 table3')
|
|
111
|
+
subject
|
|
112
|
+
end
|
|
113
|
+
end
|
|
114
|
+
context 'without_database' do
|
|
115
|
+
before do
|
|
116
|
+
de['mysql_data_entry_preference'].delete('database')
|
|
117
|
+
end
|
|
118
|
+
it { expect { subject }.to raise_error }
|
|
119
|
+
end
|
|
120
|
+
context 'with empty database' do
|
|
121
|
+
before do
|
|
122
|
+
de['mysql_data_entry_preference']['database'] = ""
|
|
123
|
+
end
|
|
124
|
+
it { expect { subject }.to raise_error }
|
|
125
|
+
end
|
|
126
|
+
context 'with empty tables' do
|
|
127
|
+
before do
|
|
128
|
+
de['mysql_data_entry_preference']['tables'] = []
|
|
129
|
+
end
|
|
130
|
+
it { expect { subject }.to raise_error }
|
|
131
|
+
end
|
|
132
|
+
end
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
end
|
|
137
|
+
end
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
require 'flydata/source'
|
|
2
|
+
|
|
3
|
+
module Flydata
|
|
4
|
+
|
|
5
|
+
describe Source do
|
|
6
|
+
let(:subject_object) { described_class.create(de) }
|
|
7
|
+
|
|
8
|
+
let(:target_class) { double('target_class') }
|
|
9
|
+
let(:type_class) { double('type_class') }
|
|
10
|
+
|
|
11
|
+
let(:expected_instance) { double('expected_instance') }
|
|
12
|
+
let(:de) { double('de') }
|
|
13
|
+
let(:opts) { double('opts') }
|
|
14
|
+
let(:class_name) { "replace this" }
|
|
15
|
+
let(:type_class_name) { "replace this" }
|
|
16
|
+
let(:data_entry_type) { "RedshiftMysqlDataEntry" }
|
|
17
|
+
|
|
18
|
+
before do
|
|
19
|
+
allow(target_class).to receive(:name).and_return(class_name)
|
|
20
|
+
allow(type_class).to receive(:name).and_return(type_class_name)
|
|
21
|
+
end
|
|
22
|
+
before do
|
|
23
|
+
allow(de).to receive(:[]).with('type').and_return(data_entry_type)
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
after do
|
|
27
|
+
if described_class.instance_variable_defined? :@component_classes
|
|
28
|
+
described_class.remove_instance_variable :@component_classes
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
describe '#method_missing' do
|
|
33
|
+
subject { subject_object.send(method_name, opts) }
|
|
34
|
+
|
|
35
|
+
before do
|
|
36
|
+
described_class.register(target_class, type_class)
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
context 'when the target data source exists' do
|
|
40
|
+
let(:data_entry_type) { "RedshiftMysqlDataEntry" }
|
|
41
|
+
let(:class_name) { "Flydata::SourceMysql::MySR" }
|
|
42
|
+
|
|
43
|
+
context 'when the target component exists' do
|
|
44
|
+
let(:method_name) { :sync_reset }
|
|
45
|
+
let(:type_class_name) { "Flydata::Source::SyncReset" }
|
|
46
|
+
|
|
47
|
+
it "returns an instance of the target_class" do
|
|
48
|
+
expect(target_class).to receive(:new).with(subject_object, opts).
|
|
49
|
+
and_return expected_instance
|
|
50
|
+
|
|
51
|
+
expect(subject).to eq expected_instance
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
context 'when the target component does not exist' do
|
|
55
|
+
let(:method_name) { :sync_start }
|
|
56
|
+
let(:type_class_name) { "Flydata::Source::SyncReset" }
|
|
57
|
+
|
|
58
|
+
it "raises an error" do
|
|
59
|
+
expect{ subject }.to raise_error /Component '.*?' is not defined/
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
context 'when the target data source does not exist' do
|
|
64
|
+
let(:data_entry_type) { "FooBarDataEntry" }
|
|
65
|
+
let(:class_name) { "Flydata::SourceFooBar::MySR" }
|
|
66
|
+
|
|
67
|
+
context 'when the target component exists' do
|
|
68
|
+
let(:method_name) { :sync_reset }
|
|
69
|
+
let(:type_class_name) { "Flydata::Source::SyncReset" }
|
|
70
|
+
|
|
71
|
+
it "raises an error" do
|
|
72
|
+
expect{ subject }.to raise_error /No source components are available/
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
describe '.underscored_source_module_sym' do
|
|
79
|
+
subject { described_class.send(:underscored_source_module_sym, target_class) }
|
|
80
|
+
|
|
81
|
+
context 'with a standard name' do
|
|
82
|
+
let(:class_name) { "Flydata::SourceMysql::SyncReset" }
|
|
83
|
+
it { is_expected.to eq :source_mysql }
|
|
84
|
+
end
|
|
85
|
+
context 'with a nested path' do
|
|
86
|
+
let(:class_name) { "Flydata::SourceMysql::Command::SyncReset" }
|
|
87
|
+
it { is_expected.to eq :source_mysql }
|
|
88
|
+
end
|
|
89
|
+
context 'with a nested path (not recommended)' do
|
|
90
|
+
let(:class_name) { "Flydata::Foo::Bar::SourceMysql::SyncReset" }
|
|
91
|
+
it { is_expected.to eq :source_mysql }
|
|
92
|
+
end
|
|
93
|
+
context 'when none of elements starts with "Source"' do
|
|
94
|
+
let(:class_name) { "Flydata::Foo::Bar::SyncReset" }
|
|
95
|
+
it { is_expected.to eq nil }
|
|
96
|
+
end
|
|
97
|
+
context 'when more than one element start with "Source"' do
|
|
98
|
+
let(:class_name) { "Flydata::SourceMysql::SourceFiles::SyncReset" }
|
|
99
|
+
it "uses the first one" do
|
|
100
|
+
is_expected.to eq :source_mysql
|
|
101
|
+
end
|
|
102
|
+
end
|
|
103
|
+
end
|
|
104
|
+
describe '.command_dir_paths' do
|
|
105
|
+
subject { described_class.command_dir_paths }
|
|
106
|
+
|
|
107
|
+
before do
|
|
108
|
+
allow(File).to receive(:exists?).and_return false
|
|
109
|
+
end
|
|
110
|
+
shared_examples 'returning the expected result' do
|
|
111
|
+
before do
|
|
112
|
+
expect(File).to receive(:exists?).with(/source_mysql\/command/).
|
|
113
|
+
and_return source_mysql_command_dir_exists
|
|
114
|
+
expect(File).to receive(:exists?).with(/source_file\/command/).
|
|
115
|
+
and_return source_file_command_dir_exists
|
|
116
|
+
end
|
|
117
|
+
it { is_expected.to eq expected_result }
|
|
118
|
+
end
|
|
119
|
+
let(:source_file_command_dir_exists) { false }
|
|
120
|
+
|
|
121
|
+
context 'with no source dir' do
|
|
122
|
+
let(:source_mysql_command_dir_exists) { false }
|
|
123
|
+
let(:expected_result) { [] }
|
|
124
|
+
it_behaves_like 'returning the expected result'
|
|
125
|
+
end
|
|
126
|
+
context 'with command dir' do
|
|
127
|
+
let(:source_mysql_command_dir_exists) { true }
|
|
128
|
+
let(:expected_result) { ["flydata/source_mysql/command"] }
|
|
129
|
+
it_behaves_like 'returning the expected result'
|
|
130
|
+
end
|
|
131
|
+
context 'with two source dirs with command dir' do
|
|
132
|
+
let(:source_mysql_command_dir_exists) { true }
|
|
133
|
+
let(:source_file_command_dir_exists) { true }
|
|
134
|
+
let(:expected_result) { %w(flydata/source_mysql/command flydata/source_file/command) }
|
|
135
|
+
it_behaves_like 'returning the expected result'
|
|
136
|
+
end
|
|
137
|
+
end
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
end
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
require 'flydata/source_zendesk/sync_generate_table_ddl'
|
|
2
|
+
|
|
3
|
+
module Flydata
|
|
4
|
+
module SourceZendesk
|
|
5
|
+
|
|
6
|
+
describe SyncGenerateTableDdl do
|
|
7
|
+
let(:subject_object) { described_class.new(source, dp, options) }
|
|
8
|
+
|
|
9
|
+
let(:source) { double('source') }
|
|
10
|
+
let(:dp) { double('dp') }
|
|
11
|
+
let(:options) { {} }
|
|
12
|
+
|
|
13
|
+
describe '#run_compatibility_check' do
|
|
14
|
+
subject { subject_object.run_compatibility_check }
|
|
15
|
+
it { subject }
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
describe '#generate_flydata_tabledef' do
|
|
19
|
+
subject { subject_object.generate_flydata_tabledef(tables, options) }
|
|
20
|
+
|
|
21
|
+
let(:supported_tables) { %w(tickets ticket_comments ticket_audits users organizations) }
|
|
22
|
+
let(:options) { double('options') }
|
|
23
|
+
|
|
24
|
+
context 'with all supported tables' do
|
|
25
|
+
let(:tables) { supported_tables }
|
|
26
|
+
it { subject }
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
end
|
|
33
|
+
end
|
|
@@ -1,15 +1,19 @@
|
|
|
1
1
|
# coding: utf-8
|
|
2
2
|
require 'spec_helper'
|
|
3
3
|
require 'flydata/sync_file_manager'
|
|
4
|
-
require 'flydata/parser/
|
|
4
|
+
require 'flydata/parser/source_table'
|
|
5
|
+
require 'flydata/command/base'
|
|
5
6
|
|
|
6
7
|
module Flydata
|
|
7
8
|
describe SyncFileManager do
|
|
8
|
-
let(:subject_object) { described_class.new(default_data_entry) }
|
|
9
|
+
let(:subject_object) { described_class.new(default_data_entry, source) }
|
|
9
10
|
|
|
10
|
-
let(:
|
|
11
|
+
let(:default_dump_dir) do
|
|
11
12
|
File.join('/tmp', "sync_dump_#{Time.now.to_i}")
|
|
12
13
|
end
|
|
14
|
+
let(:default_dump_dir2) do
|
|
15
|
+
File.join('/tmp', "sync_dump2_#{Time.now.to_i}")
|
|
16
|
+
end
|
|
13
17
|
let(:default_data_entry) do
|
|
14
18
|
{"id"=>93,
|
|
15
19
|
"name"=>"flydata_sync_mysql",
|
|
@@ -33,7 +37,7 @@ module Flydata
|
|
|
33
37
|
"mysql_data_entry_preference" =>
|
|
34
38
|
{ "host"=>"localhost", "port"=>3306, "username"=>"masashi",
|
|
35
39
|
"password"=>"welcome", "database"=>"sync_test", "tables"=>nil,
|
|
36
|
-
"
|
|
40
|
+
"dump_dir"=>default_dump_dir, "forwarder" => "tcpforwarder",
|
|
37
41
|
"data_servers"=>"localhost:9905" }
|
|
38
42
|
}
|
|
39
43
|
end
|
|
@@ -42,156 +46,188 @@ module Flydata
|
|
|
42
46
|
let (:table_name) { 'test_table' }
|
|
43
47
|
let (:table_name2) { 'test_table2' }
|
|
44
48
|
let (:last_pos) { 9999 }
|
|
45
|
-
let (:binlog_pos) { {binfile: 'mysqlbin.00001', pos: 111} }
|
|
46
49
|
let (:state) { 'CREATE_TABLE' }
|
|
47
50
|
let (:substate) { nil }
|
|
48
51
|
|
|
52
|
+
let(:source_pos_str) { "a position\t\tcould include many tabs\t456" }
|
|
53
|
+
let(:source_pos_org) { double ('source_pos_org') }
|
|
54
|
+
let(:source_pos_obj) { double('source_pos_obj') }
|
|
55
|
+
let(:source) { double('source') }
|
|
56
|
+
let(:context) { double('context') }
|
|
57
|
+
let(:columns_hash) do
|
|
58
|
+
{ 'id' => { column_name: 'id', format_type: 'int' },
|
|
59
|
+
'value' => { column_name: 'value', format_type: 'text' } }
|
|
60
|
+
end
|
|
61
|
+
let(:dump_contents_hash) do
|
|
62
|
+
{ status: status, table_name: table_name, last_pos: last_pos,
|
|
63
|
+
source_pos: source_pos_obj, state: state, substate: substate }
|
|
64
|
+
end
|
|
65
|
+
|
|
49
66
|
after :each do
|
|
50
|
-
if Dir.exists?(
|
|
51
|
-
Dir.delete(
|
|
67
|
+
if Dir.exists?(default_dump_dir)
|
|
68
|
+
Dir.delete(default_dump_dir) rescue nil
|
|
52
69
|
end
|
|
53
|
-
if File.exists?(
|
|
54
|
-
File.delete(
|
|
70
|
+
if File.exists?(default_dump_dir)
|
|
71
|
+
File.delete(default_dump_dir) rescue nil
|
|
55
72
|
end
|
|
56
73
|
end
|
|
57
74
|
|
|
58
75
|
describe '#dump_file_path' do
|
|
59
|
-
|
|
76
|
+
subject { subject_object.dump_file_path }
|
|
77
|
+
context 'when dump_dir param is nil' do
|
|
60
78
|
before do
|
|
61
|
-
default_data_entry['mysql_data_entry_preference'].delete('
|
|
79
|
+
default_data_entry['mysql_data_entry_preference'].delete('dump_dir')
|
|
62
80
|
end
|
|
63
81
|
it do
|
|
64
82
|
stub_const('Flydata::FileUtil::SyncFileManager::DUMP_DIR', File.join(Flydata::FLYDATA_HOME, 'dump'))
|
|
65
|
-
|
|
66
|
-
File.join(Flydata::FLYDATA_HOME, 'dump', 'flydata_sync_mysql.dump'))
|
|
83
|
+
is_expected.to eq(File.join(Flydata::FLYDATA_HOME, 'dump', 'flydata_sync_mysql.dump'))
|
|
67
84
|
end
|
|
68
85
|
end
|
|
69
86
|
context 'when file exists' do
|
|
70
|
-
before { `touch #{
|
|
87
|
+
before { `touch #{default_dump_dir}`}
|
|
71
88
|
it do
|
|
72
|
-
expect{
|
|
89
|
+
expect{ subject }.to raise_error
|
|
73
90
|
end
|
|
74
91
|
end
|
|
75
92
|
context 'when directory exists' do
|
|
76
|
-
before { `mkdir -p #{
|
|
93
|
+
before { `mkdir -p #{default_dump_dir}`}
|
|
77
94
|
it do
|
|
78
|
-
expect(FileUtils).to receive(:mkdir_p).with(
|
|
79
|
-
|
|
80
|
-
File.join(default_mysqldump_dir, 'flydata_sync_mysql.dump'))
|
|
95
|
+
expect(FileUtils).to receive(:mkdir_p).with(default_dump_dir).never
|
|
96
|
+
is_expected.to eq(File.join(default_dump_dir, 'flydata_sync_mysql.dump'))
|
|
81
97
|
end
|
|
82
98
|
end
|
|
83
99
|
context 'when directory or file does not exist' do
|
|
84
100
|
it do
|
|
85
|
-
expect(FileUtils).to receive(:mkdir_p).with(
|
|
86
|
-
|
|
87
|
-
File.join(default_mysqldump_dir, 'flydata_sync_mysql.dump'))
|
|
101
|
+
expect(FileUtils).to receive(:mkdir_p).with(default_dump_dir).once
|
|
102
|
+
is_expected.to eq(File.join(default_dump_dir, 'flydata_sync_mysql.dump'))
|
|
88
103
|
end
|
|
89
104
|
end
|
|
90
105
|
context 'when file name includes "~"' do
|
|
91
|
-
let(:
|
|
106
|
+
let(:default_dump_dir) { "~/tmp/dump/sync_spec_#{Time.now.to_i}" }
|
|
92
107
|
it do
|
|
93
|
-
expected_dir = File.join(ENV['HOME'],
|
|
108
|
+
expected_dir = File.join(ENV['HOME'], default_dump_dir[1..-1])
|
|
94
109
|
expect(FileUtils).to receive(:mkdir_p).with(expected_dir).once
|
|
95
|
-
|
|
96
|
-
File.join(expected_dir, 'flydata_sync_mysql.dump'))
|
|
110
|
+
is_expected.to eq(File.join(expected_dir, 'flydata_sync_mysql.dump'))
|
|
97
111
|
end
|
|
98
112
|
end
|
|
99
113
|
end
|
|
100
114
|
|
|
101
115
|
describe '#dump_pos_path' do
|
|
102
|
-
|
|
103
|
-
|
|
116
|
+
subject { subject_object.dump_pos_path }
|
|
117
|
+
it { is_expected.to eq(File.join(default_dump_dir, 'flydata_sync_mysql.dump.pos')) }
|
|
104
118
|
end
|
|
105
119
|
|
|
106
120
|
describe '#save_dump_pos' do
|
|
107
|
-
|
|
121
|
+
subject { subject_object.save_dump_pos(status, table_name, last_pos, source_pos_org,
|
|
122
|
+
state, substate) }
|
|
123
|
+
context 'without source marshal data' do
|
|
108
124
|
it do
|
|
109
|
-
expect
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
125
|
+
expect(source_pos_org).to receive(:to_s).and_return(source_pos_str)
|
|
126
|
+
expect{ subject }.not_to raise_error
|
|
127
|
+
|
|
128
|
+
expect(source).to receive(:source_pos).and_return(context)
|
|
129
|
+
expect(context).to receive(:create_source_pos).with(source_pos_str).
|
|
130
|
+
and_return(source_pos_obj)
|
|
131
|
+
|
|
132
|
+
expect(subject_object.load_dump_pos).to eq( dump_contents_hash.merge( { source_table: nil } ) )
|
|
116
133
|
end
|
|
117
134
|
end
|
|
118
135
|
end
|
|
119
136
|
|
|
120
137
|
describe '#load_dump_pos' do
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
)
|
|
125
|
-
end
|
|
138
|
+
subject { subject_object.load_dump_pos }
|
|
139
|
+
|
|
140
|
+
context 'with source marshal data' do
|
|
141
|
+
let(:source_table_obj) { double('source_table_obj') }
|
|
126
142
|
|
|
127
|
-
|
|
143
|
+
let(:source_table) { Flydata::Parser::SourceTable.new( table_name, columns_hash) }
|
|
128
144
|
before do
|
|
129
|
-
|
|
130
|
-
subject_object.
|
|
145
|
+
allow(source_pos_org).to receive(:to_s).and_return(source_pos_str)
|
|
146
|
+
subject_object.save_source_table_marshal_dump(source_table)
|
|
147
|
+
subject_object.save_dump_pos(status, table_name, last_pos, source_pos_org, state, substate)
|
|
131
148
|
end
|
|
132
149
|
it do
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
expect(
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
})
|
|
139
|
-
expect(mt.table_name).to eq(table_name)
|
|
140
|
-
expect(mt.columns).to eq({
|
|
141
|
-
'id' => { column_name: 'id', format_type: 'int' },
|
|
142
|
-
'value' => { column_name: 'value', format_type: 'text' },
|
|
143
|
-
})
|
|
150
|
+
expect(source).to receive(:source_pos).and_return(context)
|
|
151
|
+
expect(context).to receive(:create_source_pos).with(source_pos_str).and_return(source_pos_obj)
|
|
152
|
+
expect(subject_object).to receive(:load_source_table_marshal_dump).and_return(source_table_obj)
|
|
153
|
+
|
|
154
|
+
is_expected.to eq( dump_contents_hash.merge({source_table: source_table_obj}) )
|
|
144
155
|
end
|
|
145
156
|
end
|
|
146
157
|
end
|
|
147
158
|
|
|
148
|
-
describe '#
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
let(:
|
|
159
|
+
describe '#load_source_table_marshal_dump' do
|
|
160
|
+
subject { subject_object.send(:load_source_table_marshal_dump) }
|
|
161
|
+
|
|
162
|
+
let(:source_table) { Flydata::Parser::SourceTable.new( table_name, columns_hash) }
|
|
163
|
+
before do
|
|
164
|
+
subject_object.save_source_table_marshal_dump(source_table)
|
|
165
|
+
end
|
|
152
166
|
it do
|
|
153
|
-
|
|
154
|
-
expect(
|
|
167
|
+
expect(subject.table_name).to eq(table_name)
|
|
168
|
+
expect(subject.columns).to eq(columns_hash)
|
|
155
169
|
end
|
|
156
170
|
end
|
|
157
171
|
|
|
158
|
-
describe '#
|
|
159
|
-
|
|
160
|
-
let(:pos) { 107 }
|
|
161
|
-
let(:binlog_pos) { {binfile: binfile, pos: pos} }
|
|
172
|
+
describe '#save_source_pos' do
|
|
173
|
+
subject { subject_object.save_source_pos(source_pos) }
|
|
162
174
|
|
|
163
|
-
|
|
175
|
+
let(:source_pos) { double('source_pos') }
|
|
176
|
+
let(:source_pos_str) { "a position" }
|
|
177
|
+
it do
|
|
178
|
+
expect(source_pos).to receive(:to_s).and_return(source_pos_str)
|
|
179
|
+
subject
|
|
180
|
+
expect(`cat #{subject_object.source_pos_path}`).to eq(source_pos_str)
|
|
181
|
+
end
|
|
182
|
+
end
|
|
183
|
+
|
|
184
|
+
describe '#load_source_pos' do
|
|
185
|
+
subject { subject_object.load_source_pos }
|
|
164
186
|
|
|
165
|
-
|
|
187
|
+
before do
|
|
188
|
+
allow(subject_object).to receive(:source).and_return(source)
|
|
189
|
+
end
|
|
190
|
+
|
|
191
|
+
context 'when source_pos file does not exist' do
|
|
166
192
|
before do
|
|
167
|
-
File.delete(subject_object.
|
|
193
|
+
File.delete(subject_object.source_pos_path) if File.exist?(subject_object.source_pos_path)
|
|
168
194
|
end
|
|
169
195
|
it { is_expected.to be_nil }
|
|
170
196
|
end
|
|
171
197
|
|
|
172
|
-
context 'when
|
|
198
|
+
context 'when source_pos file exists' do
|
|
173
199
|
before do
|
|
174
|
-
subject_object.
|
|
200
|
+
File.open(subject_object.source_pos_path, "w"){|f| f.write(source_pos_str)}
|
|
201
|
+
end
|
|
202
|
+
after do
|
|
203
|
+
File.delete(subject_object.source_pos_path)
|
|
204
|
+
end
|
|
205
|
+
it do
|
|
206
|
+
expect(source).to receive(:source_pos).and_return(context)
|
|
207
|
+
expect(context).to receive(:create_source_pos).with(source_pos_str).
|
|
208
|
+
and_return(source_pos_obj)
|
|
209
|
+
|
|
210
|
+
is_expected.to eq source_pos_obj
|
|
175
211
|
end
|
|
176
|
-
it { is_expected.to eq(binlog_pos) }
|
|
177
212
|
end
|
|
178
213
|
end
|
|
179
214
|
|
|
180
|
-
describe '#
|
|
181
|
-
|
|
215
|
+
describe '#source_pos_path' do
|
|
216
|
+
subject { subject_object.source_pos_path }
|
|
217
|
+
it { is_expected.to eq("#{FLYDATA_HOME}/flydata_sync_mysql.binlog.pos") }
|
|
182
218
|
end
|
|
183
219
|
|
|
184
|
-
describe '#
|
|
220
|
+
describe '#sent_source_pos_path' do
|
|
185
221
|
context 'with no args' do
|
|
186
|
-
subject { subject_object.
|
|
222
|
+
subject { subject_object.sent_source_pos_path }
|
|
187
223
|
it { is_expected.to eq("#{FLYDATA_HOME}/flydata_sync_mysql.binlog.sent.pos") }
|
|
188
224
|
end
|
|
189
225
|
context 'with invalid args' do
|
|
190
|
-
subject { subject_object.
|
|
226
|
+
subject { subject_object.sent_source_pos_path('/home/ec2-user/.flydata/flydata_sync.pos') }
|
|
191
227
|
it { expect{subject}.to raise_error(ArgumentError) }
|
|
192
228
|
end
|
|
193
229
|
context 'with valid args' do
|
|
194
|
-
subject { subject_object.
|
|
230
|
+
subject { subject_object.sent_source_pos_path('/home/ec2-user/.flydata/flydata_sync.binlog.pos') }
|
|
195
231
|
it { is_expected.to eq('/home/ec2-user/.flydata/flydata_sync.binlog.sent.pos') }
|
|
196
232
|
end
|
|
197
233
|
end
|
|
@@ -329,5 +365,49 @@ module Flydata
|
|
|
329
365
|
end
|
|
330
366
|
end
|
|
331
367
|
end
|
|
368
|
+
describe '#dump_dir' do
|
|
369
|
+
subject { subject_object.send(:dump_dir) }
|
|
370
|
+
|
|
371
|
+
before do
|
|
372
|
+
default_data_entry["mysql_data_entry_preference"].delete("dump_dir")
|
|
373
|
+
end
|
|
374
|
+
|
|
375
|
+
context 'when the new property is defined' do
|
|
376
|
+
before do
|
|
377
|
+
default_data_entry["mysql_data_entry_preference"]["dump_dir"] =
|
|
378
|
+
default_dump_dir
|
|
379
|
+
end
|
|
380
|
+
context 'when the old property is defined' do
|
|
381
|
+
before do
|
|
382
|
+
default_data_entry["mysql_data_entry_preference"]["mysqldump_dir"] =
|
|
383
|
+
default_dump_dir2
|
|
384
|
+
end
|
|
385
|
+
it 'uses the dir specified by the new property' do
|
|
386
|
+
is_expected.to eq default_dump_dir
|
|
387
|
+
end
|
|
388
|
+
end
|
|
389
|
+
context 'when the old property is not defined' do
|
|
390
|
+
it 'uses the dir specified by the new property' do
|
|
391
|
+
is_expected.to eq default_dump_dir
|
|
392
|
+
end
|
|
393
|
+
end
|
|
394
|
+
end
|
|
395
|
+
context 'when the new property is not defined' do
|
|
396
|
+
context 'when the old property is defined' do
|
|
397
|
+
before do
|
|
398
|
+
default_data_entry["mysql_data_entry_preference"]["mysqldump_dir"] =
|
|
399
|
+
default_dump_dir2
|
|
400
|
+
end
|
|
401
|
+
it 'uses the dir specified by the old property' do
|
|
402
|
+
is_expected.to eq default_dump_dir2
|
|
403
|
+
end
|
|
404
|
+
end
|
|
405
|
+
context 'when the old property is not defined' do
|
|
406
|
+
it 'uses the default dir' do
|
|
407
|
+
is_expected.to eq File.join(FLYDATA_HOME, "dump")
|
|
408
|
+
end
|
|
409
|
+
end
|
|
410
|
+
end
|
|
411
|
+
end
|
|
332
412
|
end
|
|
333
413
|
end
|