flydata 0.6.3 → 0.6.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. checksums.yaml +4 -4
  2. data/Rakefile +2 -2
  3. data/VERSION +1 -1
  4. data/bin/fdredshift +78 -0
  5. data/circle.yml +1 -1
  6. data/ext/flydata/{parser/mysql → source_mysql/parser}/.gitignore +0 -0
  7. data/ext/flydata/{parser/mysql → source_mysql/parser}/dump_parser_ext.cpp +3 -3
  8. data/ext/flydata/source_mysql/parser/extconf.rb +3 -0
  9. data/ext/flydata/{parser/mysql → source_mysql/parser}/parser.txt +0 -0
  10. data/ext/flydata/{parser/mysql → source_mysql/parser}/sql_parser.cpp +0 -0
  11. data/ext/flydata/{parser/mysql → source_mysql/parser}/sql_parser.h +0 -0
  12. data/flydata-core/lib/flydata-core/mysql/binlog_pos.rb +34 -32
  13. data/flydata-core/lib/flydata-core/mysql/compatibility_checker.rb +20 -0
  14. data/flydata-core/lib/flydata-core/table_def/mysql_table_def.rb +12 -4
  15. data/flydata-core/lib/flydata-core/table_def/redshift_table_def.rb +60 -6
  16. data/flydata-core/spec/mysql/binlog_pos_spec.rb +474 -0
  17. data/flydata-core/spec/table_def/mysql_table_def_spec.rb +57 -0
  18. data/flydata-core/spec/table_def/mysql_to_redshift_table_def_spec.rb +174 -20
  19. data/flydata-core/spec/table_def/mysqldump_test_col_comment_with_AUTO_INCREMENT_keyword.dump +43 -0
  20. data/flydata-core/spec/table_def/mysqldump_test_col_comment_with_not_null_keyword.dump +43 -0
  21. data/flydata-core/spec/table_def/mysqldump_test_col_comment_with_unique_keyword.dump +43 -0
  22. data/flydata-core/spec/table_def/mysqldump_test_col_comment_with_unsigned_keyword.dump +43 -0
  23. data/flydata-core/spec/table_def/redshift_table_def_spec.rb +41 -8
  24. data/flydata.gemspec +0 -0
  25. data/lib/flydata/cli.rb +11 -5
  26. data/lib/flydata/command/base.rb +14 -1
  27. data/lib/flydata/command/exclusive_runnable.rb +42 -12
  28. data/lib/flydata/command/helper.rb +6 -6
  29. data/lib/flydata/command/sender.rb +4 -3
  30. data/lib/flydata/command/setup.rb +30 -381
  31. data/lib/flydata/command/stop.rb +1 -0
  32. data/lib/flydata/command/sync.rb +273 -301
  33. data/lib/flydata/compatibility_check.rb +24 -117
  34. data/lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb +3 -3
  35. data/lib/flydata/fluent-plugins/mysql/alter_table_query_handler.rb +2 -2
  36. data/lib/flydata/fluent-plugins/mysql/binlog_record_handler.rb +6 -6
  37. data/lib/flydata/fluent-plugins/mysql/truncate_table_query_handler.rb +0 -1
  38. data/lib/flydata/parser.rb +14 -0
  39. data/lib/flydata/{parser_provider.rb → parser/parser_provider.rb} +6 -4
  40. data/lib/flydata/parser/source_table.rb +33 -0
  41. data/lib/flydata/source.rb +105 -0
  42. data/lib/flydata/source/component.rb +21 -0
  43. data/lib/flydata/source/errors.rb +7 -0
  44. data/lib/flydata/source/generate_source_dump.rb +72 -0
  45. data/lib/flydata/source/parse_dump_and_send.rb +52 -0
  46. data/lib/flydata/source/setup.rb +31 -0
  47. data/lib/flydata/source/source_pos.rb +45 -0
  48. data/lib/flydata/source/sync.rb +56 -0
  49. data/lib/flydata/source/sync_generate_table_ddl.rb +43 -0
  50. data/lib/flydata/source_file/setup.rb +17 -0
  51. data/lib/flydata/source_file/sync.rb +14 -0
  52. data/lib/flydata/{command → source_mysql/command}/mysql.rb +2 -1
  53. data/lib/flydata/{command → source_mysql/command}/mysql_command_base.rb +2 -4
  54. data/lib/flydata/{command → source_mysql/command}/mysqlbinlog.rb +2 -1
  55. data/lib/flydata/{command → source_mysql/command}/mysqldump.rb +2 -1
  56. data/lib/flydata/source_mysql/generate_source_dump.rb +53 -0
  57. data/lib/flydata/source_mysql/mysql_compatibility_check.rb +114 -0
  58. data/lib/flydata/source_mysql/parse_dump_and_send.rb +28 -0
  59. data/lib/flydata/{parser/mysql → source_mysql/parser}/.gitignore +0 -0
  60. data/lib/flydata/{parser/mysql → source_mysql/parser}/dump_parser.rb +32 -67
  61. data/lib/flydata/{parser/mysql → source_mysql/parser}/mysql_alter_table.treetop +0 -0
  62. data/lib/flydata/source_mysql/setup.rb +24 -0
  63. data/lib/flydata/source_mysql/source_pos.rb +21 -0
  64. data/lib/flydata/source_mysql/sync.rb +45 -0
  65. data/lib/flydata/source_mysql/sync_generate_table_ddl.rb +40 -0
  66. data/lib/flydata/{mysql → source_mysql}/table_ddl.rb +6 -17
  67. data/lib/flydata/source_zendesk/sync_generate_table_ddl.rb +30 -0
  68. data/lib/flydata/source_zendesk/zendesk_flydata_tabledefs.rb +133 -0
  69. data/lib/flydata/sync_file_manager.rb +132 -73
  70. data/lib/flydata/table_ddl.rb +18 -0
  71. data/spec/flydata/cli_spec.rb +1 -0
  72. data/spec/flydata/command/exclusive_runnable_spec.rb +19 -8
  73. data/spec/flydata/command/sender_spec.rb +1 -1
  74. data/spec/flydata/command/setup_spec.rb +4 -4
  75. data/spec/flydata/command/sync_spec.rb +97 -134
  76. data/spec/flydata/compatibility_check_spec.rb +16 -289
  77. data/spec/flydata/fluent-plugins/mysql/alter_table_query_handler_spec.rb +3 -3
  78. data/spec/flydata/fluent-plugins/mysql/dml_record_handler_spec.rb +1 -1
  79. data/spec/flydata/fluent-plugins/mysql/shared_query_handler_context.rb +4 -2
  80. data/spec/flydata/fluent-plugins/mysql/truncate_query_handler_spec.rb +1 -1
  81. data/spec/flydata/source_mysql/generate_source_dump_spec.rb +69 -0
  82. data/spec/flydata/source_mysql/mysql_compatibility_check_spec.rb +280 -0
  83. data/spec/flydata/{parser/mysql → source_mysql/parser}/alter_table_parser_spec.rb +2 -2
  84. data/spec/flydata/{parser/mysql → source_mysql/parser}/dump_parser_spec.rb +75 -70
  85. data/spec/flydata/source_mysql/sync_generate_table_ddl_spec.rb +137 -0
  86. data/spec/flydata/{mysql → source_mysql}/table_ddl_spec.rb +2 -2
  87. data/spec/flydata/source_spec.rb +140 -0
  88. data/spec/flydata/source_zendesk/sync_generate_table_ddl_spec.rb +33 -0
  89. data/spec/flydata/sync_file_manager_spec.rb +157 -77
  90. data/tmpl/redshift_mysql_data_entry.conf.tmpl +1 -1
  91. metadata +56 -23
  92. data/ext/flydata/parser/mysql/extconf.rb +0 -3
  93. data/lib/flydata/mysql/binlog_position.rb +0 -22
  94. data/spec/flydata/mysql/binlog_position_spec.rb +0 -35
@@ -0,0 +1,137 @@
1
+ require 'flydata/source_mysql/sync_generate_table_ddl'
2
+
3
+ module Flydata
4
+ module SourceMysql
5
+
6
+ describe SyncGenerateTableDdl do
7
+ let(:subject_object) { described_class.new(source, dp, options) }
8
+
9
+ let(:dp) { double('dp') }
10
+ let(:de) { {
11
+ 'mysql_data_entry_preference' => {
12
+ 'host' => 'localhost',
13
+ 'port' => 3306,
14
+ 'username' => 'masashi',
15
+ 'password' => 'welcome',
16
+ 'database' => 'sync_test',
17
+ }
18
+ } }
19
+ let(:options) { {} }
20
+
21
+ let(:source) { double('source') }
22
+ before do
23
+ allow(source).to receive(:de).and_return de
24
+ end
25
+
26
+ describe '#run_compatibility_check' do
27
+ subject { subject_object.run_compatibility_check }
28
+
29
+ let(:checker) { double("checker") }
30
+ it do
31
+ expect(MysqlCompatibilityCheck).to receive(:new).and_return checker
32
+ expect(checker).to receive(:check)
33
+
34
+ subject
35
+ end
36
+ end
37
+ describe '#generate_flydata_tabledef' do
38
+ subject { subject_object.generate_flydata_tabledef(tables, options) }
39
+
40
+ let(:tables) { %w| table1 table2 table4 table3 | }
41
+
42
+ context 'with full options' do
43
+ it 'issues mysqldump command with expected parameters' do
44
+ expect(Open3).to receive(:popen3).with(
45
+ 'mysqldump -h localhost -P 3306 -umasashi -pwelcome --default-character-set=utf8 --protocol=tcp -d sync_test table1 table2 table4 table3')
46
+
47
+ subject
48
+ end
49
+ end
50
+ context 'without_host' do
51
+ before do
52
+ de['mysql_data_entry_preference'].delete('host')
53
+ end
54
+ it { expect { subject }.to raise_error }
55
+ end
56
+ context 'with empty host' do
57
+ before do
58
+ de['mysql_data_entry_preference']['host'] = ""
59
+ end
60
+ it { expect { subject }.to raise_error }
61
+ end
62
+ context 'without port' do
63
+ before do
64
+ de['mysql_data_entry_preference'].delete('port')
65
+ end
66
+ it "uses the default port" do
67
+ expect(Open3).to receive(:popen3).with(
68
+ 'mysqldump -h localhost -umasashi -pwelcome --default-character-set=utf8 --protocol=tcp -d sync_test table1 table2 table4 table3')
69
+ subject
70
+ end
71
+ end
72
+ context 'with port override' do
73
+ before do
74
+ de['mysql_data_entry_preference']['port'] = 1234
75
+ end
76
+ it "uses the specified port" do
77
+ expect(Open3).to receive(:popen3).with(
78
+ 'mysqldump -h localhost -P 1234 -umasashi -pwelcome --default-character-set=utf8 --protocol=tcp -d sync_test table1 table2 table4 table3')
79
+ subject
80
+ end
81
+ end
82
+ context 'without username' do
83
+ before do
84
+ de['mysql_data_entry_preference'].delete('username')
85
+ end
86
+ it { expect { subject }.to raise_error }
87
+ end
88
+ context 'with empty username' do
89
+ before do
90
+ de['mysql_data_entry_preference']['username'] = ""
91
+ end
92
+ it { expect { subject }.to raise_error }
93
+ end
94
+ context 'without_password' do
95
+ before do
96
+ de['mysql_data_entry_preference'].delete('password')
97
+ end
98
+ it "call mysqldump without MYSQL_PW set" do
99
+ expect(Open3).to receive(:popen3).with(
100
+ 'mysqldump -h localhost -P 3306 -umasashi --default-character-set=utf8 --protocol=tcp -d sync_test table1 table2 table4 table3')
101
+ subject
102
+ end
103
+ end
104
+ context 'with password containing symbols' do
105
+ before do
106
+ de['mysql_data_entry_preference']['password']="welcome&!@^@#^"
107
+ end
108
+ it "call mysqldump with MYSQL_PW set with correct symbols" do
109
+ expect(Open3).to receive(:popen3).with(
110
+ 'mysqldump -h localhost -P 3306 -umasashi -pwelcome\\&\\!@\\^@\\#\\^ --default-character-set=utf8 --protocol=tcp -d sync_test table1 table2 table4 table3')
111
+ subject
112
+ end
113
+ end
114
+ context 'without_database' do
115
+ before do
116
+ de['mysql_data_entry_preference'].delete('database')
117
+ end
118
+ it { expect { subject }.to raise_error }
119
+ end
120
+ context 'with empty database' do
121
+ before do
122
+ de['mysql_data_entry_preference']['database'] = ""
123
+ end
124
+ it { expect { subject }.to raise_error }
125
+ end
126
+ context 'with empty tables' do
127
+ before do
128
+ de['mysql_data_entry_preference']['tables'] = []
129
+ end
130
+ it { expect { subject }.to raise_error }
131
+ end
132
+ end
133
+ end
134
+
135
+
136
+ end
137
+ end
@@ -1,8 +1,8 @@
1
1
  require 'spec_helper'
2
- require 'flydata/mysql/table_ddl'
2
+ require 'flydata/source_mysql/table_ddl'
3
3
 
4
4
  module Flydata
5
- module Mysql
5
+ module SourceMysql
6
6
 
7
7
  describe TableDdl do
8
8
  let(:table_name1) { "items" }
@@ -0,0 +1,140 @@
1
+ require 'flydata/source'
2
+
3
+ module Flydata
4
+
5
+ describe Source do
6
+ let(:subject_object) { described_class.create(de) }
7
+
8
+ let(:target_class) { double('target_class') }
9
+ let(:type_class) { double('type_class') }
10
+
11
+ let(:expected_instance) { double('expected_instance') }
12
+ let(:de) { double('de') }
13
+ let(:opts) { double('opts') }
14
+ let(:class_name) { "replace this" }
15
+ let(:type_class_name) { "replace this" }
16
+ let(:data_entry_type) { "RedshiftMysqlDataEntry" }
17
+
18
+ before do
19
+ allow(target_class).to receive(:name).and_return(class_name)
20
+ allow(type_class).to receive(:name).and_return(type_class_name)
21
+ end
22
+ before do
23
+ allow(de).to receive(:[]).with('type').and_return(data_entry_type)
24
+ end
25
+
26
+ after do
27
+ if described_class.instance_variable_defined? :@component_classes
28
+ described_class.remove_instance_variable :@component_classes
29
+ end
30
+ end
31
+
32
+ describe '#method_missing' do
33
+ subject { subject_object.send(method_name, opts) }
34
+
35
+ before do
36
+ described_class.register(target_class, type_class)
37
+ end
38
+
39
+ context 'when the target data source exists' do
40
+ let(:data_entry_type) { "RedshiftMysqlDataEntry" }
41
+ let(:class_name) { "Flydata::SourceMysql::MySR" }
42
+
43
+ context 'when the target component exists' do
44
+ let(:method_name) { :sync_reset }
45
+ let(:type_class_name) { "Flydata::Source::SyncReset" }
46
+
47
+ it "returns an instance of the target_class" do
48
+ expect(target_class).to receive(:new).with(subject_object, opts).
49
+ and_return expected_instance
50
+
51
+ expect(subject).to eq expected_instance
52
+ end
53
+ end
54
+ context 'when the target component does not exist' do
55
+ let(:method_name) { :sync_start }
56
+ let(:type_class_name) { "Flydata::Source::SyncReset" }
57
+
58
+ it "raises an error" do
59
+ expect{ subject }.to raise_error /Component '.*?' is not defined/
60
+ end
61
+ end
62
+ end
63
+ context 'when the target data source does not exist' do
64
+ let(:data_entry_type) { "FooBarDataEntry" }
65
+ let(:class_name) { "Flydata::SourceFooBar::MySR" }
66
+
67
+ context 'when the target component exists' do
68
+ let(:method_name) { :sync_reset }
69
+ let(:type_class_name) { "Flydata::Source::SyncReset" }
70
+
71
+ it "raises an error" do
72
+ expect{ subject }.to raise_error /No source components are available/
73
+ end
74
+ end
75
+ end
76
+ end
77
+
78
+ describe '.underscored_source_module_sym' do
79
+ subject { described_class.send(:underscored_source_module_sym, target_class) }
80
+
81
+ context 'with a standard name' do
82
+ let(:class_name) { "Flydata::SourceMysql::SyncReset" }
83
+ it { is_expected.to eq :source_mysql }
84
+ end
85
+ context 'with a nested path' do
86
+ let(:class_name) { "Flydata::SourceMysql::Command::SyncReset" }
87
+ it { is_expected.to eq :source_mysql }
88
+ end
89
+ context 'with a nested path (not recommended)' do
90
+ let(:class_name) { "Flydata::Foo::Bar::SourceMysql::SyncReset" }
91
+ it { is_expected.to eq :source_mysql }
92
+ end
93
+ context 'when none of elements starts with "Source"' do
94
+ let(:class_name) { "Flydata::Foo::Bar::SyncReset" }
95
+ it { is_expected.to eq nil }
96
+ end
97
+ context 'when more than one element start with "Source"' do
98
+ let(:class_name) { "Flydata::SourceMysql::SourceFiles::SyncReset" }
99
+ it "uses the first one" do
100
+ is_expected.to eq :source_mysql
101
+ end
102
+ end
103
+ end
104
+ describe '.command_dir_paths' do
105
+ subject { described_class.command_dir_paths }
106
+
107
+ before do
108
+ allow(File).to receive(:exists?).and_return false
109
+ end
110
+ shared_examples 'returning the expected result' do
111
+ before do
112
+ expect(File).to receive(:exists?).with(/source_mysql\/command/).
113
+ and_return source_mysql_command_dir_exists
114
+ expect(File).to receive(:exists?).with(/source_file\/command/).
115
+ and_return source_file_command_dir_exists
116
+ end
117
+ it { is_expected.to eq expected_result }
118
+ end
119
+ let(:source_file_command_dir_exists) { false }
120
+
121
+ context 'with no source dir' do
122
+ let(:source_mysql_command_dir_exists) { false }
123
+ let(:expected_result) { [] }
124
+ it_behaves_like 'returning the expected result'
125
+ end
126
+ context 'with command dir' do
127
+ let(:source_mysql_command_dir_exists) { true }
128
+ let(:expected_result) { ["flydata/source_mysql/command"] }
129
+ it_behaves_like 'returning the expected result'
130
+ end
131
+ context 'with two source dirs with command dir' do
132
+ let(:source_mysql_command_dir_exists) { true }
133
+ let(:source_file_command_dir_exists) { true }
134
+ let(:expected_result) { %w(flydata/source_mysql/command flydata/source_file/command) }
135
+ it_behaves_like 'returning the expected result'
136
+ end
137
+ end
138
+ end
139
+
140
+ end
@@ -0,0 +1,33 @@
1
+ require 'flydata/source_zendesk/sync_generate_table_ddl'
2
+
3
+ module Flydata
4
+ module SourceZendesk
5
+
6
+ describe SyncGenerateTableDdl do
7
+ let(:subject_object) { described_class.new(source, dp, options) }
8
+
9
+ let(:source) { double('source') }
10
+ let(:dp) { double('dp') }
11
+ let(:options) { {} }
12
+
13
+ describe '#run_compatibility_check' do
14
+ subject { subject_object.run_compatibility_check }
15
+ it { subject }
16
+ end
17
+
18
+ describe '#generate_flydata_tabledef' do
19
+ subject { subject_object.generate_flydata_tabledef(tables, options) }
20
+
21
+ let(:supported_tables) { %w(tickets ticket_comments ticket_audits users organizations) }
22
+ let(:options) { double('options') }
23
+
24
+ context 'with all supported tables' do
25
+ let(:tables) { supported_tables }
26
+ it { subject }
27
+ end
28
+ end
29
+ end
30
+
31
+
32
+ end
33
+ end
@@ -1,15 +1,19 @@
1
1
  # coding: utf-8
2
2
  require 'spec_helper'
3
3
  require 'flydata/sync_file_manager'
4
- require 'flydata/parser/mysql/dump_parser'
4
+ require 'flydata/parser/source_table'
5
+ require 'flydata/command/base'
5
6
 
6
7
  module Flydata
7
8
  describe SyncFileManager do
8
- let(:subject_object) { described_class.new(default_data_entry) }
9
+ let(:subject_object) { described_class.new(default_data_entry, source) }
9
10
 
10
- let(:default_mysqldump_dir) do
11
+ let(:default_dump_dir) do
11
12
  File.join('/tmp', "sync_dump_#{Time.now.to_i}")
12
13
  end
14
+ let(:default_dump_dir2) do
15
+ File.join('/tmp', "sync_dump2_#{Time.now.to_i}")
16
+ end
13
17
  let(:default_data_entry) do
14
18
  {"id"=>93,
15
19
  "name"=>"flydata_sync_mysql",
@@ -33,7 +37,7 @@ module Flydata
33
37
  "mysql_data_entry_preference" =>
34
38
  { "host"=>"localhost", "port"=>3306, "username"=>"masashi",
35
39
  "password"=>"welcome", "database"=>"sync_test", "tables"=>nil,
36
- "mysqldump_dir"=>default_mysqldump_dir, "forwarder" => "tcpforwarder",
40
+ "dump_dir"=>default_dump_dir, "forwarder" => "tcpforwarder",
37
41
  "data_servers"=>"localhost:9905" }
38
42
  }
39
43
  end
@@ -42,156 +46,188 @@ module Flydata
42
46
  let (:table_name) { 'test_table' }
43
47
  let (:table_name2) { 'test_table2' }
44
48
  let (:last_pos) { 9999 }
45
- let (:binlog_pos) { {binfile: 'mysqlbin.00001', pos: 111} }
46
49
  let (:state) { 'CREATE_TABLE' }
47
50
  let (:substate) { nil }
48
51
 
52
+ let(:source_pos_str) { "a position\t\tcould include many tabs\t456" }
53
+ let(:source_pos_org) { double ('source_pos_org') }
54
+ let(:source_pos_obj) { double('source_pos_obj') }
55
+ let(:source) { double('source') }
56
+ let(:context) { double('context') }
57
+ let(:columns_hash) do
58
+ { 'id' => { column_name: 'id', format_type: 'int' },
59
+ 'value' => { column_name: 'value', format_type: 'text' } }
60
+ end
61
+ let(:dump_contents_hash) do
62
+ { status: status, table_name: table_name, last_pos: last_pos,
63
+ source_pos: source_pos_obj, state: state, substate: substate }
64
+ end
65
+
49
66
  after :each do
50
- if Dir.exists?(default_mysqldump_dir)
51
- Dir.delete(default_mysqldump_dir) rescue nil
67
+ if Dir.exists?(default_dump_dir)
68
+ Dir.delete(default_dump_dir) rescue nil
52
69
  end
53
- if File.exists?(default_mysqldump_dir)
54
- File.delete(default_mysqldump_dir) rescue nil
70
+ if File.exists?(default_dump_dir)
71
+ File.delete(default_dump_dir) rescue nil
55
72
  end
56
73
  end
57
74
 
58
75
  describe '#dump_file_path' do
59
- context 'when mysqldump_dir param is nil' do
76
+ subject { subject_object.dump_file_path }
77
+ context 'when dump_dir param is nil' do
60
78
  before do
61
- default_data_entry['mysql_data_entry_preference'].delete('mysqldump_dir')
79
+ default_data_entry['mysql_data_entry_preference'].delete('dump_dir')
62
80
  end
63
81
  it do
64
82
  stub_const('Flydata::FileUtil::SyncFileManager::DUMP_DIR', File.join(Flydata::FLYDATA_HOME, 'dump'))
65
- expect(subject_object.dump_file_path).to eq(
66
- File.join(Flydata::FLYDATA_HOME, 'dump', 'flydata_sync_mysql.dump'))
83
+ is_expected.to eq(File.join(Flydata::FLYDATA_HOME, 'dump', 'flydata_sync_mysql.dump'))
67
84
  end
68
85
  end
69
86
  context 'when file exists' do
70
- before { `touch #{default_mysqldump_dir}`}
87
+ before { `touch #{default_dump_dir}`}
71
88
  it do
72
- expect{subject_object.dump_file_path}.to raise_error
89
+ expect{ subject }.to raise_error
73
90
  end
74
91
  end
75
92
  context 'when directory exists' do
76
- before { `mkdir -p #{default_mysqldump_dir}`}
93
+ before { `mkdir -p #{default_dump_dir}`}
77
94
  it do
78
- expect(FileUtils).to receive(:mkdir_p).with(default_mysqldump_dir).never
79
- expect(subject_object.dump_file_path).to eq(
80
- File.join(default_mysqldump_dir, 'flydata_sync_mysql.dump'))
95
+ expect(FileUtils).to receive(:mkdir_p).with(default_dump_dir).never
96
+ is_expected.to eq(File.join(default_dump_dir, 'flydata_sync_mysql.dump'))
81
97
  end
82
98
  end
83
99
  context 'when directory or file does not exist' do
84
100
  it do
85
- expect(FileUtils).to receive(:mkdir_p).with(default_mysqldump_dir).once
86
- expect(subject_object.dump_file_path).to eq(
87
- File.join(default_mysqldump_dir, 'flydata_sync_mysql.dump'))
101
+ expect(FileUtils).to receive(:mkdir_p).with(default_dump_dir).once
102
+ is_expected.to eq(File.join(default_dump_dir, 'flydata_sync_mysql.dump'))
88
103
  end
89
104
  end
90
105
  context 'when file name includes "~"' do
91
- let(:default_mysqldump_dir) { "~/tmp/dump/sync_spec_#{Time.now.to_i}" }
106
+ let(:default_dump_dir) { "~/tmp/dump/sync_spec_#{Time.now.to_i}" }
92
107
  it do
93
- expected_dir = File.join(ENV['HOME'], default_mysqldump_dir[1..-1])
108
+ expected_dir = File.join(ENV['HOME'], default_dump_dir[1..-1])
94
109
  expect(FileUtils).to receive(:mkdir_p).with(expected_dir).once
95
- expect(subject_object.dump_file_path).to eq(
96
- File.join(expected_dir, 'flydata_sync_mysql.dump'))
110
+ is_expected.to eq(File.join(expected_dir, 'flydata_sync_mysql.dump'))
97
111
  end
98
112
  end
99
113
  end
100
114
 
101
115
  describe '#dump_pos_path' do
102
- it { expect(subject_object.dump_pos_path).to eq(
103
- File.join(default_mysqldump_dir, 'flydata_sync_mysql.dump.pos')) }
116
+ subject { subject_object.dump_pos_path }
117
+ it { is_expected.to eq(File.join(default_dump_dir, 'flydata_sync_mysql.dump.pos')) }
104
118
  end
105
119
 
106
120
  describe '#save_dump_pos' do
107
- context 'without mysql marshal data' do
121
+ subject { subject_object.save_dump_pos(status, table_name, last_pos, source_pos_org,
122
+ state, substate) }
123
+ context 'without source marshal data' do
108
124
  it do
109
- expect{subject_object.save_dump_pos(
110
- status, table_name, last_pos, binlog_pos, state, substate)}.not_to raise_error
111
- expect(subject_object.load_dump_pos).to eq({
112
- status: status, table_name: table_name, last_pos: last_pos,
113
- binlog_pos: binlog_pos, state: state, substate: substate,
114
- mysql_table: nil
115
- })
125
+ expect(source_pos_org).to receive(:to_s).and_return(source_pos_str)
126
+ expect{ subject }.not_to raise_error
127
+
128
+ expect(source).to receive(:source_pos).and_return(context)
129
+ expect(context).to receive(:create_source_pos).with(source_pos_str).
130
+ and_return(source_pos_obj)
131
+
132
+ expect(subject_object.load_dump_pos).to eq( dump_contents_hash.merge( { source_table: nil } ) )
116
133
  end
117
134
  end
118
135
  end
119
136
 
120
137
  describe '#load_dump_pos' do
121
- let (:mysql_table) do
122
- Flydata::Parser::Mysql::MysqlTable.new(
123
- table_name, { 'id' => { column_name: 'id', format_type: 'int' }, 'value' => { column_name: 'value', format_type: 'text' } }
124
- )
125
- end
138
+ subject { subject_object.load_dump_pos }
139
+
140
+ context 'with source marshal data' do
141
+ let(:source_table_obj) { double('source_table_obj') }
126
142
 
127
- context 'with mysql marshal data' do
143
+ let(:source_table) { Flydata::Parser::SourceTable.new( table_name, columns_hash) }
128
144
  before do
129
- subject_object.save_mysql_table_marshal_dump(mysql_table)
130
- subject_object.save_dump_pos(status, table_name, last_pos, binlog_pos, state, substate)
145
+ allow(source_pos_org).to receive(:to_s).and_return(source_pos_str)
146
+ subject_object.save_source_table_marshal_dump(source_table)
147
+ subject_object.save_dump_pos(status, table_name, last_pos, source_pos_org, state, substate)
131
148
  end
132
149
  it do
133
- ret = subject_object.load_dump_pos
134
- mt = ret.delete(:mysql_table)
135
- expect(ret).to eq({
136
- status: status, table_name: table_name, last_pos: last_pos,
137
- binlog_pos: binlog_pos, state: state, substate: substate,
138
- })
139
- expect(mt.table_name).to eq(table_name)
140
- expect(mt.columns).to eq({
141
- 'id' => { column_name: 'id', format_type: 'int' },
142
- 'value' => { column_name: 'value', format_type: 'text' },
143
- })
150
+ expect(source).to receive(:source_pos).and_return(context)
151
+ expect(context).to receive(:create_source_pos).with(source_pos_str).and_return(source_pos_obj)
152
+ expect(subject_object).to receive(:load_source_table_marshal_dump).and_return(source_table_obj)
153
+
154
+ is_expected.to eq( dump_contents_hash.merge({source_table: source_table_obj}) )
144
155
  end
145
156
  end
146
157
  end
147
158
 
148
- describe '#save_binlog' do
149
- let(:binfile) { 'mysqlbinlog.000001' }
150
- let(:pos) { 107 }
151
- let(:binlog_pos) { {binfile: binfile, pos: pos} }
159
+ describe '#load_source_table_marshal_dump' do
160
+ subject { subject_object.send(:load_source_table_marshal_dump) }
161
+
162
+ let(:source_table) { Flydata::Parser::SourceTable.new( table_name, columns_hash) }
163
+ before do
164
+ subject_object.save_source_table_marshal_dump(source_table)
165
+ end
152
166
  it do
153
- subject_object.save_binlog(binlog_pos)
154
- expect(`cat #{subject_object.binlog_path}`).to eq("#{binfile}\t#{pos}")
167
+ expect(subject.table_name).to eq(table_name)
168
+ expect(subject.columns).to eq(columns_hash)
155
169
  end
156
170
  end
157
171
 
158
- describe '#load_binlog' do
159
- let(:binfile) { 'mysqlbinlog.000001' }
160
- let(:pos) { 107 }
161
- let(:binlog_pos) { {binfile: binfile, pos: pos} }
172
+ describe '#save_source_pos' do
173
+ subject { subject_object.save_source_pos(source_pos) }
162
174
 
163
- subject { subject_object.load_binlog }
175
+ let(:source_pos) { double('source_pos') }
176
+ let(:source_pos_str) { "a position" }
177
+ it do
178
+ expect(source_pos).to receive(:to_s).and_return(source_pos_str)
179
+ subject
180
+ expect(`cat #{subject_object.source_pos_path}`).to eq(source_pos_str)
181
+ end
182
+ end
183
+
184
+ describe '#load_source_pos' do
185
+ subject { subject_object.load_source_pos }
164
186
 
165
- context 'when binlog pos does not exist' do
187
+ before do
188
+ allow(subject_object).to receive(:source).and_return(source)
189
+ end
190
+
191
+ context 'when source_pos file does not exist' do
166
192
  before do
167
- File.delete(subject_object.binlog_path) if File.exist?(subject_object.binlog_path)
193
+ File.delete(subject_object.source_pos_path) if File.exist?(subject_object.source_pos_path)
168
194
  end
169
195
  it { is_expected.to be_nil }
170
196
  end
171
197
 
172
- context 'when binlog pos exists' do
198
+ context 'when source_pos file exists' do
173
199
  before do
174
- subject_object.save_binlog(binlog_pos)
200
+ File.open(subject_object.source_pos_path, "w"){|f| f.write(source_pos_str)}
201
+ end
202
+ after do
203
+ File.delete(subject_object.source_pos_path)
204
+ end
205
+ it do
206
+ expect(source).to receive(:source_pos).and_return(context)
207
+ expect(context).to receive(:create_source_pos).with(source_pos_str).
208
+ and_return(source_pos_obj)
209
+
210
+ is_expected.to eq source_pos_obj
175
211
  end
176
- it { is_expected.to eq(binlog_pos) }
177
212
  end
178
213
  end
179
214
 
180
- describe '#binlog_path' do
181
- it { expect(subject_object.binlog_path).to eq("#{FLYDATA_HOME}/flydata_sync_mysql.binlog.pos") }
215
+ describe '#source_pos_path' do
216
+ subject { subject_object.source_pos_path }
217
+ it { is_expected.to eq("#{FLYDATA_HOME}/flydata_sync_mysql.binlog.pos") }
182
218
  end
183
219
 
184
- describe '#sent_binlog_path' do
220
+ describe '#sent_source_pos_path' do
185
221
  context 'with no args' do
186
- subject { subject_object.sent_binlog_path }
222
+ subject { subject_object.sent_source_pos_path }
187
223
  it { is_expected.to eq("#{FLYDATA_HOME}/flydata_sync_mysql.binlog.sent.pos") }
188
224
  end
189
225
  context 'with invalid args' do
190
- subject { subject_object.sent_binlog_path('/home/ec2-user/.flydata/flydata_sync.pos') }
226
+ subject { subject_object.sent_source_pos_path('/home/ec2-user/.flydata/flydata_sync.pos') }
191
227
  it { expect{subject}.to raise_error(ArgumentError) }
192
228
  end
193
229
  context 'with valid args' do
194
- subject { subject_object.sent_binlog_path('/home/ec2-user/.flydata/flydata_sync.binlog.pos') }
230
+ subject { subject_object.sent_source_pos_path('/home/ec2-user/.flydata/flydata_sync.binlog.pos') }
195
231
  it { is_expected.to eq('/home/ec2-user/.flydata/flydata_sync.binlog.sent.pos') }
196
232
  end
197
233
  end
@@ -329,5 +365,49 @@ module Flydata
329
365
  end
330
366
  end
331
367
  end
368
+ describe '#dump_dir' do
369
+ subject { subject_object.send(:dump_dir) }
370
+
371
+ before do
372
+ default_data_entry["mysql_data_entry_preference"].delete("dump_dir")
373
+ end
374
+
375
+ context 'when the new property is defined' do
376
+ before do
377
+ default_data_entry["mysql_data_entry_preference"]["dump_dir"] =
378
+ default_dump_dir
379
+ end
380
+ context 'when the old property is defined' do
381
+ before do
382
+ default_data_entry["mysql_data_entry_preference"]["mysqldump_dir"] =
383
+ default_dump_dir2
384
+ end
385
+ it 'uses the dir specified by the new property' do
386
+ is_expected.to eq default_dump_dir
387
+ end
388
+ end
389
+ context 'when the old property is not defined' do
390
+ it 'uses the dir specified by the new property' do
391
+ is_expected.to eq default_dump_dir
392
+ end
393
+ end
394
+ end
395
+ context 'when the new property is not defined' do
396
+ context 'when the old property is defined' do
397
+ before do
398
+ default_data_entry["mysql_data_entry_preference"]["mysqldump_dir"] =
399
+ default_dump_dir2
400
+ end
401
+ it 'uses the dir specified by the old property' do
402
+ is_expected.to eq default_dump_dir2
403
+ end
404
+ end
405
+ context 'when the old property is not defined' do
406
+ it 'uses the default dir' do
407
+ is_expected.to eq File.join(FLYDATA_HOME, "dump")
408
+ end
409
+ end
410
+ end
411
+ end
332
412
  end
333
413
  end