flydata 0.8.8 → 0.8.9
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/VERSION +1 -1
- data/flydata-core/lib/flydata-core/mysql/command_generator.rb +14 -0
- data/flydata-core/lib/flydata-core/mysql/compatibility_checker.rb +28 -1
- data/flydata-core/spec/mysql/command_generator_spec.rb +35 -1
- data/flydata-core/spec/mysql/compatibility_checker_spec.rb +30 -0
- data/flydata.gemspec +0 -0
- data/spec/flydata/source_mysql/mysql_compatibility_check_spec.rb +15 -2
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 18f14f5aa7cb056405bcddd04145128861746e81
|
4
|
+
data.tar.gz: 7137aa3faffbc50cdadf71a54e22f50c59f7beb4
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 515cc0f4a0520fc9c6f2573eb457d903dd83c43c6ff8ef114659b8e6050d67c3984e2ecc4acbfeba8307b12e4db383b1039df4647ede956c0d52799a465b6a6c
|
7
|
+
data.tar.gz: e2ea17f5471e190d8bcc718db2da8136bff4677337f60fb5e1f7da4011fb41de5081c9efad491bd8ebe1043296e26a969a55845a6b3446764e573d022cfad3f8
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.8.
|
1
|
+
0.8.9
|
@@ -1,13 +1,18 @@
|
|
1
1
|
require 'open3'
|
2
2
|
require 'shellwords'
|
3
3
|
require 'flydata-core/table_def/mysql_table_def'
|
4
|
+
require 'flydata-core/mysql/compatibility_checker'
|
4
5
|
|
5
6
|
module FlydataCore
|
6
7
|
module Mysql
|
7
8
|
class CommandGenerator
|
8
9
|
DEFAULT_OPTION = "--default-character-set=utf8 --protocol=tcp"
|
9
10
|
DEFAULT_MYSQL_OPTION = "#{DEFAULT_OPTION} --skip-auto-rehash"
|
11
|
+
DEFAULT_MYSQLDUMP57_OPTION = "#{DEFAULT_OPTION} --set-gtid-purged=OFF" # If you face GTID issue when start initial sync, try this option for mysqldump
|
10
12
|
|
13
|
+
# Get default command options
|
14
|
+
# Arguments:
|
15
|
+
# command - String ('mysql' or 'mysqldump')
|
11
16
|
def self.default_cmd_option(command)
|
12
17
|
case command
|
13
18
|
when 'mysql'
|
@@ -48,6 +53,15 @@ module FlydataCore
|
|
48
53
|
default_option = option[:no_default_option] ? "" : default_cmd_option(command)
|
49
54
|
default_option += " --ssl-ca #{ssl_ca}" if ssl_ca
|
50
55
|
default_option += " --ssl-cipher=#{ssl_cipher}" unless ssl_cipher.to_s.empty?
|
56
|
+
|
57
|
+
if command == 'mysqldump'
|
58
|
+
version = MysqlCompatibilityChecker.new(option).mysql_server_version
|
59
|
+
|
60
|
+
if !(version =~ /Maria/) && MysqlCompatibilityChecker.server_version_eq_or_higher?(version, "5.6.9")
|
61
|
+
default_option += " --set-gtid-purged=OFF"
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
51
65
|
default_option = nil if default_option == ''
|
52
66
|
|
53
67
|
custom_option = option[:custom_option]
|
@@ -29,6 +29,33 @@ module FlydataCore
|
|
29
29
|
"database"
|
30
30
|
end
|
31
31
|
|
32
|
+
# Utility methods
|
33
|
+
|
34
|
+
VERSION_QUERY = "SHOW VARIABLES LIKE 'version'"
|
35
|
+
def mysql_server_version
|
36
|
+
result = exec_query(VERSION_QUERY)
|
37
|
+
result.first['Value']
|
38
|
+
end
|
39
|
+
|
40
|
+
# Check if the version of MySQL server is equal or higher than the given version
|
41
|
+
# Argument:
|
42
|
+
# version: (String)
|
43
|
+
# Example:
|
44
|
+
# - MySQL server version: "5.6.10-log"
|
45
|
+
# server_version_eq_or_higher?("5.6.10-log", "5.6.9") #=> true
|
46
|
+
# - MySQL server version: "5.6.24-2+deb.sury.org~precise+2-log"
|
47
|
+
# server_version_eq_or_higher?("5.6.24-2+deb.sury.org~precise+2-log", "5.6.24") #=> true
|
48
|
+
def self.server_version_eq_or_higher?(server_version, version_to_compare)
|
49
|
+
(version_to_array(server_version) <=> version_to_array(version_to_compare)) >= 0
|
50
|
+
end
|
51
|
+
|
52
|
+
# Converting version string to Array of Integers
|
53
|
+
# Example: version_to_array("5.6.9-log") #=> [5, 6, 9]
|
54
|
+
# version_to_array("5.6.24-2+deb.sury.org~precise+2-log") #=> [5, 6, 24, 0, 0]
|
55
|
+
def self.version_to_array(version)
|
56
|
+
version.split('.').map(&:to_i)
|
57
|
+
end
|
58
|
+
|
32
59
|
# This command works only on RDS MySQL
|
33
60
|
RDS_CHECK_QUERY = "call mysql.rds_show_configuration;"
|
34
61
|
def rds?(hostname = @option[:host])
|
@@ -324,7 +351,7 @@ EOT
|
|
324
351
|
class NonRdsRetentionChecker < MysqlCompatibilityChecker
|
325
352
|
include MysqlVariablesHandling
|
326
353
|
|
327
|
-
BINLOG_RETENTION_HOURS =
|
354
|
+
BINLOG_RETENTION_HOURS = 96
|
328
355
|
EXPIRE_LOGS_DAYS_LIMIT = BINLOG_RETENTION_HOURS / 24
|
329
356
|
|
330
357
|
def check_result(result, option = @option)
|
@@ -15,6 +15,12 @@ module FlydataCore
|
|
15
15
|
} }
|
16
16
|
let(:option) { default_option }
|
17
17
|
let(:cmd_pswd_check) { 'mysqldump -h test-host -P 3306 -utestuser -p%s --default-character-set=utf8 --protocol=tcp testdb' }
|
18
|
+
let(:mysql_compatibility_checker) { instance_double(MysqlCompatibilityChecker) }
|
19
|
+
|
20
|
+
before do
|
21
|
+
allow(MysqlCompatibilityChecker).to receive(:new).and_return(mysql_compatibility_checker)
|
22
|
+
allow(mysql_compatibility_checker).to receive(:mysql_server_version).and_return('5.6.8')
|
23
|
+
end
|
18
24
|
|
19
25
|
describe '.generate_mysql_cmd' do
|
20
26
|
subject { described_class.generate_mysql_cmd(option) }
|
@@ -35,7 +41,7 @@ module FlydataCore
|
|
35
41
|
) }
|
36
42
|
end
|
37
43
|
|
38
|
-
context 'when option
|
44
|
+
context 'when option contains string key' do
|
39
45
|
before {
|
40
46
|
option[:database] = nil
|
41
47
|
option['database'] = 'another_db'
|
@@ -121,6 +127,34 @@ module FlydataCore
|
|
121
127
|
'mysqldump -h test-host -P 3306 -utestuser -ptestpassword --default-character-set=utf8 --protocol=tcp testdb test_table'
|
122
128
|
) }
|
123
129
|
end
|
130
|
+
|
131
|
+
context 'when mysql version is 5.6.9' do
|
132
|
+
before do
|
133
|
+
allow(mysql_compatibility_checker).to receive(:mysql_server_version).and_return('5.6.9')
|
134
|
+
end
|
135
|
+
|
136
|
+
context 'when command is not \'mysqldump\'' do
|
137
|
+
before { option[:command] = nil }
|
138
|
+
|
139
|
+
it { is_expected.to eq(
|
140
|
+
'mysql -h test-host -P 3306 -utestuser -ptestpassword --default-character-set=utf8 --protocol=tcp --skip-auto-rehash testdb'
|
141
|
+
) }
|
142
|
+
end
|
143
|
+
|
144
|
+
context 'when server is MariaDB' do
|
145
|
+
before do
|
146
|
+
allow(mysql_compatibility_checker).to receive(:mysql_server_version).and_return('5.6.9-MariaDB')
|
147
|
+
end
|
148
|
+
|
149
|
+
it { is_expected.to eq(
|
150
|
+
'mysqldump -h test-host -P 3306 -utestuser -ptestpassword --default-character-set=utf8 --protocol=tcp testdb'
|
151
|
+
) }
|
152
|
+
end
|
153
|
+
|
154
|
+
it { is_expected.to eq(
|
155
|
+
'mysqldump -h test-host -P 3306 -utestuser -ptestpassword --default-character-set=utf8 --protocol=tcp --set-gtid-purged=OFF testdb'
|
156
|
+
) }
|
157
|
+
end
|
124
158
|
end
|
125
159
|
|
126
160
|
describe '.generate_mysql_ddl_dump_cmd' do
|
@@ -19,6 +19,36 @@ module FlydataCore
|
|
19
19
|
describe MysqlCompatibilityChecker do
|
20
20
|
let(:subject_object) { described_class.new(option) }
|
21
21
|
|
22
|
+
describe '#mysql_server_version' do
|
23
|
+
before do
|
24
|
+
version = [{"Variable_name"=>"version", "Value"=>"5.5.57-log"}]
|
25
|
+
allow(subject_object).to receive(:exec_query).with("SHOW VARIABLES LIKE 'version'").and_return(version)
|
26
|
+
end
|
27
|
+
|
28
|
+
it 'returns the mysql version' do
|
29
|
+
expect(subject_object.mysql_server_version).to eq('5.5.57-log')
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
describe '.server_version_eq_or_higher' do
|
34
|
+
it 'returns true if versions are equal' do
|
35
|
+
expect(described_class.server_version_eq_or_higher?("5.6.9-log", "5.6.9")).to be_truthy
|
36
|
+
end
|
37
|
+
it 'returns true if version is higher' do
|
38
|
+
expect(described_class.server_version_eq_or_higher?("5.6.10-log", "5.6.9")).to be_truthy
|
39
|
+
end
|
40
|
+
it 'returns false if version is lower' do
|
41
|
+
expect(described_class.server_version_eq_or_higher?("5.6.8-log", "5.6.9")).to be_falsey
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
describe '.version_to_array' do
|
46
|
+
it 'converts string to array' do
|
47
|
+
expect(described_class.version_to_array('5.6.9-log')).to eq([5, 6, 9])
|
48
|
+
expect(described_class.version_to_array('5.6.24-2+deb.sury.org~precise+2-log')).to eq([5, 6, 24, 0, 0])
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
22
52
|
describe '#rds?' do
|
23
53
|
subject { subject_object.rds? }
|
24
54
|
|
data/flydata.gemspec
CHANGED
Binary file
|
@@ -170,15 +170,23 @@ module SourceMysql
|
|
170
170
|
end
|
171
171
|
context "where retention period is too low" do
|
172
172
|
before do
|
173
|
-
allow(client).to receive(:query).with("call mysql.rds_show_configuration;").and_return([{"name"=>"binlog retention hours", "value"=>
|
173
|
+
allow(client).to receive(:query).with("call mysql.rds_show_configuration;").and_return([{"name"=>"binlog retention hours", "value"=>95}])
|
174
174
|
end
|
175
175
|
it do
|
176
176
|
expect{subject}.to raise_error(FlydataCore::MysqlCompatibilityError, /rds_set_config/)
|
177
177
|
end
|
178
178
|
end
|
179
|
+
context "where retention period is equal to the recommended limit" do
|
180
|
+
before do
|
181
|
+
allow(client).to receive(:query).with("call mysql.rds_show_configuration;").and_return([{"name"=>"binlog retention hours", "value"=>96}])
|
182
|
+
end
|
183
|
+
it do
|
184
|
+
expect{subject}.to_not raise_error
|
185
|
+
end
|
186
|
+
end
|
179
187
|
context "where retention period is over recommended limit" do
|
180
188
|
before do
|
181
|
-
allow(client).to receive(:query).with("call mysql.rds_show_configuration;").and_return([{"name"=>"binlog retention hours", "value"=>
|
189
|
+
allow(client).to receive(:query).with("call mysql.rds_show_configuration;").and_return([{"name"=>"binlog retention hours", "value"=>97}])
|
182
190
|
end
|
183
191
|
it do
|
184
192
|
expect{subject}.to_not raise_error
|
@@ -191,6 +199,11 @@ module SourceMysql
|
|
191
199
|
it do
|
192
200
|
expect{subject}.to_not raise_error
|
193
201
|
end
|
202
|
+
it "logs the correct message" do
|
203
|
+
message = "[WARNING]Cannot verify RDS retention period on current MySQL user account.\nTo see retention period, please run this on your RDS:\n $> call mysql.rds_show_configuration;\nPlease verify that the hours is not nil and is at least 96 hours\nTo set binlog retention hours, you can run this on your RDS:\n $> call mysql.rds_set_configuration('binlog retention hours', 96);\n"
|
204
|
+
expect(subject_object).to receive(:log_warn_stderr).with(message)
|
205
|
+
subject
|
206
|
+
end
|
194
207
|
end
|
195
208
|
end
|
196
209
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: flydata
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.8.
|
4
|
+
version: 0.8.9
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Koichi Fujikawa
|
@@ -13,7 +13,7 @@ authors:
|
|
13
13
|
autorequire:
|
14
14
|
bindir: bin
|
15
15
|
cert_chain: []
|
16
|
-
date:
|
16
|
+
date: 2018-02-13 00:00:00.000000000 Z
|
17
17
|
dependencies:
|
18
18
|
- !ruby/object:Gem::Dependency
|
19
19
|
name: rest-client
|
@@ -950,7 +950,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
950
950
|
version: '0'
|
951
951
|
requirements: []
|
952
952
|
rubyforge_project:
|
953
|
-
rubygems_version: 2.4.
|
953
|
+
rubygems_version: 2.4.8
|
954
954
|
signing_key:
|
955
955
|
specification_version: 4
|
956
956
|
summary: FlyData Agent
|