flydata 0.3.18 → 0.3.19
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/VERSION +1 -1
- data/flydata-core/lib/flydata-core/redshift/string.rb +25 -0
- data/flydata-core/lib/flydata-core/table_def/redshift_table_def.rb +4 -2
- data/flydata-core/spec/redshift/string_spec.rb +106 -0
- data/flydata-core/spec/table_def/redshift_table_def_spec.rb +13 -4
- data/flydata.gemspec +5 -3
- data/lib/flydata/command/sync.rb +1 -1
- data/lib/flydata/compatibility_check.rb +37 -11
- data/lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb +10 -2
- data/lib/flydata/fluent-plugins/mysql/alter_table_query_handler.rb +1 -0
- data/lib/flydata/parser/mysql/dump_parser.rb +1 -1
- data/spec/flydata/compatibility_check_spec.rb +47 -24
- metadata +4 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: fb027d1283d4bc0ba51a57444fd0c864d8611675
|
4
|
+
data.tar.gz: 2d2b13d509dfffb4903b5d47291c8d981c166148
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 7fb220f5741f887dbef704517432a62a7843bdfc26918a7cfd2553e3bea49fdd93a346aac977dacf5ccf97ddb4a010704f15eea1ef0644dca4ec35d5027b097b
|
7
|
+
data.tar.gz: 302b30c6205a8289b1ef4ad6ed983bc0f5f6b13b7486426276f8f9efe3f99dfd2dc225067db6464a69974c72fbf1f5aeb25264e80ec645250405fb16c146e708
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.3.
|
1
|
+
0.3.19
|
@@ -0,0 +1,25 @@
|
|
1
|
+
module FlydataCore
|
2
|
+
module Redshift
|
3
|
+
|
4
|
+
class String
|
5
|
+
# Redshift supports UTF-8 but it enforces stricter rule than other
|
6
|
+
# implementations such as MySQL or Ruby. This method returns a
|
7
|
+
# Redshift-safe string from the given string.
|
8
|
+
def self.encode(string, options = {})
|
9
|
+
result = string.encoding == Encoding::UTF_8 ?
|
10
|
+
string.encode(Encoding::UTF_16, options).encode(Encoding::UTF_8) :
|
11
|
+
string.encode(Encoding::UTF_8, options)
|
12
|
+
rep_str = options[:replace] || "\uFFFD"
|
13
|
+
result.each_codepoint.with_index(0) do |cp, i|
|
14
|
+
# Per Redshift document
|
15
|
+
# http://docs.aws.amazon.com/redshift/latest/dg/multi-byte-character-load-errors.html
|
16
|
+
if cp >= 0xFDD0 && cp <= 0xFDEF || cp == 0xFFFE || cp == 0xFFFF
|
17
|
+
result[i] = rep_str
|
18
|
+
end
|
19
|
+
end
|
20
|
+
result
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
end
|
25
|
+
end
|
@@ -322,11 +322,13 @@ EOS
|
|
322
322
|
end
|
323
323
|
|
324
324
|
def self.parse_date(value)
|
325
|
-
|
325
|
+
return nil if value.nil?
|
326
|
+
value_str = value.to_s
|
327
|
+
dt = Date.parse(convert_year_into_date(value_str))
|
326
328
|
dt.strftime('%Y-%m-%d')
|
327
329
|
rescue ArgumentError => ae
|
328
330
|
# '0000-00-00' is valid for mysql date column
|
329
|
-
return '0001-01-01' if
|
331
|
+
return '0001-01-01' if value_str == '0000-00-00'
|
330
332
|
raise ae
|
331
333
|
end
|
332
334
|
|
@@ -0,0 +1,106 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'flydata-core/redshift/string'
|
3
|
+
|
4
|
+
module FlydataCore
|
5
|
+
module Redshift
|
6
|
+
|
7
|
+
describe String do
|
8
|
+
let(:subject_object) { described_class }
|
9
|
+
|
10
|
+
describe '.encode' do
|
11
|
+
subject { subject_object.encode(string, options) }
|
12
|
+
let(:result) { string.encode(Encoding::UTF_8) }
|
13
|
+
|
14
|
+
shared_examples "converting string to a Redshift friendly string" do
|
15
|
+
before do
|
16
|
+
string.encode!(src_encoding)
|
17
|
+
end
|
18
|
+
it do
|
19
|
+
is_expected.to eq result
|
20
|
+
end
|
21
|
+
end
|
22
|
+
shared_examples "converting an invalid string to raise an error" do
|
23
|
+
before do
|
24
|
+
string.encode!(src_encoding)
|
25
|
+
end
|
26
|
+
it do
|
27
|
+
expect{subject}.to raise_error error
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
context "with replace option" do
|
32
|
+
let(:options) { { invalid: :replace, undef: :replace } }
|
33
|
+
context 'UTF-8 string' do
|
34
|
+
let(:src_encoding) { Encoding::UTF_8 }
|
35
|
+
context 'valid char' do
|
36
|
+
let(:string) { "Straße" }
|
37
|
+
|
38
|
+
it_behaves_like "converting string to a Redshift friendly string"
|
39
|
+
end
|
40
|
+
context 'invalid char' do
|
41
|
+
let(:string) { "\xeb\x13\x00" }
|
42
|
+
let(:result) { "\uFFFD\u0013\u0000".encode(Encoding::UTF_8) }
|
43
|
+
|
44
|
+
it_behaves_like "converting string to a Redshift friendly string"
|
45
|
+
end
|
46
|
+
context 'invalid char for Redshift (error 6)' do
|
47
|
+
let(:string) { "\xef\xb7\x91" }
|
48
|
+
let(:result) { "\uFFFD".encode(Encoding::UTF_8) }
|
49
|
+
|
50
|
+
it_behaves_like "converting string to a Redshift friendly string"
|
51
|
+
end
|
52
|
+
context 'invalid char for Redshift (error 7)' do
|
53
|
+
let(:string) { "\xef\xbf\xbf" }
|
54
|
+
let(:result) { "\uFFFD".encode(Encoding::UTF_8) }
|
55
|
+
|
56
|
+
it_behaves_like "converting string to a Redshift friendly string"
|
57
|
+
end
|
58
|
+
end
|
59
|
+
context 'Shift JIS string' do
|
60
|
+
let(:src_encoding) { Encoding::SHIFT_JIS }
|
61
|
+
let(:string) { "テスト" }
|
62
|
+
it_behaves_like "converting string to a Redshift friendly string"
|
63
|
+
end
|
64
|
+
end
|
65
|
+
context 'without replace option' do
|
66
|
+
let(:options) { {} }
|
67
|
+
context 'UTF-8 string' do
|
68
|
+
let(:src_encoding) { Encoding::UTF_8 }
|
69
|
+
context 'valid char' do
|
70
|
+
let(:string) { "Straße" }
|
71
|
+
|
72
|
+
it_behaves_like "converting string to a Redshift friendly string"
|
73
|
+
end
|
74
|
+
context 'invalid char' do
|
75
|
+
let(:string) { "\xeb\x13\x00" }
|
76
|
+
let(:error) { Encoding::InvalidByteSequenceError }
|
77
|
+
|
78
|
+
it_behaves_like "converting an invalid string to raise an error"
|
79
|
+
end
|
80
|
+
context 'invalid char for Redshift (error 6)' do
|
81
|
+
let(:string) { "\xef\xb7\x91" }
|
82
|
+
let(:result) { "\uFFFD".encode(Encoding::UTF_8) }
|
83
|
+
|
84
|
+
# Redshift invalid characters are replaced regardless of options
|
85
|
+
it_behaves_like "converting string to a Redshift friendly string"
|
86
|
+
end
|
87
|
+
context 'invalid char for Redshift (error 7)' do
|
88
|
+
let(:string) { "\xef\xbf\xbf" }
|
89
|
+
let(:result) { "\uFFFD".encode(Encoding::UTF_8) }
|
90
|
+
|
91
|
+
# Redshift invalid characters are replaced regardless of options
|
92
|
+
it_behaves_like "converting string to a Redshift friendly string"
|
93
|
+
end
|
94
|
+
end
|
95
|
+
context 'Shift JIS string' do
|
96
|
+
let(:src_encoding) { Encoding::SHIFT_JIS }
|
97
|
+
let(:string) { "テスト" }
|
98
|
+
it_behaves_like "converting string to a Redshift friendly string"
|
99
|
+
end
|
100
|
+
end
|
101
|
+
end
|
102
|
+
end
|
103
|
+
|
104
|
+
end
|
105
|
+
end
|
106
|
+
|
@@ -625,16 +625,21 @@ EOT
|
|
625
625
|
end
|
626
626
|
end
|
627
627
|
|
628
|
-
describe '.
|
628
|
+
describe '.parse_date' do
|
629
629
|
let(:value) { nil }
|
630
|
-
subject { described_class.
|
630
|
+
subject { described_class.parse_date(value) }
|
631
631
|
|
632
632
|
context 'with year values' do
|
633
|
-
context 'with value 0' do
|
633
|
+
context 'with value string 0' do
|
634
634
|
let(:value){ '0' }
|
635
635
|
it { is_expected.to eq('0001-01-01') }
|
636
636
|
end
|
637
637
|
|
638
|
+
context 'with value int 0' do
|
639
|
+
let(:value){ 0 }
|
640
|
+
it { is_expected.to eq('0001-01-01') }
|
641
|
+
end
|
642
|
+
|
638
643
|
context 'with value 0000' do
|
639
644
|
let(:value){ '0000' }
|
640
645
|
it { is_expected.to eq('0001-01-01') }
|
@@ -672,9 +677,13 @@ EOT
|
|
672
677
|
|
673
678
|
context 'with zero date' do
|
674
679
|
let(:value){ '0000-00-00' }
|
675
|
-
|
680
|
+
it { is_expected.to eq('0001-01-01') }
|
676
681
|
end
|
677
682
|
end
|
683
|
+
|
684
|
+
context 'with nil' do
|
685
|
+
it { is_expected.to eq(nil) }
|
686
|
+
end
|
678
687
|
end
|
679
688
|
end
|
680
689
|
|
data/flydata.gemspec
CHANGED
@@ -2,16 +2,16 @@
|
|
2
2
|
# DO NOT EDIT THIS FILE DIRECTLY
|
3
3
|
# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
|
4
4
|
# -*- encoding: utf-8 -*-
|
5
|
-
# stub: flydata 0.3.
|
5
|
+
# stub: flydata 0.3.19 ruby lib
|
6
6
|
|
7
7
|
Gem::Specification.new do |s|
|
8
8
|
s.name = "flydata"
|
9
|
-
s.version = "0.3.
|
9
|
+
s.version = "0.3.19"
|
10
10
|
|
11
11
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
12
12
|
s.require_paths = ["lib"]
|
13
13
|
s.authors = ["Koichi Fujikawa", "Masashi Miyazaki", "Matthew Luu", "Mak Inada", "Sriram NS"]
|
14
|
-
s.date = "2015-
|
14
|
+
s.date = "2015-05-12"
|
15
15
|
s.description = "FlyData Agent"
|
16
16
|
s.email = "sysadmin@flydata.com"
|
17
17
|
s.executables = ["fdmysqldump", "flydata", "serverinfo"]
|
@@ -48,6 +48,7 @@ Gem::Specification.new do |s|
|
|
48
48
|
"flydata-core/lib/flydata-core/fluent/config_helper.rb",
|
49
49
|
"flydata-core/lib/flydata-core/logger.rb",
|
50
50
|
"flydata-core/lib/flydata-core/record/record.rb",
|
51
|
+
"flydata-core/lib/flydata-core/redshift/string.rb",
|
51
52
|
"flydata-core/lib/flydata-core/table_def.rb",
|
52
53
|
"flydata-core/lib/flydata-core/table_def/mysql_table_def.rb",
|
53
54
|
"flydata-core/lib/flydata-core/table_def/redshift_table_def.rb",
|
@@ -55,6 +56,7 @@ Gem::Specification.new do |s|
|
|
55
56
|
"flydata-core/spec/config/user_maintenance_spec.rb",
|
56
57
|
"flydata-core/spec/fluent/config_helper_spec.rb",
|
57
58
|
"flydata-core/spec/logger_spec.rb",
|
59
|
+
"flydata-core/spec/redshift/string_spec.rb",
|
58
60
|
"flydata-core/spec/spec_helper.rb",
|
59
61
|
"flydata-core/spec/table_def/mysql_table_def_spec.rb",
|
60
62
|
"flydata-core/spec/table_def/mysql_to_redshift_table_def_spec.rb",
|
data/lib/flydata/command/sync.rb
CHANGED
@@ -420,7 +420,7 @@ EOM
|
|
420
420
|
end
|
421
421
|
end
|
422
422
|
|
423
|
-
log_info_stdout("
|
423
|
+
log_info_stdout("Setting binary log position and exporting data from the database.")
|
424
424
|
log_info_stdout("This process can take hours depending on data size and load on your database. Please be patient...")
|
425
425
|
if file_dump
|
426
426
|
begin
|
@@ -100,26 +100,43 @@ module Flydata
|
|
100
100
|
end
|
101
101
|
|
102
102
|
def check_mysql_user_compat
|
103
|
-
|
103
|
+
databases = ['mysql', @db_opts[:database]]
|
104
|
+
get_grant_regex = /GRANT (?<privs>.*) ON (`)?(?<db_name>[^`]*)(`)?\.\* TO '#{@db_opts[:username]}/
|
105
|
+
necessary_permission_fields = ["SELECT","RELOAD","LOCK TABLES","REPLICATION SLAVE","REPLICATION CLIENT"]
|
106
|
+
all_privileges_field = ["ALL PRIVILEGES"]
|
107
|
+
|
108
|
+
# Do not catch MySQL connection problem because check should stop if no MySQL connection can be made.
|
104
109
|
grants_sql = "SHOW GRANTS"
|
105
|
-
|
106
|
-
necessary_permission_fields= ["SELECT","RELOAD","LOCK TABLES","REPLICATION SLAVE","REPLICATION CLIENT"]
|
107
|
-
all_privileges_field= ["ALL PRIVILEGES"]
|
110
|
+
client = Mysql2::Client.new(@db_opts)
|
108
111
|
result = client.query(grants_sql)
|
109
|
-
# Do not catch MySQL connection problem because check should stop if no MySQL connection can be made.
|
110
112
|
client.close
|
111
|
-
|
113
|
+
|
114
|
+
found_priv = Hash[databases.map {|d| [d,[]]}]
|
115
|
+
missing_priv = {}
|
116
|
+
|
112
117
|
result.each do |res|
|
113
118
|
# SHOW GRANTS should only return one column
|
114
119
|
res_value = res.values.first
|
115
|
-
|
116
|
-
|
117
|
-
|
120
|
+
matched_values = res_value.match(get_grant_regex)
|
121
|
+
next unless matched_values
|
122
|
+
line_priv = matched_values["privs"].split(", ")
|
123
|
+
if matched_values["db_name"] == "*"
|
124
|
+
return true if (all_privileges_field - line_priv).empty?
|
125
|
+
databases.each {|d| found_priv[d] << line_priv }
|
126
|
+
elsif databases.include? matched_values["db_name"]
|
127
|
+
if (all_privileges_field - line_priv).empty?
|
128
|
+
found_priv[matched_values["db_name"]] = necessary_permission_fields
|
129
|
+
else
|
130
|
+
found_priv[matched_values["db_name"]] << line_priv
|
118
131
|
end
|
119
|
-
return true if (necessary_permission_fields-found_priv).empty? or all_privileges_field.all? {|d| res_value.match(d)}
|
120
132
|
end
|
133
|
+
missing_priv = get_missing_privileges(found_priv, necessary_permission_fields)
|
134
|
+
return true if missing_priv.empty?
|
121
135
|
end
|
122
|
-
|
136
|
+
error_text = "The user '#{@db_opts[:username]}' does not have the correct permissions to run FlyData Sync\n"
|
137
|
+
error_text << " * These privileges are missing...\n"
|
138
|
+
missing_priv.each_key {|db| error_text << " for the database '#{db}': #{missing_priv[db].join(", ")}\n"}
|
139
|
+
raise MysqlCompatibilityError, error_text
|
123
140
|
end
|
124
141
|
|
125
142
|
def check_mysql_protocol_tcp_compat
|
@@ -221,6 +238,15 @@ module Flydata
|
|
221
238
|
end
|
222
239
|
end
|
223
240
|
|
241
|
+
def get_missing_privileges(found_priv, all_priv)
|
242
|
+
return_hash = {}
|
243
|
+
found_priv.each_key do |key|
|
244
|
+
missing_priv = all_priv - found_priv[key].flatten.uniq
|
245
|
+
return_hash[key] = missing_priv unless missing_priv.empty?
|
246
|
+
end
|
247
|
+
return_hash
|
248
|
+
end
|
249
|
+
|
224
250
|
def run_mysql_retention_check(mysql_client)
|
225
251
|
expire_logs_days_limit = BINLOG_RETENTION_HOURS / 24
|
226
252
|
sel_query = SELECT_QUERY_TMPLT % '@@expire_logs_days'
|
@@ -80,8 +80,16 @@ class MysqlBinlogFlydataInput < MysqlBinlogInput
|
|
80
80
|
|
81
81
|
$log.info "mysql host:\"#{@host}\" port:\"#{@port}\" username:\"#{@username}\" database:\"#{@database}\" tables:\"#{@tables}\" tables_append_only:\"#{tables_append_only}\""
|
82
82
|
$log.info "mysql client version: #{`mysql -V`}"
|
83
|
-
|
84
|
-
|
83
|
+
server_msg = `echo 'select version();' | #{Flydata::Mysql::MysqlUtil.generate_mysql_cmd(@db_opts)} 2>&1`
|
84
|
+
server_msg = server_msg.each_line.select{|l| l.start_with?('ERROR ')}.join("\n")
|
85
|
+
if ($?.exitstatus == 0)
|
86
|
+
$log.info "mysql server version: #{server_msg.strip}"
|
87
|
+
else
|
88
|
+
err_msg = "Failed to access mysql server... #{server_msg.strip}"
|
89
|
+
$log.error err_msg
|
90
|
+
$stderr.puts err_msg
|
91
|
+
#exit 1 # causes retry loop
|
92
|
+
end
|
85
93
|
|
86
94
|
@tables = @tables.split(/,\s*/)
|
87
95
|
@omit_events = Hash.new
|
@@ -35,6 +35,12 @@ module Flydata
|
|
35
35
|
end
|
36
36
|
|
37
37
|
describe MysqlCompatibilityCheck do
|
38
|
+
let(:default_data_port) do
|
39
|
+
{
|
40
|
+
"servers"=>["sample-test-site.com"]
|
41
|
+
}
|
42
|
+
end
|
43
|
+
|
38
44
|
let(:default_mysql_cred) do
|
39
45
|
{
|
40
46
|
"host" => "test",
|
@@ -44,69 +50,84 @@ module Flydata
|
|
44
50
|
"database" => "test_db"
|
45
51
|
}
|
46
52
|
end
|
53
|
+
|
47
54
|
describe "#check_mysql_user_compat" do
|
48
55
|
let(:client) { double('client') }
|
49
|
-
subject { MysqlCompatibilityCheck.new(
|
56
|
+
subject { MysqlCompatibilityCheck.new(default_data_port, default_mysql_cred) }
|
50
57
|
before do
|
51
58
|
allow(Mysql2::Client).to receive(:new).and_return(client)
|
52
59
|
allow(client).to receive(:close)
|
53
60
|
end
|
54
61
|
|
55
|
-
context "with all privileges
|
62
|
+
context "with all privileges in all databases" do
|
56
63
|
before do
|
57
|
-
allow(client).to receive(:query).and_return([{"Grants for test"=>"GRANT
|
64
|
+
allow(client).to receive(:query).and_return([{"Grants for test"=>"GRANT ALL PRIVILEGES ON *.* TO 'test'@'host"}])
|
58
65
|
end
|
59
66
|
it do
|
60
|
-
expect{subject.check_mysql_user_compat}
|
67
|
+
expect{subject.check_mysql_user_compat}.to_not raise_error
|
61
68
|
end
|
62
69
|
end
|
63
|
-
context "with missing privileges
|
70
|
+
context "with missing privileges in one database" do
|
64
71
|
before do
|
65
|
-
allow(client).to receive(:query).and_return([{"Grants for test"=>"GRANT
|
72
|
+
allow(client).to receive(:query).and_return([{"Grants for test"=>"GRANT ALL PRIVILEGES ON 'mysql'.* TO 'test'@'host"},
|
73
|
+
{"Grants for test"=>"GRANT SELECT, RELOAD, REPLICATION CLIENT ON `test_db`.* TO 'test'@'host"}])
|
66
74
|
end
|
67
75
|
it do
|
68
|
-
expect{subject.check_mysql_user_compat}.to raise_error(Flydata::MysqlCompatibilityCheck::MysqlCompatibilityError, /LOCK TABLES, REPLICATION SLAVE/)
|
76
|
+
expect{subject.check_mysql_user_compat}.to raise_error(Flydata::MysqlCompatibilityCheck::MysqlCompatibilityError, /test_db': LOCK TABLES, REPLICATION SLAVE/)
|
69
77
|
end
|
70
78
|
end
|
71
|
-
context "with all
|
79
|
+
context "with all required privileges in between all and specific databases" do
|
72
80
|
before do
|
73
81
|
allow(client).to receive(:query).and_return([{"Grants for test"=>"GRANT RELOAD, REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO 'test'@'host"},
|
74
|
-
{"Grants for test"=>"GRANT SELECT, LOCK TABLES ON `test_db`.* TO 'test'@'host"}
|
82
|
+
{"Grants for test"=>"GRANT SELECT, LOCK TABLES ON `test_db`.* TO 'test'@'host"},
|
83
|
+
{"Grants for test"=>"GRANT SELECT, LOCK TABLES ON `mysql`.* TO 'test'@'host"}])
|
75
84
|
end
|
76
85
|
it do
|
77
|
-
expect{subject.check_mysql_user_compat}
|
86
|
+
expect{subject.check_mysql_user_compat}.to_not raise_error
|
78
87
|
end
|
79
88
|
end
|
80
|
-
context "with missing privileges
|
89
|
+
context "with missing privileges in each database" do
|
81
90
|
before do
|
82
91
|
allow(client).to receive(:query).and_return([{"Grants for test"=>"GRANT REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO 'test'@'host"},
|
83
|
-
{"Grants for test"=>"GRANT SELECT, LOCK TABLES ON `test_db`.* TO 'test'@'host"}
|
92
|
+
{"Grants for test"=>"GRANT SELECT, LOCK TABLES ON `test_db`.* TO 'test'@'host"},
|
93
|
+
{"Grants for test"=>"GRANT SELECT, LOCK TABLES ON `mysql`.* TO 'test'@'host"}])
|
84
94
|
end
|
85
95
|
it do
|
86
|
-
expect{subject.check_mysql_user_compat}.to raise_error(Flydata::MysqlCompatibilityCheck::MysqlCompatibilityError, /RELOAD/)
|
96
|
+
expect{subject.check_mysql_user_compat}.to raise_error(Flydata::MysqlCompatibilityCheck::MysqlCompatibilityError, /mysql': RELOAD\n.*test_db': RELOAD/)
|
87
97
|
end
|
88
98
|
end
|
89
|
-
context "with all
|
99
|
+
context "with all required privileges in all databases" do
|
90
100
|
before do
|
91
101
|
allow(client).to receive(:query).and_return([{"Grants for test"=>"GRANT SELECT, LOCK TABLES, RELOAD, REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO 'test'@'host"}])
|
92
102
|
end
|
93
103
|
it do
|
94
|
-
expect{subject.check_mysql_user_compat}
|
104
|
+
expect{subject.check_mysql_user_compat}.to_not raise_error
|
95
105
|
end
|
96
106
|
end
|
97
|
-
context "with missing privileges
|
107
|
+
context "with missing privileges in all databases" do
|
98
108
|
before do
|
99
109
|
allow(client).to receive(:query).and_return([{"Grants for test"=>"GRANT RELOAD, REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO 'test'@'host"}])
|
100
110
|
end
|
101
111
|
it do
|
102
|
-
expect{subject.check_mysql_user_compat}.to raise_error(Flydata::MysqlCompatibilityCheck::MysqlCompatibilityError, /SELECT, LOCK TABLES/)
|
112
|
+
expect{subject.check_mysql_user_compat}.to raise_error(Flydata::MysqlCompatibilityCheck::MysqlCompatibilityError, /mysql': SELECT, LOCK TABLES\n.*test_db': SELECT, LOCK TABLES/)
|
113
|
+
end
|
114
|
+
end
|
115
|
+
context "with privileges for other databases only" do
|
116
|
+
before do
|
117
|
+
allow(client).to receive(:query).and_return([{"Grants for test"=>"GRANT REPLICATION CLIENT ON `test_db_01`.* TO 'test'@'host"},
|
118
|
+
{"Grants for test"=>"GRANT LOCK TABLES ON `test_db_02`.* TO 'test'@'host"},
|
119
|
+
{"Grants for test"=>"GRANT SELECT ON `text_db_03`.* TO 'test'@'host"}])
|
120
|
+
end
|
121
|
+
it do
|
122
|
+
expect{subject.check_mysql_user_compat}.to raise_error(Flydata::MysqlCompatibilityCheck::MysqlCompatibilityError, /mysql': SELECT, RELOAD, LOCK TABLES, REPLICATION SLAVE, REPLICATION CLIENT\n.*test_db': SELECT, RELOAD, LOCK TABLES, REPLICATION SLAVE, REPLICATION CLIENT/)
|
103
123
|
end
|
104
124
|
end
|
105
125
|
|
106
126
|
end
|
127
|
+
|
107
128
|
describe "#check_mysql_binlog_retention" do
|
108
129
|
context "on on-premise mysql server" do
|
109
|
-
subject { MysqlCompatibilityCheck.new(
|
130
|
+
subject { MysqlCompatibilityCheck.new(default_data_port, default_mysql_cred) }
|
110
131
|
context "where retention is below limit" do
|
111
132
|
let(:client) { double('client') }
|
112
133
|
before do
|
@@ -124,10 +145,11 @@ module Flydata
|
|
124
145
|
before do
|
125
146
|
allow(Mysql2::Client).to receive(:new).and_return(client)
|
126
147
|
allow(client).to receive(:query).and_return([{"@@expire_logs_days"=>0}])
|
148
|
+
allow(client).to receive(:close)
|
127
149
|
allow(subject).to receive(:is_rds?).and_return(false)
|
128
150
|
end
|
129
151
|
it do
|
130
|
-
expect{subject.check_mysql_binlog_retention}
|
152
|
+
expect{subject.check_mysql_binlog_retention}.to_not raise_error
|
131
153
|
end
|
132
154
|
end
|
133
155
|
context "where retention is above limit" do
|
@@ -135,15 +157,16 @@ module Flydata
|
|
135
157
|
before do
|
136
158
|
allow(Mysql2::Client).to receive(:new).and_return(client)
|
137
159
|
allow(client).to receive(:query).and_return([{"@@expire_logs_days"=>11}])
|
160
|
+
allow(client).to receive(:close)
|
138
161
|
allow(subject).to receive(:is_rds?).and_return(false)
|
139
162
|
end
|
140
163
|
it do
|
141
|
-
expect{subject.check_mysql_binlog_retention}
|
164
|
+
expect{subject.check_mysql_binlog_retention}.to_not raise_error
|
142
165
|
end
|
143
166
|
end
|
144
167
|
end
|
145
168
|
context "on RDS" do
|
146
|
-
subject { MysqlCompatibilityCheck.new(
|
169
|
+
subject { MysqlCompatibilityCheck.new(default_data_port, default_mysql_cred) }
|
147
170
|
context "where retention period is nil" do
|
148
171
|
let(:client) { double('client') }
|
149
172
|
before do
|
@@ -180,7 +203,7 @@ module Flydata
|
|
180
203
|
allow(subject).to receive(:is_rds?).and_return(true)
|
181
204
|
end
|
182
205
|
it do
|
183
|
-
expect{subject.check_mysql_binlog_retention}
|
206
|
+
expect{subject.check_mysql_binlog_retention}.to_not raise_error
|
184
207
|
end
|
185
208
|
end
|
186
209
|
context "where user has no access to rds configuration" do
|
@@ -193,7 +216,7 @@ module Flydata
|
|
193
216
|
allow(subject).to receive(:is_rds?).and_return(true)
|
194
217
|
end
|
195
218
|
it do
|
196
|
-
expect{subject.check_mysql_binlog_retention}
|
219
|
+
expect{subject.check_mysql_binlog_retention}.to_not raise_error
|
197
220
|
end
|
198
221
|
end
|
199
222
|
end
|
@@ -212,7 +235,7 @@ module Flydata
|
|
212
235
|
let(:engine_table) { {"table_name"=>"engine_table", "table_type"=>"BASE TABLE", "engine"=>"MEMORY"} }
|
213
236
|
let(:view) { {"table_name"=>"view_table", "table_type"=>"VIEW", "engine"=>nil} }
|
214
237
|
let(:client) { double('client') }
|
215
|
-
let(:subject_object) { MysqlCompatibilityCheck.new(
|
238
|
+
let(:subject_object) { MysqlCompatibilityCheck.new(default_data_port,test_data_entry, {}) }
|
216
239
|
let(:error) { Flydata::MysqlCompatibilityCheck::MysqlCompatibilityError }
|
217
240
|
let(:base_error_msg) { "FlyData does not support VIEW and MEMORY ENGINE table. Remove following tables from data entry: %s" }
|
218
241
|
subject { subject_object.check_mysql_table_types }
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: flydata
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.3.
|
4
|
+
version: 0.3.19
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Koichi Fujikawa
|
@@ -12,7 +12,7 @@ authors:
|
|
12
12
|
autorequire:
|
13
13
|
bindir: bin
|
14
14
|
cert_chain: []
|
15
|
-
date: 2015-
|
15
|
+
date: 2015-05-12 00:00:00.000000000 Z
|
16
16
|
dependencies:
|
17
17
|
- !ruby/object:Gem::Dependency
|
18
18
|
name: rest-client
|
@@ -471,6 +471,7 @@ files:
|
|
471
471
|
- flydata-core/lib/flydata-core/fluent/config_helper.rb
|
472
472
|
- flydata-core/lib/flydata-core/logger.rb
|
473
473
|
- flydata-core/lib/flydata-core/record/record.rb
|
474
|
+
- flydata-core/lib/flydata-core/redshift/string.rb
|
474
475
|
- flydata-core/lib/flydata-core/table_def.rb
|
475
476
|
- flydata-core/lib/flydata-core/table_def/mysql_table_def.rb
|
476
477
|
- flydata-core/lib/flydata-core/table_def/redshift_table_def.rb
|
@@ -478,6 +479,7 @@ files:
|
|
478
479
|
- flydata-core/spec/config/user_maintenance_spec.rb
|
479
480
|
- flydata-core/spec/fluent/config_helper_spec.rb
|
480
481
|
- flydata-core/spec/logger_spec.rb
|
482
|
+
- flydata-core/spec/redshift/string_spec.rb
|
481
483
|
- flydata-core/spec/spec_helper.rb
|
482
484
|
- flydata-core/spec/table_def/mysql_table_def_spec.rb
|
483
485
|
- flydata-core/spec/table_def/mysql_to_redshift_table_def_spec.rb
|