flydata 0.2.11 → 0.2.12
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile +1 -1
- data/Gemfile.lock +2 -2
- data/VERSION +1 -1
- data/flydata.gemspec +10 -7
- data/lib/flydata/command/sender.rb +5 -0
- data/lib/flydata/command/sync.rb +20 -4
- data/lib/flydata/compatibility_check.rb +178 -0
- data/lib/flydata/parser/mysql/dump_parser.rb +0 -122
- data/spec/flydata/command/sender_spec.rb +3 -0
- data/spec/flydata/compatibility_check_spec.rb +25 -0
- metadata +7 -5
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 834f55f013a932c4bbd53408f020fe886ab2acf5
|
4
|
+
data.tar.gz: 702f346a6a072648956489fab6bf75776adb3351
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 3cafe705f68a2b86801e0af7bd93f5b450adaceb1f183876b44d01c3b921c437c78376b093b2e516e80446bc84fb18f18dfeeacc88ee1b4b6ff2922e0020fd06
|
7
|
+
data.tar.gz: 9aa3192f6524da454763d808941fdb847806166cfd369712835d37c36d33f2d5f307ac98fe31f44cea46a212c90b12b4c2c78594ebb691623369ab6705947cf5
|
data/Gemfile
CHANGED
@@ -7,7 +7,7 @@ gem "activesupport", '~> 4.0.0'
|
|
7
7
|
gem "json", '~> 1.8', '>= 1.8.0'
|
8
8
|
gem "highline", '~> 1.6', '>= 1.6.19'
|
9
9
|
gem "fluentd", "0.10.46"
|
10
|
-
gem "ruby-binlog", '~> 1.0', '>= 1.0.
|
10
|
+
gem "ruby-binlog", '~> 1.0', '>= 1.0.2'
|
11
11
|
gem "fluent-plugin-mysql-binlog", '~> 0.0', '>= 0.0.2'
|
12
12
|
gem "mysql2", '~> 0.3', '>= 0.3.11'
|
13
13
|
gem "slop", '~> 3.4', '>= 3.4.6'
|
data/Gemfile.lock
CHANGED
@@ -103,7 +103,7 @@ GEM
|
|
103
103
|
rspec-mocks (3.0.3)
|
104
104
|
rspec-support (~> 3.0.0)
|
105
105
|
rspec-support (3.0.3)
|
106
|
-
ruby-binlog (1.0.
|
106
|
+
ruby-binlog (1.0.2)
|
107
107
|
ruby-prof (0.15.1)
|
108
108
|
sigdump (0.2.2)
|
109
109
|
slop (3.6.0)
|
@@ -133,7 +133,7 @@ DEPENDENCIES
|
|
133
133
|
pry
|
134
134
|
rest-client (~> 1.6, >= 1.6.7)
|
135
135
|
rspec (~> 3.0)
|
136
|
-
ruby-binlog (~> 1.0, >= 1.0.
|
136
|
+
ruby-binlog (~> 1.0, >= 1.0.2)
|
137
137
|
ruby-prof (~> 0.15, >= 0.15.1)
|
138
138
|
slop (~> 3.4, >= 3.4.6)
|
139
139
|
sqlite3 (~> 1.3, >= 1.3.9)
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.2.
|
1
|
+
0.2.12
|
data/flydata.gemspec
CHANGED
@@ -2,14 +2,16 @@
|
|
2
2
|
# DO NOT EDIT THIS FILE DIRECTLY
|
3
3
|
# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
|
4
4
|
# -*- encoding: utf-8 -*-
|
5
|
+
# stub: flydata 0.2.12 ruby lib
|
5
6
|
|
6
7
|
Gem::Specification.new do |s|
|
7
8
|
s.name = "flydata"
|
8
|
-
s.version = "0.2.
|
9
|
+
s.version = "0.2.12"
|
9
10
|
|
10
11
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
12
|
+
s.require_paths = ["lib"]
|
11
13
|
s.authors = ["Koichi Fujikawa", "Masashi Miyazaki", "Matthew Luu", "Mak Inada", "Sriram NS"]
|
12
|
-
s.date = "2014-
|
14
|
+
s.date = "2014-11-03"
|
13
15
|
s.description = "FlyData Agent"
|
14
16
|
s.email = "sysadmin@flydata.com"
|
15
17
|
s.executables = ["fdmysqldump", "flydata", "serverinfo"]
|
@@ -53,6 +55,7 @@ Gem::Specification.new do |s|
|
|
53
55
|
"lib/flydata/command/stop.rb",
|
54
56
|
"lib/flydata/command/sync.rb",
|
55
57
|
"lib/flydata/command/version.rb",
|
58
|
+
"lib/flydata/compatibility_check.rb",
|
56
59
|
"lib/flydata/credentials.rb",
|
57
60
|
"lib/flydata/cron.rb",
|
58
61
|
"lib/flydata/errors.rb",
|
@@ -92,6 +95,7 @@ Gem::Specification.new do |s|
|
|
92
95
|
"spec/flydata/cli_spec.rb",
|
93
96
|
"spec/flydata/command/sender_spec.rb",
|
94
97
|
"spec/flydata/command/sync_spec.rb",
|
98
|
+
"spec/flydata/compatibility_check_spec.rb",
|
95
99
|
"spec/flydata/fluent-plugins/in_mysql_binlog_flydata_spec.rb",
|
96
100
|
"spec/flydata/fluent-plugins/mysql/alter_table_query_handler_spec.rb",
|
97
101
|
"spec/flydata/fluent-plugins/mysql/binlog_position_spec.rb",
|
@@ -118,8 +122,7 @@ Gem::Specification.new do |s|
|
|
118
122
|
]
|
119
123
|
s.homepage = "http://flydata.com/"
|
120
124
|
s.licenses = ["All right reserved."]
|
121
|
-
s.
|
122
|
-
s.rubygems_version = "2.0.14"
|
125
|
+
s.rubygems_version = "2.2.2"
|
123
126
|
s.summary = "FlyData Agent"
|
124
127
|
|
125
128
|
if s.respond_to? :specification_version then
|
@@ -132,7 +135,7 @@ Gem::Specification.new do |s|
|
|
132
135
|
s.add_runtime_dependency(%q<json>, [">= 1.8.0", "~> 1.8"])
|
133
136
|
s.add_runtime_dependency(%q<highline>, [">= 1.6.19", "~> 1.6"])
|
134
137
|
s.add_runtime_dependency(%q<fluentd>, ["= 0.10.46"])
|
135
|
-
s.add_runtime_dependency(%q<ruby-binlog>, [">= 1.0.
|
138
|
+
s.add_runtime_dependency(%q<ruby-binlog>, [">= 1.0.2", "~> 1.0"])
|
136
139
|
s.add_runtime_dependency(%q<fluent-plugin-mysql-binlog>, [">= 0.0.2", "~> 0.0"])
|
137
140
|
s.add_runtime_dependency(%q<mysql2>, [">= 0.3.11", "~> 0.3"])
|
138
141
|
s.add_runtime_dependency(%q<slop>, [">= 3.4.6", "~> 3.4"])
|
@@ -153,7 +156,7 @@ Gem::Specification.new do |s|
|
|
153
156
|
s.add_dependency(%q<json>, [">= 1.8.0", "~> 1.8"])
|
154
157
|
s.add_dependency(%q<highline>, [">= 1.6.19", "~> 1.6"])
|
155
158
|
s.add_dependency(%q<fluentd>, ["= 0.10.46"])
|
156
|
-
s.add_dependency(%q<ruby-binlog>, [">= 1.0.
|
159
|
+
s.add_dependency(%q<ruby-binlog>, [">= 1.0.2", "~> 1.0"])
|
157
160
|
s.add_dependency(%q<fluent-plugin-mysql-binlog>, [">= 0.0.2", "~> 0.0"])
|
158
161
|
s.add_dependency(%q<mysql2>, [">= 0.3.11", "~> 0.3"])
|
159
162
|
s.add_dependency(%q<slop>, [">= 3.4.6", "~> 3.4"])
|
@@ -175,7 +178,7 @@ Gem::Specification.new do |s|
|
|
175
178
|
s.add_dependency(%q<json>, [">= 1.8.0", "~> 1.8"])
|
176
179
|
s.add_dependency(%q<highline>, [">= 1.6.19", "~> 1.6"])
|
177
180
|
s.add_dependency(%q<fluentd>, ["= 0.10.46"])
|
178
|
-
s.add_dependency(%q<ruby-binlog>, [">= 1.0.
|
181
|
+
s.add_dependency(%q<ruby-binlog>, [">= 1.0.2", "~> 1.0"])
|
179
182
|
s.add_dependency(%q<fluent-plugin-mysql-binlog>, [">= 0.0.2", "~> 0.0"])
|
180
183
|
s.add_dependency(%q<mysql2>, [">= 0.3.11", "~> 0.3"])
|
181
184
|
s.add_dependency(%q<slop>, [">= 3.4.6", "~> 3.4"])
|
@@ -1,3 +1,5 @@
|
|
1
|
+
require 'flydata/compatibility_check'
|
2
|
+
|
1
3
|
module Flydata
|
2
4
|
module Command
|
3
5
|
class Sender < Base
|
@@ -20,6 +22,9 @@ module Flydata
|
|
20
22
|
|
21
23
|
wait_until_server_ready(options)
|
22
24
|
|
25
|
+
dp = flydata.data_port.get
|
26
|
+
AgentCompatibilityCheck.new(dp).check
|
27
|
+
|
23
28
|
# Start sender(fluentd) process
|
24
29
|
say('Starting sender process.') unless options[:quiet]
|
25
30
|
Dir.chdir(FLYDATA_HOME){
|
data/lib/flydata/command/sync.rb
CHANGED
@@ -4,6 +4,7 @@ require 'open3'
|
|
4
4
|
require 'mysql2'
|
5
5
|
require 'flydata/sync_file_manager'
|
6
6
|
require 'flydata/errors'
|
7
|
+
require 'flydata/compatibility_check'
|
7
8
|
require 'flydata/table_def'
|
8
9
|
require 'flydata/output/forwarder'
|
9
10
|
require 'flydata/parser/mysql/dump_parser'
|
@@ -89,6 +90,7 @@ module Flydata
|
|
89
90
|
delete_files.flatten.each do |path|
|
90
91
|
FileUtils.rm(path) if File.exists?(path)
|
91
92
|
end
|
93
|
+
puts "Reset completed successfully."
|
92
94
|
end
|
93
95
|
|
94
96
|
def wait_for_server_buffer
|
@@ -153,7 +155,8 @@ module Flydata
|
|
153
155
|
|
154
156
|
def generate_table_ddl(*tables)
|
155
157
|
de = retrieve_data_entry
|
156
|
-
|
158
|
+
dp = flydata.data_port.get
|
159
|
+
Flydata::MysqlCompatibilityCheck.new(dp, de['mysql_data_entry_preference']).check
|
157
160
|
do_generate_table_ddl(override_tables(de, tables))
|
158
161
|
end
|
159
162
|
|
@@ -186,8 +189,20 @@ module Flydata
|
|
186
189
|
end
|
187
190
|
|
188
191
|
def cleanup_sync_server(de, tables = [])
|
189
|
-
|
190
|
-
|
192
|
+
print "Cleaning the server."
|
193
|
+
worker = Thread.new do
|
194
|
+
begin
|
195
|
+
flydata.data_entry.cleanup_sync(de['id'], tables)
|
196
|
+
rescue RestClient::RequestTimeout
|
197
|
+
# server is taking time to cleanup. Try again
|
198
|
+
retry
|
199
|
+
end
|
200
|
+
end
|
201
|
+
until worker.join(5)
|
202
|
+
print "."
|
203
|
+
end
|
204
|
+
puts
|
205
|
+
puts "Done."
|
191
206
|
end
|
192
207
|
|
193
208
|
def do_check(de)
|
@@ -295,6 +310,7 @@ module Flydata
|
|
295
310
|
return if dump_pos_info[:status] == STATUS_WAITING || dump_pos_info[:status] == STATUS_COMPLETE
|
296
311
|
|
297
312
|
# mysqldump file exists -> skip dump
|
313
|
+
dp = flydata.data_port.get
|
298
314
|
fp = sync_fm.dump_file_path
|
299
315
|
if file_dump && File.exists?(fp) && File.size(fp) > 0
|
300
316
|
puts " -> Skip"
|
@@ -326,8 +342,8 @@ EOM
|
|
326
342
|
db_bytesize = Flydata::Parser::Mysql::DatabaseSizeCheck.new(de['mysql_data_entry_preference']).get_db_bytesize
|
327
343
|
puts " -> #{as_size(db_bytesize)} (#{db_bytesize} byte)"
|
328
344
|
puts "Exporting data from database..."
|
345
|
+
Flydata::MysqlCompatibilityCheck.new(dp, de['mysql_data_entry_preference'], fp).check
|
329
346
|
if file_dump
|
330
|
-
Flydata::Parser::Mysql::CompatibilityCheck.new(de['mysql_data_entry_preference'], fp).check
|
331
347
|
Flydata::Parser::Mysql::MysqlDumpGeneratorNoMasterData.
|
332
348
|
new(de['mysql_data_entry_preference']).dump(fp)
|
333
349
|
puts " -> Done"
|
@@ -0,0 +1,178 @@
|
|
1
|
+
module Flydata
|
2
|
+
|
3
|
+
class CompatibilityCheck
|
4
|
+
class CompatibilityError < StandardError
|
5
|
+
end
|
6
|
+
|
7
|
+
def initialize(dp_hash, de_hash=nil, options={})
|
8
|
+
@dp = dp_hash
|
9
|
+
@errors=[]
|
10
|
+
end
|
11
|
+
|
12
|
+
def check
|
13
|
+
self.methods.grep(/^check_/).each do |m|
|
14
|
+
begin
|
15
|
+
send(m)
|
16
|
+
rescue CompatibilityError => e
|
17
|
+
@errors << e
|
18
|
+
end
|
19
|
+
end
|
20
|
+
print_errors
|
21
|
+
end
|
22
|
+
|
23
|
+
def print_errors
|
24
|
+
return if @errors.empty?
|
25
|
+
puts "There may be some compatibility issues with this current server : "
|
26
|
+
@errors.each do |error|
|
27
|
+
puts " * #{error.message}"
|
28
|
+
end
|
29
|
+
raise "Please correct these errors if you wish to run FlyData Agent"
|
30
|
+
end
|
31
|
+
|
32
|
+
end
|
33
|
+
|
34
|
+
class AgentCompatibilityCheck < CompatibilityCheck
|
35
|
+
class AgentCompatibilityError < StandardError
|
36
|
+
end
|
37
|
+
|
38
|
+
TCP_PORT=45326
|
39
|
+
SSL_PORT=45327
|
40
|
+
|
41
|
+
def check_outgoing_ports
|
42
|
+
unless can_connect_to_port?(TCP_PORT) and
|
43
|
+
can_connect_to_port?(SSL_PORT)
|
44
|
+
raise AgentCompatibilityError,
|
45
|
+
"Cannot connect to outisde ports. Please check to make sure you have these outgoing ports open: #{TCP_PORT},#{SSL_PORT}"
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
private
|
50
|
+
def can_connect_to_port?(port)
|
51
|
+
url = @dp["servers"].first
|
52
|
+
begin
|
53
|
+
e = TCPSocket.new(url, port)
|
54
|
+
e.close
|
55
|
+
return true
|
56
|
+
rescue Errno::ETIMEDOUT => e
|
57
|
+
return false
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
class MysqlCompatibilityCheck < CompatibilityCheck
|
63
|
+
|
64
|
+
class MysqlCompatibilityError < StandardError
|
65
|
+
end
|
66
|
+
|
67
|
+
SELECT_QUERY_TMPLT = "SELECT %s"
|
68
|
+
|
69
|
+
#def initialize(de_hash, dump_dir=nil)
|
70
|
+
def initialize(dp_hash, de_hash, options={})
|
71
|
+
super
|
72
|
+
@db_opts = [:host, :port, :username, :password, :database].inject({}) {|h, sym| h[sym] = de_hash[sym.to_s]; h}
|
73
|
+
@dump_dir = options['dump_dir'] || nil
|
74
|
+
end
|
75
|
+
|
76
|
+
def print_errors
|
77
|
+
return if @errors.empty?
|
78
|
+
puts "There may be some compatibility issues with your MySQL credentials: "
|
79
|
+
@errors.each do |error|
|
80
|
+
puts " * #{error.message}"
|
81
|
+
end
|
82
|
+
raise "Please correct these errors if you wish to run FlyData Sync"
|
83
|
+
end
|
84
|
+
|
85
|
+
def check_mysql_user_compat
|
86
|
+
client = Mysql2::Client.new(@db_opts)
|
87
|
+
grants_sql = "SHOW GRANTS"
|
88
|
+
correct_db = ["ON (\\*|#{@db_opts[:database]})","TO '#{@db_opts[:username]}"]
|
89
|
+
necessary_permission_fields= ["SELECT","RELOAD","LOCK TABLES","REPLICATION SLAVE","REPLICATION CLIENT"]
|
90
|
+
all_privileges_field= ["ALL PRIVILEGES"]
|
91
|
+
result = client.query(grants_sql)
|
92
|
+
# Do not catch MySQL connection problem because check should stop if no MySQL connection can be made.
|
93
|
+
client.close
|
94
|
+
missing_priv = []
|
95
|
+
result.each do |res|
|
96
|
+
# SHOW GRANTS should only return one column
|
97
|
+
res_value = res.values.first
|
98
|
+
if correct_db.all? {|perm| res_value.match(perm)}
|
99
|
+
necessary_permission_fields.each do |priv|
|
100
|
+
missing_priv << priv unless res_value.match(priv)
|
101
|
+
end
|
102
|
+
return true if missing_priv.empty? or all_privileges_field.all? {|d| res_value.match(d)}
|
103
|
+
end
|
104
|
+
end
|
105
|
+
raise MysqlCompatibilityError, "The user '#{@db_opts[:username]}' does not have the correct permissions to run FlyData Sync\n * These privileges are missing: #{missing_priv.join(", ")}"
|
106
|
+
end
|
107
|
+
|
108
|
+
def check_mysql_protocol_tcp_compat
|
109
|
+
query = "mysql -u #{@db_opts[:username]} -h #{@db_opts[:host]} -P #{@db_opts[:port]} #{@db_opts[:database]} -e \"SHOW GRANTS;\" --protocol=tcp"
|
110
|
+
query << " -p#{@db_opts[:password]}" unless @db_opts[:password].to_s.empty?
|
111
|
+
|
112
|
+
Open3.popen3(query) do |stdin, stdout, stderr|
|
113
|
+
stdin.close
|
114
|
+
while !stderr.eof?
|
115
|
+
line = stderr.gets
|
116
|
+
unless /Warning: Using a password on the command line interface can be insecure./ === line
|
117
|
+
raise MysqlCompatibilityError, "Cannot connect to MySQL database. Please make sure you can connect with this command:\n $ mysql -u #{@db_opts[:username]} -h #{@db_opts[:host]} -P #{@db_opts[:port]} #{@db_opts[:database]} --protocol=tcp -p"
|
118
|
+
end
|
119
|
+
end
|
120
|
+
end
|
121
|
+
end
|
122
|
+
|
123
|
+
def check_mysql_row_mode_compat
|
124
|
+
sys_var_to_check = {'@@binlog_format'=>'ROW', '@@binlog_checksum'=>'NONE', '@@log_bin_use_v1_row_events'=>1}
|
125
|
+
errors={}
|
126
|
+
|
127
|
+
client = Mysql2::Client.new(@db_opts)
|
128
|
+
|
129
|
+
begin
|
130
|
+
sys_var_to_check.each_key do |sys_var|
|
131
|
+
sel_query = SELECT_QUERY_TMPLT % sys_var
|
132
|
+
begin
|
133
|
+
result = client.query(sel_query)
|
134
|
+
unless result.first[sys_var] == sys_var_to_check[sys_var]
|
135
|
+
errors[sys_var]=result.first[sys_var]
|
136
|
+
end
|
137
|
+
rescue Mysql2::Error => e
|
138
|
+
if e.message =~ /Unknown system variable/
|
139
|
+
unless e.message =~ /(binlog_checksum|log_bin_use_v1_row_events)/
|
140
|
+
errors[sys_var] = false
|
141
|
+
end
|
142
|
+
else
|
143
|
+
raise e
|
144
|
+
end
|
145
|
+
end
|
146
|
+
end
|
147
|
+
ensure
|
148
|
+
client.close
|
149
|
+
end
|
150
|
+
unless errors.empty?
|
151
|
+
error_explanation = ""
|
152
|
+
errors.each_key do |err_key|
|
153
|
+
error_explanation << "\n * #{err_key} is #{errors[err_key]} but should be #{sys_var_to_check[err_key]}"
|
154
|
+
end
|
155
|
+
raise MysqlCompatibilityError, "These system variable(s) are not the correct value: #{error_explanation}\n Please change these system variables for FlyData Sync to run correctly"
|
156
|
+
end
|
157
|
+
end
|
158
|
+
|
159
|
+
def check_writing_permissions
|
160
|
+
write_errors = []
|
161
|
+
paths_to_check = ["~/.flydata"]
|
162
|
+
paths_to_check << @dump_dir unless @dump_dir.to_s.empty?
|
163
|
+
paths_to_check.each do |path|
|
164
|
+
full_path = File.expand_path(path)
|
165
|
+
full_path = File.dirname(full_path) unless File.directory?(full_path)
|
166
|
+
write_errors << full_path unless File.writable?(full_path)
|
167
|
+
end
|
168
|
+
unless write_errors.empty?
|
169
|
+
error_dir = write_errors.join(", ")
|
170
|
+
raise MysqlCompatibilityError, "We cannot access the directories: #{error_dir}"
|
171
|
+
end
|
172
|
+
end
|
173
|
+
end
|
174
|
+
|
175
|
+
end
|
176
|
+
|
177
|
+
|
178
|
+
|
@@ -572,128 +572,6 @@ EOS
|
|
572
572
|
end
|
573
573
|
end
|
574
574
|
end
|
575
|
-
class CompatibilityCheck
|
576
|
-
|
577
|
-
class CompatibilityError < StandardError
|
578
|
-
end
|
579
|
-
|
580
|
-
SELECT_QUERY_TMPLT = "SELECT %s"
|
581
|
-
|
582
|
-
def initialize(de_hash, dump_dir=nil)
|
583
|
-
@db_opts = [:host, :port, :username, :password, :database].inject({}) {|h, sym| h[sym] = de_hash[sym.to_s]; h}
|
584
|
-
@dump_dir = dump_dir
|
585
|
-
@errors=[]
|
586
|
-
end
|
587
|
-
|
588
|
-
def check
|
589
|
-
self.methods.grep(/^check_/).each do |m|
|
590
|
-
begin
|
591
|
-
send(m)
|
592
|
-
rescue CompatibilityError => e
|
593
|
-
@errors << e
|
594
|
-
end
|
595
|
-
end
|
596
|
-
print_errors
|
597
|
-
end
|
598
|
-
|
599
|
-
def print_errors
|
600
|
-
return if @errors.empty?
|
601
|
-
puts "There may be some compatibility issues with your MySQL credentials: "
|
602
|
-
@errors.each do |error|
|
603
|
-
puts " * #{error.message}"
|
604
|
-
end
|
605
|
-
raise "Please correct these errors if you wish to run FlyData Sync"
|
606
|
-
end
|
607
|
-
|
608
|
-
def check_mysql_user_compat
|
609
|
-
client = Mysql2::Client.new(@db_opts)
|
610
|
-
grants_sql = "SHOW GRANTS"
|
611
|
-
correct_db = ["ON (\\*|#{@db_opts[:database]})","TO '#{@db_opts[:username]}"]
|
612
|
-
necessary_permission_fields= ["SELECT","RELOAD","LOCK TABLES","REPLICATION SLAVE","REPLICATION CLIENT"]
|
613
|
-
all_privileges_field= ["ALL PRIVILEGES"]
|
614
|
-
result = client.query(grants_sql)
|
615
|
-
# Do not catch MySQL connection problem because check should stop if no MySQL connection can be made.
|
616
|
-
client.close
|
617
|
-
missing_priv = []
|
618
|
-
result.each do |res|
|
619
|
-
# SHOW GRANTS should only return one column
|
620
|
-
res_value = res.values.first
|
621
|
-
if correct_db.all? {|perm| res_value.match(perm)}
|
622
|
-
necessary_permission_fields.each do |priv|
|
623
|
-
missing_priv << priv unless res_value.match(priv)
|
624
|
-
end
|
625
|
-
return true if missing_priv.empty? or all_privileges_field.all? {|d| res_value.match(d)}
|
626
|
-
end
|
627
|
-
end
|
628
|
-
raise CompatibilityError, "The user '#{@db_opts[:username]}' does not have the correct permissions to run FlyData Sync\n * These privileges are missing: #{missing_priv.join(", ")}"
|
629
|
-
end
|
630
|
-
|
631
|
-
def check_mysql_protocol_tcp_compat
|
632
|
-
query = "mysql -u #{@db_opts[:username]} -h #{@db_opts[:host]} -P #{@db_opts[:port]} #{@db_opts[:database]} -e \"SHOW GRANTS;\" --protocol=tcp"
|
633
|
-
query << " -p#{@db_opts[:password]}" unless @db_opts[:password].to_s.empty?
|
634
|
-
|
635
|
-
Open3.popen3(query) do |stdin, stdout, stderr|
|
636
|
-
stdin.close
|
637
|
-
while !stderr.eof?
|
638
|
-
line = stderr.gets
|
639
|
-
unless /Warning: Using a password on the command line interface can be insecure./ === line
|
640
|
-
raise CompatibilityError, "Cannot connect to MySQL database. Please make sure you can connect with this command:\n $ mysql -u #{@db_opts[:username]} -h #{@db_opts[:host]} -P #{@db_opts[:port]} #{@db_opts[:database]} --protocol=tcp -p"
|
641
|
-
end
|
642
|
-
end
|
643
|
-
end
|
644
|
-
end
|
645
|
-
|
646
|
-
def check_mysql_row_mode_compat
|
647
|
-
sys_var_to_check = {'@@binlog_format'=>'ROW', '@@binlog_checksum'=>'NONE', '@@log_bin_use_v1_row_events'=>1}
|
648
|
-
errors={}
|
649
|
-
|
650
|
-
client = Mysql2::Client.new(@db_opts)
|
651
|
-
|
652
|
-
begin
|
653
|
-
sys_var_to_check.each_key do |sys_var|
|
654
|
-
sel_query = SELECT_QUERY_TMPLT % sys_var
|
655
|
-
begin
|
656
|
-
result = client.query(sel_query)
|
657
|
-
unless result.first[sys_var] == sys_var_to_check[sys_var]
|
658
|
-
errors[sys_var]=result.first[sys_var]
|
659
|
-
end
|
660
|
-
rescue Mysql2::Error => e
|
661
|
-
if e.message =~ /Unknown system variable/
|
662
|
-
unless e.message =~ /(binlog_checksum|log_bin_use_v1_row_events)/
|
663
|
-
errors[sys_var] = false
|
664
|
-
end
|
665
|
-
else
|
666
|
-
raise e
|
667
|
-
end
|
668
|
-
end
|
669
|
-
end
|
670
|
-
ensure
|
671
|
-
client.close
|
672
|
-
end
|
673
|
-
unless errors.empty?
|
674
|
-
error_explanation = ""
|
675
|
-
errors.each_key do |err_key|
|
676
|
-
error_explanation << "\n * #{err_key} is #{errors[err_key]} but should be #{sys_var_to_check[err_key]}"
|
677
|
-
end
|
678
|
-
raise CompatibilityError, "These system variable(s) are not the correct value: #{error_explanation}\n Please change these system variables for FlyData Sync to run correctly"
|
679
|
-
end
|
680
|
-
end
|
681
|
-
|
682
|
-
def check_writing_permissions
|
683
|
-
write_errors = []
|
684
|
-
paths_to_check = ["~/.flydata"]
|
685
|
-
paths_to_check << @dump_dir unless @dump_dir.to_s.empty?
|
686
|
-
paths_to_check.each do |path|
|
687
|
-
full_path = File.expand_path(path)
|
688
|
-
full_path = File.dirname(full_path) unless File.directory?(full_path)
|
689
|
-
write_errors << full_path unless File.writable?(full_path)
|
690
|
-
end
|
691
|
-
unless write_errors.empty?
|
692
|
-
error_dir = write_errors.join(", ")
|
693
|
-
raise CompatibilityError, "We cannot access the directories: #{error_dir}"
|
694
|
-
end
|
695
|
-
end
|
696
|
-
end
|
697
575
|
|
698
576
|
class DatabaseSizeCheck
|
699
577
|
include MysqlAccessible
|
@@ -16,7 +16,10 @@ module Flydata
|
|
16
16
|
expect(subject).to receive(:wait_until_server_ready)
|
17
17
|
expect(subject).to receive(:wait_until_client_ready)
|
18
18
|
allow(Kernel).to receive(:sleep)
|
19
|
+
allow_any_instance_of(Flydata::Api::DataPort).to receive(:get).and_return("Wibble")
|
20
|
+
allow_any_instance_of(Flydata::AgentCompatibilityCheck).to receive(:check).and_return(true)
|
19
21
|
end
|
22
|
+
|
20
23
|
context "as daemon" do
|
21
24
|
let(:args) { [] }
|
22
25
|
it "starts fluend with daemon option" do
|
@@ -0,0 +1,25 @@
|
|
1
|
+
module Flydata
|
2
|
+
describe AgentCompatibilityCheck do
|
3
|
+
let(:default_data_port) do
|
4
|
+
{
|
5
|
+
"servers"=>["sample-test-site.com"]
|
6
|
+
}
|
7
|
+
end
|
8
|
+
|
9
|
+
describe "#check" do
|
10
|
+
subject { AgentCompatibilityCheck.new(:default_data_port) }
|
11
|
+
context "runs all check methods" do
|
12
|
+
it "successfully checks open ports" do
|
13
|
+
expect(subject).to receive(:can_connect_to_port?).with(Flydata::AgentCompatibilityCheck::TCP_PORT).and_return true
|
14
|
+
expect(subject).to receive(:can_connect_to_port?).with(Flydata::AgentCompatibilityCheck::SSL_PORT).and_return true
|
15
|
+
subject.check
|
16
|
+
end
|
17
|
+
it "successfully catches errors on open ports" do
|
18
|
+
expect(subject).to receive(:can_connect_to_port?).with(Flydata::AgentCompatibilityCheck::TCP_PORT).and_return true
|
19
|
+
expect(subject).to receive(:can_connect_to_port?).with(Flydata::AgentCompatibilityCheck::SSL_PORT).and_return false
|
20
|
+
expect{subject.check_outgoing_ports}.to raise_error(Flydata::AgentCompatibilityCheck::AgentCompatibilityError, /ports/)
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: flydata
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.12
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Koichi Fujikawa
|
@@ -12,7 +12,7 @@ authors:
|
|
12
12
|
autorequire:
|
13
13
|
bindir: bin
|
14
14
|
cert_chain: []
|
15
|
-
date: 2014-
|
15
|
+
date: 2014-11-03 00:00:00.000000000 Z
|
16
16
|
dependencies:
|
17
17
|
- !ruby/object:Gem::Dependency
|
18
18
|
name: rest-client
|
@@ -128,7 +128,7 @@ dependencies:
|
|
128
128
|
requirements:
|
129
129
|
- - '>='
|
130
130
|
- !ruby/object:Gem::Version
|
131
|
-
version: 1.0.
|
131
|
+
version: 1.0.2
|
132
132
|
- - ~>
|
133
133
|
- !ruby/object:Gem::Version
|
134
134
|
version: '1.0'
|
@@ -138,7 +138,7 @@ dependencies:
|
|
138
138
|
requirements:
|
139
139
|
- - '>='
|
140
140
|
- !ruby/object:Gem::Version
|
141
|
-
version: 1.0.
|
141
|
+
version: 1.0.2
|
142
142
|
- - ~>
|
143
143
|
- !ruby/object:Gem::Version
|
144
144
|
version: '1.0'
|
@@ -438,6 +438,7 @@ files:
|
|
438
438
|
- lib/flydata/command/stop.rb
|
439
439
|
- lib/flydata/command/sync.rb
|
440
440
|
- lib/flydata/command/version.rb
|
441
|
+
- lib/flydata/compatibility_check.rb
|
441
442
|
- lib/flydata/credentials.rb
|
442
443
|
- lib/flydata/cron.rb
|
443
444
|
- lib/flydata/errors.rb
|
@@ -477,6 +478,7 @@ files:
|
|
477
478
|
- spec/flydata/cli_spec.rb
|
478
479
|
- spec/flydata/command/sender_spec.rb
|
479
480
|
- spec/flydata/command/sync_spec.rb
|
481
|
+
- spec/flydata/compatibility_check_spec.rb
|
480
482
|
- spec/flydata/fluent-plugins/in_mysql_binlog_flydata_spec.rb
|
481
483
|
- spec/flydata/fluent-plugins/mysql/alter_table_query_handler_spec.rb
|
482
484
|
- spec/flydata/fluent-plugins/mysql/binlog_position_spec.rb
|
@@ -520,7 +522,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
520
522
|
version: '0'
|
521
523
|
requirements: []
|
522
524
|
rubyforge_project:
|
523
|
-
rubygems_version: 2.
|
525
|
+
rubygems_version: 2.2.2
|
524
526
|
signing_key:
|
525
527
|
specification_version: 4
|
526
528
|
summary: FlyData Agent
|