flydata 0.2.11 → 0.2.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile +1 -1
- data/Gemfile.lock +2 -2
- data/VERSION +1 -1
- data/flydata.gemspec +10 -7
- data/lib/flydata/command/sender.rb +5 -0
- data/lib/flydata/command/sync.rb +20 -4
- data/lib/flydata/compatibility_check.rb +178 -0
- data/lib/flydata/parser/mysql/dump_parser.rb +0 -122
- data/spec/flydata/command/sender_spec.rb +3 -0
- data/spec/flydata/compatibility_check_spec.rb +25 -0
- metadata +7 -5
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 834f55f013a932c4bbd53408f020fe886ab2acf5
|
|
4
|
+
data.tar.gz: 702f346a6a072648956489fab6bf75776adb3351
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 3cafe705f68a2b86801e0af7bd93f5b450adaceb1f183876b44d01c3b921c437c78376b093b2e516e80446bc84fb18f18dfeeacc88ee1b4b6ff2922e0020fd06
|
|
7
|
+
data.tar.gz: 9aa3192f6524da454763d808941fdb847806166cfd369712835d37c36d33f2d5f307ac98fe31f44cea46a212c90b12b4c2c78594ebb691623369ab6705947cf5
|
data/Gemfile
CHANGED
|
@@ -7,7 +7,7 @@ gem "activesupport", '~> 4.0.0'
|
|
|
7
7
|
gem "json", '~> 1.8', '>= 1.8.0'
|
|
8
8
|
gem "highline", '~> 1.6', '>= 1.6.19'
|
|
9
9
|
gem "fluentd", "0.10.46"
|
|
10
|
-
gem "ruby-binlog", '~> 1.0', '>= 1.0.
|
|
10
|
+
gem "ruby-binlog", '~> 1.0', '>= 1.0.2'
|
|
11
11
|
gem "fluent-plugin-mysql-binlog", '~> 0.0', '>= 0.0.2'
|
|
12
12
|
gem "mysql2", '~> 0.3', '>= 0.3.11'
|
|
13
13
|
gem "slop", '~> 3.4', '>= 3.4.6'
|
data/Gemfile.lock
CHANGED
|
@@ -103,7 +103,7 @@ GEM
|
|
|
103
103
|
rspec-mocks (3.0.3)
|
|
104
104
|
rspec-support (~> 3.0.0)
|
|
105
105
|
rspec-support (3.0.3)
|
|
106
|
-
ruby-binlog (1.0.
|
|
106
|
+
ruby-binlog (1.0.2)
|
|
107
107
|
ruby-prof (0.15.1)
|
|
108
108
|
sigdump (0.2.2)
|
|
109
109
|
slop (3.6.0)
|
|
@@ -133,7 +133,7 @@ DEPENDENCIES
|
|
|
133
133
|
pry
|
|
134
134
|
rest-client (~> 1.6, >= 1.6.7)
|
|
135
135
|
rspec (~> 3.0)
|
|
136
|
-
ruby-binlog (~> 1.0, >= 1.0.
|
|
136
|
+
ruby-binlog (~> 1.0, >= 1.0.2)
|
|
137
137
|
ruby-prof (~> 0.15, >= 0.15.1)
|
|
138
138
|
slop (~> 3.4, >= 3.4.6)
|
|
139
139
|
sqlite3 (~> 1.3, >= 1.3.9)
|
data/VERSION
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
0.2.
|
|
1
|
+
0.2.12
|
data/flydata.gemspec
CHANGED
|
@@ -2,14 +2,16 @@
|
|
|
2
2
|
# DO NOT EDIT THIS FILE DIRECTLY
|
|
3
3
|
# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
|
|
4
4
|
# -*- encoding: utf-8 -*-
|
|
5
|
+
# stub: flydata 0.2.12 ruby lib
|
|
5
6
|
|
|
6
7
|
Gem::Specification.new do |s|
|
|
7
8
|
s.name = "flydata"
|
|
8
|
-
s.version = "0.2.
|
|
9
|
+
s.version = "0.2.12"
|
|
9
10
|
|
|
10
11
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
|
12
|
+
s.require_paths = ["lib"]
|
|
11
13
|
s.authors = ["Koichi Fujikawa", "Masashi Miyazaki", "Matthew Luu", "Mak Inada", "Sriram NS"]
|
|
12
|
-
s.date = "2014-
|
|
14
|
+
s.date = "2014-11-03"
|
|
13
15
|
s.description = "FlyData Agent"
|
|
14
16
|
s.email = "sysadmin@flydata.com"
|
|
15
17
|
s.executables = ["fdmysqldump", "flydata", "serverinfo"]
|
|
@@ -53,6 +55,7 @@ Gem::Specification.new do |s|
|
|
|
53
55
|
"lib/flydata/command/stop.rb",
|
|
54
56
|
"lib/flydata/command/sync.rb",
|
|
55
57
|
"lib/flydata/command/version.rb",
|
|
58
|
+
"lib/flydata/compatibility_check.rb",
|
|
56
59
|
"lib/flydata/credentials.rb",
|
|
57
60
|
"lib/flydata/cron.rb",
|
|
58
61
|
"lib/flydata/errors.rb",
|
|
@@ -92,6 +95,7 @@ Gem::Specification.new do |s|
|
|
|
92
95
|
"spec/flydata/cli_spec.rb",
|
|
93
96
|
"spec/flydata/command/sender_spec.rb",
|
|
94
97
|
"spec/flydata/command/sync_spec.rb",
|
|
98
|
+
"spec/flydata/compatibility_check_spec.rb",
|
|
95
99
|
"spec/flydata/fluent-plugins/in_mysql_binlog_flydata_spec.rb",
|
|
96
100
|
"spec/flydata/fluent-plugins/mysql/alter_table_query_handler_spec.rb",
|
|
97
101
|
"spec/flydata/fluent-plugins/mysql/binlog_position_spec.rb",
|
|
@@ -118,8 +122,7 @@ Gem::Specification.new do |s|
|
|
|
118
122
|
]
|
|
119
123
|
s.homepage = "http://flydata.com/"
|
|
120
124
|
s.licenses = ["All right reserved."]
|
|
121
|
-
s.
|
|
122
|
-
s.rubygems_version = "2.0.14"
|
|
125
|
+
s.rubygems_version = "2.2.2"
|
|
123
126
|
s.summary = "FlyData Agent"
|
|
124
127
|
|
|
125
128
|
if s.respond_to? :specification_version then
|
|
@@ -132,7 +135,7 @@ Gem::Specification.new do |s|
|
|
|
132
135
|
s.add_runtime_dependency(%q<json>, [">= 1.8.0", "~> 1.8"])
|
|
133
136
|
s.add_runtime_dependency(%q<highline>, [">= 1.6.19", "~> 1.6"])
|
|
134
137
|
s.add_runtime_dependency(%q<fluentd>, ["= 0.10.46"])
|
|
135
|
-
s.add_runtime_dependency(%q<ruby-binlog>, [">= 1.0.
|
|
138
|
+
s.add_runtime_dependency(%q<ruby-binlog>, [">= 1.0.2", "~> 1.0"])
|
|
136
139
|
s.add_runtime_dependency(%q<fluent-plugin-mysql-binlog>, [">= 0.0.2", "~> 0.0"])
|
|
137
140
|
s.add_runtime_dependency(%q<mysql2>, [">= 0.3.11", "~> 0.3"])
|
|
138
141
|
s.add_runtime_dependency(%q<slop>, [">= 3.4.6", "~> 3.4"])
|
|
@@ -153,7 +156,7 @@ Gem::Specification.new do |s|
|
|
|
153
156
|
s.add_dependency(%q<json>, [">= 1.8.0", "~> 1.8"])
|
|
154
157
|
s.add_dependency(%q<highline>, [">= 1.6.19", "~> 1.6"])
|
|
155
158
|
s.add_dependency(%q<fluentd>, ["= 0.10.46"])
|
|
156
|
-
s.add_dependency(%q<ruby-binlog>, [">= 1.0.
|
|
159
|
+
s.add_dependency(%q<ruby-binlog>, [">= 1.0.2", "~> 1.0"])
|
|
157
160
|
s.add_dependency(%q<fluent-plugin-mysql-binlog>, [">= 0.0.2", "~> 0.0"])
|
|
158
161
|
s.add_dependency(%q<mysql2>, [">= 0.3.11", "~> 0.3"])
|
|
159
162
|
s.add_dependency(%q<slop>, [">= 3.4.6", "~> 3.4"])
|
|
@@ -175,7 +178,7 @@ Gem::Specification.new do |s|
|
|
|
175
178
|
s.add_dependency(%q<json>, [">= 1.8.0", "~> 1.8"])
|
|
176
179
|
s.add_dependency(%q<highline>, [">= 1.6.19", "~> 1.6"])
|
|
177
180
|
s.add_dependency(%q<fluentd>, ["= 0.10.46"])
|
|
178
|
-
s.add_dependency(%q<ruby-binlog>, [">= 1.0.
|
|
181
|
+
s.add_dependency(%q<ruby-binlog>, [">= 1.0.2", "~> 1.0"])
|
|
179
182
|
s.add_dependency(%q<fluent-plugin-mysql-binlog>, [">= 0.0.2", "~> 0.0"])
|
|
180
183
|
s.add_dependency(%q<mysql2>, [">= 0.3.11", "~> 0.3"])
|
|
181
184
|
s.add_dependency(%q<slop>, [">= 3.4.6", "~> 3.4"])
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
require 'flydata/compatibility_check'
|
|
2
|
+
|
|
1
3
|
module Flydata
|
|
2
4
|
module Command
|
|
3
5
|
class Sender < Base
|
|
@@ -20,6 +22,9 @@ module Flydata
|
|
|
20
22
|
|
|
21
23
|
wait_until_server_ready(options)
|
|
22
24
|
|
|
25
|
+
dp = flydata.data_port.get
|
|
26
|
+
AgentCompatibilityCheck.new(dp).check
|
|
27
|
+
|
|
23
28
|
# Start sender(fluentd) process
|
|
24
29
|
say('Starting sender process.') unless options[:quiet]
|
|
25
30
|
Dir.chdir(FLYDATA_HOME){
|
data/lib/flydata/command/sync.rb
CHANGED
|
@@ -4,6 +4,7 @@ require 'open3'
|
|
|
4
4
|
require 'mysql2'
|
|
5
5
|
require 'flydata/sync_file_manager'
|
|
6
6
|
require 'flydata/errors'
|
|
7
|
+
require 'flydata/compatibility_check'
|
|
7
8
|
require 'flydata/table_def'
|
|
8
9
|
require 'flydata/output/forwarder'
|
|
9
10
|
require 'flydata/parser/mysql/dump_parser'
|
|
@@ -89,6 +90,7 @@ module Flydata
|
|
|
89
90
|
delete_files.flatten.each do |path|
|
|
90
91
|
FileUtils.rm(path) if File.exists?(path)
|
|
91
92
|
end
|
|
93
|
+
puts "Reset completed successfully."
|
|
92
94
|
end
|
|
93
95
|
|
|
94
96
|
def wait_for_server_buffer
|
|
@@ -153,7 +155,8 @@ module Flydata
|
|
|
153
155
|
|
|
154
156
|
def generate_table_ddl(*tables)
|
|
155
157
|
de = retrieve_data_entry
|
|
156
|
-
|
|
158
|
+
dp = flydata.data_port.get
|
|
159
|
+
Flydata::MysqlCompatibilityCheck.new(dp, de['mysql_data_entry_preference']).check
|
|
157
160
|
do_generate_table_ddl(override_tables(de, tables))
|
|
158
161
|
end
|
|
159
162
|
|
|
@@ -186,8 +189,20 @@ module Flydata
|
|
|
186
189
|
end
|
|
187
190
|
|
|
188
191
|
def cleanup_sync_server(de, tables = [])
|
|
189
|
-
|
|
190
|
-
|
|
192
|
+
print "Cleaning the server."
|
|
193
|
+
worker = Thread.new do
|
|
194
|
+
begin
|
|
195
|
+
flydata.data_entry.cleanup_sync(de['id'], tables)
|
|
196
|
+
rescue RestClient::RequestTimeout
|
|
197
|
+
# server is taking time to cleanup. Try again
|
|
198
|
+
retry
|
|
199
|
+
end
|
|
200
|
+
end
|
|
201
|
+
until worker.join(5)
|
|
202
|
+
print "."
|
|
203
|
+
end
|
|
204
|
+
puts
|
|
205
|
+
puts "Done."
|
|
191
206
|
end
|
|
192
207
|
|
|
193
208
|
def do_check(de)
|
|
@@ -295,6 +310,7 @@ module Flydata
|
|
|
295
310
|
return if dump_pos_info[:status] == STATUS_WAITING || dump_pos_info[:status] == STATUS_COMPLETE
|
|
296
311
|
|
|
297
312
|
# mysqldump file exists -> skip dump
|
|
313
|
+
dp = flydata.data_port.get
|
|
298
314
|
fp = sync_fm.dump_file_path
|
|
299
315
|
if file_dump && File.exists?(fp) && File.size(fp) > 0
|
|
300
316
|
puts " -> Skip"
|
|
@@ -326,8 +342,8 @@ EOM
|
|
|
326
342
|
db_bytesize = Flydata::Parser::Mysql::DatabaseSizeCheck.new(de['mysql_data_entry_preference']).get_db_bytesize
|
|
327
343
|
puts " -> #{as_size(db_bytesize)} (#{db_bytesize} byte)"
|
|
328
344
|
puts "Exporting data from database..."
|
|
345
|
+
Flydata::MysqlCompatibilityCheck.new(dp, de['mysql_data_entry_preference'], fp).check
|
|
329
346
|
if file_dump
|
|
330
|
-
Flydata::Parser::Mysql::CompatibilityCheck.new(de['mysql_data_entry_preference'], fp).check
|
|
331
347
|
Flydata::Parser::Mysql::MysqlDumpGeneratorNoMasterData.
|
|
332
348
|
new(de['mysql_data_entry_preference']).dump(fp)
|
|
333
349
|
puts " -> Done"
|
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
module Flydata
|
|
2
|
+
|
|
3
|
+
class CompatibilityCheck
|
|
4
|
+
class CompatibilityError < StandardError
|
|
5
|
+
end
|
|
6
|
+
|
|
7
|
+
def initialize(dp_hash, de_hash=nil, options={})
|
|
8
|
+
@dp = dp_hash
|
|
9
|
+
@errors=[]
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
def check
|
|
13
|
+
self.methods.grep(/^check_/).each do |m|
|
|
14
|
+
begin
|
|
15
|
+
send(m)
|
|
16
|
+
rescue CompatibilityError => e
|
|
17
|
+
@errors << e
|
|
18
|
+
end
|
|
19
|
+
end
|
|
20
|
+
print_errors
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def print_errors
|
|
24
|
+
return if @errors.empty?
|
|
25
|
+
puts "There may be some compatibility issues with this current server : "
|
|
26
|
+
@errors.each do |error|
|
|
27
|
+
puts " * #{error.message}"
|
|
28
|
+
end
|
|
29
|
+
raise "Please correct these errors if you wish to run FlyData Agent"
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
class AgentCompatibilityCheck < CompatibilityCheck
|
|
35
|
+
class AgentCompatibilityError < StandardError
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
TCP_PORT=45326
|
|
39
|
+
SSL_PORT=45327
|
|
40
|
+
|
|
41
|
+
def check_outgoing_ports
|
|
42
|
+
unless can_connect_to_port?(TCP_PORT) and
|
|
43
|
+
can_connect_to_port?(SSL_PORT)
|
|
44
|
+
raise AgentCompatibilityError,
|
|
45
|
+
"Cannot connect to outisde ports. Please check to make sure you have these outgoing ports open: #{TCP_PORT},#{SSL_PORT}"
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
private
|
|
50
|
+
def can_connect_to_port?(port)
|
|
51
|
+
url = @dp["servers"].first
|
|
52
|
+
begin
|
|
53
|
+
e = TCPSocket.new(url, port)
|
|
54
|
+
e.close
|
|
55
|
+
return true
|
|
56
|
+
rescue Errno::ETIMEDOUT => e
|
|
57
|
+
return false
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
class MysqlCompatibilityCheck < CompatibilityCheck
|
|
63
|
+
|
|
64
|
+
class MysqlCompatibilityError < StandardError
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
SELECT_QUERY_TMPLT = "SELECT %s"
|
|
68
|
+
|
|
69
|
+
#def initialize(de_hash, dump_dir=nil)
|
|
70
|
+
def initialize(dp_hash, de_hash, options={})
|
|
71
|
+
super
|
|
72
|
+
@db_opts = [:host, :port, :username, :password, :database].inject({}) {|h, sym| h[sym] = de_hash[sym.to_s]; h}
|
|
73
|
+
@dump_dir = options['dump_dir'] || nil
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def print_errors
|
|
77
|
+
return if @errors.empty?
|
|
78
|
+
puts "There may be some compatibility issues with your MySQL credentials: "
|
|
79
|
+
@errors.each do |error|
|
|
80
|
+
puts " * #{error.message}"
|
|
81
|
+
end
|
|
82
|
+
raise "Please correct these errors if you wish to run FlyData Sync"
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
def check_mysql_user_compat
|
|
86
|
+
client = Mysql2::Client.new(@db_opts)
|
|
87
|
+
grants_sql = "SHOW GRANTS"
|
|
88
|
+
correct_db = ["ON (\\*|#{@db_opts[:database]})","TO '#{@db_opts[:username]}"]
|
|
89
|
+
necessary_permission_fields= ["SELECT","RELOAD","LOCK TABLES","REPLICATION SLAVE","REPLICATION CLIENT"]
|
|
90
|
+
all_privileges_field= ["ALL PRIVILEGES"]
|
|
91
|
+
result = client.query(grants_sql)
|
|
92
|
+
# Do not catch MySQL connection problem because check should stop if no MySQL connection can be made.
|
|
93
|
+
client.close
|
|
94
|
+
missing_priv = []
|
|
95
|
+
result.each do |res|
|
|
96
|
+
# SHOW GRANTS should only return one column
|
|
97
|
+
res_value = res.values.first
|
|
98
|
+
if correct_db.all? {|perm| res_value.match(perm)}
|
|
99
|
+
necessary_permission_fields.each do |priv|
|
|
100
|
+
missing_priv << priv unless res_value.match(priv)
|
|
101
|
+
end
|
|
102
|
+
return true if missing_priv.empty? or all_privileges_field.all? {|d| res_value.match(d)}
|
|
103
|
+
end
|
|
104
|
+
end
|
|
105
|
+
raise MysqlCompatibilityError, "The user '#{@db_opts[:username]}' does not have the correct permissions to run FlyData Sync\n * These privileges are missing: #{missing_priv.join(", ")}"
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
def check_mysql_protocol_tcp_compat
|
|
109
|
+
query = "mysql -u #{@db_opts[:username]} -h #{@db_opts[:host]} -P #{@db_opts[:port]} #{@db_opts[:database]} -e \"SHOW GRANTS;\" --protocol=tcp"
|
|
110
|
+
query << " -p#{@db_opts[:password]}" unless @db_opts[:password].to_s.empty?
|
|
111
|
+
|
|
112
|
+
Open3.popen3(query) do |stdin, stdout, stderr|
|
|
113
|
+
stdin.close
|
|
114
|
+
while !stderr.eof?
|
|
115
|
+
line = stderr.gets
|
|
116
|
+
unless /Warning: Using a password on the command line interface can be insecure./ === line
|
|
117
|
+
raise MysqlCompatibilityError, "Cannot connect to MySQL database. Please make sure you can connect with this command:\n $ mysql -u #{@db_opts[:username]} -h #{@db_opts[:host]} -P #{@db_opts[:port]} #{@db_opts[:database]} --protocol=tcp -p"
|
|
118
|
+
end
|
|
119
|
+
end
|
|
120
|
+
end
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
def check_mysql_row_mode_compat
|
|
124
|
+
sys_var_to_check = {'@@binlog_format'=>'ROW', '@@binlog_checksum'=>'NONE', '@@log_bin_use_v1_row_events'=>1}
|
|
125
|
+
errors={}
|
|
126
|
+
|
|
127
|
+
client = Mysql2::Client.new(@db_opts)
|
|
128
|
+
|
|
129
|
+
begin
|
|
130
|
+
sys_var_to_check.each_key do |sys_var|
|
|
131
|
+
sel_query = SELECT_QUERY_TMPLT % sys_var
|
|
132
|
+
begin
|
|
133
|
+
result = client.query(sel_query)
|
|
134
|
+
unless result.first[sys_var] == sys_var_to_check[sys_var]
|
|
135
|
+
errors[sys_var]=result.first[sys_var]
|
|
136
|
+
end
|
|
137
|
+
rescue Mysql2::Error => e
|
|
138
|
+
if e.message =~ /Unknown system variable/
|
|
139
|
+
unless e.message =~ /(binlog_checksum|log_bin_use_v1_row_events)/
|
|
140
|
+
errors[sys_var] = false
|
|
141
|
+
end
|
|
142
|
+
else
|
|
143
|
+
raise e
|
|
144
|
+
end
|
|
145
|
+
end
|
|
146
|
+
end
|
|
147
|
+
ensure
|
|
148
|
+
client.close
|
|
149
|
+
end
|
|
150
|
+
unless errors.empty?
|
|
151
|
+
error_explanation = ""
|
|
152
|
+
errors.each_key do |err_key|
|
|
153
|
+
error_explanation << "\n * #{err_key} is #{errors[err_key]} but should be #{sys_var_to_check[err_key]}"
|
|
154
|
+
end
|
|
155
|
+
raise MysqlCompatibilityError, "These system variable(s) are not the correct value: #{error_explanation}\n Please change these system variables for FlyData Sync to run correctly"
|
|
156
|
+
end
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
def check_writing_permissions
|
|
160
|
+
write_errors = []
|
|
161
|
+
paths_to_check = ["~/.flydata"]
|
|
162
|
+
paths_to_check << @dump_dir unless @dump_dir.to_s.empty?
|
|
163
|
+
paths_to_check.each do |path|
|
|
164
|
+
full_path = File.expand_path(path)
|
|
165
|
+
full_path = File.dirname(full_path) unless File.directory?(full_path)
|
|
166
|
+
write_errors << full_path unless File.writable?(full_path)
|
|
167
|
+
end
|
|
168
|
+
unless write_errors.empty?
|
|
169
|
+
error_dir = write_errors.join(", ")
|
|
170
|
+
raise MysqlCompatibilityError, "We cannot access the directories: #{error_dir}"
|
|
171
|
+
end
|
|
172
|
+
end
|
|
173
|
+
end
|
|
174
|
+
|
|
175
|
+
end
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
|
|
@@ -572,128 +572,6 @@ EOS
|
|
|
572
572
|
end
|
|
573
573
|
end
|
|
574
574
|
end
|
|
575
|
-
class CompatibilityCheck
|
|
576
|
-
|
|
577
|
-
class CompatibilityError < StandardError
|
|
578
|
-
end
|
|
579
|
-
|
|
580
|
-
SELECT_QUERY_TMPLT = "SELECT %s"
|
|
581
|
-
|
|
582
|
-
def initialize(de_hash, dump_dir=nil)
|
|
583
|
-
@db_opts = [:host, :port, :username, :password, :database].inject({}) {|h, sym| h[sym] = de_hash[sym.to_s]; h}
|
|
584
|
-
@dump_dir = dump_dir
|
|
585
|
-
@errors=[]
|
|
586
|
-
end
|
|
587
|
-
|
|
588
|
-
def check
|
|
589
|
-
self.methods.grep(/^check_/).each do |m|
|
|
590
|
-
begin
|
|
591
|
-
send(m)
|
|
592
|
-
rescue CompatibilityError => e
|
|
593
|
-
@errors << e
|
|
594
|
-
end
|
|
595
|
-
end
|
|
596
|
-
print_errors
|
|
597
|
-
end
|
|
598
|
-
|
|
599
|
-
def print_errors
|
|
600
|
-
return if @errors.empty?
|
|
601
|
-
puts "There may be some compatibility issues with your MySQL credentials: "
|
|
602
|
-
@errors.each do |error|
|
|
603
|
-
puts " * #{error.message}"
|
|
604
|
-
end
|
|
605
|
-
raise "Please correct these errors if you wish to run FlyData Sync"
|
|
606
|
-
end
|
|
607
|
-
|
|
608
|
-
def check_mysql_user_compat
|
|
609
|
-
client = Mysql2::Client.new(@db_opts)
|
|
610
|
-
grants_sql = "SHOW GRANTS"
|
|
611
|
-
correct_db = ["ON (\\*|#{@db_opts[:database]})","TO '#{@db_opts[:username]}"]
|
|
612
|
-
necessary_permission_fields= ["SELECT","RELOAD","LOCK TABLES","REPLICATION SLAVE","REPLICATION CLIENT"]
|
|
613
|
-
all_privileges_field= ["ALL PRIVILEGES"]
|
|
614
|
-
result = client.query(grants_sql)
|
|
615
|
-
# Do not catch MySQL connection problem because check should stop if no MySQL connection can be made.
|
|
616
|
-
client.close
|
|
617
|
-
missing_priv = []
|
|
618
|
-
result.each do |res|
|
|
619
|
-
# SHOW GRANTS should only return one column
|
|
620
|
-
res_value = res.values.first
|
|
621
|
-
if correct_db.all? {|perm| res_value.match(perm)}
|
|
622
|
-
necessary_permission_fields.each do |priv|
|
|
623
|
-
missing_priv << priv unless res_value.match(priv)
|
|
624
|
-
end
|
|
625
|
-
return true if missing_priv.empty? or all_privileges_field.all? {|d| res_value.match(d)}
|
|
626
|
-
end
|
|
627
|
-
end
|
|
628
|
-
raise CompatibilityError, "The user '#{@db_opts[:username]}' does not have the correct permissions to run FlyData Sync\n * These privileges are missing: #{missing_priv.join(", ")}"
|
|
629
|
-
end
|
|
630
|
-
|
|
631
|
-
def check_mysql_protocol_tcp_compat
|
|
632
|
-
query = "mysql -u #{@db_opts[:username]} -h #{@db_opts[:host]} -P #{@db_opts[:port]} #{@db_opts[:database]} -e \"SHOW GRANTS;\" --protocol=tcp"
|
|
633
|
-
query << " -p#{@db_opts[:password]}" unless @db_opts[:password].to_s.empty?
|
|
634
|
-
|
|
635
|
-
Open3.popen3(query) do |stdin, stdout, stderr|
|
|
636
|
-
stdin.close
|
|
637
|
-
while !stderr.eof?
|
|
638
|
-
line = stderr.gets
|
|
639
|
-
unless /Warning: Using a password on the command line interface can be insecure./ === line
|
|
640
|
-
raise CompatibilityError, "Cannot connect to MySQL database. Please make sure you can connect with this command:\n $ mysql -u #{@db_opts[:username]} -h #{@db_opts[:host]} -P #{@db_opts[:port]} #{@db_opts[:database]} --protocol=tcp -p"
|
|
641
|
-
end
|
|
642
|
-
end
|
|
643
|
-
end
|
|
644
|
-
end
|
|
645
|
-
|
|
646
|
-
def check_mysql_row_mode_compat
|
|
647
|
-
sys_var_to_check = {'@@binlog_format'=>'ROW', '@@binlog_checksum'=>'NONE', '@@log_bin_use_v1_row_events'=>1}
|
|
648
|
-
errors={}
|
|
649
|
-
|
|
650
|
-
client = Mysql2::Client.new(@db_opts)
|
|
651
|
-
|
|
652
|
-
begin
|
|
653
|
-
sys_var_to_check.each_key do |sys_var|
|
|
654
|
-
sel_query = SELECT_QUERY_TMPLT % sys_var
|
|
655
|
-
begin
|
|
656
|
-
result = client.query(sel_query)
|
|
657
|
-
unless result.first[sys_var] == sys_var_to_check[sys_var]
|
|
658
|
-
errors[sys_var]=result.first[sys_var]
|
|
659
|
-
end
|
|
660
|
-
rescue Mysql2::Error => e
|
|
661
|
-
if e.message =~ /Unknown system variable/
|
|
662
|
-
unless e.message =~ /(binlog_checksum|log_bin_use_v1_row_events)/
|
|
663
|
-
errors[sys_var] = false
|
|
664
|
-
end
|
|
665
|
-
else
|
|
666
|
-
raise e
|
|
667
|
-
end
|
|
668
|
-
end
|
|
669
|
-
end
|
|
670
|
-
ensure
|
|
671
|
-
client.close
|
|
672
|
-
end
|
|
673
|
-
unless errors.empty?
|
|
674
|
-
error_explanation = ""
|
|
675
|
-
errors.each_key do |err_key|
|
|
676
|
-
error_explanation << "\n * #{err_key} is #{errors[err_key]} but should be #{sys_var_to_check[err_key]}"
|
|
677
|
-
end
|
|
678
|
-
raise CompatibilityError, "These system variable(s) are not the correct value: #{error_explanation}\n Please change these system variables for FlyData Sync to run correctly"
|
|
679
|
-
end
|
|
680
|
-
end
|
|
681
|
-
|
|
682
|
-
def check_writing_permissions
|
|
683
|
-
write_errors = []
|
|
684
|
-
paths_to_check = ["~/.flydata"]
|
|
685
|
-
paths_to_check << @dump_dir unless @dump_dir.to_s.empty?
|
|
686
|
-
paths_to_check.each do |path|
|
|
687
|
-
full_path = File.expand_path(path)
|
|
688
|
-
full_path = File.dirname(full_path) unless File.directory?(full_path)
|
|
689
|
-
write_errors << full_path unless File.writable?(full_path)
|
|
690
|
-
end
|
|
691
|
-
unless write_errors.empty?
|
|
692
|
-
error_dir = write_errors.join(", ")
|
|
693
|
-
raise CompatibilityError, "We cannot access the directories: #{error_dir}"
|
|
694
|
-
end
|
|
695
|
-
end
|
|
696
|
-
end
|
|
697
575
|
|
|
698
576
|
class DatabaseSizeCheck
|
|
699
577
|
include MysqlAccessible
|
|
@@ -16,7 +16,10 @@ module Flydata
|
|
|
16
16
|
expect(subject).to receive(:wait_until_server_ready)
|
|
17
17
|
expect(subject).to receive(:wait_until_client_ready)
|
|
18
18
|
allow(Kernel).to receive(:sleep)
|
|
19
|
+
allow_any_instance_of(Flydata::Api::DataPort).to receive(:get).and_return("Wibble")
|
|
20
|
+
allow_any_instance_of(Flydata::AgentCompatibilityCheck).to receive(:check).and_return(true)
|
|
19
21
|
end
|
|
22
|
+
|
|
20
23
|
context "as daemon" do
|
|
21
24
|
let(:args) { [] }
|
|
22
25
|
it "starts fluend with daemon option" do
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
module Flydata
|
|
2
|
+
describe AgentCompatibilityCheck do
|
|
3
|
+
let(:default_data_port) do
|
|
4
|
+
{
|
|
5
|
+
"servers"=>["sample-test-site.com"]
|
|
6
|
+
}
|
|
7
|
+
end
|
|
8
|
+
|
|
9
|
+
describe "#check" do
|
|
10
|
+
subject { AgentCompatibilityCheck.new(:default_data_port) }
|
|
11
|
+
context "runs all check methods" do
|
|
12
|
+
it "successfully checks open ports" do
|
|
13
|
+
expect(subject).to receive(:can_connect_to_port?).with(Flydata::AgentCompatibilityCheck::TCP_PORT).and_return true
|
|
14
|
+
expect(subject).to receive(:can_connect_to_port?).with(Flydata::AgentCompatibilityCheck::SSL_PORT).and_return true
|
|
15
|
+
subject.check
|
|
16
|
+
end
|
|
17
|
+
it "successfully catches errors on open ports" do
|
|
18
|
+
expect(subject).to receive(:can_connect_to_port?).with(Flydata::AgentCompatibilityCheck::TCP_PORT).and_return true
|
|
19
|
+
expect(subject).to receive(:can_connect_to_port?).with(Flydata::AgentCompatibilityCheck::SSL_PORT).and_return false
|
|
20
|
+
expect{subject.check_outgoing_ports}.to raise_error(Flydata::AgentCompatibilityCheck::AgentCompatibilityError, /ports/)
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
end
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: flydata
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.2.
|
|
4
|
+
version: 0.2.12
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Koichi Fujikawa
|
|
@@ -12,7 +12,7 @@ authors:
|
|
|
12
12
|
autorequire:
|
|
13
13
|
bindir: bin
|
|
14
14
|
cert_chain: []
|
|
15
|
-
date: 2014-
|
|
15
|
+
date: 2014-11-03 00:00:00.000000000 Z
|
|
16
16
|
dependencies:
|
|
17
17
|
- !ruby/object:Gem::Dependency
|
|
18
18
|
name: rest-client
|
|
@@ -128,7 +128,7 @@ dependencies:
|
|
|
128
128
|
requirements:
|
|
129
129
|
- - '>='
|
|
130
130
|
- !ruby/object:Gem::Version
|
|
131
|
-
version: 1.0.
|
|
131
|
+
version: 1.0.2
|
|
132
132
|
- - ~>
|
|
133
133
|
- !ruby/object:Gem::Version
|
|
134
134
|
version: '1.0'
|
|
@@ -138,7 +138,7 @@ dependencies:
|
|
|
138
138
|
requirements:
|
|
139
139
|
- - '>='
|
|
140
140
|
- !ruby/object:Gem::Version
|
|
141
|
-
version: 1.0.
|
|
141
|
+
version: 1.0.2
|
|
142
142
|
- - ~>
|
|
143
143
|
- !ruby/object:Gem::Version
|
|
144
144
|
version: '1.0'
|
|
@@ -438,6 +438,7 @@ files:
|
|
|
438
438
|
- lib/flydata/command/stop.rb
|
|
439
439
|
- lib/flydata/command/sync.rb
|
|
440
440
|
- lib/flydata/command/version.rb
|
|
441
|
+
- lib/flydata/compatibility_check.rb
|
|
441
442
|
- lib/flydata/credentials.rb
|
|
442
443
|
- lib/flydata/cron.rb
|
|
443
444
|
- lib/flydata/errors.rb
|
|
@@ -477,6 +478,7 @@ files:
|
|
|
477
478
|
- spec/flydata/cli_spec.rb
|
|
478
479
|
- spec/flydata/command/sender_spec.rb
|
|
479
480
|
- spec/flydata/command/sync_spec.rb
|
|
481
|
+
- spec/flydata/compatibility_check_spec.rb
|
|
480
482
|
- spec/flydata/fluent-plugins/in_mysql_binlog_flydata_spec.rb
|
|
481
483
|
- spec/flydata/fluent-plugins/mysql/alter_table_query_handler_spec.rb
|
|
482
484
|
- spec/flydata/fluent-plugins/mysql/binlog_position_spec.rb
|
|
@@ -520,7 +522,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
520
522
|
version: '0'
|
|
521
523
|
requirements: []
|
|
522
524
|
rubyforge_project:
|
|
523
|
-
rubygems_version: 2.
|
|
525
|
+
rubygems_version: 2.2.2
|
|
524
526
|
signing_key:
|
|
525
527
|
specification_version: 4
|
|
526
528
|
summary: FlyData Agent
|