flydata 0.2.19 → 0.2.20
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/VERSION +1 -1
- data/flydata-core/lib/flydata-core/logger.rb +37 -25
- data/flydata.gemspec +3 -3
- data/lib/flydata.rb +8 -0
- data/lib/flydata/cli.rb +15 -2
- data/lib/flydata/command/base.rb +14 -6
- data/lib/flydata/command/sender.rb +26 -26
- data/lib/flydata/command/sync.rb +39 -42
- data/lib/flydata/command/version.rb +3 -2
- data/lib/flydata/command_logger.rb +26 -0
- data/lib/flydata/compatibility_check.rb +16 -6
- data/lib/flydata/helpers.rb +14 -5
- data/lib/flydata/output/forwarder.rb +9 -5
- data/lib/flydata/parser/mysql/dump_parser.rb +12 -2
- data/spec/flydata/command/sender_spec.rb +6 -6
- data/spec/flydata/compatibility_check_spec.rb +3 -0
- data/spec/flydata/output/forwarder_spec.rb +1 -0
- data/spec/flydata/parser/mysql/dump_parser_spec.rb +135 -69
- metadata +3 -3
- data/lib/flydata/agent_errors.rb +0 -8
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f4d29cd6b59f9acb406d62017d559296f0fa27b7
|
4
|
+
data.tar.gz: ab8d2cdcebbaab49345f62da3aef259e5a977e02
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 6f590f2046f0c016a2beae67555a0544b919b9ff5873376ee190f6a2065ae30e6e961178926127537a97baab5140dc1e1c9c5e5051895a5661d4cf93b90ccc0a
|
7
|
+
data.tar.gz: 6af1ee2c5597ed08e133650a5882ca6a4531ff47b0a25bf82566a3c3e0de4ecb3172c6c6da62449a853af49785197c15ae012ce59a4303e4a11d1826c02e7cd6
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.2.
|
1
|
+
0.2.20
|
@@ -100,8 +100,43 @@ module FlydataCore
|
|
100
100
|
log_params
|
101
101
|
end
|
102
102
|
|
103
|
+
# hook method
|
104
|
+
def before_logging(level, raw_msg, built_msg, log_params, options)
|
105
|
+
end
|
106
|
+
|
103
107
|
# log methods
|
104
|
-
def log_common(level, message, log_params
|
108
|
+
def log_common(level, message, log_params, options)
|
109
|
+
msg = build_log_message(level, message, log_params, options)
|
110
|
+
before_logging(level, message, msg, log_params, options)
|
111
|
+
get_logger(options).send(level, msg)
|
112
|
+
end
|
113
|
+
|
114
|
+
def log_debug(message, log_params = {}, options = {})
|
115
|
+
log_common(:debug, message, log_params, options)
|
116
|
+
end
|
117
|
+
|
118
|
+
def log_info(message, log_params = {}, options = {})
|
119
|
+
log_common(:info, message, log_params, options)
|
120
|
+
end
|
121
|
+
|
122
|
+
def log_warn(message, log_params = {}, options = {})
|
123
|
+
log_common(:warn, message, log_params, options)
|
124
|
+
end
|
125
|
+
|
126
|
+
def log_error(message, log_params = {}, options = {})
|
127
|
+
log_common(:error, message, log_params, options)
|
128
|
+
end
|
129
|
+
|
130
|
+
def log_error_with_backtrace(message, log_params = {}, options = {})
|
131
|
+
log_common(:error, message, log_params, options.merge(backtrace: true))
|
132
|
+
end
|
133
|
+
|
134
|
+
def get_logger(options = {})
|
135
|
+
options[:logger] || logger || log_context_logger || $log
|
136
|
+
end
|
137
|
+
|
138
|
+
def build_log_message(level, raw_message, log_params = {}, options = {})
|
139
|
+
message = raw_message.dup
|
105
140
|
# check keys
|
106
141
|
if e = log_params[:error]
|
107
142
|
message += " error_class=#{e.class.to_s} error=\"#{e.to_s.strip.gsub(/\n/, ' ')}\""
|
@@ -133,30 +168,7 @@ module FlydataCore
|
|
133
168
|
backtrace = e.backtrace ? e.backtrace.join("\n") : 'backtrace is empty.'
|
134
169
|
msg += " backtrace:\n#{backtrace}"
|
135
170
|
end
|
136
|
-
|
137
|
-
# get logger
|
138
|
-
lg = options[:logger] || logger || log_context_logger || $log
|
139
|
-
lg.send(level, msg)
|
140
|
-
end
|
141
|
-
|
142
|
-
def log_debug(message, log_params = {}, options = {})
|
143
|
-
log_common(:debug, message, log_params, options)
|
144
|
-
end
|
145
|
-
|
146
|
-
def log_info(message, log_params = {}, options = {})
|
147
|
-
log_common(:info, message, log_params, options)
|
148
|
-
end
|
149
|
-
|
150
|
-
def log_warn(message, log_params = {}, options = {})
|
151
|
-
log_common(:warn, message, log_params, options)
|
152
|
-
end
|
153
|
-
|
154
|
-
def log_error(message, log_params = {}, options = {})
|
155
|
-
log_common(:error, message, log_params, options)
|
156
|
-
end
|
157
|
-
|
158
|
-
def log_error_with_backtrace(message, log_params = {}, options = {})
|
159
|
-
log_common(:error, message, log_params, options.merge(backtrace: true))
|
171
|
+
msg
|
160
172
|
end
|
161
173
|
|
162
174
|
def build_log_text(items = log_context_items)
|
data/flydata.gemspec
CHANGED
@@ -5,11 +5,11 @@
|
|
5
5
|
|
6
6
|
Gem::Specification.new do |s|
|
7
7
|
s.name = "flydata"
|
8
|
-
s.version = "0.2.
|
8
|
+
s.version = "0.2.20"
|
9
9
|
|
10
10
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
11
|
s.authors = ["Koichi Fujikawa", "Masashi Miyazaki", "Matthew Luu", "Mak Inada", "Sriram NS"]
|
12
|
-
s.date = "2014-12-
|
12
|
+
s.date = "2014-12-05"
|
13
13
|
s.description = "FlyData Agent"
|
14
14
|
s.email = "sysadmin@flydata.com"
|
15
15
|
s.executables = ["fdmysqldump", "flydata", "serverinfo"]
|
@@ -63,7 +63,6 @@ Gem::Specification.new do |s|
|
|
63
63
|
"flydata.gemspec",
|
64
64
|
"lib/fly_data_model.rb",
|
65
65
|
"lib/flydata.rb",
|
66
|
-
"lib/flydata/agent_errors.rb",
|
67
66
|
"lib/flydata/api/base.rb",
|
68
67
|
"lib/flydata/api/data_entry.rb",
|
69
68
|
"lib/flydata/api/data_port.rb",
|
@@ -86,6 +85,7 @@ Gem::Specification.new do |s|
|
|
86
85
|
"lib/flydata/command/stop.rb",
|
87
86
|
"lib/flydata/command/sync.rb",
|
88
87
|
"lib/flydata/command/version.rb",
|
88
|
+
"lib/flydata/command_logger.rb",
|
89
89
|
"lib/flydata/compatibility_check.rb",
|
90
90
|
"lib/flydata/credentials.rb",
|
91
91
|
"lib/flydata/cron.rb",
|
data/lib/flydata.rb
CHANGED
@@ -25,6 +25,14 @@ module Flydata
|
|
25
25
|
FLYDATA_DEBUG = !!(ENV['FLYDATA_DEBUG'])
|
26
26
|
FLYDATA_HOME = ENV['FLYDATA_HOME'] || "#{ENV['HOME']}/.flydata"
|
27
27
|
FLYDATA_GEM_HOME = File.expand_path(File.join(File.dirname(__FILE__), '..'))
|
28
|
+
FLYDATA_GEM_BIN = File.join(FLYDATA_GEM_HOME, 'bin')
|
28
29
|
FLYDATA_TMPL_DIR = File.join(FLYDATA_GEM_HOME, 'tmpl')
|
30
|
+
FLYDATA_FLUENT_PLUGIN_DIR = File.join(FLYDATA_GEM_HOME, 'lib', 'flydata', 'fluent-plugins')
|
31
|
+
|
32
|
+
FLYDATA_SERVERINFO = File.join(FLYDATA_GEM_BIN, 'serverinfo')
|
33
|
+
FLYDATA_LOG = File.join(FLYDATA_HOME, 'flydata.log')
|
34
|
+
FLYDATA_CONF = File.join(FLYDATA_HOME, 'flydata.conf')
|
35
|
+
|
36
|
+
VERSION_PATH = File.join(FLYDATA_GEM_HOME, 'VERSION')
|
29
37
|
include Flydata::Heroku
|
30
38
|
end
|
data/lib/flydata/cli.rb
CHANGED
@@ -1,15 +1,26 @@
|
|
1
1
|
require 'slop'
|
2
|
+
require_relative 'command_logger'
|
2
3
|
|
3
4
|
module Flydata
|
4
5
|
class Cli
|
5
6
|
include Helpers
|
7
|
+
include CommandLoggable
|
6
8
|
def initialize(args)
|
7
9
|
@args = args
|
10
|
+
unless $log
|
11
|
+
$log = Logger.new(FLYDATA_LOG)
|
12
|
+
# 2014-10-24 14:05:21 -0700 command.info: message
|
13
|
+
$log.datetime_format = "%Y-%m-%d %H:%M:%S %z "
|
14
|
+
$log.formatter = proc do |severity, datetime, progname, msg|
|
15
|
+
"#{datetime} command.#{severity.to_s.downcase}: #{msg}\n"
|
16
|
+
end
|
17
|
+
end
|
18
|
+
log_info("Start command.", {cmd: @args.join(' '), ver:flydata_version})
|
8
19
|
end
|
9
20
|
|
10
21
|
def run
|
11
22
|
unless check_environment
|
12
|
-
|
23
|
+
log_error_stderr("Sorry, you have to run the installation command to use flydata. Please go to https://console.flydata.com/ and sign up.")
|
13
24
|
return
|
14
25
|
end
|
15
26
|
begin
|
@@ -28,7 +39,8 @@ module Flydata
|
|
28
39
|
cmd_obj = cmd_cls.new(options)
|
29
40
|
sub_cmd ? cmd_obj.send(sub_cmd,*@args) : cmd_obj.run(*@args)
|
30
41
|
else
|
31
|
-
|
42
|
+
$stderr.puts usage_text(false)
|
43
|
+
return
|
32
44
|
end
|
33
45
|
rescue => e
|
34
46
|
#raise e
|
@@ -36,6 +48,7 @@ module Flydata
|
|
36
48
|
$stderr.puts
|
37
49
|
$stderr.puts
|
38
50
|
$stderr.puts usage_text
|
51
|
+
log_error_with_backtrace('Error occured during command.', {error: e, args: @args})
|
39
52
|
raise e if FLYDATA_DEBUG
|
40
53
|
exit 1
|
41
54
|
end
|
data/lib/flydata/command/base.rb
CHANGED
@@ -1,6 +1,8 @@
|
|
1
1
|
module Flydata
|
2
2
|
module Command
|
3
3
|
class Base
|
4
|
+
include CommandLoggable
|
5
|
+
|
4
6
|
def initialize(options = Slop.new)
|
5
7
|
@api_client = ApiClient.instance
|
6
8
|
@opts = options
|
@@ -37,7 +39,7 @@ module Flydata
|
|
37
39
|
suffix = default_yes ? "(Y/n)" : "(y/n)"
|
38
40
|
prompt = "#{message} #{suffix}: "
|
39
41
|
if opts && opts.yes? # Yes option
|
40
|
-
|
42
|
+
log_info_stdout("#{prompt}Yes")
|
41
43
|
return true
|
42
44
|
end
|
43
45
|
|
@@ -46,18 +48,22 @@ module Flydata
|
|
46
48
|
return true if default_yes and ans == ''
|
47
49
|
if ans.size > 0
|
48
50
|
case ans[0].downcase
|
49
|
-
when 'y'
|
50
|
-
|
51
|
+
when 'y'
|
52
|
+
log_info("#{prompt}Yes")
|
53
|
+
return true
|
54
|
+
when 'n'
|
55
|
+
log_info("#{prompt}No")
|
56
|
+
return false
|
51
57
|
end
|
52
58
|
end
|
53
|
-
|
59
|
+
log_warn_stderr(" ! Please answer y[es] or n[o]")
|
54
60
|
newline
|
55
61
|
end
|
56
62
|
end
|
57
63
|
def choose_one(message, prompt, menu_list, value_list=[])
|
58
64
|
choice = nil
|
59
65
|
value_list = menu_list unless menu_list.size == value_list.size
|
60
|
-
|
66
|
+
log_info_stdout(message) if message
|
61
67
|
choose do |menu|
|
62
68
|
menu.index = :number
|
63
69
|
menu.index_suffix = ") "
|
@@ -65,12 +71,13 @@ module Flydata
|
|
65
71
|
menu.prompt = prompt ? prompt : ">> "
|
66
72
|
menu.choices(*menu_list) {|item| choice = value_list[menu_list.index(item)]}
|
67
73
|
end
|
74
|
+
log_info_stdout(" -> #{choice}")
|
68
75
|
choice
|
69
76
|
end
|
70
77
|
def ask_input_table_name
|
71
78
|
input = nil
|
72
79
|
loop do
|
73
|
-
|
80
|
+
log_info_stdout("Input a table name.")
|
74
81
|
say(">> ")
|
75
82
|
input = gets.strip
|
76
83
|
if input =~ /^[a-zA-Z0-9_]+$/
|
@@ -79,6 +86,7 @@ module Flydata
|
|
79
86
|
puts "Please enter a valid table name."
|
80
87
|
end
|
81
88
|
end
|
89
|
+
log_info(">> #{input}")
|
82
90
|
input
|
83
91
|
end
|
84
92
|
end
|
@@ -16,7 +16,7 @@ module Flydata
|
|
16
16
|
end
|
17
17
|
# Check if process exist
|
18
18
|
if process_exist?
|
19
|
-
|
19
|
+
log_info_stdout("Process is still running. Please stop process first.") unless options[:quiet]
|
20
20
|
return
|
21
21
|
end
|
22
22
|
|
@@ -31,11 +31,11 @@ module Flydata
|
|
31
31
|
AgentCompatibilityCheck.new(dp).check
|
32
32
|
|
33
33
|
# Start sender(fluentd) process
|
34
|
-
|
34
|
+
log_info_stdout("Starting sender process.") unless options[:quiet]
|
35
35
|
Dir.chdir(FLYDATA_HOME){
|
36
|
-
Kernel.system("bash #{
|
36
|
+
Kernel.system("bash #{FLYDATA_SERVERINFO}", :out => [FLYDATA_LOG,'a'], :err => [FLYDATA_LOG,'a'])
|
37
37
|
daemon_opt = opts.no_daemon? ? "" : daemon_option
|
38
|
-
Kernel.system("ruby `which fluentd` #{daemon_opt} -l #{
|
38
|
+
Kernel.system("ruby `which fluentd` #{daemon_opt} -l #{FLYDATA_LOG} -c #{FLYDATA_CONF} -p #{FLYDATA_FLUENT_PLUGIN_DIR}")
|
39
39
|
}
|
40
40
|
Kernel.sleep 5
|
41
41
|
|
@@ -43,21 +43,21 @@ module Flydata
|
|
43
43
|
#wait_until_logs_uploaded
|
44
44
|
if options[:show_final_message] && !options[:quiet]
|
45
45
|
data_port = flydata.data_port.get
|
46
|
-
|
47
|
-
|
46
|
+
log_info_stdout("Go to your Dashboard! #{flydata.flydata_api_host}/data_ports/#{data_port['id']}")
|
47
|
+
log_info_stdout <<EOF
|
48
48
|
Please Note: Records and Total Size are updated every 10-20 minutes.
|
49
49
|
EOF
|
50
50
|
end
|
51
51
|
end
|
52
52
|
def stop(options = {})
|
53
53
|
unless process_exist?
|
54
|
-
|
54
|
+
log_info_stdout("Process doesn't exist.") unless options[:quiet]
|
55
55
|
return true
|
56
56
|
end
|
57
57
|
|
58
|
-
|
58
|
+
log_info_stdout('Stopping sender process.') unless options[:quiet]
|
59
59
|
if Kernel.system("kill `cat #{pid_file}`") and wait_until_client_stop(options)
|
60
|
-
|
60
|
+
log_info_stdout('Done.') unless options[:quiet]
|
61
61
|
return true
|
62
62
|
end
|
63
63
|
raise 'Something has gone wrong..'
|
@@ -65,17 +65,17 @@ EOF
|
|
65
65
|
def flush_client_buffer(options = {})
|
66
66
|
unless process_exist?
|
67
67
|
return true if client_buffer_empty?
|
68
|
-
|
68
|
+
log_info_stdout("Process doesn't exist. But, the client buffer is not empty!!") unless options[:quiet]
|
69
69
|
start(options)
|
70
70
|
end
|
71
71
|
|
72
|
-
|
72
|
+
log_info_stdout('Stopping input plugins and flushing the client buffer.') unless options[:quiet]
|
73
73
|
Kernel.system("kill -USR1 `cat #{pid_file}`")
|
74
74
|
|
75
75
|
retry_count = 12
|
76
76
|
1.upto(retry_count) do |i|
|
77
77
|
return true if client_buffer_empty?
|
78
|
-
|
78
|
+
log_info_stdout("Waiting for the buffer to get empty... (#{i}/#{retry_count})") unless options[:quiet]
|
79
79
|
Kernel.sleep 5
|
80
80
|
end
|
81
81
|
|
@@ -83,23 +83,23 @@ EOF
|
|
83
83
|
end
|
84
84
|
def restart(options = {})
|
85
85
|
if process_exist?
|
86
|
-
|
86
|
+
log_info_stdout('Restarting sender process.') unless options[:quiet]
|
87
87
|
if Kernel.system("kill -HUP `cat #{pid_file}`")
|
88
|
-
|
88
|
+
log_info_stdout('Done.') unless options[:quiet]
|
89
89
|
return true
|
90
90
|
else
|
91
91
|
raise 'Something has gone wrong..'
|
92
92
|
end
|
93
93
|
else
|
94
|
-
|
94
|
+
log_info_stdout("Process doesn't exist.") unless options[:quiet]
|
95
95
|
start(options)
|
96
96
|
end
|
97
97
|
end
|
98
98
|
def status
|
99
99
|
if process_exist?
|
100
|
-
|
100
|
+
log_info_stdout("Process is running.")
|
101
101
|
else
|
102
|
-
|
102
|
+
log_info_stdout("Process is not running.")
|
103
103
|
end
|
104
104
|
end
|
105
105
|
def process_exist?
|
@@ -108,7 +108,7 @@ EOF
|
|
108
108
|
end
|
109
109
|
def kill_all(optiosn = {})
|
110
110
|
if Kernel.system("ps ax | grep 'flydata' | grep -v grep | awk '{print \"kill \" $1}' | sh")
|
111
|
-
|
111
|
+
log_info_stdout("Done.") unless options[:quiet]
|
112
112
|
return true
|
113
113
|
else
|
114
114
|
raise 'Something has gone wrong...'
|
@@ -133,7 +133,7 @@ EOF
|
|
133
133
|
retry_count = 10
|
134
134
|
1.upto(retry_count) do |i|
|
135
135
|
return true if server_ready?
|
136
|
-
|
136
|
+
log_info_stdout("Waiting for the server side to become active... (#{i}/#{retry_count})") unless options[:quiet]
|
137
137
|
Kernel.sleep 30
|
138
138
|
end
|
139
139
|
false
|
@@ -142,13 +142,13 @@ EOF
|
|
142
142
|
retry_count = 10
|
143
143
|
1.upto(retry_count) do |i|
|
144
144
|
if client_ready?
|
145
|
-
|
145
|
+
log_info_stdout("Done! Client is ready now.") unless options[:quiet]
|
146
146
|
return true
|
147
147
|
end
|
148
148
|
if process_died?
|
149
149
|
raise "Client could not been launched. Detail here #{FLYDATA_HOME}/flydata.log"
|
150
150
|
end
|
151
|
-
|
151
|
+
log_info_stdout("Waiting for the client side to become active... (#{i}/#{retry_count})") unless options[:quiet]
|
152
152
|
Kernel.sleep 10
|
153
153
|
end
|
154
154
|
raise "Somthing has gone wrong... Please try setup command again."
|
@@ -157,23 +157,23 @@ EOF
|
|
157
157
|
retry_count = 5
|
158
158
|
1.upto(retry_count) do |i|
|
159
159
|
return true unless process_exist?
|
160
|
-
|
160
|
+
log_info_stdout("Waiting for the client to stop... (#{i}/#{retry_count})") unless options[:quiet]
|
161
161
|
Kernel.sleep 3
|
162
162
|
end
|
163
163
|
false
|
164
164
|
end
|
165
165
|
def wait_until_logs_uploaded(options = {})
|
166
|
-
|
166
|
+
log_info_stdout('Starting to check the upload from your server.') unless options[:quiet]
|
167
167
|
data_port = flydata.data_port.get
|
168
168
|
data_port_id = data_port['id']
|
169
169
|
|
170
170
|
retry_count = 10
|
171
171
|
1.upto(retry_count) do |i|
|
172
172
|
if uploaded_successfully?(data_port_id)
|
173
|
-
|
173
|
+
log_info_stdout("Uploading your logs correctly.") unless options[:quiet]
|
174
174
|
return true
|
175
175
|
end
|
176
|
-
|
176
|
+
log_info_stdout("Waiting logs uploading... (#{i}/#{retry_count})") unless options[:quiet]
|
177
177
|
Kernel.sleep 30
|
178
178
|
end
|
179
179
|
raise 'Cannot confirm that your logs exist on the FlyData server. Something has gone wrong..'
|
@@ -196,7 +196,7 @@ EOF
|
|
196
196
|
end
|
197
197
|
def client_buffer_empty?(options = {})
|
198
198
|
client_buffer = File.join(FLYDATA_HOME, 'buffer')
|
199
|
-
|
199
|
+
log_info_stdout("Checking the client buffer #{client_buffer}") unless options[:quiet]
|
200
200
|
Dir.glob("#{client_buffer}/*").empty?
|
201
201
|
end
|
202
202
|
def pid_file
|
data/lib/flydata/command/sync.rb
CHANGED
@@ -3,7 +3,6 @@ require 'msgpack'
|
|
3
3
|
require 'open3'
|
4
4
|
require 'mysql2'
|
5
5
|
require 'flydata/sync_file_manager'
|
6
|
-
require 'flydata/agent_errors'
|
7
6
|
require 'flydata/compatibility_check'
|
8
7
|
require 'flydata/output/forwarder'
|
9
8
|
require 'flydata/parser/mysql/dump_parser'
|
@@ -35,10 +34,10 @@ module Flydata
|
|
35
34
|
if (sender.process_exist?)
|
36
35
|
if tables.empty?
|
37
36
|
# full sync
|
38
|
-
|
37
|
+
log_warn_stderr("FlyData Agent is already running. If you'd like to restart FlyData Sync from scratch, run 'flydata sync:reset' first.")
|
39
38
|
else
|
40
39
|
# per-table sync
|
41
|
-
|
40
|
+
log_warn_stderr("Flydata Agent is already running. If you'd like to Sync the table(s), run 'flydata sync:flush' first.")
|
42
41
|
end
|
43
42
|
exit 1
|
44
43
|
end
|
@@ -58,7 +57,7 @@ module Flydata
|
|
58
57
|
|
59
58
|
def flush
|
60
59
|
flush_buffer_and_stop
|
61
|
-
|
60
|
+
log_info_stdout("Buffers have been flushed and the sender process has been stopped.")
|
62
61
|
end
|
63
62
|
|
64
63
|
def self.slop_reset
|
@@ -91,11 +90,11 @@ module Flydata
|
|
91
90
|
delete_files.flatten.each do |path|
|
92
91
|
FileUtils.rm(path) if File.exists?(path)
|
93
92
|
end
|
94
|
-
|
93
|
+
log_info_stdout("Reset completed successfully.")
|
95
94
|
end
|
96
95
|
|
97
96
|
def wait_for_server_buffer
|
98
|
-
|
97
|
+
log_info_stdout("Waiting for the server buffer to get empty.")
|
99
98
|
while (status = check) && (status['state'] == 'processing')
|
100
99
|
print_progress(status)
|
101
100
|
sleep 10
|
@@ -104,24 +103,24 @@ module Flydata
|
|
104
103
|
|
105
104
|
def wait_for_server_data_processing
|
106
105
|
state = :PROCESS
|
107
|
-
|
106
|
+
log_info_stdout("Uploading data to Redshift...")
|
108
107
|
sleep 10
|
109
108
|
status = nil
|
110
109
|
while (status = check)
|
111
110
|
if state == :PROCESS && status['state'] == 'uploading'
|
112
|
-
|
111
|
+
log_info_stdout(" -> Done")
|
113
112
|
state = :UPLOAD
|
114
|
-
|
113
|
+
log_info_stdout("Finishing data upload...")
|
115
114
|
end
|
116
115
|
print_progress(status)
|
117
116
|
sleep 10
|
118
117
|
end
|
119
118
|
if (state == :PROCESS)
|
120
119
|
# :UPLOAD state was skipped due to no data
|
121
|
-
|
122
|
-
|
120
|
+
log_info_stdout(" -> Done")
|
121
|
+
log_info_stdout("Finishing data upload...")
|
123
122
|
end
|
124
|
-
|
123
|
+
log_info_stdout(" -> Done")
|
125
124
|
end
|
126
125
|
|
127
126
|
def check
|
@@ -142,9 +141,9 @@ module Flydata
|
|
142
141
|
sync_fm = Flydata::FileUtil::SyncFileManager.new(de)
|
143
142
|
binlog_path = sync_fm.binlog_path
|
144
143
|
`touch #{binlog_path}`
|
145
|
-
|
146
|
-
|
147
|
-
|
144
|
+
log_info_stdout("Created an empty binlog position file.")
|
145
|
+
log_info_stdout("-> #{binlog_path}")
|
146
|
+
log_info_stdout("Run 'flydata start' to start continuous sync.")
|
148
147
|
end
|
149
148
|
|
150
149
|
def self.slop_generate_table_ddl
|
@@ -192,6 +191,7 @@ module Flydata
|
|
192
191
|
|
193
192
|
def cleanup_sync_server(de, tables = [])
|
194
193
|
print "Cleaning the server."
|
194
|
+
log_info("Cleaning the server.")
|
195
195
|
worker = Thread.new do
|
196
196
|
begin
|
197
197
|
flydata.data_entry.cleanup_sync(de['id'], tables)
|
@@ -204,7 +204,7 @@ module Flydata
|
|
204
204
|
print "."
|
205
205
|
end
|
206
206
|
puts
|
207
|
-
|
207
|
+
log_info_stdout("Done.")
|
208
208
|
end
|
209
209
|
|
210
210
|
def do_check(de)
|
@@ -213,7 +213,7 @@ module Flydata
|
|
213
213
|
|
214
214
|
def print_progress(buffer_stat)
|
215
215
|
message = buffer_stat['message']
|
216
|
-
|
216
|
+
log_info_stdout(message) unless message.nil? || message.empty?
|
217
217
|
end
|
218
218
|
|
219
219
|
DDL_DUMP_CMD_TEMPLATE = "MYSQL_PWD=\"%s\" mysqldump --protocol=tcp -d -h %s -P %s -u %s %s %s"
|
@@ -264,15 +264,15 @@ module Flydata
|
|
264
264
|
raise errors unless errors.empty?
|
265
265
|
end
|
266
266
|
unless error_list.empty?
|
267
|
-
|
267
|
+
log_error_stderr("We have noticed the following error(s):")
|
268
268
|
group_error = error_list.group_by {|d| d[:error]}
|
269
269
|
group_error.each_key do |a|
|
270
|
-
|
270
|
+
log_error_stderr("The following table(s) have #{a}:")
|
271
271
|
group_error[a].each do |hash|
|
272
|
-
|
272
|
+
log_error_stderr(" - #{hash[:table]}") if hash[:table]
|
273
273
|
end
|
274
274
|
end
|
275
|
-
|
275
|
+
log_error_stderr("Please fix the above error(s) to try to sync those table(s) or contact us for further help.")
|
276
276
|
end
|
277
277
|
end
|
278
278
|
|
@@ -315,7 +315,7 @@ module Flydata
|
|
315
315
|
dp = flydata.data_port.get
|
316
316
|
fp = sync_fm.dump_file_path
|
317
317
|
if file_dump && File.exists?(fp) && File.size(fp) > 0
|
318
|
-
|
318
|
+
log_info_stdout(" -> Skip")
|
319
319
|
return call_block_or_return_io(fp, &block)
|
320
320
|
end
|
321
321
|
|
@@ -338,30 +338,27 @@ EOM
|
|
338
338
|
Dump file saves contents of your tables temporarily. Make sure you have enough disk space.
|
339
339
|
EOM
|
340
340
|
print confirmation_text
|
341
|
+
log_info confirmation_text.strip
|
341
342
|
|
342
343
|
if ask_yes_no('Start Sync?')
|
343
|
-
|
344
|
+
log_info_stdout("Checking database size(not same as mysqldump data size)...")
|
345
|
+
|
344
346
|
db_bytesize = Flydata::Parser::Mysql::DatabaseSizeCheck.new(de['mysql_data_entry_preference']).get_db_bytesize
|
345
|
-
|
346
|
-
|
347
|
-
|
347
|
+
log_info_stdout(" -> #{as_size(db_bytesize)} (#{db_bytesize} byte)")
|
348
|
+
log_info_stdout("Exporting data from database.")
|
349
|
+
log_info_stdout("This process can take hours depending on data size and load on your database. Please be patient...")
|
348
350
|
Flydata::MysqlCompatibilityCheck.new(dp, de['mysql_data_entry_preference'], fp).check
|
349
351
|
if file_dump
|
350
352
|
Flydata::Parser::Mysql::MysqlDumpGeneratorNoMasterData.
|
351
353
|
new(de['mysql_data_entry_preference']).dump(fp)
|
352
|
-
|
354
|
+
log_info_stdout(" -> Done")
|
353
355
|
call_block_or_return_io(fp, &block)
|
354
356
|
else
|
355
357
|
Flydata::Parser::Mysql::MysqlDumpGeneratorNoMasterData.
|
356
358
|
new(de['mysql_data_entry_preference']).dump {|io| block.call(io, db_bytesize)}
|
357
359
|
end
|
358
360
|
else
|
359
|
-
|
360
|
-
puts "You can change the dump file path with 'mysqldump_path' property in the following conf file."
|
361
|
-
puts
|
362
|
-
puts " #{Flydata::Preference::DataEntryPreference.conf_path(de)}"
|
363
|
-
puts
|
364
|
-
return nil
|
361
|
+
exit 1
|
365
362
|
end
|
366
363
|
end
|
367
364
|
|
@@ -419,12 +416,12 @@ EOM
|
|
419
416
|
dump_pos_info = sync_fm.load_dump_pos
|
420
417
|
option = dump_pos_info || {}
|
421
418
|
if option[:table_name] && option[:last_pos].to_i != -1
|
422
|
-
|
419
|
+
log_info_stdout("Resuming... Last processed table: #{option[:table_name]}")
|
423
420
|
else
|
424
421
|
#If its a new sync, ensure server side resources are clean
|
425
422
|
cleanup_sync_server(de, de['mysql_data_entry_preference']['tables'].split(',')) unless opts.skip_cleanup?
|
426
423
|
end
|
427
|
-
|
424
|
+
log_info_stdout("Sending data to FlyData Server...")
|
428
425
|
|
429
426
|
bench_start_time = Time.now
|
430
427
|
|
@@ -461,9 +458,9 @@ EOM
|
|
461
458
|
|
462
459
|
# show the current progress
|
463
460
|
if last_pos.to_i == -1 # stream dump
|
464
|
-
|
461
|
+
log_info_stdout(" -> #{as_size(bytesize)} (#{bytesize} byte) completed...")
|
465
462
|
else
|
466
|
-
|
463
|
+
log_info_stdout(" -> #{(last_pos.to_f/dump_file_size * 100).round(1)}% (#{last_pos}/#{dump_file_size}) completed...")
|
467
464
|
end
|
468
465
|
|
469
466
|
# save check point
|
@@ -472,13 +469,13 @@ EOM
|
|
472
469
|
}
|
473
470
|
)
|
474
471
|
forwarder.close
|
475
|
-
|
472
|
+
log_info_stdout(" -> Done")
|
476
473
|
sync_fm.save_dump_pos(STATUS_WAITING, '', dump_file_size, binlog_pos)
|
477
474
|
|
478
475
|
if ENV['FLYDATA_BENCHMARK']
|
479
476
|
bench_end_time = Time.now
|
480
477
|
elapsed_time = bench_end_time.to_i - bench_start_time.to_i
|
481
|
-
|
478
|
+
log_info_stdout("Elapsed:#{elapsed_time}sec start:#{bench_start_time} end:#{bench_end_time}")
|
482
479
|
end
|
483
480
|
end
|
484
481
|
|
@@ -524,14 +521,14 @@ Thank you for using FlyData!
|
|
524
521
|
sync_fm.reset_table_position_files(de['mysql_data_entry_preference']['tables'].split(','))
|
525
522
|
sync_fm.backup_dump_dir
|
526
523
|
unless opts.no_flydata_start?
|
527
|
-
|
524
|
+
log_info_stdout("Starting FlyData Agent...")
|
528
525
|
Flydata::Command::Sender.new.start(quiet: true)
|
529
|
-
|
526
|
+
log_info_stdout(" -> Done")
|
530
527
|
end
|
531
528
|
dashboard_url = "#{flydata.flydata_api_host}/dashboard"
|
532
529
|
redshift_console_url = "#{flydata.flydata_api_host}/redshift_clusters/query/new"
|
533
530
|
last_message = ALL_DONE_MESSAGE_TEMPLATE % [redshift_console_url, dashboard_url]
|
534
|
-
|
531
|
+
log_info_stdout(last_message)
|
535
532
|
else
|
536
533
|
raise "Initial sync status is not complete. Try running 'flydata sync'."
|
537
534
|
end
|
@@ -1,9 +1,10 @@
|
|
1
1
|
module Flydata
|
2
2
|
module Command
|
3
3
|
class Version < Base
|
4
|
-
|
4
|
+
include Helpers
|
5
|
+
|
5
6
|
def run
|
6
|
-
|
7
|
+
log_info_stdout("flydata version \"#{flydata_version}\"")
|
7
8
|
end
|
8
9
|
end
|
9
10
|
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
require 'flydata-core/core_ext'
|
2
|
+
require 'flydata-core/logger'
|
3
|
+
|
4
|
+
module Flydata
|
5
|
+
module CommandLoggable
|
6
|
+
include_unless_included FlydataCore::Logger
|
7
|
+
|
8
|
+
# hook
|
9
|
+
def before_logging(level, raw_msg, built_msg, log_params, options)
|
10
|
+
$stdout.puts raw_msg if options[:stdout]
|
11
|
+
$stderr.puts raw_msg if options[:stderr]
|
12
|
+
end
|
13
|
+
|
14
|
+
def log_info_stdout(message, log_params = {})
|
15
|
+
log_info(message, log_params, {stdout: true})
|
16
|
+
end
|
17
|
+
|
18
|
+
def log_warn_stderr(message, log_params = {})
|
19
|
+
log_warn(message, log_params, {stderr: true})
|
20
|
+
end
|
21
|
+
|
22
|
+
def log_error_stderr(message)
|
23
|
+
log_error(message, log_params, {stderr: true})
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -1,6 +1,10 @@
|
|
1
|
+
require 'flydata/command_logger'
|
2
|
+
|
1
3
|
module Flydata
|
2
4
|
|
3
5
|
class CompatibilityCheck
|
6
|
+
include CommandLoggable
|
7
|
+
|
4
8
|
class CompatibilityError < StandardError
|
5
9
|
end
|
6
10
|
|
@@ -13,6 +17,8 @@ module Flydata
|
|
13
17
|
self.methods.grep(/^check_/).each do |m|
|
14
18
|
begin
|
15
19
|
send(m)
|
20
|
+
rescue ArgumentError => e
|
21
|
+
# ignore
|
16
22
|
rescue CompatibilityError => e
|
17
23
|
@errors << e
|
18
24
|
end
|
@@ -22,9 +28,9 @@ module Flydata
|
|
22
28
|
|
23
29
|
def print_errors
|
24
30
|
return if @errors.empty?
|
25
|
-
|
31
|
+
log_error_stderr "There may be some compatibility issues with this current server : "
|
26
32
|
@errors.each do |error|
|
27
|
-
|
33
|
+
log_error_stderr " * #{error.message}"
|
28
34
|
end
|
29
35
|
raise "Please correct these errors if you wish to run FlyData Agent"
|
30
36
|
end
|
@@ -79,9 +85,9 @@ module Flydata
|
|
79
85
|
|
80
86
|
def print_errors
|
81
87
|
return if @errors.empty?
|
82
|
-
|
88
|
+
log_error_stderr "There may be some compatibility issues with your MySQL credentials: "
|
83
89
|
@errors.each do |error|
|
84
|
-
|
90
|
+
log_error_stderr " * #{error.message}"
|
85
91
|
end
|
86
92
|
raise "Please correct these errors if you wish to run FlyData Sync"
|
87
93
|
end
|
@@ -115,8 +121,12 @@ module Flydata
|
|
115
121
|
Open3.popen3(query) do |stdin, stdout, stderr|
|
116
122
|
stdin.close
|
117
123
|
while !stderr.eof?
|
118
|
-
|
119
|
-
|
124
|
+
lines = []
|
125
|
+
while line = stderr.gets; lines << line.strip; end
|
126
|
+
err_reason = lines.join(" ")
|
127
|
+
log_error("Error occured during access to mysql server.", {err: err_reason})
|
128
|
+
|
129
|
+
unless /Warning: Using a password on the command line interface can be insecure/ === err_reason
|
120
130
|
raise MysqlCompatibilityError, "Cannot connect to MySQL database. Please make sure you can connect with this command:\n $ mysql -u #{@db_opts[:username]} -h #{@db_opts[:host]} -P #{@db_opts[:port]} #{@db_opts[:database]} --protocol=tcp -p"
|
121
131
|
end
|
122
132
|
end
|
data/lib/flydata/helpers.rb
CHANGED
@@ -12,19 +12,24 @@ module Flydata
|
|
12
12
|
[klass, method]
|
13
13
|
end
|
14
14
|
|
15
|
-
def usage_text
|
15
|
+
def usage_text(err = true)
|
16
16
|
text = ""
|
17
|
-
|
17
|
+
|
18
|
+
if err
|
19
|
+
text += <<-EOM
|
18
20
|
Fix the issue and try again. If the problem continues, please contact support@flydata.com
|
19
21
|
|
20
22
|
EOM
|
21
|
-
|
22
|
-
|
23
|
-
|
23
|
+
|
24
|
+
flydata_log = File.join(FLYDATA_HOME, 'flydata.log')
|
25
|
+
if File.exists?(flydata_log)
|
26
|
+
text += <<-EOM
|
24
27
|
Also check the Agent log.
|
25
28
|
Log path: #{flydata_log}
|
26
29
|
EOM
|
30
|
+
end
|
27
31
|
end
|
32
|
+
|
28
33
|
text += ""
|
29
34
|
text += '-' * 64
|
30
35
|
text += "\n"
|
@@ -45,6 +50,10 @@ Usage: flydata COMMAND
|
|
45
50
|
text
|
46
51
|
end
|
47
52
|
|
53
|
+
def flydata_version
|
54
|
+
IO.read(VERSION_PATH).strip
|
55
|
+
end
|
56
|
+
|
48
57
|
def to_command_class(class_name)
|
49
58
|
eval("Flydata::Command::#{class_name.camelcase}")
|
50
59
|
end
|
@@ -1,15 +1,19 @@
|
|
1
1
|
require 'socket'
|
2
|
+
require 'msgpack'
|
3
|
+
require 'flydata/command_logger'
|
2
4
|
|
3
5
|
module Flydata
|
4
6
|
module Output
|
5
7
|
class ForwarderFactory
|
8
|
+
include CommandLoggable
|
9
|
+
|
6
10
|
def self.create(forwarder_key, tag, servers, options = {})
|
7
11
|
case forwarder_key
|
8
12
|
when nil, "tcpforwarder"
|
9
|
-
puts
|
13
|
+
puts("Creating TCP connection") if FLYDATA_DEBUG
|
10
14
|
forward = TcpForwarder.new(tag, servers, options)
|
11
15
|
when "sslforwarder"
|
12
|
-
puts
|
16
|
+
puts("Creating SSL connection") if FLYDATA_DEBUG
|
13
17
|
forward = SslForwarder.new(tag, servers, options)
|
14
18
|
else
|
15
19
|
raise "Unsupported Forwarding type #{forwarder_key}"
|
@@ -88,12 +92,12 @@ module Flydata
|
|
88
92
|
rescue => e
|
89
93
|
retry_count += 1
|
90
94
|
if retry_count > RETRY_LIMIT
|
91
|
-
|
95
|
+
log_error_stderr("! Error: Failed to send data. Exceeded the retry limit. retry_count:#{retry_count}")
|
92
96
|
raise e
|
93
97
|
end
|
94
|
-
|
98
|
+
log_warn_stderr("! Warn: Retring to send data. retry_count:#{retry_count} error=#{e.to_s}")
|
95
99
|
wait_time = RETRY_INTERVAL ** retry_count
|
96
|
-
|
100
|
+
log_warn_stderr(" Now waiting for next retry. time=#{wait_time}sec")
|
97
101
|
sleep wait_time
|
98
102
|
retry
|
99
103
|
ensure
|
@@ -1,3 +1,5 @@
|
|
1
|
+
require 'fiber'
|
2
|
+
|
1
3
|
module Flydata
|
2
4
|
module Parser
|
3
5
|
module Mysql
|
@@ -106,7 +108,7 @@ EOS
|
|
106
108
|
rd_io.extend(DumpStreamIO)
|
107
109
|
|
108
110
|
# start mysqldump
|
109
|
-
Open3.popen3 @dump_cmd do |cmd_in, cmd_out, cmd_err|
|
111
|
+
Open3.popen3 @dump_cmd do |cmd_in, cmd_out, cmd_err, wait_thr|
|
110
112
|
cmd_in.close_write
|
111
113
|
cmd_out.set_encoding("utf-8") # mysqldump output must be in UTF-8
|
112
114
|
|
@@ -126,9 +128,10 @@ EOS
|
|
126
128
|
end
|
127
129
|
|
128
130
|
# show err message
|
131
|
+
errors = ""
|
129
132
|
threads << Thread.new do
|
130
133
|
cmd_err.each_line do |line|
|
131
|
-
|
134
|
+
errors << line unless /^Warning:/ === line
|
132
135
|
end
|
133
136
|
end
|
134
137
|
|
@@ -141,7 +144,14 @@ EOS
|
|
141
144
|
end
|
142
145
|
|
143
146
|
threads.each(&:join)
|
147
|
+
unless wait_thr.value == 0
|
148
|
+
#Raise error if the process exited with status != 0
|
149
|
+
#(Even if there was no message on stderr stream)
|
150
|
+
errors = "Failed to run mysqldump command." if errors.empty?
|
151
|
+
end
|
152
|
+
raise errors unless errors.empty?
|
144
153
|
end
|
154
|
+
true
|
145
155
|
rescue
|
146
156
|
# Cleanup
|
147
157
|
FileUtils.rm(file_path) if file_path && File.exists?(file_path)
|
@@ -24,9 +24,9 @@ module Flydata
|
|
24
24
|
let(:args) { [] }
|
25
25
|
it "starts fluend with daemon option" do
|
26
26
|
expect(Kernel).to receive(:system).with( Regexp.new(
|
27
|
-
"bash
|
27
|
+
"bash .+/bin/serverinfo"), :out => [Regexp.new(".+/flydata.log"),'a'], :err => [Regexp.new(".+/flydata.log"),'a'])
|
28
28
|
expect(Kernel).to receive(:system).with( Regexp.new(
|
29
|
-
"ruby `which fluentd` -d .+/flydata.pid -l .+/flydata.log -c .+/flydata.conf -p
|
29
|
+
"ruby `which fluentd` -d .+/flydata.pid -l .+/flydata.log -c .+/flydata.conf -p .+/lib/flydata/fluent-plugins"))
|
30
30
|
subject.start(false)
|
31
31
|
end
|
32
32
|
end
|
@@ -34,9 +34,9 @@ module Flydata
|
|
34
34
|
let(:args) { ["-n"] }
|
35
35
|
it "starts fluentd with no daemon option" do
|
36
36
|
expect(Kernel).to receive(:system).with( Regexp.new(
|
37
|
-
"bash
|
37
|
+
"bash .+/bin/serverinfo"), :out => [Regexp.new(".+/flydata.log"),'a'], :err => [Regexp.new(".+/flydata.log"),'a'])
|
38
38
|
expect(Kernel).to receive(:system).with(Regexp.new(
|
39
|
-
"ruby `which fluentd` -l .+/flydata.log -c .+/flydata.conf -p
|
39
|
+
"ruby `which fluentd` -l .+/flydata.log -c .+/flydata.conf -p .+/lib/flydata/fluent-plugins"))
|
40
40
|
subject.start(false)
|
41
41
|
end
|
42
42
|
end
|
@@ -44,9 +44,9 @@ module Flydata
|
|
44
44
|
let(:args) { ["--no-daemon"] }
|
45
45
|
it "starts fluentd with no daemon option" do
|
46
46
|
expect(Kernel).to receive(:system).with( Regexp.new(
|
47
|
-
"bash
|
47
|
+
"bash .+/bin/serverinfo"), :out => [Regexp.new(".+/flydata.log"),'a'], :err => [Regexp.new(".+/flydata.log"),'a'])
|
48
48
|
expect(Kernel).to receive(:system).with(Regexp.new(
|
49
|
-
"ruby `which fluentd` -l .+/flydata.log -c .+/flydata.conf -p
|
49
|
+
"ruby `which fluentd` -l .+/flydata.log -c .+/flydata.conf -p .+/lib/flydata/fluent-plugins"))
|
50
50
|
subject.start(false)
|
51
51
|
end
|
52
52
|
end
|
@@ -1,91 +1,157 @@
|
|
1
1
|
# coding: utf-8
|
2
2
|
require 'spec_helper'
|
3
|
+
require 'flydata/parser/mysql/dump_parser'
|
4
|
+
require 'open3'
|
5
|
+
require 'mysql2'
|
3
6
|
|
4
7
|
module Flydata
|
5
8
|
module Parser
|
6
9
|
module Mysql
|
7
|
-
|
8
|
-
let(:
|
9
|
-
|
10
|
-
|
10
|
+
context "Test Dump Generators" do
|
11
|
+
let(:stdin) do
|
12
|
+
s = double(:stdin)
|
13
|
+
allow(s).to receive(:close_write)
|
14
|
+
s
|
15
|
+
end
|
16
|
+
let(:stdout) do
|
17
|
+
s = double(:stdout)
|
18
|
+
allow(s).to receive(:set_encoding)
|
19
|
+
allow(s).to receive(:gets).and_return("first line")
|
20
|
+
allow(s).to receive(:each_line).and_yield("another line")
|
21
|
+
s
|
22
|
+
end
|
11
23
|
let(:file_path) { File.join('/tmp', "flydata_sync_spec_mysqldump_#{Time.now.to_i}") }
|
12
|
-
let(:
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
let(:default_dump_generator) { MysqlDumpGeneratorMasterData.new(default_conf) }
|
18
|
-
|
19
|
-
describe '#initialize' do
|
20
|
-
context 'with password' do
|
21
|
-
subject { default_dump_generator.instance_variable_get(:@dump_cmd) }
|
22
|
-
it { is_expected.to eq('MYSQL_PWD="pass" mysqldump --default-character-set=utf8 --protocol=tcp -h localhost -P 3306 -uadmin --skip-lock-tables ' +
|
23
|
-
'--single-transaction --hex-blob --flush-logs --master-data=2 dev users groups') }
|
24
|
-
end
|
25
|
-
context 'without password' do
|
26
|
-
let (:dump_generator) do
|
27
|
-
MysqlDumpGeneratorMasterData.new(default_conf.merge({'password' => ''}))
|
28
|
-
end
|
29
|
-
subject { dump_generator.instance_variable_get(:@dump_cmd) }
|
30
|
-
it { is_expected.to eq('MYSQL_PWD="" mysqldump --default-character-set=utf8 --protocol=tcp -h localhost -P 3306 -uadmin --skip-lock-tables ' +
|
31
|
-
'--single-transaction --hex-blob --flush-logs --master-data=2 dev users groups') }
|
32
|
-
end
|
24
|
+
let(:default_conf) do
|
25
|
+
{
|
26
|
+
'host' => 'localhost', 'port' => 3306, 'username' => 'admin',
|
27
|
+
'password' => 'pass', 'database' => 'dev', 'tables' => 'users,groups',
|
28
|
+
}
|
33
29
|
end
|
34
30
|
|
35
|
-
describe
|
36
|
-
|
31
|
+
describe MysqlDumpGeneratorNoMasterData do
|
32
|
+
let(:stderr) { double(:stderr) }
|
33
|
+
let(:wait_thr) { double(:wait_thr) }
|
34
|
+
|
35
|
+
let(:default_dump_generator) { MysqlDumpGeneratorNoMasterData.new(default_conf) }
|
36
|
+
let(:mysql_client) do
|
37
|
+
m = double(:mysql_client)
|
38
|
+
allow(m).to receive(:query).with(/TABLES/)
|
39
|
+
allow(m).to receive(:query).with("SHOW VARIABLES LIKE 'version'").
|
40
|
+
and_return([{'Value' => '5.1.40-0ubuntu0.12.04.1-log'}])
|
41
|
+
allow(m).to receive(:query).with("SHOW MASTER STATUS;").
|
42
|
+
and_return([{'File' => 'mysql-bin.000451', 'Position' => 89872}])
|
43
|
+
allow(m).to receive(:close)
|
44
|
+
m
|
45
|
+
end
|
46
|
+
|
47
|
+
describe '#dump' do
|
37
48
|
before do
|
38
|
-
|
39
|
-
expect(
|
40
|
-
|
41
|
-
|
42
|
-
|
49
|
+
expect(Mysql2::Client).to receive(:new).and_return(mysql_client)
|
50
|
+
expect(Open3).to receive(:popen3).and_yield(stdin, stdout, stderr, wait_thr)
|
51
|
+
end
|
52
|
+
context "when mysqldump exits with status 0" do
|
53
|
+
it do
|
54
|
+
expect(wait_thr).to receive(:value).and_return(0)
|
55
|
+
expect(stderr).to receive(:each_line).and_yield("")
|
56
|
+
expect(default_dump_generator.dump(file_path)).to be_truthy
|
57
|
+
expect(File.exists?(file_path)).to be_truthy
|
58
|
+
end
|
43
59
|
end
|
44
|
-
|
45
|
-
|
46
|
-
|
60
|
+
context "when mysqldump exits with status 1" do
|
61
|
+
it do
|
62
|
+
expect(wait_thr).to receive(:value).and_return(1)
|
63
|
+
expect(stderr).to receive(:each_line).and_yield("")
|
64
|
+
expect{ default_dump_generator.dump(file_path) }.to raise_error
|
65
|
+
end
|
47
66
|
end
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
)
|
67
|
+
context "when mysqldump exits with status 0, but there are error in stderr" do
|
68
|
+
it do
|
69
|
+
expect(wait_thr).to receive(:value).and_return(0)
|
70
|
+
expect(stderr).to receive(:each_line).and_yield("mysqldump error")
|
71
|
+
expect{ default_dump_generator.dump(file_path) }.to raise_error
|
72
|
+
end
|
55
73
|
end
|
56
|
-
|
57
|
-
|
58
|
-
expect(File.exists?(file_path)).to be_falsey
|
74
|
+
after :each do
|
75
|
+
File.delete(file_path) if File.exists?(file_path)
|
59
76
|
end
|
60
77
|
end
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
78
|
+
end
|
79
|
+
describe MysqlDumpGeneratorMasterData do
|
80
|
+
let(:status) { double(:status) }
|
81
|
+
let(:dump_io) { File.open(file_path, 'r', encoding: "utf-8") }
|
82
|
+
let(:default_dump_generator) { MysqlDumpGeneratorMasterData.new(default_conf) }
|
83
|
+
|
84
|
+
describe '#initialize' do
|
85
|
+
context 'with password' do
|
86
|
+
subject { default_dump_generator.instance_variable_get(:@dump_cmd) }
|
87
|
+
it { is_expected.to eq('MYSQL_PWD="pass" mysqldump --default-character-set=utf8 --protocol=tcp -h localhost -P 3306 -uadmin --skip-lock-tables ' +
|
88
|
+
'--single-transaction --hex-blob --flush-logs --master-data=2 dev users groups') }
|
89
|
+
end
|
90
|
+
context 'without password' do
|
91
|
+
let (:dump_generator) do
|
92
|
+
MysqlDumpGeneratorMasterData.new(default_conf.merge({'password' => ''}))
|
93
|
+
end
|
94
|
+
subject { dump_generator.instance_variable_get(:@dump_cmd) }
|
95
|
+
it { is_expected.to eq('MYSQL_PWD="" mysqldump --default-character-set=utf8 --protocol=tcp -h localhost -P 3306 -uadmin --skip-lock-tables ' +
|
96
|
+
'--single-transaction --hex-blob --flush-logs --master-data=2 dev users groups') }
|
72
97
|
end
|
73
98
|
end
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
99
|
+
|
100
|
+
describe '#dump' do
|
101
|
+
context 'when exit status is not 0' do
|
102
|
+
before do
|
103
|
+
`touch #{file_path}`
|
104
|
+
expect(status).to receive(:exitstatus).and_return 1
|
105
|
+
expect(Open3).to receive(:capture3).and_return(
|
106
|
+
['(dummy std out)', '(dummy std err)', status]
|
107
|
+
)
|
108
|
+
end
|
109
|
+
it do
|
110
|
+
expect{ default_dump_generator.dump(file_path) }.to raise_error
|
111
|
+
expect(File.exists?(file_path)).to be_falsey
|
112
|
+
end
|
81
113
|
end
|
82
|
-
|
83
|
-
|
84
|
-
|
114
|
+
context 'when exit status is 0 but no file' do
|
115
|
+
before do
|
116
|
+
expect(status).to receive(:exitstatus).and_return 0
|
117
|
+
expect(Open3).to receive(:capture3).and_return(
|
118
|
+
['(dummy std out)', '(dummy std err)', status]
|
119
|
+
)
|
120
|
+
end
|
121
|
+
it do
|
122
|
+
expect{ default_dump_generator.dump(file_path) }.to raise_error
|
123
|
+
expect(File.exists?(file_path)).to be_falsey
|
124
|
+
end
|
125
|
+
end
|
126
|
+
context 'when exit status is 0 but file size is 0' do
|
127
|
+
before do
|
128
|
+
`touch #{file_path}`
|
129
|
+
expect(status).to receive(:exitstatus).and_return 0
|
130
|
+
expect(Open3).to receive(:capture3).and_return(
|
131
|
+
['(dummy std out)', '(dummy std err)', status]
|
132
|
+
)
|
133
|
+
end
|
134
|
+
it do
|
135
|
+
expect{ default_dump_generator.dump(file_path) }.to raise_error
|
136
|
+
expect(File.exists?(file_path)).to be_truthy
|
137
|
+
end
|
138
|
+
end
|
139
|
+
context 'when exit status is 0' do
|
140
|
+
before do
|
141
|
+
`echo "something..." > #{file_path}`
|
142
|
+
expect(status).to receive(:exitstatus).and_return 0
|
143
|
+
expect(Open3).to receive(:capture3).and_return(
|
144
|
+
['(dummy std out)', '(dummy std err)', status]
|
145
|
+
)
|
146
|
+
end
|
147
|
+
it do
|
148
|
+
expect(default_dump_generator.dump(file_path)).to be_truthy
|
149
|
+
expect(File.exists?(file_path)).to be_truthy
|
150
|
+
end
|
151
|
+
end
|
152
|
+
after :each do
|
153
|
+
File.delete(file_path) if File.exists?(file_path)
|
85
154
|
end
|
86
|
-
end
|
87
|
-
after :each do
|
88
|
-
File.delete(file_path) if File.exists?(file_path)
|
89
155
|
end
|
90
156
|
end
|
91
157
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: flydata
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.20
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Koichi Fujikawa
|
@@ -12,7 +12,7 @@ authors:
|
|
12
12
|
autorequire:
|
13
13
|
bindir: bin
|
14
14
|
cert_chain: []
|
15
|
-
date: 2014-12-
|
15
|
+
date: 2014-12-05 00:00:00.000000000 Z
|
16
16
|
dependencies:
|
17
17
|
- !ruby/object:Gem::Dependency
|
18
18
|
name: rest-client
|
@@ -448,7 +448,6 @@ files:
|
|
448
448
|
- flydata.gemspec
|
449
449
|
- lib/fly_data_model.rb
|
450
450
|
- lib/flydata.rb
|
451
|
-
- lib/flydata/agent_errors.rb
|
452
451
|
- lib/flydata/api/base.rb
|
453
452
|
- lib/flydata/api/data_entry.rb
|
454
453
|
- lib/flydata/api/data_port.rb
|
@@ -471,6 +470,7 @@ files:
|
|
471
470
|
- lib/flydata/command/stop.rb
|
472
471
|
- lib/flydata/command/sync.rb
|
473
472
|
- lib/flydata/command/version.rb
|
473
|
+
- lib/flydata/command_logger.rb
|
474
474
|
- lib/flydata/compatibility_check.rb
|
475
475
|
- lib/flydata/credentials.rb
|
476
476
|
- lib/flydata/cron.rb
|