flydata 0.2.19 → 0.2.20

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: eb04da6c165e8ace6ac6b7089c73eca79d2d1fc1
4
- data.tar.gz: 77794c3f97103bd4090a70bdceb6036eb8e488ab
3
+ metadata.gz: f4d29cd6b59f9acb406d62017d559296f0fa27b7
4
+ data.tar.gz: ab8d2cdcebbaab49345f62da3aef259e5a977e02
5
5
  SHA512:
6
- metadata.gz: 305c9ace6887dcf12a1fa4dd62830e41c11a87b42acd0efa06a0032b28dabccdacafc9945e9fd7b4b6d23eb872db347481ca7f605b814363a2d39c34a425340c
7
- data.tar.gz: 76a51648a3e4ba7b363072e99c397b7b8542a0b57135cc494de51d5a4d65fda5f81cc263d0ffbe2a006e3f1e0591073187580e65f6e5ac05cebdedb22356b736
6
+ metadata.gz: 6f590f2046f0c016a2beae67555a0544b919b9ff5873376ee190f6a2065ae30e6e961178926127537a97baab5140dc1e1c9c5e5051895a5661d4cf93b90ccc0a
7
+ data.tar.gz: 6af1ee2c5597ed08e133650a5882ca6a4531ff47b0a25bf82566a3c3e0de4ecb3172c6c6da62449a853af49785197c15ae012ce59a4303e4a11d1826c02e7cd6
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.2.19
1
+ 0.2.20
@@ -100,8 +100,43 @@ module FlydataCore
100
100
  log_params
101
101
  end
102
102
 
103
+ # hook method
104
+ def before_logging(level, raw_msg, built_msg, log_params, options)
105
+ end
106
+
103
107
  # log methods
104
- def log_common(level, message, log_params = {}, options = {})
108
+ def log_common(level, message, log_params, options)
109
+ msg = build_log_message(level, message, log_params, options)
110
+ before_logging(level, message, msg, log_params, options)
111
+ get_logger(options).send(level, msg)
112
+ end
113
+
114
+ def log_debug(message, log_params = {}, options = {})
115
+ log_common(:debug, message, log_params, options)
116
+ end
117
+
118
+ def log_info(message, log_params = {}, options = {})
119
+ log_common(:info, message, log_params, options)
120
+ end
121
+
122
+ def log_warn(message, log_params = {}, options = {})
123
+ log_common(:warn, message, log_params, options)
124
+ end
125
+
126
+ def log_error(message, log_params = {}, options = {})
127
+ log_common(:error, message, log_params, options)
128
+ end
129
+
130
+ def log_error_with_backtrace(message, log_params = {}, options = {})
131
+ log_common(:error, message, log_params, options.merge(backtrace: true))
132
+ end
133
+
134
+ def get_logger(options = {})
135
+ options[:logger] || logger || log_context_logger || $log
136
+ end
137
+
138
+ def build_log_message(level, raw_message, log_params = {}, options = {})
139
+ message = raw_message.dup
105
140
  # check keys
106
141
  if e = log_params[:error]
107
142
  message += " error_class=#{e.class.to_s} error=\"#{e.to_s.strip.gsub(/\n/, ' ')}\""
@@ -133,30 +168,7 @@ module FlydataCore
133
168
  backtrace = e.backtrace ? e.backtrace.join("\n") : 'backtrace is empty.'
134
169
  msg += " backtrace:\n#{backtrace}"
135
170
  end
136
-
137
- # get logger
138
- lg = options[:logger] || logger || log_context_logger || $log
139
- lg.send(level, msg)
140
- end
141
-
142
- def log_debug(message, log_params = {}, options = {})
143
- log_common(:debug, message, log_params, options)
144
- end
145
-
146
- def log_info(message, log_params = {}, options = {})
147
- log_common(:info, message, log_params, options)
148
- end
149
-
150
- def log_warn(message, log_params = {}, options = {})
151
- log_common(:warn, message, log_params, options)
152
- end
153
-
154
- def log_error(message, log_params = {}, options = {})
155
- log_common(:error, message, log_params, options)
156
- end
157
-
158
- def log_error_with_backtrace(message, log_params = {}, options = {})
159
- log_common(:error, message, log_params, options.merge(backtrace: true))
171
+ msg
160
172
  end
161
173
 
162
174
  def build_log_text(items = log_context_items)
data/flydata.gemspec CHANGED
@@ -5,11 +5,11 @@
5
5
 
6
6
  Gem::Specification.new do |s|
7
7
  s.name = "flydata"
8
- s.version = "0.2.19"
8
+ s.version = "0.2.20"
9
9
 
10
10
  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
11
11
  s.authors = ["Koichi Fujikawa", "Masashi Miyazaki", "Matthew Luu", "Mak Inada", "Sriram NS"]
12
- s.date = "2014-12-03"
12
+ s.date = "2014-12-05"
13
13
  s.description = "FlyData Agent"
14
14
  s.email = "sysadmin@flydata.com"
15
15
  s.executables = ["fdmysqldump", "flydata", "serverinfo"]
@@ -63,7 +63,6 @@ Gem::Specification.new do |s|
63
63
  "flydata.gemspec",
64
64
  "lib/fly_data_model.rb",
65
65
  "lib/flydata.rb",
66
- "lib/flydata/agent_errors.rb",
67
66
  "lib/flydata/api/base.rb",
68
67
  "lib/flydata/api/data_entry.rb",
69
68
  "lib/flydata/api/data_port.rb",
@@ -86,6 +85,7 @@ Gem::Specification.new do |s|
86
85
  "lib/flydata/command/stop.rb",
87
86
  "lib/flydata/command/sync.rb",
88
87
  "lib/flydata/command/version.rb",
88
+ "lib/flydata/command_logger.rb",
89
89
  "lib/flydata/compatibility_check.rb",
90
90
  "lib/flydata/credentials.rb",
91
91
  "lib/flydata/cron.rb",
data/lib/flydata.rb CHANGED
@@ -25,6 +25,14 @@ module Flydata
25
25
  FLYDATA_DEBUG = !!(ENV['FLYDATA_DEBUG'])
26
26
  FLYDATA_HOME = ENV['FLYDATA_HOME'] || "#{ENV['HOME']}/.flydata"
27
27
  FLYDATA_GEM_HOME = File.expand_path(File.join(File.dirname(__FILE__), '..'))
28
+ FLYDATA_GEM_BIN = File.join(FLYDATA_GEM_HOME, 'bin')
28
29
  FLYDATA_TMPL_DIR = File.join(FLYDATA_GEM_HOME, 'tmpl')
30
+ FLYDATA_FLUENT_PLUGIN_DIR = File.join(FLYDATA_GEM_HOME, 'lib', 'flydata', 'fluent-plugins')
31
+
32
+ FLYDATA_SERVERINFO = File.join(FLYDATA_GEM_BIN, 'serverinfo')
33
+ FLYDATA_LOG = File.join(FLYDATA_HOME, 'flydata.log')
34
+ FLYDATA_CONF = File.join(FLYDATA_HOME, 'flydata.conf')
35
+
36
+ VERSION_PATH = File.join(FLYDATA_GEM_HOME, 'VERSION')
29
37
  include Flydata::Heroku
30
38
  end
data/lib/flydata/cli.rb CHANGED
@@ -1,15 +1,26 @@
1
1
  require 'slop'
2
+ require_relative 'command_logger'
2
3
 
3
4
  module Flydata
4
5
  class Cli
5
6
  include Helpers
7
+ include CommandLoggable
6
8
  def initialize(args)
7
9
  @args = args
10
+ unless $log
11
+ $log = Logger.new(FLYDATA_LOG)
12
+ # 2014-10-24 14:05:21 -0700 command.info: message
13
+ $log.datetime_format = "%Y-%m-%d %H:%M:%S %z "
14
+ $log.formatter = proc do |severity, datetime, progname, msg|
15
+ "#{datetime} command.#{severity.to_s.downcase}: #{msg}\n"
16
+ end
17
+ end
18
+ log_info("Start command.", {cmd: @args.join(' '), ver:flydata_version})
8
19
  end
9
20
 
10
21
  def run
11
22
  unless check_environment
12
- puts "Sorry, you have to run the installation command to use flydata. Please go to https://console.flydata.com/ and sign up."
23
+ log_error_stderr("Sorry, you have to run the installation command to use flydata. Please go to https://console.flydata.com/ and sign up.")
13
24
  return
14
25
  end
15
26
  begin
@@ -28,7 +39,8 @@ module Flydata
28
39
  cmd_obj = cmd_cls.new(options)
29
40
  sub_cmd ? cmd_obj.send(sub_cmd,*@args) : cmd_obj.run(*@args)
30
41
  else
31
- raise 'no command given'
42
+ $stderr.puts usage_text(false)
43
+ return
32
44
  end
33
45
  rescue => e
34
46
  #raise e
@@ -36,6 +48,7 @@ module Flydata
36
48
  $stderr.puts
37
49
  $stderr.puts
38
50
  $stderr.puts usage_text
51
+ log_error_with_backtrace('Error occured during command.', {error: e, args: @args})
39
52
  raise e if FLYDATA_DEBUG
40
53
  exit 1
41
54
  end
@@ -1,6 +1,8 @@
1
1
  module Flydata
2
2
  module Command
3
3
  class Base
4
+ include CommandLoggable
5
+
4
6
  def initialize(options = Slop.new)
5
7
  @api_client = ApiClient.instance
6
8
  @opts = options
@@ -37,7 +39,7 @@ module Flydata
37
39
  suffix = default_yes ? "(Y/n)" : "(y/n)"
38
40
  prompt = "#{message} #{suffix}: "
39
41
  if opts && opts.yes? # Yes option
40
- puts "#{prompt}Yes"
42
+ log_info_stdout("#{prompt}Yes")
41
43
  return true
42
44
  end
43
45
 
@@ -46,18 +48,22 @@ module Flydata
46
48
  return true if default_yes and ans == ''
47
49
  if ans.size > 0
48
50
  case ans[0].downcase
49
- when 'y'; return true
50
- when 'n'; return false
51
+ when 'y'
52
+ log_info("#{prompt}Yes")
53
+ return true
54
+ when 'n'
55
+ log_info("#{prompt}No")
56
+ return false
51
57
  end
52
58
  end
53
- say(" ! Please answer y[es] or n[o]")
59
+ log_warn_stderr(" ! Please answer y[es] or n[o]")
54
60
  newline
55
61
  end
56
62
  end
57
63
  def choose_one(message, prompt, menu_list, value_list=[])
58
64
  choice = nil
59
65
  value_list = menu_list unless menu_list.size == value_list.size
60
- say(message) if message
66
+ log_info_stdout(message) if message
61
67
  choose do |menu|
62
68
  menu.index = :number
63
69
  menu.index_suffix = ") "
@@ -65,12 +71,13 @@ module Flydata
65
71
  menu.prompt = prompt ? prompt : ">> "
66
72
  menu.choices(*menu_list) {|item| choice = value_list[menu_list.index(item)]}
67
73
  end
74
+ log_info_stdout(" -> #{choice}")
68
75
  choice
69
76
  end
70
77
  def ask_input_table_name
71
78
  input = nil
72
79
  loop do
73
- say("Input a table name.")
80
+ log_info_stdout("Input a table name.")
74
81
  say(">> ")
75
82
  input = gets.strip
76
83
  if input =~ /^[a-zA-Z0-9_]+$/
@@ -79,6 +86,7 @@ module Flydata
79
86
  puts "Please enter a valid table name."
80
87
  end
81
88
  end
89
+ log_info(">> #{input}")
82
90
  input
83
91
  end
84
92
  end
@@ -16,7 +16,7 @@ module Flydata
16
16
  end
17
17
  # Check if process exist
18
18
  if process_exist?
19
- say("Process is still running. Please stop process first.") unless options[:quiet]
19
+ log_info_stdout("Process is still running. Please stop process first.") unless options[:quiet]
20
20
  return
21
21
  end
22
22
 
@@ -31,11 +31,11 @@ module Flydata
31
31
  AgentCompatibilityCheck.new(dp).check
32
32
 
33
33
  # Start sender(fluentd) process
34
- say('Starting sender process.') unless options[:quiet]
34
+ log_info_stdout("Starting sender process.") unless options[:quiet]
35
35
  Dir.chdir(FLYDATA_HOME){
36
- Kernel.system("bash #{File.dirname(__FILE__)}/../../../bin/serverinfo", :out => ["#{FLYDATA_HOME}/flydata.log",'a'], :err => ["#{FLYDATA_HOME}/flydata.log",'a'])
36
+ Kernel.system("bash #{FLYDATA_SERVERINFO}", :out => [FLYDATA_LOG,'a'], :err => [FLYDATA_LOG,'a'])
37
37
  daemon_opt = opts.no_daemon? ? "" : daemon_option
38
- Kernel.system("ruby `which fluentd` #{daemon_opt} -l #{FLYDATA_HOME}/flydata.log -c #{FLYDATA_HOME}/flydata.conf -p #{File.dirname(__FILE__)}/../fluent-plugins")
38
+ Kernel.system("ruby `which fluentd` #{daemon_opt} -l #{FLYDATA_LOG} -c #{FLYDATA_CONF} -p #{FLYDATA_FLUENT_PLUGIN_DIR}")
39
39
  }
40
40
  Kernel.sleep 5
41
41
 
@@ -43,21 +43,21 @@ module Flydata
43
43
  #wait_until_logs_uploaded
44
44
  if options[:show_final_message] && !options[:quiet]
45
45
  data_port = flydata.data_port.get
46
- say("Go to your Dashboard! #{flydata.flydata_api_host}/data_ports/#{data_port['id']}")
47
- say <<EOF
46
+ log_info_stdout("Go to your Dashboard! #{flydata.flydata_api_host}/data_ports/#{data_port['id']}")
47
+ log_info_stdout <<EOF
48
48
  Please Note: Records and Total Size are updated every 10-20 minutes.
49
49
  EOF
50
50
  end
51
51
  end
52
52
  def stop(options = {})
53
53
  unless process_exist?
54
- say("Process doesn't exist.") unless options[:quiet]
54
+ log_info_stdout("Process doesn't exist.") unless options[:quiet]
55
55
  return true
56
56
  end
57
57
 
58
- say('Stopping sender process.') unless options[:quiet]
58
+ log_info_stdout('Stopping sender process.') unless options[:quiet]
59
59
  if Kernel.system("kill `cat #{pid_file}`") and wait_until_client_stop(options)
60
- say('Done.') unless options[:quiet]
60
+ log_info_stdout('Done.') unless options[:quiet]
61
61
  return true
62
62
  end
63
63
  raise 'Something has gone wrong..'
@@ -65,17 +65,17 @@ EOF
65
65
  def flush_client_buffer(options = {})
66
66
  unless process_exist?
67
67
  return true if client_buffer_empty?
68
- say("Process doesn't exist. But, the client buffer is not empty!!") unless options[:quiet]
68
+ log_info_stdout("Process doesn't exist. But, the client buffer is not empty!!") unless options[:quiet]
69
69
  start(options)
70
70
  end
71
71
 
72
- say('Stopping input plugins and flushing the client buffer.') unless options[:quiet]
72
+ log_info_stdout('Stopping input plugins and flushing the client buffer.') unless options[:quiet]
73
73
  Kernel.system("kill -USR1 `cat #{pid_file}`")
74
74
 
75
75
  retry_count = 12
76
76
  1.upto(retry_count) do |i|
77
77
  return true if client_buffer_empty?
78
- say("Waiting for the buffer to get empty... (#{i}/#{retry_count})") unless options[:quiet]
78
+ log_info_stdout("Waiting for the buffer to get empty... (#{i}/#{retry_count})") unless options[:quiet]
79
79
  Kernel.sleep 5
80
80
  end
81
81
 
@@ -83,23 +83,23 @@ EOF
83
83
  end
84
84
  def restart(options = {})
85
85
  if process_exist?
86
- say('Restarting sender process.') unless options[:quiet]
86
+ log_info_stdout('Restarting sender process.') unless options[:quiet]
87
87
  if Kernel.system("kill -HUP `cat #{pid_file}`")
88
- say('Done.') unless options[:quiet]
88
+ log_info_stdout('Done.') unless options[:quiet]
89
89
  return true
90
90
  else
91
91
  raise 'Something has gone wrong..'
92
92
  end
93
93
  else
94
- say("Process doesn't exist.") unless options[:quiet]
94
+ log_info_stdout("Process doesn't exist.") unless options[:quiet]
95
95
  start(options)
96
96
  end
97
97
  end
98
98
  def status
99
99
  if process_exist?
100
- say("Process is running.")
100
+ log_info_stdout("Process is running.")
101
101
  else
102
- say("Process is not running.")
102
+ log_info_stdout("Process is not running.")
103
103
  end
104
104
  end
105
105
  def process_exist?
@@ -108,7 +108,7 @@ EOF
108
108
  end
109
109
  def kill_all(optiosn = {})
110
110
  if Kernel.system("ps ax | grep 'flydata' | grep -v grep | awk '{print \"kill \" $1}' | sh")
111
- say("Done.") unless options[:quiet]
111
+ log_info_stdout("Done.") unless options[:quiet]
112
112
  return true
113
113
  else
114
114
  raise 'Something has gone wrong...'
@@ -133,7 +133,7 @@ EOF
133
133
  retry_count = 10
134
134
  1.upto(retry_count) do |i|
135
135
  return true if server_ready?
136
- say("Waiting for the server side to become active... (#{i}/#{retry_count})") unless options[:quiet]
136
+ log_info_stdout("Waiting for the server side to become active... (#{i}/#{retry_count})") unless options[:quiet]
137
137
  Kernel.sleep 30
138
138
  end
139
139
  false
@@ -142,13 +142,13 @@ EOF
142
142
  retry_count = 10
143
143
  1.upto(retry_count) do |i|
144
144
  if client_ready?
145
- say("Done! Client is ready now.") unless options[:quiet]
145
+ log_info_stdout("Done! Client is ready now.") unless options[:quiet]
146
146
  return true
147
147
  end
148
148
  if process_died?
149
149
  raise "Client could not been launched. Detail here #{FLYDATA_HOME}/flydata.log"
150
150
  end
151
- say("Waiting for the client side to become active... (#{i}/#{retry_count})") unless options[:quiet]
151
+ log_info_stdout("Waiting for the client side to become active... (#{i}/#{retry_count})") unless options[:quiet]
152
152
  Kernel.sleep 10
153
153
  end
154
154
  raise "Somthing has gone wrong... Please try setup command again."
@@ -157,23 +157,23 @@ EOF
157
157
  retry_count = 5
158
158
  1.upto(retry_count) do |i|
159
159
  return true unless process_exist?
160
- say("Waiting for the client to stop... (#{i}/#{retry_count})") unless options[:quiet]
160
+ log_info_stdout("Waiting for the client to stop... (#{i}/#{retry_count})") unless options[:quiet]
161
161
  Kernel.sleep 3
162
162
  end
163
163
  false
164
164
  end
165
165
  def wait_until_logs_uploaded(options = {})
166
- say('Starting to check the upload from your server.') unless options[:quiet]
166
+ log_info_stdout('Starting to check the upload from your server.') unless options[:quiet]
167
167
  data_port = flydata.data_port.get
168
168
  data_port_id = data_port['id']
169
169
 
170
170
  retry_count = 10
171
171
  1.upto(retry_count) do |i|
172
172
  if uploaded_successfully?(data_port_id)
173
- say("Uploading your logs correctly.") unless options[:quiet]
173
+ log_info_stdout("Uploading your logs correctly.") unless options[:quiet]
174
174
  return true
175
175
  end
176
- say("Waiting logs uploading... (#{i}/#{retry_count})") unless options[:quiet]
176
+ log_info_stdout("Waiting logs uploading... (#{i}/#{retry_count})") unless options[:quiet]
177
177
  Kernel.sleep 30
178
178
  end
179
179
  raise 'Cannot confirm that your logs exist on the FlyData server. Something has gone wrong..'
@@ -196,7 +196,7 @@ EOF
196
196
  end
197
197
  def client_buffer_empty?(options = {})
198
198
  client_buffer = File.join(FLYDATA_HOME, 'buffer')
199
- say("Checking the client buffer #{client_buffer}") unless options[:quiet]
199
+ log_info_stdout("Checking the client buffer #{client_buffer}") unless options[:quiet]
200
200
  Dir.glob("#{client_buffer}/*").empty?
201
201
  end
202
202
  def pid_file
@@ -3,7 +3,6 @@ require 'msgpack'
3
3
  require 'open3'
4
4
  require 'mysql2'
5
5
  require 'flydata/sync_file_manager'
6
- require 'flydata/agent_errors'
7
6
  require 'flydata/compatibility_check'
8
7
  require 'flydata/output/forwarder'
9
8
  require 'flydata/parser/mysql/dump_parser'
@@ -35,10 +34,10 @@ module Flydata
35
34
  if (sender.process_exist?)
36
35
  if tables.empty?
37
36
  # full sync
38
- puts "FlyData Agent is already running. If you'd like to restart FlyData Sync from scratch, run 'flydata sync:reset' first."
37
+ log_warn_stderr("FlyData Agent is already running. If you'd like to restart FlyData Sync from scratch, run 'flydata sync:reset' first.")
39
38
  else
40
39
  # per-table sync
41
- puts "Flydata Agent is already running. If you'd like to Sync the table(s), run 'flydata sync:flush' first."
40
+ log_warn_stderr("Flydata Agent is already running. If you'd like to Sync the table(s), run 'flydata sync:flush' first.")
42
41
  end
43
42
  exit 1
44
43
  end
@@ -58,7 +57,7 @@ module Flydata
58
57
 
59
58
  def flush
60
59
  flush_buffer_and_stop
61
- puts "Buffers have been flushed and the sender process has been stopped."
60
+ log_info_stdout("Buffers have been flushed and the sender process has been stopped.")
62
61
  end
63
62
 
64
63
  def self.slop_reset
@@ -91,11 +90,11 @@ module Flydata
91
90
  delete_files.flatten.each do |path|
92
91
  FileUtils.rm(path) if File.exists?(path)
93
92
  end
94
- puts "Reset completed successfully."
93
+ log_info_stdout("Reset completed successfully.")
95
94
  end
96
95
 
97
96
  def wait_for_server_buffer
98
- puts "Waiting for the server buffer to get empty"
97
+ log_info_stdout("Waiting for the server buffer to get empty.")
99
98
  while (status = check) && (status['state'] == 'processing')
100
99
  print_progress(status)
101
100
  sleep 10
@@ -104,24 +103,24 @@ module Flydata
104
103
 
105
104
  def wait_for_server_data_processing
106
105
  state = :PROCESS
107
- puts "Uploading data to Redshift..."
106
+ log_info_stdout("Uploading data to Redshift...")
108
107
  sleep 10
109
108
  status = nil
110
109
  while (status = check)
111
110
  if state == :PROCESS && status['state'] == 'uploading'
112
- puts " -> Done"
111
+ log_info_stdout(" -> Done")
113
112
  state = :UPLOAD
114
- puts "Finishing data upload..."
113
+ log_info_stdout("Finishing data upload...")
115
114
  end
116
115
  print_progress(status)
117
116
  sleep 10
118
117
  end
119
118
  if (state == :PROCESS)
120
119
  # :UPLOAD state was skipped due to no data
121
- puts " -> Done"
122
- puts "Finishing data upload..."
120
+ log_info_stdout(" -> Done")
121
+ log_info_stdout("Finishing data upload...")
123
122
  end
124
- puts " -> Done"
123
+ log_info_stdout(" -> Done")
125
124
  end
126
125
 
127
126
  def check
@@ -142,9 +141,9 @@ module Flydata
142
141
  sync_fm = Flydata::FileUtil::SyncFileManager.new(de)
143
142
  binlog_path = sync_fm.binlog_path
144
143
  `touch #{binlog_path}`
145
- puts "Created an empty binlog position file."
146
- puts "-> #{binlog_path}"
147
- puts "Run 'flydata start' to start continuous sync."
144
+ log_info_stdout("Created an empty binlog position file.")
145
+ log_info_stdout("-> #{binlog_path}")
146
+ log_info_stdout("Run 'flydata start' to start continuous sync.")
148
147
  end
149
148
 
150
149
  def self.slop_generate_table_ddl
@@ -192,6 +191,7 @@ module Flydata
192
191
 
193
192
  def cleanup_sync_server(de, tables = [])
194
193
  print "Cleaning the server."
194
+ log_info("Cleaning the server.")
195
195
  worker = Thread.new do
196
196
  begin
197
197
  flydata.data_entry.cleanup_sync(de['id'], tables)
@@ -204,7 +204,7 @@ module Flydata
204
204
  print "."
205
205
  end
206
206
  puts
207
- puts "Done."
207
+ log_info_stdout("Done.")
208
208
  end
209
209
 
210
210
  def do_check(de)
@@ -213,7 +213,7 @@ module Flydata
213
213
 
214
214
  def print_progress(buffer_stat)
215
215
  message = buffer_stat['message']
216
- puts message unless message.nil? || message.empty?
216
+ log_info_stdout(message) unless message.nil? || message.empty?
217
217
  end
218
218
 
219
219
  DDL_DUMP_CMD_TEMPLATE = "MYSQL_PWD=\"%s\" mysqldump --protocol=tcp -d -h %s -P %s -u %s %s %s"
@@ -264,15 +264,15 @@ module Flydata
264
264
  raise errors unless errors.empty?
265
265
  end
266
266
  unless error_list.empty?
267
- $stderr.puts "We have noticed the following error(s):"
267
+ log_error_stderr("We have noticed the following error(s):")
268
268
  group_error = error_list.group_by {|d| d[:error]}
269
269
  group_error.each_key do |a|
270
- $stderr.puts "The following table(s) have #{a}:"
270
+ log_error_stderr("The following table(s) have #{a}:")
271
271
  group_error[a].each do |hash|
272
- $stderr.puts " - #{hash[:table]}" if hash[:table]
272
+ log_error_stderr(" - #{hash[:table]}") if hash[:table]
273
273
  end
274
274
  end
275
- $stderr.puts "Please fix the above error(s) to try to sync those table(s) or contact us for further help."
275
+ log_error_stderr("Please fix the above error(s) to try to sync those table(s) or contact us for further help.")
276
276
  end
277
277
  end
278
278
 
@@ -315,7 +315,7 @@ module Flydata
315
315
  dp = flydata.data_port.get
316
316
  fp = sync_fm.dump_file_path
317
317
  if file_dump && File.exists?(fp) && File.size(fp) > 0
318
- puts " -> Skip"
318
+ log_info_stdout(" -> Skip")
319
319
  return call_block_or_return_io(fp, &block)
320
320
  end
321
321
 
@@ -338,30 +338,27 @@ EOM
338
338
  Dump file saves contents of your tables temporarily. Make sure you have enough disk space.
339
339
  EOM
340
340
  print confirmation_text
341
+ log_info confirmation_text.strip
341
342
 
342
343
  if ask_yes_no('Start Sync?')
343
- puts "Checking database size(not same as mysqldump data size)..."
344
+ log_info_stdout("Checking database size(not same as mysqldump data size)...")
345
+
344
346
  db_bytesize = Flydata::Parser::Mysql::DatabaseSizeCheck.new(de['mysql_data_entry_preference']).get_db_bytesize
345
- puts " -> #{as_size(db_bytesize)} (#{db_bytesize} byte)"
346
- puts "Exporting data from database."
347
- puts "This process can take hours depending on data size and load on your database. Please be patient..."
347
+ log_info_stdout(" -> #{as_size(db_bytesize)} (#{db_bytesize} byte)")
348
+ log_info_stdout("Exporting data from database.")
349
+ log_info_stdout("This process can take hours depending on data size and load on your database. Please be patient...")
348
350
  Flydata::MysqlCompatibilityCheck.new(dp, de['mysql_data_entry_preference'], fp).check
349
351
  if file_dump
350
352
  Flydata::Parser::Mysql::MysqlDumpGeneratorNoMasterData.
351
353
  new(de['mysql_data_entry_preference']).dump(fp)
352
- puts " -> Done"
354
+ log_info_stdout(" -> Done")
353
355
  call_block_or_return_io(fp, &block)
354
356
  else
355
357
  Flydata::Parser::Mysql::MysqlDumpGeneratorNoMasterData.
356
358
  new(de['mysql_data_entry_preference']).dump {|io| block.call(io, db_bytesize)}
357
359
  end
358
360
  else
359
- newline
360
- puts "You can change the dump file path with 'mysqldump_path' property in the following conf file."
361
- puts
362
- puts " #{Flydata::Preference::DataEntryPreference.conf_path(de)}"
363
- puts
364
- return nil
361
+ exit 1
365
362
  end
366
363
  end
367
364
 
@@ -419,12 +416,12 @@ EOM
419
416
  dump_pos_info = sync_fm.load_dump_pos
420
417
  option = dump_pos_info || {}
421
418
  if option[:table_name] && option[:last_pos].to_i != -1
422
- puts "Resuming... Last processed table: #{option[:table_name]}"
419
+ log_info_stdout("Resuming... Last processed table: #{option[:table_name]}")
423
420
  else
424
421
  #If its a new sync, ensure server side resources are clean
425
422
  cleanup_sync_server(de, de['mysql_data_entry_preference']['tables'].split(',')) unless opts.skip_cleanup?
426
423
  end
427
- puts "Sending data to FlyData Server..."
424
+ log_info_stdout("Sending data to FlyData Server...")
428
425
 
429
426
  bench_start_time = Time.now
430
427
 
@@ -461,9 +458,9 @@ EOM
461
458
 
462
459
  # show the current progress
463
460
  if last_pos.to_i == -1 # stream dump
464
- puts " -> #{as_size(bytesize)} (#{bytesize} byte) completed..."
461
+ log_info_stdout(" -> #{as_size(bytesize)} (#{bytesize} byte) completed...")
465
462
  else
466
- puts " -> #{(last_pos.to_f/dump_file_size * 100).round(1)}% (#{last_pos}/#{dump_file_size}) completed..."
463
+ log_info_stdout(" -> #{(last_pos.to_f/dump_file_size * 100).round(1)}% (#{last_pos}/#{dump_file_size}) completed...")
467
464
  end
468
465
 
469
466
  # save check point
@@ -472,13 +469,13 @@ EOM
472
469
  }
473
470
  )
474
471
  forwarder.close
475
- puts " -> Done"
472
+ log_info_stdout(" -> Done")
476
473
  sync_fm.save_dump_pos(STATUS_WAITING, '', dump_file_size, binlog_pos)
477
474
 
478
475
  if ENV['FLYDATA_BENCHMARK']
479
476
  bench_end_time = Time.now
480
477
  elapsed_time = bench_end_time.to_i - bench_start_time.to_i
481
- puts "Elapsed:#{elapsed_time}sec start:#{bench_start_time} end:#{bench_end_time}"
478
+ log_info_stdout("Elapsed:#{elapsed_time}sec start:#{bench_start_time} end:#{bench_end_time}")
482
479
  end
483
480
  end
484
481
 
@@ -524,14 +521,14 @@ Thank you for using FlyData!
524
521
  sync_fm.reset_table_position_files(de['mysql_data_entry_preference']['tables'].split(','))
525
522
  sync_fm.backup_dump_dir
526
523
  unless opts.no_flydata_start?
527
- puts "Starting FlyData Agent..."
524
+ log_info_stdout("Starting FlyData Agent...")
528
525
  Flydata::Command::Sender.new.start(quiet: true)
529
- puts " -> Done"
526
+ log_info_stdout(" -> Done")
530
527
  end
531
528
  dashboard_url = "#{flydata.flydata_api_host}/dashboard"
532
529
  redshift_console_url = "#{flydata.flydata_api_host}/redshift_clusters/query/new"
533
530
  last_message = ALL_DONE_MESSAGE_TEMPLATE % [redshift_console_url, dashboard_url]
534
- puts last_message
531
+ log_info_stdout(last_message)
535
532
  else
536
533
  raise "Initial sync status is not complete. Try running 'flydata sync'."
537
534
  end
@@ -1,9 +1,10 @@
1
1
  module Flydata
2
2
  module Command
3
3
  class Version < Base
4
- VERSION_PATH = File.join(FLYDATA_GEM_HOME, 'VERSION')
4
+ include Helpers
5
+
5
6
  def run
6
- puts "flydata version \"#{File.open(VERSION_PATH, 'r').readline.strip}\""
7
+ log_info_stdout("flydata version \"#{flydata_version}\"")
7
8
  end
8
9
  end
9
10
  end
@@ -0,0 +1,26 @@
1
+ require 'flydata-core/core_ext'
2
+ require 'flydata-core/logger'
3
+
4
+ module Flydata
5
+ module CommandLoggable
6
+ include_unless_included FlydataCore::Logger
7
+
8
+ # hook
9
+ def before_logging(level, raw_msg, built_msg, log_params, options)
10
+ $stdout.puts raw_msg if options[:stdout]
11
+ $stderr.puts raw_msg if options[:stderr]
12
+ end
13
+
14
+ def log_info_stdout(message, log_params = {})
15
+ log_info(message, log_params, {stdout: true})
16
+ end
17
+
18
+ def log_warn_stderr(message, log_params = {})
19
+ log_warn(message, log_params, {stderr: true})
20
+ end
21
+
22
+ def log_error_stderr(message)
23
+ log_error(message, log_params, {stderr: true})
24
+ end
25
+ end
26
+ end
@@ -1,6 +1,10 @@
1
+ require 'flydata/command_logger'
2
+
1
3
  module Flydata
2
4
 
3
5
  class CompatibilityCheck
6
+ include CommandLoggable
7
+
4
8
  class CompatibilityError < StandardError
5
9
  end
6
10
 
@@ -13,6 +17,8 @@ module Flydata
13
17
  self.methods.grep(/^check_/).each do |m|
14
18
  begin
15
19
  send(m)
20
+ rescue ArgumentError => e
21
+ # ignore
16
22
  rescue CompatibilityError => e
17
23
  @errors << e
18
24
  end
@@ -22,9 +28,9 @@ module Flydata
22
28
 
23
29
  def print_errors
24
30
  return if @errors.empty?
25
- puts "There may be some compatibility issues with this current server : "
31
+ log_error_stderr "There may be some compatibility issues with this current server : "
26
32
  @errors.each do |error|
27
- puts " * #{error.message}"
33
+ log_error_stderr " * #{error.message}"
28
34
  end
29
35
  raise "Please correct these errors if you wish to run FlyData Agent"
30
36
  end
@@ -79,9 +85,9 @@ module Flydata
79
85
 
80
86
  def print_errors
81
87
  return if @errors.empty?
82
- puts "There may be some compatibility issues with your MySQL credentials: "
88
+ log_error_stderr "There may be some compatibility issues with your MySQL credentials: "
83
89
  @errors.each do |error|
84
- puts " * #{error.message}"
90
+ log_error_stderr " * #{error.message}"
85
91
  end
86
92
  raise "Please correct these errors if you wish to run FlyData Sync"
87
93
  end
@@ -115,8 +121,12 @@ module Flydata
115
121
  Open3.popen3(query) do |stdin, stdout, stderr|
116
122
  stdin.close
117
123
  while !stderr.eof?
118
- line = stderr.gets
119
- unless /Warning: Using a password on the command line interface can be insecure./ === line
124
+ lines = []
125
+ while line = stderr.gets; lines << line.strip; end
126
+ err_reason = lines.join(" ")
127
+ log_error("Error occured during access to mysql server.", {err: err_reason})
128
+
129
+ unless /Warning: Using a password on the command line interface can be insecure/ === err_reason
120
130
  raise MysqlCompatibilityError, "Cannot connect to MySQL database. Please make sure you can connect with this command:\n $ mysql -u #{@db_opts[:username]} -h #{@db_opts[:host]} -P #{@db_opts[:port]} #{@db_opts[:database]} --protocol=tcp -p"
121
131
  end
122
132
  end
@@ -12,19 +12,24 @@ module Flydata
12
12
  [klass, method]
13
13
  end
14
14
 
15
- def usage_text
15
+ def usage_text(err = true)
16
16
  text = ""
17
- text += <<-EOM
17
+
18
+ if err
19
+ text += <<-EOM
18
20
  Fix the issue and try again. If the problem continues, please contact support@flydata.com
19
21
 
20
22
  EOM
21
- flydata_log = File.join(FLYDATA_HOME, 'flydata.log')
22
- if File.exists?(flydata_log)
23
- text += <<-EOM
23
+
24
+ flydata_log = File.join(FLYDATA_HOME, 'flydata.log')
25
+ if File.exists?(flydata_log)
26
+ text += <<-EOM
24
27
  Also check the Agent log.
25
28
  Log path: #{flydata_log}
26
29
  EOM
30
+ end
27
31
  end
32
+
28
33
  text += ""
29
34
  text += '-' * 64
30
35
  text += "\n"
@@ -45,6 +50,10 @@ Usage: flydata COMMAND
45
50
  text
46
51
  end
47
52
 
53
+ def flydata_version
54
+ IO.read(VERSION_PATH).strip
55
+ end
56
+
48
57
  def to_command_class(class_name)
49
58
  eval("Flydata::Command::#{class_name.camelcase}")
50
59
  end
@@ -1,15 +1,19 @@
1
1
  require 'socket'
2
+ require 'msgpack'
3
+ require 'flydata/command_logger'
2
4
 
3
5
  module Flydata
4
6
  module Output
5
7
  class ForwarderFactory
8
+ include CommandLoggable
9
+
6
10
  def self.create(forwarder_key, tag, servers, options = {})
7
11
  case forwarder_key
8
12
  when nil, "tcpforwarder"
9
- puts "Creating TCP connection" if FLYDATA_DEBUG
13
+ puts("Creating TCP connection") if FLYDATA_DEBUG
10
14
  forward = TcpForwarder.new(tag, servers, options)
11
15
  when "sslforwarder"
12
- puts "Creating SSL connection" if FLYDATA_DEBUG
16
+ puts("Creating SSL connection") if FLYDATA_DEBUG
13
17
  forward = SslForwarder.new(tag, servers, options)
14
18
  else
15
19
  raise "Unsupported Forwarding type #{forwarder_key}"
@@ -88,12 +92,12 @@ module Flydata
88
92
  rescue => e
89
93
  retry_count += 1
90
94
  if retry_count > RETRY_LIMIT
91
- puts "! Error: Failed to send data. Exceeded the retry limit. retry_count:#{retry_count}"
95
+ log_error_stderr("! Error: Failed to send data. Exceeded the retry limit. retry_count:#{retry_count}")
92
96
  raise e
93
97
  end
94
- puts "! Warn: Retring to send data. retry_count:#{retry_count} error=#{e.to_s}"
98
+ log_warn_stderr("! Warn: Retring to send data. retry_count:#{retry_count} error=#{e.to_s}")
95
99
  wait_time = RETRY_INTERVAL ** retry_count
96
- puts " Now waiting for next retry. time=#{wait_time}sec"
100
+ log_warn_stderr(" Now waiting for next retry. time=#{wait_time}sec")
97
101
  sleep wait_time
98
102
  retry
99
103
  ensure
@@ -1,3 +1,5 @@
1
+ require 'fiber'
2
+
1
3
  module Flydata
2
4
  module Parser
3
5
  module Mysql
@@ -106,7 +108,7 @@ EOS
106
108
  rd_io.extend(DumpStreamIO)
107
109
 
108
110
  # start mysqldump
109
- Open3.popen3 @dump_cmd do |cmd_in, cmd_out, cmd_err|
111
+ Open3.popen3 @dump_cmd do |cmd_in, cmd_out, cmd_err, wait_thr|
110
112
  cmd_in.close_write
111
113
  cmd_out.set_encoding("utf-8") # mysqldump output must be in UTF-8
112
114
 
@@ -126,9 +128,10 @@ EOS
126
128
  end
127
129
 
128
130
  # show err message
131
+ errors = ""
129
132
  threads << Thread.new do
130
133
  cmd_err.each_line do |line|
131
- $stderr.print line unless /^Warning:/ === line
134
+ errors << line unless /^Warning:/ === line
132
135
  end
133
136
  end
134
137
 
@@ -141,7 +144,14 @@ EOS
141
144
  end
142
145
 
143
146
  threads.each(&:join)
147
+ unless wait_thr.value == 0
148
+ #Raise error if the process exited with status != 0
149
+ #(Even if there was no message on stderr stream)
150
+ errors = "Failed to run mysqldump command." if errors.empty?
151
+ end
152
+ raise errors unless errors.empty?
144
153
  end
154
+ true
145
155
  rescue
146
156
  # Cleanup
147
157
  FileUtils.rm(file_path) if file_path && File.exists?(file_path)
@@ -24,9 +24,9 @@ module Flydata
24
24
  let(:args) { [] }
25
25
  it "starts fluend with daemon option" do
26
26
  expect(Kernel).to receive(:system).with( Regexp.new(
27
- "bash .+/../../../bin/serverinfo"), :out => [Regexp.new(".+/flydata.log"),'a'], :err => [Regexp.new(".+/flydata.log"),'a'])
27
+ "bash .+/bin/serverinfo"), :out => [Regexp.new(".+/flydata.log"),'a'], :err => [Regexp.new(".+/flydata.log"),'a'])
28
28
  expect(Kernel).to receive(:system).with( Regexp.new(
29
- "ruby `which fluentd` -d .+/flydata.pid -l .+/flydata.log -c .+/flydata.conf -p .+/\.\./fluent-plugins"))
29
+ "ruby `which fluentd` -d .+/flydata.pid -l .+/flydata.log -c .+/flydata.conf -p .+/lib/flydata/fluent-plugins"))
30
30
  subject.start(false)
31
31
  end
32
32
  end
@@ -34,9 +34,9 @@ module Flydata
34
34
  let(:args) { ["-n"] }
35
35
  it "starts fluentd with no daemon option" do
36
36
  expect(Kernel).to receive(:system).with( Regexp.new(
37
- "bash .+/../../../bin/serverinfo"), :out => [Regexp.new(".+/flydata.log"),'a'], :err => [Regexp.new(".+/flydata.log"),'a'])
37
+ "bash .+/bin/serverinfo"), :out => [Regexp.new(".+/flydata.log"),'a'], :err => [Regexp.new(".+/flydata.log"),'a'])
38
38
  expect(Kernel).to receive(:system).with(Regexp.new(
39
- "ruby `which fluentd` -l .+/flydata.log -c .+/flydata.conf -p .+/\.\./fluent-plugins"))
39
+ "ruby `which fluentd` -l .+/flydata.log -c .+/flydata.conf -p .+/lib/flydata/fluent-plugins"))
40
40
  subject.start(false)
41
41
  end
42
42
  end
@@ -44,9 +44,9 @@ module Flydata
44
44
  let(:args) { ["--no-daemon"] }
45
45
  it "starts fluentd with no daemon option" do
46
46
  expect(Kernel).to receive(:system).with( Regexp.new(
47
- "bash .+/../../../bin/serverinfo"), :out => [Regexp.new(".+/flydata.log"),'a'], :err => [Regexp.new(".+/flydata.log"),'a'])
47
+ "bash .+/bin/serverinfo"), :out => [Regexp.new(".+/flydata.log"),'a'], :err => [Regexp.new(".+/flydata.log"),'a'])
48
48
  expect(Kernel).to receive(:system).with(Regexp.new(
49
- "ruby `which fluentd` -l .+/flydata.log -c .+/flydata.conf -p .+/\.\./fluent-plugins"))
49
+ "ruby `which fluentd` -l .+/flydata.log -c .+/flydata.conf -p .+/lib/flydata/fluent-plugins"))
50
50
  subject.start(false)
51
51
  end
52
52
  end
@@ -1,3 +1,6 @@
1
+ require 'spec_helper'
2
+ require 'flydata/compatibility_check'
3
+
1
4
  module Flydata
2
5
  describe AgentCompatibilityCheck do
3
6
  let(:default_data_port) do
@@ -1,5 +1,6 @@
1
1
  # coding: utf-8
2
2
  require 'spec_helper'
3
+ require 'flydata/output/forwarder'
3
4
 
4
5
  module Flydata
5
6
  module Output
@@ -1,91 +1,157 @@
1
1
  # coding: utf-8
2
2
  require 'spec_helper'
3
+ require 'flydata/parser/mysql/dump_parser'
4
+ require 'open3'
5
+ require 'mysql2'
3
6
 
4
7
  module Flydata
5
8
  module Parser
6
9
  module Mysql
7
- describe MysqlDumpGeneratorMasterData do
8
- let(:stdout) { double(:stdout) }
9
- let(:stderr) { double(:stderr) }
10
- let(:status) { double(:status) }
10
+ context "Test Dump Generators" do
11
+ let(:stdin) do
12
+ s = double(:stdin)
13
+ allow(s).to receive(:close_write)
14
+ s
15
+ end
16
+ let(:stdout) do
17
+ s = double(:stdout)
18
+ allow(s).to receive(:set_encoding)
19
+ allow(s).to receive(:gets).and_return("first line")
20
+ allow(s).to receive(:each_line).and_yield("another line")
21
+ s
22
+ end
11
23
  let(:file_path) { File.join('/tmp', "flydata_sync_spec_mysqldump_#{Time.now.to_i}") }
12
- let(:dump_io) { File.open(file_path, 'r', encoding: "utf-8") }
13
- let(:default_conf) { {
14
- 'host' => 'localhost', 'port' => 3306, 'username' => 'admin',
15
- 'password' => 'pass', 'database' => 'dev', 'tables' => 'users,groups',
16
- } }
17
- let(:default_dump_generator) { MysqlDumpGeneratorMasterData.new(default_conf) }
18
-
19
- describe '#initialize' do
20
- context 'with password' do
21
- subject { default_dump_generator.instance_variable_get(:@dump_cmd) }
22
- it { is_expected.to eq('MYSQL_PWD="pass" mysqldump --default-character-set=utf8 --protocol=tcp -h localhost -P 3306 -uadmin --skip-lock-tables ' +
23
- '--single-transaction --hex-blob --flush-logs --master-data=2 dev users groups') }
24
- end
25
- context 'without password' do
26
- let (:dump_generator) do
27
- MysqlDumpGeneratorMasterData.new(default_conf.merge({'password' => ''}))
28
- end
29
- subject { dump_generator.instance_variable_get(:@dump_cmd) }
30
- it { is_expected.to eq('MYSQL_PWD="" mysqldump --default-character-set=utf8 --protocol=tcp -h localhost -P 3306 -uadmin --skip-lock-tables ' +
31
- '--single-transaction --hex-blob --flush-logs --master-data=2 dev users groups') }
32
- end
24
+ let(:default_conf) do
25
+ {
26
+ 'host' => 'localhost', 'port' => 3306, 'username' => 'admin',
27
+ 'password' => 'pass', 'database' => 'dev', 'tables' => 'users,groups',
28
+ }
33
29
  end
34
30
 
35
- describe '#dump' do
36
- context 'when exit status is not 0' do
31
+ describe MysqlDumpGeneratorNoMasterData do
32
+ let(:stderr) { double(:stderr) }
33
+ let(:wait_thr) { double(:wait_thr) }
34
+
35
+ let(:default_dump_generator) { MysqlDumpGeneratorNoMasterData.new(default_conf) }
36
+ let(:mysql_client) do
37
+ m = double(:mysql_client)
38
+ allow(m).to receive(:query).with(/TABLES/)
39
+ allow(m).to receive(:query).with("SHOW VARIABLES LIKE 'version'").
40
+ and_return([{'Value' => '5.1.40-0ubuntu0.12.04.1-log'}])
41
+ allow(m).to receive(:query).with("SHOW MASTER STATUS;").
42
+ and_return([{'File' => 'mysql-bin.000451', 'Position' => 89872}])
43
+ allow(m).to receive(:close)
44
+ m
45
+ end
46
+
47
+ describe '#dump' do
37
48
  before do
38
- `touch #{file_path}`
39
- expect(status).to receive(:exitstatus).and_return 1
40
- expect(Open3).to receive(:capture3).and_return(
41
- ['(dummy std out)', '(dummy std err)', status]
42
- )
49
+ expect(Mysql2::Client).to receive(:new).and_return(mysql_client)
50
+ expect(Open3).to receive(:popen3).and_yield(stdin, stdout, stderr, wait_thr)
51
+ end
52
+ context "when mysqldump exits with status 0" do
53
+ it do
54
+ expect(wait_thr).to receive(:value).and_return(0)
55
+ expect(stderr).to receive(:each_line).and_yield("")
56
+ expect(default_dump_generator.dump(file_path)).to be_truthy
57
+ expect(File.exists?(file_path)).to be_truthy
58
+ end
43
59
  end
44
- it do
45
- expect{ default_dump_generator.dump(file_path) }.to raise_error
46
- expect(File.exists?(file_path)).to be_falsey
60
+ context "when mysqldump exits with status 1" do
61
+ it do
62
+ expect(wait_thr).to receive(:value).and_return(1)
63
+ expect(stderr).to receive(:each_line).and_yield("")
64
+ expect{ default_dump_generator.dump(file_path) }.to raise_error
65
+ end
47
66
  end
48
- end
49
- context 'when exit status is 0 but no file' do
50
- before do
51
- expect(status).to receive(:exitstatus).and_return 0
52
- expect(Open3).to receive(:capture3).and_return(
53
- ['(dummy std out)', '(dummy std err)', status]
54
- )
67
+ context "when mysqldump exits with status 0, but there are error in stderr" do
68
+ it do
69
+ expect(wait_thr).to receive(:value).and_return(0)
70
+ expect(stderr).to receive(:each_line).and_yield("mysqldump error")
71
+ expect{ default_dump_generator.dump(file_path) }.to raise_error
72
+ end
55
73
  end
56
- it do
57
- expect{ default_dump_generator.dump(file_path) }.to raise_error
58
- expect(File.exists?(file_path)).to be_falsey
74
+ after :each do
75
+ File.delete(file_path) if File.exists?(file_path)
59
76
  end
60
77
  end
61
- context 'when exit status is 0 but file size is 0' do
62
- before do
63
- `touch #{file_path}`
64
- expect(status).to receive(:exitstatus).and_return 0
65
- expect(Open3).to receive(:capture3).and_return(
66
- ['(dummy std out)', '(dummy std err)', status]
67
- )
68
- end
69
- it do
70
- expect{ default_dump_generator.dump(file_path) }.to raise_error
71
- expect(File.exists?(file_path)).to be_truthy
78
+ end
79
+ describe MysqlDumpGeneratorMasterData do
80
+ let(:status) { double(:status) }
81
+ let(:dump_io) { File.open(file_path, 'r', encoding: "utf-8") }
82
+ let(:default_dump_generator) { MysqlDumpGeneratorMasterData.new(default_conf) }
83
+
84
+ describe '#initialize' do
85
+ context 'with password' do
86
+ subject { default_dump_generator.instance_variable_get(:@dump_cmd) }
87
+ it { is_expected.to eq('MYSQL_PWD="pass" mysqldump --default-character-set=utf8 --protocol=tcp -h localhost -P 3306 -uadmin --skip-lock-tables ' +
88
+ '--single-transaction --hex-blob --flush-logs --master-data=2 dev users groups') }
89
+ end
90
+ context 'without password' do
91
+ let (:dump_generator) do
92
+ MysqlDumpGeneratorMasterData.new(default_conf.merge({'password' => ''}))
93
+ end
94
+ subject { dump_generator.instance_variable_get(:@dump_cmd) }
95
+ it { is_expected.to eq('MYSQL_PWD="" mysqldump --default-character-set=utf8 --protocol=tcp -h localhost -P 3306 -uadmin --skip-lock-tables ' +
96
+ '--single-transaction --hex-blob --flush-logs --master-data=2 dev users groups') }
72
97
  end
73
98
  end
74
- context 'when exit status is 0' do
75
- before do
76
- `echo "something..." > #{file_path}`
77
- expect(status).to receive(:exitstatus).and_return 0
78
- expect(Open3).to receive(:capture3).and_return(
79
- ['(dummy std out)', '(dummy std err)', status]
80
- )
99
+
100
+ describe '#dump' do
101
+ context 'when exit status is not 0' do
102
+ before do
103
+ `touch #{file_path}`
104
+ expect(status).to receive(:exitstatus).and_return 1
105
+ expect(Open3).to receive(:capture3).and_return(
106
+ ['(dummy std out)', '(dummy std err)', status]
107
+ )
108
+ end
109
+ it do
110
+ expect{ default_dump_generator.dump(file_path) }.to raise_error
111
+ expect(File.exists?(file_path)).to be_falsey
112
+ end
81
113
  end
82
- it do
83
- expect(default_dump_generator.dump(file_path)).to be_truthy
84
- expect(File.exists?(file_path)).to be_truthy
114
+ context 'when exit status is 0 but no file' do
115
+ before do
116
+ expect(status).to receive(:exitstatus).and_return 0
117
+ expect(Open3).to receive(:capture3).and_return(
118
+ ['(dummy std out)', '(dummy std err)', status]
119
+ )
120
+ end
121
+ it do
122
+ expect{ default_dump_generator.dump(file_path) }.to raise_error
123
+ expect(File.exists?(file_path)).to be_falsey
124
+ end
125
+ end
126
+ context 'when exit status is 0 but file size is 0' do
127
+ before do
128
+ `touch #{file_path}`
129
+ expect(status).to receive(:exitstatus).and_return 0
130
+ expect(Open3).to receive(:capture3).and_return(
131
+ ['(dummy std out)', '(dummy std err)', status]
132
+ )
133
+ end
134
+ it do
135
+ expect{ default_dump_generator.dump(file_path) }.to raise_error
136
+ expect(File.exists?(file_path)).to be_truthy
137
+ end
138
+ end
139
+ context 'when exit status is 0' do
140
+ before do
141
+ `echo "something..." > #{file_path}`
142
+ expect(status).to receive(:exitstatus).and_return 0
143
+ expect(Open3).to receive(:capture3).and_return(
144
+ ['(dummy std out)', '(dummy std err)', status]
145
+ )
146
+ end
147
+ it do
148
+ expect(default_dump_generator.dump(file_path)).to be_truthy
149
+ expect(File.exists?(file_path)).to be_truthy
150
+ end
151
+ end
152
+ after :each do
153
+ File.delete(file_path) if File.exists?(file_path)
85
154
  end
86
- end
87
- after :each do
88
- File.delete(file_path) if File.exists?(file_path)
89
155
  end
90
156
  end
91
157
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: flydata
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.19
4
+ version: 0.2.20
5
5
  platform: ruby
6
6
  authors:
7
7
  - Koichi Fujikawa
@@ -12,7 +12,7 @@ authors:
12
12
  autorequire:
13
13
  bindir: bin
14
14
  cert_chain: []
15
- date: 2014-12-03 00:00:00.000000000 Z
15
+ date: 2014-12-05 00:00:00.000000000 Z
16
16
  dependencies:
17
17
  - !ruby/object:Gem::Dependency
18
18
  name: rest-client
@@ -448,7 +448,6 @@ files:
448
448
  - flydata.gemspec
449
449
  - lib/fly_data_model.rb
450
450
  - lib/flydata.rb
451
- - lib/flydata/agent_errors.rb
452
451
  - lib/flydata/api/base.rb
453
452
  - lib/flydata/api/data_entry.rb
454
453
  - lib/flydata/api/data_port.rb
@@ -471,6 +470,7 @@ files:
471
470
  - lib/flydata/command/stop.rb
472
471
  - lib/flydata/command/sync.rb
473
472
  - lib/flydata/command/version.rb
473
+ - lib/flydata/command_logger.rb
474
474
  - lib/flydata/compatibility_check.rb
475
475
  - lib/flydata/credentials.rb
476
476
  - lib/flydata/cron.rb
@@ -1,8 +0,0 @@
1
- module Flydata
2
- class TableDefError < StandardError
3
- attr_reader :err_hash
4
- def initialize(err_hash)
5
- @err_hash = err_hash
6
- end
7
- end
8
- end