flydata 0.2.19 → 0.2.20

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: eb04da6c165e8ace6ac6b7089c73eca79d2d1fc1
-  data.tar.gz: 77794c3f97103bd4090a70bdceb6036eb8e488ab
+  metadata.gz: f4d29cd6b59f9acb406d62017d559296f0fa27b7
+  data.tar.gz: ab8d2cdcebbaab49345f62da3aef259e5a977e02
 SHA512:
-  metadata.gz: 305c9ace6887dcf12a1fa4dd62830e41c11a87b42acd0efa06a0032b28dabccdacafc9945e9fd7b4b6d23eb872db347481ca7f605b814363a2d39c34a425340c
-  data.tar.gz: 76a51648a3e4ba7b363072e99c397b7b8542a0b57135cc494de51d5a4d65fda5f81cc263d0ffbe2a006e3f1e0591073187580e65f6e5ac05cebdedb22356b736
+  metadata.gz: 6f590f2046f0c016a2beae67555a0544b919b9ff5873376ee190f6a2065ae30e6e961178926127537a97baab5140dc1e1c9c5e5051895a5661d4cf93b90ccc0a
+  data.tar.gz: 6af1ee2c5597ed08e133650a5882ca6a4531ff47b0a25bf82566a3c3e0de4ecb3172c6c6da62449a853af49785197c15ae012ce59a4303e4a11d1826c02e7cd6

data/VERSION

@@ -1 +1 @@
-0.2.19
+0.2.20

data/flydata-core/lib/flydata-core/logger.rb

@@ -100,8 +100,43 @@ module FlydataCore
       log_params
     end
 
+    # hook method
+    def before_logging(level, raw_msg, built_msg, log_params, options)
+    end
+
     # log methods
-    def log_common(level, message, log_params = {}, options = {})
+    def log_common(level, message, log_params, options)
+      msg = build_log_message(level, message, log_params, options)
+      before_logging(level, message, msg, log_params, options)
+      get_logger(options).send(level, msg)
+    end
+
+    def log_debug(message, log_params = {}, options = {})
+      log_common(:debug, message, log_params, options)
+    end
+
+    def log_info(message, log_params = {}, options = {})
+      log_common(:info, message, log_params, options)
+    end
+
+    def log_warn(message, log_params = {}, options = {})
+      log_common(:warn, message, log_params, options)
+    end
+
+    def log_error(message, log_params = {}, options = {})
+      log_common(:error, message, log_params, options)
+    end
+
+    def log_error_with_backtrace(message, log_params = {}, options = {})
+      log_common(:error, message, log_params, options.merge(backtrace: true))
+    end
+
+    def get_logger(options = {})
+      options[:logger] || logger || log_context_logger || $log
+    end
+
+    def build_log_message(level, raw_message, log_params = {}, options = {})
+      message = raw_message.dup
       # check keys
       if e = log_params[:error]
         message += " error_class=#{e.class.to_s} error=\"#{e.to_s.strip.gsub(/\n/, ' ')}\""
@@ -133,30 +168,7 @@ module FlydataCore
         backtrace = e.backtrace ? e.backtrace.join("\n") : 'backtrace is empty.'
         msg += " backtrace:\n#{backtrace}"
       end
-
-      # get logger
-      lg = options[:logger] || logger || log_context_logger || $log
-      lg.send(level, msg)
-    end
-
-    def log_debug(message, log_params = {}, options = {})
-      log_common(:debug, message, log_params, options)
-    end
-
-    def log_info(message, log_params = {}, options = {})
-      log_common(:info, message, log_params, options)
-    end
-
-    def log_warn(message, log_params = {}, options = {})
-      log_common(:warn, message, log_params, options)
-    end
-
-    def log_error(message, log_params = {}, options = {})
-      log_common(:error, message, log_params, options)
-    end
-
-    def log_error_with_backtrace(message, log_params = {}, options = {})
-      log_common(:error, message, log_params, options.merge(backtrace: true))
+      msg
     end
 
     def build_log_text(items = log_context_items)

data/flydata.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "flydata"
-  s.version = "0.2.19"
+  s.version = "0.2.20"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Koichi Fujikawa", "Masashi Miyazaki", "Matthew Luu", "Mak Inada", "Sriram NS"]
-  s.date = "2014-12-03"
+  s.date = "2014-12-05"
   s.description = "FlyData Agent"
   s.email = "sysadmin@flydata.com"
   s.executables = ["fdmysqldump", "flydata", "serverinfo"]
@@ -63,7 +63,6 @@ Gem::Specification.new do |s|
     "flydata.gemspec",
     "lib/fly_data_model.rb",
     "lib/flydata.rb",
-    "lib/flydata/agent_errors.rb",
     "lib/flydata/api/base.rb",
     "lib/flydata/api/data_entry.rb",
     "lib/flydata/api/data_port.rb",
@@ -86,6 +85,7 @@ Gem::Specification.new do |s|
     "lib/flydata/command/stop.rb",
     "lib/flydata/command/sync.rb",
     "lib/flydata/command/version.rb",
+    "lib/flydata/command_logger.rb",
     "lib/flydata/compatibility_check.rb",
     "lib/flydata/credentials.rb",
     "lib/flydata/cron.rb",

data/lib/flydata.rb

@@ -25,6 +25,14 @@ module Flydata
   FLYDATA_DEBUG = !!(ENV['FLYDATA_DEBUG'])
   FLYDATA_HOME = ENV['FLYDATA_HOME'] || "#{ENV['HOME']}/.flydata"
   FLYDATA_GEM_HOME = File.expand_path(File.join(File.dirname(__FILE__), '..'))
+  FLYDATA_GEM_BIN = File.join(FLYDATA_GEM_HOME, 'bin')
   FLYDATA_TMPL_DIR = File.join(FLYDATA_GEM_HOME, 'tmpl')
+  FLYDATA_FLUENT_PLUGIN_DIR = File.join(FLYDATA_GEM_HOME, 'lib', 'flydata', 'fluent-plugins')
+
+  FLYDATA_SERVERINFO = File.join(FLYDATA_GEM_BIN, 'serverinfo')
+  FLYDATA_LOG = File.join(FLYDATA_HOME, 'flydata.log')
+  FLYDATA_CONF = File.join(FLYDATA_HOME, 'flydata.conf')
+
+  VERSION_PATH = File.join(FLYDATA_GEM_HOME, 'VERSION')
   include Flydata::Heroku
 end

data/lib/flydata/cli.rb

@@ -1,15 +1,26 @@
 require 'slop'
+require_relative 'command_logger'
 
 module Flydata
   class Cli
     include Helpers
+    include CommandLoggable
     def initialize(args)
       @args = args
+      unless $log
+        $log = Logger.new(FLYDATA_LOG)
+        # 2014-10-24 14:05:21 -0700 command.info: message
+        $log.datetime_format = "%Y-%m-%d %H:%M:%S %z "
+        $log.formatter = proc do |severity, datetime, progname, msg|
+          "#{datetime} command.#{severity.to_s.downcase}: #{msg}\n"
+        end
+      end
+      log_info("Start command.", {cmd: @args.join(' '), ver:flydata_version})
     end
 
     def run
       unless check_environment
-        puts "Sorry, you have to run the installation command to use flydata. Please go to https://console.flydata.com/ and sign up."
+        log_error_stderr("Sorry, you have to run the installation command to use flydata. Please go to https://console.flydata.com/ and sign up.")
         return
       end
       begin
@@ -28,7 +39,8 @@ module Flydata
           cmd_obj = cmd_cls.new(options)
           sub_cmd ? cmd_obj.send(sub_cmd,*@args) : cmd_obj.run(*@args)
         else
-          raise 'no command given'
+          $stderr.puts usage_text(false)
+          return
         end
       rescue => e
         #raise e
@@ -36,6 +48,7 @@ module Flydata
         $stderr.puts
         $stderr.puts
         $stderr.puts usage_text
+        log_error_with_backtrace('Error occured during command.', {error: e, args: @args})
         raise e if FLYDATA_DEBUG
         exit 1
       end

data/lib/flydata/command/base.rb

@@ -1,6 +1,8 @@
 module Flydata
   module Command
     class Base
+      include CommandLoggable
+
       def initialize(options = Slop.new)
         @api_client = ApiClient.instance
         @opts = options
@@ -37,7 +39,7 @@ module Flydata
         suffix = default_yes ? "(Y/n)" : "(y/n)"
         prompt = "#{message} #{suffix}:  "
         if opts && opts.yes? # Yes option
-          puts "#{prompt}Yes"
+          log_info_stdout("#{prompt}Yes")
           return true
         end
 
@@ -46,18 +48,22 @@ module Flydata
           return true if default_yes and ans == ''
           if ans.size > 0
             case ans[0].downcase
-            when 'y'; return true
-            when 'n'; return false
+            when 'y'
+              log_info("#{prompt}Yes")
+              return true
+            when 'n'
+              log_info("#{prompt}No")
+              return false
             end
           end
-          say(" ! Please answer y[es] or n[o]")
+          log_warn_stderr(" ! Please answer y[es] or n[o]")
           newline
         end
       end
       def choose_one(message, prompt, menu_list, value_list=[])
         choice = nil
         value_list = menu_list unless menu_list.size == value_list.size
-        say(message) if message
+        log_info_stdout(message) if message
         choose do |menu|
           menu.index = :number
           menu.index_suffix = ") "
@@ -65,12 +71,13 @@ module Flydata
           menu.prompt = prompt ? prompt : ">> "
           menu.choices(*menu_list) {|item| choice = value_list[menu_list.index(item)]}
         end
+        log_info_stdout("  -> #{choice}")
         choice
       end
       def ask_input_table_name
         input = nil
         loop do
-          say("Input a table name.")
+          log_info_stdout("Input a table name.")
           say(">> ")
           input = gets.strip
           if input =~ /^[a-zA-Z0-9_]+$/
@@ -79,6 +86,7 @@ module Flydata
             puts "Please enter a valid table name."
           end
         end
+        log_info(">> #{input}")
         input
       end
     end

data/lib/flydata/command/sender.rb

@@ -16,7 +16,7 @@ module Flydata
         end
         # Check if process exist
         if process_exist?
-          say("Process is still running. Please stop process first.") unless options[:quiet]
+          log_info_stdout("Process is still running. Please stop process first.") unless options[:quiet]
           return
         end
 
@@ -31,11 +31,11 @@ module Flydata
         AgentCompatibilityCheck.new(dp).check
 
         # Start sender(fluentd) process
-        say('Starting sender process.') unless options[:quiet]
+        log_info_stdout("Starting sender process.") unless options[:quiet]
         Dir.chdir(FLYDATA_HOME){
-          Kernel.system("bash #{File.dirname(__FILE__)}/../../../bin/serverinfo", :out => ["#{FLYDATA_HOME}/flydata.log",'a'], :err => ["#{FLYDATA_HOME}/flydata.log",'a'])
+          Kernel.system("bash #{FLYDATA_SERVERINFO}", :out => [FLYDATA_LOG,'a'], :err => [FLYDATA_LOG,'a'])
           daemon_opt = opts.no_daemon? ? "" : daemon_option
-          Kernel.system("ruby `which fluentd` #{daemon_opt} -l #{FLYDATA_HOME}/flydata.log -c #{FLYDATA_HOME}/flydata.conf -p #{File.dirname(__FILE__)}/../fluent-plugins")
+          Kernel.system("ruby `which fluentd` #{daemon_opt} -l #{FLYDATA_LOG} -c #{FLYDATA_CONF} -p #{FLYDATA_FLUENT_PLUGIN_DIR}")
         }
         Kernel.sleep 5
 
@@ -43,21 +43,21 @@ module Flydata
         #wait_until_logs_uploaded
         if options[:show_final_message] && !options[:quiet]
           data_port = flydata.data_port.get
-          say("Go to your Dashboard! #{flydata.flydata_api_host}/data_ports/#{data_port['id']}")
-          say <<EOF
+          log_info_stdout("Go to your Dashboard! #{flydata.flydata_api_host}/data_ports/#{data_port['id']}")
+          log_info_stdout <<EOF
 Please Note: Records and Total Size are updated every 10-20 minutes.
 EOF
         end
       end
       def stop(options = {})
         unless process_exist?
-          say("Process doesn't exist.") unless options[:quiet]
+          log_info_stdout("Process doesn't exist.") unless options[:quiet]
           return true
         end
 
-        say('Stopping sender process.') unless options[:quiet]
+        log_info_stdout('Stopping sender process.') unless options[:quiet]
         if Kernel.system("kill `cat #{pid_file}`") and wait_until_client_stop(options)
-          say('Done.') unless options[:quiet]
+          log_info_stdout('Done.') unless options[:quiet]
           return true
         end
         raise 'Something has gone wrong..'
@@ -65,17 +65,17 @@ EOF
       def flush_client_buffer(options = {})
         unless process_exist?
           return true if client_buffer_empty?
-          say("Process doesn't exist. But, the client buffer is not empty!!") unless options[:quiet]
+          log_info_stdout("Process doesn't exist. But, the client buffer is not empty!!") unless options[:quiet]
           start(options)
         end
 
-        say('Stopping input plugins and flushing the client buffer.') unless options[:quiet]
+        log_info_stdout('Stopping input plugins and flushing the client buffer.') unless options[:quiet]
         Kernel.system("kill -USR1 `cat #{pid_file}`")
 
         retry_count = 12
         1.upto(retry_count) do |i|
           return true if client_buffer_empty?
-          say("Waiting for the buffer to get empty... (#{i}/#{retry_count})") unless options[:quiet]
+          log_info_stdout("Waiting for the buffer to get empty... (#{i}/#{retry_count})") unless options[:quiet]
           Kernel.sleep 5
         end
 
@@ -83,23 +83,23 @@ EOF
       end
       def restart(options = {})
         if process_exist?
-          say('Restarting sender process.') unless options[:quiet]
+          log_info_stdout('Restarting sender process.') unless options[:quiet]
           if Kernel.system("kill -HUP `cat #{pid_file}`")
-            say('Done.') unless options[:quiet]
+            log_info_stdout('Done.') unless options[:quiet]
             return true
           else
             raise 'Something has gone wrong..'
           end
         else
-          say("Process doesn't exist.") unless options[:quiet]
+          log_info_stdout("Process doesn't exist.") unless options[:quiet]
           start(options)
         end
       end
       def status
         if process_exist?
-          say("Process is running.")
+          log_info_stdout("Process is running.")
         else
-          say("Process is not running.")
+          log_info_stdout("Process is not running.")
         end
       end
       def process_exist?
@@ -108,7 +108,7 @@ EOF
       end
       def kill_all(optiosn = {})
         if Kernel.system("ps ax | grep 'flydata' | grep -v grep | awk '{print \"kill  \" $1}' | sh")
-          say("Done.") unless options[:quiet]
+          log_info_stdout("Done.") unless options[:quiet]
           return true
         else
           raise 'Something has gone wrong...'
@@ -133,7 +133,7 @@ EOF
         retry_count = 10
         1.upto(retry_count) do |i|
           return true if server_ready?
-          say("Waiting for the server side to become active... (#{i}/#{retry_count})") unless options[:quiet]
+          log_info_stdout("Waiting for the server side to become active... (#{i}/#{retry_count})") unless options[:quiet]
           Kernel.sleep 30
         end
         false
@@ -142,13 +142,13 @@ EOF
         retry_count = 10
         1.upto(retry_count) do |i|
           if client_ready?
-            say("Done! Client is ready now.") unless options[:quiet]
+            log_info_stdout("Done! Client is ready now.") unless options[:quiet]
             return true
           end
           if process_died?
             raise "Client could not been launched. Detail here #{FLYDATA_HOME}/flydata.log"
           end
-          say("Waiting for the client side to become active... (#{i}/#{retry_count})") unless options[:quiet]
+          log_info_stdout("Waiting for the client side to become active... (#{i}/#{retry_count})") unless options[:quiet]
           Kernel.sleep 10
         end
         raise "Somthing has gone wrong... Please try setup command again."
@@ -157,23 +157,23 @@ EOF
         retry_count = 5
         1.upto(retry_count) do |i|
           return true unless process_exist?
-          say("Waiting for the client to stop... (#{i}/#{retry_count})") unless options[:quiet]
+          log_info_stdout("Waiting for the client to stop... (#{i}/#{retry_count})") unless options[:quiet]
           Kernel.sleep 3
         end
         false
       end
       def wait_until_logs_uploaded(options = {})
-        say('Starting to check the upload from your server.') unless options[:quiet]
+        log_info_stdout('Starting to check the upload from your server.') unless options[:quiet]
         data_port = flydata.data_port.get
         data_port_id = data_port['id']
 
         retry_count = 10
         1.upto(retry_count) do |i|
           if uploaded_successfully?(data_port_id)
-            say("Uploading your logs correctly.") unless options[:quiet]
+            log_info_stdout("Uploading your logs correctly.") unless options[:quiet]
             return true
           end
-          say("Waiting logs uploading... (#{i}/#{retry_count})") unless options[:quiet]
+          log_info_stdout("Waiting logs uploading... (#{i}/#{retry_count})") unless options[:quiet]
           Kernel.sleep 30
         end
         raise 'Cannot confirm that your logs exist on the FlyData server. Something has gone wrong..'
@@ -196,7 +196,7 @@ EOF
       end
       def client_buffer_empty?(options = {})
         client_buffer = File.join(FLYDATA_HOME, 'buffer')
-        say("Checking the client buffer #{client_buffer}") unless options[:quiet]
+        log_info_stdout("Checking the client buffer #{client_buffer}") unless options[:quiet]
         Dir.glob("#{client_buffer}/*").empty?
       end
       def pid_file

data/lib/flydata/command/sync.rb

@@ -3,7 +3,6 @@ require 'msgpack'
 require 'open3'
 require 'mysql2'
 require 'flydata/sync_file_manager'
-require 'flydata/agent_errors'
 require 'flydata/compatibility_check'
 require 'flydata/output/forwarder'
 require 'flydata/parser/mysql/dump_parser'
@@ -35,10 +34,10 @@ module Flydata
         if (sender.process_exist?)
           if tables.empty?
             # full sync
-            puts "FlyData Agent is already running.  If you'd like to restart FlyData Sync from scratch, run 'flydata sync:reset' first."
+            log_warn_stderr("FlyData Agent is already running.  If you'd like to restart FlyData Sync from scratch, run 'flydata sync:reset' first.")
           else
             # per-table sync
-            puts "Flydata Agent is already running.  If you'd like to Sync the table(s), run 'flydata sync:flush' first."
+            log_warn_stderr("Flydata Agent is already running.  If you'd like to Sync the table(s), run 'flydata sync:flush' first.")
           end
           exit 1
         end
@@ -58,7 +57,7 @@ module Flydata
 
       def flush
         flush_buffer_and_stop
-        puts "Buffers have been flushed and the sender process has been stopped."
+        log_info_stdout("Buffers have been flushed and the sender process has been stopped.")
       end
 
       def self.slop_reset
@@ -91,11 +90,11 @@ module Flydata
         delete_files.flatten.each do |path|
           FileUtils.rm(path) if File.exists?(path)
         end
-        puts "Reset completed successfully."
+        log_info_stdout("Reset completed successfully.")
       end
 
       def wait_for_server_buffer
-        puts "Waiting for the server buffer to get empty"
+        log_info_stdout("Waiting for the server buffer to get empty.")
         while (status = check) && (status['state'] == 'processing')
           print_progress(status)
           sleep 10
@@ -104,24 +103,24 @@ module Flydata
 
       def wait_for_server_data_processing
         state = :PROCESS
-        puts "Uploading data to Redshift..."
+        log_info_stdout("Uploading data to Redshift...")
         sleep 10
         status = nil
         while (status = check)
           if state == :PROCESS && status['state'] == 'uploading'
-            puts "  -> Done"
+            log_info_stdout("  -> Done")
             state = :UPLOAD
-            puts "Finishing data upload..."
+            log_info_stdout("Finishing data upload...")
           end
           print_progress(status)
           sleep 10
         end
         if (state == :PROCESS)
           # :UPLOAD state was skipped due to no data
-          puts "  -> Done"
-          puts "Finishing data upload..."
+          log_info_stdout("  -> Done")
+          log_info_stdout("Finishing data upload...")
         end
-        puts "  -> Done"
+        log_info_stdout("  -> Done")
       end
 
       def check
@@ -142,9 +141,9 @@ module Flydata
         sync_fm = Flydata::FileUtil::SyncFileManager.new(de)
         binlog_path = sync_fm.binlog_path
         `touch #{binlog_path}`
-        puts "Created an empty binlog position file."
-        puts "-> #{binlog_path}"
-        puts "Run 'flydata start' to start continuous sync."
+        log_info_stdout("Created an empty binlog position file.")
+        log_info_stdout("-> #{binlog_path}")
+        log_info_stdout("Run 'flydata start' to start continuous sync.")
       end
 
       def self.slop_generate_table_ddl
@@ -192,6 +191,7 @@ module Flydata
 
       def cleanup_sync_server(de, tables = [])
         print "Cleaning the server."
+        log_info("Cleaning the server.")
         worker = Thread.new do
           begin
             flydata.data_entry.cleanup_sync(de['id'], tables)
@@ -204,7 +204,7 @@ module Flydata
           print "."
         end
         puts
-        puts "Done."
+        log_info_stdout("Done.")
       end
 
       def do_check(de)
@@ -213,7 +213,7 @@ module Flydata
 
       def print_progress(buffer_stat)
         message = buffer_stat['message']
-        puts message unless message.nil? || message.empty?
+        log_info_stdout(message) unless message.nil? || message.empty?
       end
 
       DDL_DUMP_CMD_TEMPLATE = "MYSQL_PWD=\"%s\" mysqldump --protocol=tcp -d -h %s -P %s -u %s %s %s"
@@ -264,15 +264,15 @@ module Flydata
           raise errors unless errors.empty?
         end
         unless error_list.empty?
-          $stderr.puts "We have noticed the following error(s):"
+          log_error_stderr("We have noticed the following error(s):")
           group_error =  error_list.group_by {|d| d[:error]}
           group_error.each_key do |a|
-            $stderr.puts "The following table(s) have #{a}:"
+            log_error_stderr("The following table(s) have #{a}:")
             group_error[a].each do |hash|
-              $stderr.puts "  - #{hash[:table]}" if hash[:table]
+              log_error_stderr("  - #{hash[:table]}") if hash[:table]
             end
           end
-          $stderr.puts "Please fix the above error(s) to try to sync those table(s) or contact us for further help."
+          log_error_stderr("Please fix the above error(s) to try to sync those table(s) or contact us for further help.")
         end
       end
 
@@ -315,7 +315,7 @@ module Flydata
         dp = flydata.data_port.get
         fp = sync_fm.dump_file_path
         if file_dump && File.exists?(fp) && File.size(fp) > 0
-          puts "  -> Skip"
+          log_info_stdout("  -> Skip")
           return call_block_or_return_io(fp, &block)
         end
 
@@ -338,30 +338,27 @@ EOM
 Dump file saves contents of your tables temporarily.  Make sure you have enough disk space.
 EOM
         print confirmation_text
+        log_info confirmation_text.strip
 
         if ask_yes_no('Start Sync?')
-          puts "Checking database size(not same as mysqldump data size)..."
+          log_info_stdout("Checking database size(not same as mysqldump data size)...")
+
           db_bytesize = Flydata::Parser::Mysql::DatabaseSizeCheck.new(de['mysql_data_entry_preference']).get_db_bytesize
-          puts "  -> #{as_size(db_bytesize)} (#{db_bytesize} byte)"
-          puts "Exporting data from database."
-          puts "This process can take hours depending on data size and load on your database.  Please be patient..."
+          log_info_stdout("  -> #{as_size(db_bytesize)} (#{db_bytesize} byte)")
+          log_info_stdout("Exporting data from database.")
+          log_info_stdout("This process can take hours depending on data size and load on your database.  Please be patient...")
           Flydata::MysqlCompatibilityCheck.new(dp, de['mysql_data_entry_preference'], fp).check
           if file_dump
             Flydata::Parser::Mysql::MysqlDumpGeneratorNoMasterData.
               new(de['mysql_data_entry_preference']).dump(fp)
-            puts "  -> Done"
+            log_info_stdout("  -> Done")
             call_block_or_return_io(fp, &block)
           else
             Flydata::Parser::Mysql::MysqlDumpGeneratorNoMasterData.
               new(de['mysql_data_entry_preference']).dump {|io| block.call(io, db_bytesize)}
           end
         else
-          newline
-          puts "You can change the dump file path with 'mysqldump_path' property in the following conf file."
-          puts
-          puts "    #{Flydata::Preference::DataEntryPreference.conf_path(de)}"
-          puts
-          return nil
+          exit 1
         end
       end
 
@@ -419,12 +416,12 @@ EOM
         dump_pos_info = sync_fm.load_dump_pos
         option = dump_pos_info || {}
         if option[:table_name] && option[:last_pos].to_i != -1
-          puts "Resuming... Last processed table: #{option[:table_name]}"
+          log_info_stdout("Resuming... Last processed table: #{option[:table_name]}")
         else
           #If its a new sync, ensure server side resources are clean
           cleanup_sync_server(de, de['mysql_data_entry_preference']['tables'].split(',')) unless opts.skip_cleanup?
         end
-        puts "Sending data to FlyData Server..."
+        log_info_stdout("Sending data to FlyData Server...")
 
         bench_start_time = Time.now
 
@@ -461,9 +458,9 @@ EOM
 
             # show the current progress
             if last_pos.to_i == -1   # stream dump
-              puts "  -> #{as_size(bytesize)} (#{bytesize} byte) completed..."
+              log_info_stdout("  -> #{as_size(bytesize)} (#{bytesize} byte) completed...")
             else
-              puts "  -> #{(last_pos.to_f/dump_file_size * 100).round(1)}% (#{last_pos}/#{dump_file_size}) completed..."
+              log_info_stdout("  -> #{(last_pos.to_f/dump_file_size * 100).round(1)}% (#{last_pos}/#{dump_file_size}) completed...")
             end
 
             # save check point
@@ -472,13 +469,13 @@ EOM
           }
         )
         forwarder.close
-        puts "  -> Done"
+        log_info_stdout("  -> Done")
         sync_fm.save_dump_pos(STATUS_WAITING, '', dump_file_size, binlog_pos)
 
         if ENV['FLYDATA_BENCHMARK']
           bench_end_time = Time.now
           elapsed_time = bench_end_time.to_i - bench_start_time.to_i
-          puts "Elapsed:#{elapsed_time}sec start:#{bench_start_time} end:#{bench_end_time}"
+          log_info_stdout("Elapsed:#{elapsed_time}sec start:#{bench_start_time} end:#{bench_end_time}")
         end
       end
 
@@ -524,14 +521,14 @@ Thank you for using FlyData!
           sync_fm.reset_table_position_files(de['mysql_data_entry_preference']['tables'].split(','))
           sync_fm.backup_dump_dir
           unless opts.no_flydata_start?
-            puts "Starting FlyData Agent..."
+            log_info_stdout("Starting FlyData Agent...")
             Flydata::Command::Sender.new.start(quiet: true)
-            puts "  -> Done"
+            log_info_stdout("  -> Done")
           end
           dashboard_url = "#{flydata.flydata_api_host}/dashboard"
           redshift_console_url = "#{flydata.flydata_api_host}/redshift_clusters/query/new"
           last_message = ALL_DONE_MESSAGE_TEMPLATE % [redshift_console_url, dashboard_url]
-          puts last_message
+          log_info_stdout(last_message)
         else
           raise "Initial sync status is not complete. Try running 'flydata sync'."
         end

data/lib/flydata/command/version.rb

@@ -1,9 +1,10 @@
 module Flydata
   module Command
     class Version < Base
-      VERSION_PATH = File.join(FLYDATA_GEM_HOME, 'VERSION')
+      include Helpers
+
       def run
-        puts "flydata version \"#{File.open(VERSION_PATH, 'r').readline.strip}\""
+        log_info_stdout("flydata version \"#{flydata_version}\"")
       end
     end
   end

data/lib/flydata/command_logger.rb

@@ -0,0 +1,26 @@
+require 'flydata-core/core_ext'
+require 'flydata-core/logger'
+
+module Flydata
+  module CommandLoggable
+    include_unless_included FlydataCore::Logger
+
+    # hook
+    def before_logging(level, raw_msg, built_msg, log_params, options)
+      $stdout.puts raw_msg if options[:stdout]
+      $stderr.puts raw_msg if options[:stderr]
+    end
+
+    def log_info_stdout(message, log_params = {})
+      log_info(message, log_params, {stdout: true})
+    end
+
+    def log_warn_stderr(message, log_params = {})
+      log_warn(message, log_params, {stderr: true})
+    end
+
+    def log_error_stderr(message)
+      log_error(message, log_params, {stderr: true})
+    end
+  end
+end

data/lib/flydata/compatibility_check.rb

@@ -1,6 +1,10 @@
+require 'flydata/command_logger'
+
 module Flydata
 
   class CompatibilityCheck
+    include CommandLoggable
+
     class CompatibilityError < StandardError
     end
 
@@ -13,6 +17,8 @@ module Flydata
       self.methods.grep(/^check_/).each do |m|
         begin
           send(m)
+        rescue ArgumentError => e
+          # ignore
         rescue CompatibilityError => e
           @errors << e
         end
@@ -22,9 +28,9 @@ module Flydata
 
     def print_errors
       return if @errors.empty?
-      puts "There may be some compatibility issues with this current server : "
+      log_error_stderr "There may be some compatibility issues with this current server : "
       @errors.each do |error|
-        puts "  * #{error.message}"
+        log_error_stderr "  * #{error.message}"
       end
       raise "Please correct these errors if you wish to run FlyData Agent"
     end
@@ -79,9 +85,9 @@ module Flydata
 
     def print_errors
       return if @errors.empty?
-      puts "There may be some compatibility issues with your MySQL credentials: "
+      log_error_stderr "There may be some compatibility issues with your MySQL credentials: "
       @errors.each do |error|
-        puts "  * #{error.message}"
+        log_error_stderr "  * #{error.message}"
       end
       raise "Please correct these errors if you wish to run FlyData Sync"
     end
@@ -115,8 +121,12 @@ module Flydata
       Open3.popen3(query) do |stdin, stdout, stderr|
         stdin.close
         while !stderr.eof?
-          line = stderr.gets
-          unless /Warning: Using a password on the command line interface can be insecure./ === line
+          lines = []
+          while line = stderr.gets; lines << line.strip; end
+          err_reason = lines.join(" ")
+          log_error("Error occured during access to mysql server.", {err: err_reason})
+
+          unless /Warning: Using a password on the command line interface can be insecure/ === err_reason
             raise MysqlCompatibilityError, "Cannot connect to MySQL database. Please make sure you can connect with this command:\n      $ mysql -u #{@db_opts[:username]} -h #{@db_opts[:host]} -P #{@db_opts[:port]} #{@db_opts[:database]} --protocol=tcp -p"
           end
         end

data/lib/flydata/helpers.rb

@@ -12,19 +12,24 @@ module Flydata
       [klass, method]
     end
 
-    def usage_text
+    def usage_text(err = true)
       text = ""
-      text += <<-EOM
+
+      if err
+        text += <<-EOM
 Fix the issue and try again.  If the problem continues, please contact support@flydata.com
 
 EOM
-      flydata_log = File.join(FLYDATA_HOME, 'flydata.log')
-      if File.exists?(flydata_log)
-        text += <<-EOM
+
+        flydata_log = File.join(FLYDATA_HOME, 'flydata.log')
+        if File.exists?(flydata_log)
+          text += <<-EOM
 Also check the Agent log.
    Log path: #{flydata_log}
 EOM
+        end
       end
+
       text += ""
       text += '-' * 64
       text += "\n"
@@ -45,6 +50,10 @@ Usage: flydata COMMAND
       text
     end
 
+    def flydata_version
+      IO.read(VERSION_PATH).strip
+    end
+
     def to_command_class(class_name)
       eval("Flydata::Command::#{class_name.camelcase}")
     end

data/lib/flydata/output/forwarder.rb

@@ -1,15 +1,19 @@
 require 'socket'
+require 'msgpack'
+require 'flydata/command_logger'
 
 module Flydata
   module Output
     class ForwarderFactory
+      include CommandLoggable
+
       def self.create(forwarder_key, tag, servers, options = {})
         case forwarder_key
         when nil, "tcpforwarder"
-          puts "Creating TCP connection" if FLYDATA_DEBUG
+          puts("Creating TCP connection") if FLYDATA_DEBUG
           forward = TcpForwarder.new(tag, servers, options)
         when "sslforwarder"
-          puts "Creating SSL connection" if FLYDATA_DEBUG
+          puts("Creating SSL connection") if FLYDATA_DEBUG
           forward = SslForwarder.new(tag, servers, options)
         else
           raise "Unsupported Forwarding type #{forwarder_key}"
@@ -88,12 +92,12 @@ module Flydata
         rescue => e
           retry_count += 1
           if retry_count > RETRY_LIMIT
-            puts "! Error: Failed to send data. Exceeded the retry limit. retry_count:#{retry_count}"
+            log_error_stderr("! Error: Failed to send data. Exceeded the retry limit. retry_count:#{retry_count}")
             raise e
           end
-          puts "! Warn: Retring to send data. retry_count:#{retry_count} error=#{e.to_s}"
+          log_warn_stderr("! Warn: Retring to send data. retry_count:#{retry_count} error=#{e.to_s}")
           wait_time = RETRY_INTERVAL ** retry_count
-          puts "  Now waiting for next retry. time=#{wait_time}sec"
+          log_warn_stderr("  Now waiting for next retry. time=#{wait_time}sec")
           sleep wait_time
           retry
         ensure

data/lib/flydata/parser/mysql/dump_parser.rb

@@ -1,3 +1,5 @@
+require 'fiber'
+
 module Flydata
   module Parser
     module Mysql
@@ -106,7 +108,7 @@ EOS
             rd_io.extend(DumpStreamIO)
 
             # start mysqldump
-            Open3.popen3 @dump_cmd do |cmd_in, cmd_out, cmd_err|
+            Open3.popen3 @dump_cmd do |cmd_in, cmd_out, cmd_err, wait_thr|
               cmd_in.close_write
               cmd_out.set_encoding("utf-8") # mysqldump output must be in UTF-8
 
@@ -126,9 +128,10 @@ EOS
               end
 
               # show err message
+              errors = ""
               threads << Thread.new do
                 cmd_err.each_line do |line|
-                  $stderr.print line unless /^Warning:/ === line
+                  errors << line unless /^Warning:/ === line
                 end
               end
 
@@ -141,7 +144,14 @@ EOS
               end
 
               threads.each(&:join)
+              unless wait_thr.value == 0
+                #Raise error if the process exited with status != 0
+                #(Even if there was no message on stderr stream)
+                errors = "Failed to run mysqldump command." if errors.empty?
+              end
+              raise errors unless errors.empty?
             end
+            true
           rescue
             # Cleanup
             FileUtils.rm(file_path) if file_path && File.exists?(file_path)

data/spec/flydata/command/sender_spec.rb

@@ -24,9 +24,9 @@ module Flydata
           let(:args) { [] }
           it "starts fluend with daemon option" do
             expect(Kernel).to receive(:system).with( Regexp.new(
-"bash .+/../../../bin/serverinfo"), :out => [Regexp.new(".+/flydata.log"),'a'], :err => [Regexp.new(".+/flydata.log"),'a'])
+"bash .+/bin/serverinfo"), :out => [Regexp.new(".+/flydata.log"),'a'], :err => [Regexp.new(".+/flydata.log"),'a'])
             expect(Kernel).to receive(:system).with( Regexp.new(
-"ruby `which fluentd` -d .+/flydata.pid -l .+/flydata.log -c .+/flydata.conf -p .+/\.\./fluent-plugins"))
+"ruby `which fluentd` -d .+/flydata.pid -l .+/flydata.log -c .+/flydata.conf -p .+/lib/flydata/fluent-plugins"))
             subject.start(false)
           end
         end
@@ -34,9 +34,9 @@ module Flydata
           let(:args) { ["-n"] }
           it "starts fluentd with no daemon option" do
             expect(Kernel).to receive(:system).with( Regexp.new(
-"bash .+/../../../bin/serverinfo"), :out => [Regexp.new(".+/flydata.log"),'a'], :err => [Regexp.new(".+/flydata.log"),'a'])
+"bash .+/bin/serverinfo"), :out => [Regexp.new(".+/flydata.log"),'a'], :err => [Regexp.new(".+/flydata.log"),'a'])
             expect(Kernel).to receive(:system).with(Regexp.new(
-"ruby `which fluentd`  -l .+/flydata.log -c .+/flydata.conf -p .+/\.\./fluent-plugins"))
+"ruby `which fluentd`  -l .+/flydata.log -c .+/flydata.conf -p .+/lib/flydata/fluent-plugins"))
             subject.start(false)
           end
         end
@@ -44,9 +44,9 @@ module Flydata
           let(:args) { ["--no-daemon"] }
           it "starts fluentd with no daemon option" do
             expect(Kernel).to receive(:system).with( Regexp.new(
-"bash .+/../../../bin/serverinfo"), :out => [Regexp.new(".+/flydata.log"),'a'], :err => [Regexp.new(".+/flydata.log"),'a'])
+"bash .+/bin/serverinfo"), :out => [Regexp.new(".+/flydata.log"),'a'], :err => [Regexp.new(".+/flydata.log"),'a'])
             expect(Kernel).to receive(:system).with(Regexp.new(
-"ruby `which fluentd`  -l .+/flydata.log -c .+/flydata.conf -p .+/\.\./fluent-plugins"))
+"ruby `which fluentd`  -l .+/flydata.log -c .+/flydata.conf -p .+/lib/flydata/fluent-plugins"))
             subject.start(false)
           end
         end

data/spec/flydata/compatibility_check_spec.rb

@@ -1,3 +1,6 @@
+require 'spec_helper'
+require 'flydata/compatibility_check'
+
 module Flydata
   describe AgentCompatibilityCheck do
     let(:default_data_port) do

data/spec/flydata/output/forwarder_spec.rb

@@ -1,5 +1,6 @@
 # coding: utf-8
 require 'spec_helper'
+require 'flydata/output/forwarder'
 
 module Flydata
   module Output

data/spec/flydata/parser/mysql/dump_parser_spec.rb

@@ -1,91 +1,157 @@
 # coding: utf-8
 require 'spec_helper'
+require 'flydata/parser/mysql/dump_parser'
+require 'open3'
+require 'mysql2'
 
 module Flydata
   module Parser
     module Mysql
-      describe MysqlDumpGeneratorMasterData do
-        let(:stdout) { double(:stdout) }
-        let(:stderr) { double(:stderr) }
-        let(:status) { double(:status) }
+      context "Test Dump Generators" do
+        let(:stdin) do
+          s = double(:stdin)
+          allow(s).to receive(:close_write)
+          s
+        end
+        let(:stdout) do
+          s = double(:stdout)
+          allow(s).to receive(:set_encoding)
+          allow(s).to receive(:gets).and_return("first line")
+          allow(s).to receive(:each_line).and_yield("another line")
+          s
+        end
         let(:file_path) { File.join('/tmp', "flydata_sync_spec_mysqldump_#{Time.now.to_i}") }
-        let(:dump_io) { File.open(file_path, 'r', encoding: "utf-8") }
-        let(:default_conf) { {
-          'host' => 'localhost', 'port' => 3306, 'username' => 'admin',
-          'password' => 'pass', 'database' => 'dev', 'tables' => 'users,groups',
-        } }
-        let(:default_dump_generator) { MysqlDumpGeneratorMasterData.new(default_conf) }
-
-        describe '#initialize' do
-          context 'with password' do
-            subject { default_dump_generator.instance_variable_get(:@dump_cmd) }
-            it { is_expected.to eq('MYSQL_PWD="pass" mysqldump --default-character-set=utf8 --protocol=tcp -h localhost -P 3306 -uadmin --skip-lock-tables ' +
-                                   '--single-transaction --hex-blob --flush-logs --master-data=2 dev users groups') }
-          end
-          context 'without password' do
-            let (:dump_generator) do
-              MysqlDumpGeneratorMasterData.new(default_conf.merge({'password' => ''}))
-            end
-            subject { dump_generator.instance_variable_get(:@dump_cmd) }
-            it { is_expected.to eq('MYSQL_PWD="" mysqldump --default-character-set=utf8 --protocol=tcp -h localhost -P 3306 -uadmin --skip-lock-tables ' +
-                                   '--single-transaction --hex-blob --flush-logs --master-data=2 dev users groups') }
-          end
+        let(:default_conf) do
+          {
+            'host' => 'localhost', 'port' => 3306, 'username' => 'admin',
+            'password' => 'pass', 'database' => 'dev', 'tables' => 'users,groups',
+          }
         end
 
-        describe '#dump' do
-          context 'when exit status is not 0' do
+        describe MysqlDumpGeneratorNoMasterData do
+          let(:stderr) { double(:stderr) }
+          let(:wait_thr) { double(:wait_thr) }
+
+          let(:default_dump_generator) { MysqlDumpGeneratorNoMasterData.new(default_conf) }
+          let(:mysql_client) do
+            m = double(:mysql_client)
+            allow(m).to receive(:query).with(/TABLES/)
+            allow(m).to receive(:query).with("SHOW VARIABLES LIKE 'version'").
+                        and_return([{'Value' => '5.1.40-0ubuntu0.12.04.1-log'}])
+            allow(m).to receive(:query).with("SHOW MASTER STATUS;").
+                        and_return([{'File' => 'mysql-bin.000451', 'Position' => 89872}])
+            allow(m).to receive(:close)
+            m
+          end
+
+          describe '#dump' do
             before do
-              `touch #{file_path}`
-              expect(status).to receive(:exitstatus).and_return 1
-              expect(Open3).to receive(:capture3).and_return(
-                ['(dummy std out)', '(dummy std err)', status]
-              )
+              expect(Mysql2::Client).to receive(:new).and_return(mysql_client)
+              expect(Open3).to receive(:popen3).and_yield(stdin, stdout, stderr, wait_thr)
+            end
+            context "when mysqldump exits with status 0" do
+              it do
+                expect(wait_thr).to receive(:value).and_return(0)
+                expect(stderr).to receive(:each_line).and_yield("")
+                expect(default_dump_generator.dump(file_path)).to be_truthy
+                expect(File.exists?(file_path)).to be_truthy
+              end
             end
-            it do
-              expect{ default_dump_generator.dump(file_path) }.to raise_error
-              expect(File.exists?(file_path)).to be_falsey
+            context "when mysqldump exits with status 1" do
+              it do
+                expect(wait_thr).to receive(:value).and_return(1)
+                expect(stderr).to receive(:each_line).and_yield("")
+                expect{ default_dump_generator.dump(file_path) }.to raise_error
+              end
             end
-          end
-          context 'when exit status is 0 but no file' do
-            before do
-              expect(status).to receive(:exitstatus).and_return 0
-              expect(Open3).to receive(:capture3).and_return(
-                ['(dummy std out)', '(dummy std err)', status]
-              )
+            context "when mysqldump exits with status 0, but there are error in stderr" do
+              it do
+                expect(wait_thr).to receive(:value).and_return(0)
+                expect(stderr).to receive(:each_line).and_yield("mysqldump error")
+                expect{ default_dump_generator.dump(file_path) }.to raise_error
+              end
             end
-            it do
-              expect{ default_dump_generator.dump(file_path) }.to raise_error
-              expect(File.exists?(file_path)).to be_falsey
+            after :each do
+              File.delete(file_path) if File.exists?(file_path)
             end
           end
-          context 'when exit status is 0 but file size is 0' do
-            before do
-              `touch #{file_path}`
-              expect(status).to receive(:exitstatus).and_return 0
-              expect(Open3).to receive(:capture3).and_return(
-                ['(dummy std out)', '(dummy std err)', status]
-              )
-            end
-            it do
-              expect{ default_dump_generator.dump(file_path) }.to raise_error
-              expect(File.exists?(file_path)).to be_truthy
+        end
+        describe MysqlDumpGeneratorMasterData do
+          let(:status) { double(:status) }
+          let(:dump_io) { File.open(file_path, 'r', encoding: "utf-8") }
+          let(:default_dump_generator) { MysqlDumpGeneratorMasterData.new(default_conf) }
+
+          describe '#initialize' do
+            context 'with password' do
+              subject { default_dump_generator.instance_variable_get(:@dump_cmd) }
+              it { is_expected.to eq('MYSQL_PWD="pass" mysqldump --default-character-set=utf8 --protocol=tcp -h localhost -P 3306 -uadmin --skip-lock-tables ' +
+                                     '--single-transaction --hex-blob --flush-logs --master-data=2 dev users groups') }
+            end
+            context 'without password' do
+              let (:dump_generator) do
+                MysqlDumpGeneratorMasterData.new(default_conf.merge({'password' => ''}))
+              end
+              subject { dump_generator.instance_variable_get(:@dump_cmd) }
+              it { is_expected.to eq('MYSQL_PWD="" mysqldump --default-character-set=utf8 --protocol=tcp -h localhost -P 3306 -uadmin --skip-lock-tables ' +
+                                     '--single-transaction --hex-blob --flush-logs --master-data=2 dev users groups') }
             end
           end
-          context 'when exit status is 0' do
-            before do
-              `echo "something..." > #{file_path}`
-              expect(status).to receive(:exitstatus).and_return 0
-              expect(Open3).to receive(:capture3).and_return(
-                ['(dummy std out)', '(dummy std err)', status]
-              )
+
+          describe '#dump' do
+            context 'when exit status is not 0' do
+              before do
+                `touch #{file_path}`
+                expect(status).to receive(:exitstatus).and_return 1
+                expect(Open3).to receive(:capture3).and_return(
+                  ['(dummy std out)', '(dummy std err)', status]
+                )
+              end
+              it do
+                expect{ default_dump_generator.dump(file_path) }.to raise_error
+                expect(File.exists?(file_path)).to be_falsey
+              end
             end
-            it do
-              expect(default_dump_generator.dump(file_path)).to be_truthy
-              expect(File.exists?(file_path)).to be_truthy
+            context 'when exit status is 0 but no file' do
+              before do
+                expect(status).to receive(:exitstatus).and_return 0
+                expect(Open3).to receive(:capture3).and_return(
+                  ['(dummy std out)', '(dummy std err)', status]
+                )
+              end
+              it do
+                expect{ default_dump_generator.dump(file_path) }.to raise_error
+                expect(File.exists?(file_path)).to be_falsey
+              end
+            end
+            context 'when exit status is 0 but file size is 0' do
+              before do
+                `touch #{file_path}`
+                expect(status).to receive(:exitstatus).and_return 0
+                expect(Open3).to receive(:capture3).and_return(
+                  ['(dummy std out)', '(dummy std err)', status]
+                )
+              end
+              it do
+                expect{ default_dump_generator.dump(file_path) }.to raise_error
+                expect(File.exists?(file_path)).to be_truthy
+              end
+            end
+            context 'when exit status is 0' do
+              before do
+                `echo "something..." > #{file_path}`
+                expect(status).to receive(:exitstatus).and_return 0
+                expect(Open3).to receive(:capture3).and_return(
+                  ['(dummy std out)', '(dummy std err)', status]
+                )
+              end
+              it do
+                expect(default_dump_generator.dump(file_path)).to be_truthy
+                expect(File.exists?(file_path)).to be_truthy
+              end
+            end
+            after :each do
+              File.delete(file_path) if File.exists?(file_path)
             end
-          end
-          after :each do
-            File.delete(file_path) if File.exists?(file_path)
           end
         end
       end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flydata
 version: !ruby/object:Gem::Version
-  version: 0.2.19
+  version: 0.2.20
 platform: ruby
 authors:
 - Koichi Fujikawa
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-12-03 00:00:00.000000000 Z
+date: 2014-12-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -448,7 +448,6 @@ files:
 - flydata.gemspec
 - lib/fly_data_model.rb
 - lib/flydata.rb
-- lib/flydata/agent_errors.rb
 - lib/flydata/api/base.rb
 - lib/flydata/api/data_entry.rb
 - lib/flydata/api/data_port.rb
@@ -471,6 +470,7 @@ files:
 - lib/flydata/command/stop.rb
 - lib/flydata/command/sync.rb
 - lib/flydata/command/version.rb
+- lib/flydata/command_logger.rb
 - lib/flydata/compatibility_check.rb
 - lib/flydata/credentials.rb
 - lib/flydata/cron.rb

data/lib/flydata/agent_errors.rb

@@ -1,8 +0,0 @@
-module Flydata
-  class TableDefError < StandardError
-    attr_reader :err_hash
-    def initialize(err_hash)
-      @err_hash = err_hash
-    end
-  end
-end