flydata 0.1.8 → 0.1.9

data/Gemfile

@@ -10,6 +10,7 @@ gem "fluentd", "0.10.46"
 gem "ruby-binlog", ">= 1.0.1"
 gem "fluent-plugin-mysql-binlog", "~> 0.0.2"
 gem "mysql2", "~> 0.3.11"
+gem "slop"
 
 group :development do
   gem "bundler"

data/Gemfile.lock

@@ -97,6 +97,7 @@ GEM
     ruby-binlog (1.0.1)
     ruby-prof (0.14.2)
     sigdump (0.2.2)
+    slop (3.4.6)
     sqlite3 (1.3.8)
     thread_safe (0.1.3)
       atomic
@@ -124,5 +125,6 @@ DEPENDENCIES
   rspec
   ruby-binlog (>= 1.0.1)
   ruby-prof
+  slop
   sqlite3
   timecop

data/VERSION

@@ -1 +1 @@
-0.1.8
+0.1.9

data/flydata.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "flydata"
-  s.version = "0.1.8"
+  s.version = "0.1.9"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Koichi Fujikawa"]
-  s.date = "2014-06-23"
+  s.date = "2014-07-01"
   s.description = "FlyData Command Line Interface"
   s.email = "sysadmin@flydata.co"
   s.executables = ["fdmysqldump", "flydata"]
@@ -77,6 +77,7 @@ Gem::Specification.new do |s|
     "spec/fluent_plugins_spec_helper.rb",
     "spec/fly_data_model_spec.rb",
     "spec/flydata/api/data_entry_spec.rb",
+    "spec/flydata/cli_spec.rb",
     "spec/flydata/command/sender_spec.rb",
     "spec/flydata/command/sync_spec.rb",
     "spec/flydata/fluent-plugins/in_mysql_binlog_flydata_spec.rb",
@@ -90,6 +91,7 @@ Gem::Specification.new do |s|
     "spec/flydata/table_def/mysqldump_test_table_enum.dump",
     "spec/flydata/table_def/mysqldump_test_table_multi_pk.dump",
     "spec/flydata/table_def/mysqldump_test_table_no_pk.dump",
+    "spec/flydata/table_def/mysqldump_test_unsigned.dump",
     "spec/flydata/table_def/redshift_table_def_spec.rb",
     "spec/flydata/util/encryptor_spec.rb",
     "spec/flydata_spec.rb",
@@ -99,7 +101,7 @@ Gem::Specification.new do |s|
   s.homepage = "http://flydata.co/"
   s.licenses = ["All right reserved."]
   s.require_paths = ["lib"]
-  s.rubygems_version = "1.8.23"
+  s.rubygems_version = "1.8.24"
   s.summary = "FlyData CLI"
 
   if s.respond_to? :specification_version then
@@ -115,6 +117,7 @@ Gem::Specification.new do |s|
       s.add_runtime_dependency(%q<ruby-binlog>, [">= 1.0.1"])
       s.add_runtime_dependency(%q<fluent-plugin-mysql-binlog>, ["~> 0.0.2"])
       s.add_runtime_dependency(%q<mysql2>, ["~> 0.3.11"])
+      s.add_runtime_dependency(%q<slop>, [">= 0"])
       s.add_development_dependency(%q<bundler>, [">= 0"])
       s.add_development_dependency(%q<jeweler>, [">= 0"])
       s.add_development_dependency(%q<rspec>, [">= 0"])
@@ -134,6 +137,7 @@ Gem::Specification.new do |s|
       s.add_dependency(%q<ruby-binlog>, [">= 1.0.1"])
       s.add_dependency(%q<fluent-plugin-mysql-binlog>, ["~> 0.0.2"])
       s.add_dependency(%q<mysql2>, ["~> 0.3.11"])
+      s.add_dependency(%q<slop>, [">= 0"])
       s.add_dependency(%q<bundler>, [">= 0"])
       s.add_dependency(%q<jeweler>, [">= 0"])
       s.add_dependency(%q<rspec>, [">= 0"])
@@ -154,6 +158,7 @@ Gem::Specification.new do |s|
     s.add_dependency(%q<ruby-binlog>, [">= 1.0.1"])
     s.add_dependency(%q<fluent-plugin-mysql-binlog>, ["~> 0.0.2"])
     s.add_dependency(%q<mysql2>, ["~> 0.3.11"])
+    s.add_dependency(%q<slop>, [">= 0"])
     s.add_dependency(%q<bundler>, [">= 0"])
     s.add_dependency(%q<jeweler>, [">= 0"])
     s.add_dependency(%q<rspec>, [">= 0"])

data/lib/flydata/cli.rb

@@ -1,3 +1,5 @@
+require 'slop'
+
 module Flydata
   class Cli
     include Helpers
@@ -15,7 +17,11 @@ module Flydata
           first_arg = @args.shift
           cmd, sub_cmd = parse_command(first_arg)
           cmd_cls = "Flydata::Command::#{cmd.capitalize}".constantize
-          cmd_obj = cmd_cls.new
+          # Command class can define options for each subcommand by defining method "slop_subcommandname"
+          slop_method = sub_cmd ? "slop_#{sub_cmd}".to_sym : :slop
+          options = cmd_cls.respond_to?(slop_method) ? cmd_cls.send(slop_method) : Slop.new(strict: true)
+          options.parse!(@args)
+          cmd_obj = cmd_cls.new(options)
           sub_cmd ? cmd_obj.send(sub_cmd,*@args) : cmd_obj.run(*@args)
         else
           raise 'no command given'

data/lib/flydata/command/base.rb

@@ -1,9 +1,11 @@
 module Flydata
   module Command
     class Base
-      def initialize
+      def initialize(options = Slop.new)
         @api_client = ApiClient.instance
+        @opts = options
       end
+      attr_reader :opts
       def flydata; @api_client end
 
       def retrieve_data_entries

data/lib/flydata/command/sender.rb

@@ -1,25 +1,36 @@
 module Flydata
   module Command
     class Sender < Base
-      def start(show_final_message = true)
+      def self.slop_start
+        Slop.new do
+          on 'n', 'no-daemon', 'Start FlyData agent as a regular program'
+        end
+      end
+      def start(options_or_show_final_message = {show_final_message: true}) # For backward compatibility. Use only as options going forward 
+        if options_or_show_final_message.kind_of? Hash
+          options = options_or_show_final_message
+        else
+          options = {show_final_message: options_or_show_final_message}
+        end
         # Check if process exist
         if process_exist?
-          say("Process is still running. Please stop process first.")
+          say("Process is still running. Please stop process first.") unless options[:quiet]
           return
         end
 
-        wait_until_server_ready
+        wait_until_server_ready(options)
 
         # Start sender(fluentd) process
-        say('Starting sender process.')
+        say('Starting sender process.') unless options[:quiet]
         Dir.chdir(FLYDATA_HOME){
-          system("fluentd -d #{FLYDATA_HOME}/flydata.pid -l #{FLYDATA_HOME}/flydata.log -c #{FLYDATA_HOME}/flydata.conf -p #{File.dirname(__FILE__)}/../fluent-plugins")
+          daemon_option = opts.no_daemon? ? "" : "-d #{FLYDATA_HOME}/flydata.pid"
+          Kernel.system("fluentd #{daemon_option} -l #{FLYDATA_HOME}/flydata.log -c #{FLYDATA_HOME}/flydata.conf -p #{File.dirname(__FILE__)}/../fluent-plugins")
         }
-        sleep 5
+        Kernel.sleep 5
 
-        wait_until_client_ready
+        wait_until_client_ready(options)
         #wait_until_logs_uploaded
-        if show_final_message
+        if options[:show_final_message] && !options[:quiet]
           data_port = flydata.data_port.get
           say("Go to your Dashboard! #{flydata.flydata_api_host}/data_ports/#{data_port['id']}")
           say <<EOF
@@ -34,43 +45,43 @@ EOF
         end
 
         say('Stopping sender process.') unless options[:quiet]
-        if system("kill `cat #{FLYDATA_HOME}/flydata.pid`") and wait_until_client_stop
+        if Kernel.system("kill `cat #{FLYDATA_HOME}/flydata.pid`") and wait_until_client_stop(options)
           say('Done.') unless options[:quiet]
           return true
         end
         raise 'Something has gone wrong..'
       end
-      def flush_client_buffer
+      def flush_client_buffer(options = {})
         unless process_exist?
           return true if client_buffer_empty?
-          say("Process doesn't exist. But, the client buffer is not empty!!")
-          start false
+          say("Process doesn't exist. But, the client buffer is not empty!!") unless options[:quiet]
+          start(options)
         end
 
-        say('Stopping input plugins and flushing the client buffer.')
-        system("kill -USR1 `cat #{FLYDATA_HOME}/flydata.pid`")
+        say('Stopping input plugins and flushing the client buffer.') unless options[:quiet]
+        Kernel.system("kill -USR1 `cat #{FLYDATA_HOME}/flydata.pid`")
 
         retry_count = 12
         1.upto(retry_count) do |i|
           return true if client_buffer_empty?
-          say("Waiting for the buffer to get empty... (#{i}/#{retry_count})")
-          sleep 5
+          say("Waiting for the buffer to get empty... (#{i}/#{retry_count})") unless options[:quiet]
+          Kernel.sleep 5
         end
 
         raise 'Something is wrong! Unable to flush client buffer'
       end
-      def restart
+      def restart(options = {})
         if process_exist?
-          say('Restarting sender process.')
-          if system("kill -HUP `cat #{FLYDATA_HOME}/flydata.pid`")
-            say('Done.')
+          say('Restarting sender process.') unless options[:quiet]
+          if Kernel.system("kill -HUP `cat #{FLYDATA_HOME}/flydata.pid`")
+            say('Done.') unless options[:quiet]
             return true
           else
             raise 'Something has gone wrong..'
           end
         else
-          say("Process doesn't exist.")
-          start
+          say("Process doesn't exist.") unless options[:quiet]
+          start(options)
         end
       end
       def status
@@ -80,9 +91,13 @@ EOF
           say("Process is not running.")
         end
       end
-      def kill_all
-        if system("ps ax | grep 'flydata' | grep -v grep | awk '{print \"kill  \" $1}' | sh")
-          say("Done.")
+      def process_exist?
+        # Returns true if the process is running
+        `[ -f #{FLYDATA_HOME}/flydata.pid ] && pgrep -P \`cat #{FLYDATA_HOME}/flydata.pid\``.to_i > 0
+      end
+      def kill_all(optiosn = {})
+        if Kernel.system("ps ax | grep 'flydata' | grep -v grep | awk '{print \"kill  \" $1}' | sh")
+          say("Done.") unless options[:quiet]
           return true
         else
           raise 'Something has gone wrong...'
@@ -90,52 +105,52 @@ EOF
       end
 
       private
-      def wait_until_server_ready
+      def wait_until_server_ready(options = {})
         retry_count = 10
         1.upto(retry_count) do |i|
           return true if server_ready?
-          say("Waiting for the server side to become active... (#{i}/#{retry_count})")
-          sleep 30
+          say("Waiting for the server side to become active... (#{i}/#{retry_count})") unless options[:quiet]
+          Kernel.sleep 30
         end
         false
       end
-      def wait_until_client_ready
+      def wait_until_client_ready(options = {})
         retry_count = 10
         1.upto(retry_count) do |i|
           if client_ready?
-            say("Done! Client is ready now.")
+            say("Done! Client is ready now.") unless options[:quiet]
             return true
           end
           if process_died?
             raise "Client could not been launched. Detail here #{FLYDATA_HOME}/flydata.log"
           end
-          say("Waiting for the client side to become active... (#{i}/#{retry_count})")
-          sleep 10
+          say("Waiting for the client side to become active... (#{i}/#{retry_count})") unless options[:quiet]
+          Kernel.sleep 10
         end
         raise "Somthing has gone wrong... Please try setup command again."
       end
-      def wait_until_client_stop
+      def wait_until_client_stop(options = {})
         retry_count = 5
         1.upto(retry_count) do |i|
           return true unless process_exist?
-          say("Waiting for the client to stop... (#{i}/#{retry_count})")
-          sleep 3
+          say("Waiting for the client to stop... (#{i}/#{retry_count})") unless options[:quiet]
+          Kernel.sleep 3
         end
         false
       end
-      def wait_until_logs_uploaded
-        say('Starting to check the upload from your server.')
+      def wait_until_logs_uploaded(options = {})
+        say('Starting to check the upload from your server.') unless options[:quiet]
         data_port = flydata.data_port.get
         data_port_id = data_port['id']
 
         retry_count = 10
         1.upto(retry_count) do |i|
           if uploaded_successfully?(data_port_id)
-            say("Uploading your logs correctly.")
+            say("Uploading your logs correctly.") unless options[:quiet]
             return true
           end
-          say("Waiting logs uploading... (#{i}/#{retry_count})")
-          sleep 30
+          say("Waiting logs uploading... (#{i}/#{retry_count})") unless options[:quiet]
+          Kernel.sleep 30
         end
         raise 'Cannot confirm that your logs exist on the FlyData server. Something has gone wrong..'
       end
@@ -147,10 +162,6 @@ EOF
       def client_ready?
         process_exist?
       end
-      def process_exist?
-        # Returns true if the process is running
-        `[ -f #{FLYDATA_HOME}/flydata.pid ] && pgrep -P \`cat #{FLYDATA_HOME}/flydata.pid\``.to_i > 0
-      end
       def process_died?
         # Returns true if the process is running
         !!(`tail -n 1 #{FLYDATA_HOME}/flydata.log` =~ /process died within/)
@@ -159,9 +170,9 @@ EOF
         res = flydata.get("/data_ports/#{data_port_id}/tail.json")
         res and res['logs'] and res['logs'].size > 0
       end
-      def client_buffer_empty?
+      def client_buffer_empty?(options = {})
         client_buffer = File.join(FLYDATA_HOME, 'buffer')
-        say("Checking the client buffer #{client_buffer}")
+        say("Checking the client buffer #{client_buffer}") unless options[:quiet]
         Dir.glob("#{client_buffer}/*").empty?
       end
     end

data/lib/flydata/command/setup.rb

@@ -19,40 +19,93 @@ module Flydata
         files.map { |f| File.expand_path(f) }.map { |f| File.directory?(f) ? f + "/" : f }
       end
 
+      ALL_DONE_MESSAGE_TEMPLATE =  <<-EOM
+Congratulations! FlyData has started uploading your data.
+
+To complete your installation and to add the `flydata` command, please run the
+following from the command line.
+
+    $ source ~/.bashrc
+
+What's next?
+
+    - Check data on Redshift (%s)
+    - Check your FlyData usage on the FlyData Dashboard (%s)
+    - To manage the FlyData Agent, use the 'flydata' command (type 'flydata' for
+      help)
+    - If you encounter an issue,
+        please check our documentation (https://www.flydata.com/docs/) or
+        contact our customer support team (support@flydata.com)
+
+Thank you for using FlyData!
+      EOM
+
+      INITIAL_SYNC_MESSAGE_TEMPLATE = <<-EOM
+FlyData Agent has been installed on your server successfully.
+
+
+What's next?
+
+1. Generate a script to create tables on Redshift by running the following command.
+
+    $ flydata sync:generate_table_ddl > create_table.sql
+
+2. Create tables on Redshift by running the 'create_table.sql' script on your Redshift cluster.
+    The script is auto-generated from your MySQL tables.  You can also edit the script to add extra Redshift parameters such as distkey and sortkey.  Once you run the script and create tables in Redshift, you are ready for the next step.  
+
+3. Start Sync
+    Run the following command on your server. The command will start synchronizing data between MySQL and Redshift!
+
+    $ flydata sync
+
+      EOM
+
+      NO_DE_SYNC_CANCEL_MESSAGE_TEMPLATE = <<-EOM
+FlyData Agent has been installed on your server successfully.
+
+
+What's next?
+
+1. Create a data entry from the dashboard (%s)
+
+2. Reinstall FlyData Agent by running the install command on the dashboard
+
+      EOM
+
       def initial_run
-        last_message = nil
-        run do
+        data_port = flydata.data_port.get
+        dashboard_url = "#{flydata.flydata_api_host}/data_ports/#{data_port['id']}"
+        redshift_console_url = "#{flydata.flydata_api_host}/redshift_clusters/query/new"
+        last_message = ALL_DONE_MESSAGE_TEMPLATE % [redshift_console_url, dashboard_url]
+
+        run(quiet: true) do
           Flydata::Command::Conf.new.copy_templates
           puts
-          shown_redshift_entries = (show_registered_redshift_entries)
-          shown_mysql_data_entries = show_registered_redshift_mysql_data_entries
-          if shown_redshift_entries
-            Flydata::Command::Sender.new.stop
+          if has_registered_redshift_entries?
+            Flydata::Command::Sender.new.stop(quiet: true)
             true
-          elsif shown_mysql_data_entries
+          elsif has_registered_redshift_mysql_data_entries?
             de = retrieve_data_entries.first
             if File.exists?(Flydata::FileUtil::SyncFileManager.new(de).binlog_path)
-              Flydata::Command::Sender.new.stop
+              Flydata::Command::Sender.new.stop(quiet: true)
               true
             else
-              last_message =  "\n! NOTE: Initial synchronization of MySQL database is required.\n" +
-                              "  Please run 'flydata sync' to start synchronization."
+              last_message = INITIAL_SYNC_MESSAGE_TEMPLATE
               false
             end
           else
-            _run
+            need_restart = _run
+            last_message = NO_DE_SYNC_CANCEL_MESSAGE_TEMPLATE % [dashboard_url] unless need_restart
+            need_restart
           end
         end
-        puts
-        print_usage
-        puts "Completed setup of FlyData!"
-        puts last_message if last_message
+        puts last_message
       end
 
-      def run(&block)
+      def run(options = {}, &block)
         Flydata::Command::Login.new.run unless flydata.credentials.authenticated?
         ret = block_given? ? yield : _run
-        Flydata::Command::Sender.new.restart if ret
+        Flydata::Command::Sender.new.restart(options) if ret
       end
 
       private
@@ -82,6 +135,10 @@ module Flydata
         end
       end
 
+      def has_registered_redshift_mysql_data_entries?
+        has_registered_entries?('RedshiftMysqlDataEntry')
+      end
+
       #### redshift backup mode
       def start_redshift_mode
         newline
@@ -99,6 +156,10 @@ module Flydata
         end
       end
 
+      def has_registered_redshift_entries?
+        has_registered_entries?('RedshiftFileDataEntry')
+      end
+
       def ask_adding_new_redshift
         puts "Start registration of a new entry:"
         newline
@@ -236,6 +297,12 @@ module Flydata
         end
       end
 
+      def has_registered_entries?(type='FlyDataEntry')
+        data_entries = retrieve_data_entries
+        data_entries = data_entries.select {|de| de['type'] == type}
+        data_entries && data_entries.size > 0
+      end
+
       def choose_log_path_from_examples
         candidates = (`ls #{LOG_PATH_EXAMPLES.join(' ')} 2>/dev/null`).split(/\s+/)
         candidates = candidates.find_all{|path| File.readable?(path)}