flydata 0.6.3 → 0.6.4

data/lib/flydata/command/sender.rb

@@ -9,6 +9,7 @@ module Flydata
         Slop.new do
           on 'n', 'no-daemon', 'Start FlyData agent as a regular program'
           on 'e', 'no-email', 'Skip sending init-sync-start notification email'
+          on 'force-run', 'Run forcefully, ignoring exclusive run info'
         end
       end
       def start(options_or_show_final_message = {show_final_message: true}) # For backward compatibility. Use only as options going forward
@@ -56,10 +57,10 @@ module Flydata
         end
 
         quiet_option = options[:quiet]
-        # surpress messages if fluentd is started in #try_mysql_sync
+        # surpress messages if fluentd is started in #try_initial_sync
         options[:quiet] = true
-        Flydata::Command::Sync.new.try_mysql_sync(
-          binlog_ready_callback: start_fluentd,
+        Flydata::Command::Sync.new.try_initial_sync(
+          source_pos_ready_callback: start_fluentd,
           no_email: opts.no_email?)
         options[:quiet] = quiet_option
         start_fluentd.call unless fluentd_started

data/lib/flydata/command/setup.rb

@@ -3,27 +3,10 @@ require 'flydata/command/conf'
 require 'flydata/command/login'
 require 'flydata/command/sender'
 require 'flydata/command/helper'
-require 'flydata/helpers'
-require 'flydata/sync_file_manager'
 
 module Flydata
   module Command
     class Setup < Base
-      include Helpers
-      LOG_PATH_EXAMPLES=
-        %w(/var/log/httpd/access_log /var/log/apache2/access.log
-           /var/log/httpd-access.log /var/log/apache2/access_log
-           /var/log/messages /var/log/maillog /var/log/mysql/error.log
-           /home/*/deploy/shared/log/*.log)
-      OTHER = '-- None of above --'
-      ENTER_TABLE_NAME = '-- Create a table on Redshift from your logs --'
-
-      # readline settings for asking log path
-      Readline.completion_append_character = nil
-      Readline.completion_proc = lambda do |prefix|
-        files = Dir["#{prefix}*"]
-        files.map { |f| File.expand_path(f) }.map { |f| File.directory?(f) ? f + "/" : f }
-      end
 
       ALL_DONE_MESSAGE_TEMPLATE =  <<-EOM
 Congratulations! FlyData has started uploading your data.
@@ -63,7 +46,7 @@ What's next?
     $ flydata sync:generate_table_ddl > create_table.sql
 
 2. Create tables on Redshift by running the 'create_table.sql' script on your Redshift cluster.
-    The script is auto-generated from your MySQL tables.  You can also edit the script to add extra Redshift parameters such as distkey and sortkey.  Once you run the script and create tables in Redshift, you are ready for the next step.  
+    The script is auto-generated from your MySQL tables.  You can also edit the script to add extra Redshift parameters such as distkey and sortkey.  Once you run the script and create tables in Redshift, you are ready for the next step.
 
 3. Start Sync
     Run the following command on your server. The command will start synchronizing data between MySQL and Redshift!
@@ -85,386 +68,52 @@ What's next?
       EOM
 
       def initial_run
-        data_port = flydata.data_port.get
-        dashboard_url = "#{flydata.flydata_api_host}/dashboard"
-        redshift_console_url = "#{flydata.flydata_api_host}/redshift_clusters/query/new"
-        last_message = ALL_DONE_MESSAGE_TEMPLATE % [redshift_console_url, dashboard_url]
-
         # Surprisingly, Helper's start method kills the process after starting
         # the helper thanks to ServerEngine.  So the command needs to run in a
         # separate process.
         fork { Flydata::Command::Helper.new.restart(quiet: true) }
-        run(quiet: true) do
-          Flydata::Command::Conf.new.copy_templates
-          puts
-          if has_registered_redshift_entries?
-            Flydata::Command::Sender.new.stop(quiet: true)
-            true
-          elsif has_registered_redshift_mysql_data_entries?
-            de = retrieve_data_entries.first
-            if File.exists?(Flydata::SyncFileManager.new(de).binlog_path)
-              sender = Flydata::Command::Sender.new
-              if sender.process_exist?
-                sender.stop(quiet: true)
-                true
+        Flydata::Command::Login.new.run unless flydata.credentials.authenticated?
+        Flydata::Command::Conf.new.copy_templates
+        puts
+        de_exists = !!retrieve_data_entries.first
+        ret = if de_exists && source.setup.initial_run_need_restart?
+                sender = Flydata::Command::Sender.new
+                if sender.process_exist?
+                  sender.stop(quiet: true)
+                  true
+                else
+                  false
+                end
               else
                 false
               end
-            else
-              last_message = INITIAL_SYNC_MESSAGE_TEMPLATE
-              false
-            end
-          else
-            last_message = NO_DE_CANCEL_MESSAGE_TEMPLATE % [dashboard_url]
-            false
-          end
+        Flydata::Command::Sender.new.restart(quiet: true) if ret
+        last_message = de_exists ? source.setup.initial_run_complete_message
+                                  : :no_de_cancel
+        msgs = standard_messages
+        if msgs.has_key?(last_message)
+          last_message = msgs[last_message]
         end
         puts last_message
       end
 
       def run(options = {}, &block)
-        Flydata::Command::Login.new.run unless flydata.credentials.authenticated?
-        ret = block_given? ? yield : _run
-        Flydata::Command::Sender.new.restart(options) if ret
-      end
-
-      private
-      def _run
-        #ask redshift
-        case ask_data_entry_type
-        when :redshift
-          start_redshift_mode
-        when :s3backup
-          start_s3_backup_mode
-        when :restart_flydata
-          true
-        else
-          false
-        end
-      end
-      def ask_data_entry_type
-        choose_one('Choose an option', nil,
-                   ["Setup Redshift and S3 Backup", "Setup S3 Backup only", "Restart FlyData", "Cancel"],
-                   [:redshift, :s3backup, :restart_flydata, :cancel])
-      end
-
-      #### flydata-sync(RedshiftMysqlDataEntry)
-      def show_registered_redshift_mysql_data_entries
-        show_registered_entries('RedshiftMysqlDataEntry') do |de|
-          say(" - #{de['display_name']}: flydata-sync (mysql -> redshift)")
-        end
-      end
-
-      def has_registered_redshift_mysql_data_entries?
-        has_registered_entries?('RedshiftMysqlDataEntry')
-      end
-
-      #### redshift backup mode
-      def start_redshift_mode
-        newline
-        show_registered_redshift_entries
-        newline
-        ask_adding_new_redshift
-      end
-
-      def show_registered_redshift_entries
-        show_registered_entries('RedshiftFileDataEntry') do |de|
-          sn = de['redshift_schema_name']
-          tn = de['redshift_table_name']
-          table_name = sn.to_s.empty? ? tn : "#{sn}.#{tn}"
-          say(" - #{de['display_name']}: #{de['log_path']} -> #{table_name} (#{table_name}_dev)")
+        unless data_entry
+          raise "No data entry exists.  Please create one from FlyData Console (#{dashboard_url})"
         end
+        # 'flydata setup' command is deprecated
+        raise "You can set up FlyData on FlyData Console (#{dashboard_url})"
       end
 
-      def has_registered_redshift_entries?
-        has_registered_entries?('RedshiftFileDataEntry')
-      end
-
-      def ask_adding_new_redshift
-        puts "Start registration of a new entry:"
-        newline
-        # select table on redshift
-        puts "[Select a Table from your Redshift cluster]"
-        table_name = ask_redshift_table_name
-        return unless table_name
-        newline
-        # enter the log path
-        puts "[Enter a Local Log Path]"
-        puts "Enter the absolute path of your local log that you want to  upload to the '#{table_name} (#{table_name}_dev)' table on Redshift."
-        log_path = ask_log_path("Log path (tab:completion, return:cancel) >> ")
-        return unless log_path and not log_path.empty?
-        newline
-        # select the log type
-        puts "[Select a Log Format]"
-        log_file_type = choose_log_file_type(log_path)
-        # confirm and save
-        newline
-        puts "[Confirm]"
-        separator
-        puts " table(redshift) -> #{table_name} (#{table_name}_dev)"
-        puts " local log path  -> #{log_path}"
-        puts " log file format -> #{log_file_type}"
-        separator
-        return unless ask_yes_no("Are you sure you want to register this new entry?", true)
-        create_redshift_log_entry(log_path, log_file_type, table_name)
-      end
-
-      def choose_log_file_type(target_path)
-        choose_one("Please select the log file format of (#{target_path})", nil,
-                   %w(csv tsv json apache_access_log))
-      end
-
-      def ask_redshift_table_name
-        ret = flydata.data_port.redshift_table_list
-
-        all_tables = ret['table_list'].collect {|tn| tn.gsub(/_dev$/, '')}.uniq
-        prod_tables = ret['table_list'].select {|tn| not tn.end_with?('_dev')}
-        dev_tables = ret['table_list'].select {|tn| tn.end_with?('_dev')}
-
-        if all_tables.size < 1
-          return ask_enter_table_name
-        end
-
-        if development?
-          menu_list = all_tables.collect do |tn|
-            menu = ["#{tn}", "( #{tn}_dev )"]
-            if dev_tables.index("#{tn}_dev").nil?
-              menu << "!WARN The '#{tn}_dev' table does not exist on your Redshift cluster"
-            else
-              menu << "OK"
-            end
-            menu
-          end
-          menu_str_list = format_menu_list(menu_list)
-          message = " !DEVELOPMENT MODE!  FlyData will only upload to tables with '_dev' suffix in the name."
-        else
-          menu_list = all_tables.collect do |tn|
-            menu = ["#{tn}", "( #{tn}_dev )"]
-            if prod_tables.index("#{tn}").nil?
-              menu << "!WARN '#{tn}' table does not exist"
-            else
-              menu << "OK"
-            end
-            menu
-          end
-          menu_str_list = format_menu_list(menu_list)
-          message = " !PRODUCTION MODE!"
-        end
-        menu_str_list << ENTER_TABLE_NAME
-        menu_str = choose_one("Please select the table on Redshift that you want to use to store your local data. \n#{message}", nil, menu_str_list)
-        if menu_str == ENTER_TABLE_NAME
-          ask_enter_table_name
-        elsif menu_str
-          menu_str.split.first
-        else
-          nil
-        end
-      end
-
-      def create_redshift_log_entry(log_path, log_file_type, table_name)
-        if table_name.include?(".")
-          schema_name, table_name = table_name.split(".")
-        end
-        data_port = flydata.data_port.get
-        flydata.data_entry.create(
-          data_port_id: data_port['id'],
-          log_path: log_path,
-          log_file_type: log_file_type,
-          redshift_table_name: table_name,
-          redshift_schema_name: schema_name,
-        )
-        say("Process added successfuly!")
-        return true
-      end
-
-      #### s3 backup mode
-      def start_s3_backup_mode
-        # choose entries
-        unless (shown = show_registered_entries)
-          list = build_recommended_entries
-          print_recommended(list)
-          register_all(list) if ask_yes_no("Register all of these common entries?")
-        end
-        unless (shown ||= show_registered_entries) and not more_entry?
-          begin
-            show_registered_entries unless shown
-            shown = false
-            choose_log_selection(list)
-            newline
-          end while more_entry?
-        end
-        return true
-      end
-
-      def show_registered_entries(type='FileDataEntry', &block)
-        data_entries = retrieve_data_entries
-        @last_fetched_entries = data_entries
-        data_entries = data_entries.select {|de| de['type'] == type}
-        if data_entries and data_entries.size > 0
-          puts('Registered entries on FlyData: ')
-          separator
-          data_entries.each { |data_entry|
-            if block_given?
-              yield data_entry
-            else
-              say(" - #{data_entry['display_name']}\t#{data_entry['log_path']}")
-            end
-          }
-          separator
-          true
-        else
-          false
-        end
-      end
-
-      def has_registered_entries?(type='FlyDataEntry')
-        data_entries = retrieve_data_entries
-        data_entries = data_entries.select {|de| de['type'] == type}
-        data_entries && data_entries.size > 0
-      end
-
-      def choose_log_path_from_examples
-        candidates = (`ls #{LOG_PATH_EXAMPLES.join(' ')} 2>/dev/null`).split(/\s+/)
-        candidates = candidates.find_all{|path| File.readable?(path)}
-        if @last_fetched_entries
-           candidates = candidates - @last_fetched_entries.map{|v| v['log_path']}
-        end
-        return OTHER unless candidates.size > 0
-        candidates << OTHER
-        choice = nil
-        say('Please select your log path for sending FlyData')
-        newline
-        choose do |menu|
-          menu.index = :letter
-          menu.index_suffix = ") "
-          menu.prompt = "Your log path:  "
-          menu.choices(*candidates) {|item| choice = item}
-        end
-        newline
-        choice
-      end
-
-      def ask_and_create_data_entry
-        path = ask_log_path
-        create_log_entry(path)
-      end
-
-      def build_recommended_entries
-        path_options=[]
-        Dir['/var/log*/**/*log'].each{|f| if (FileTest.file?(f) and
-                                        FileTest.readable?(f) and
-                                        ( f =~ /apache2|httpd|syslog|mail|auth/)) and
-                                        !(@last_fetched_entries and @last_fetched_entries.any?{|e| e['log_path'] == f})
-                                        then path_options << f end}
-        path_options
-      end
-
-      def print_recommended(list, options=false)
-        newline
-        puts " Recommended list:"
-        list.each_with_index { |value, index|
-          puts "   #{index+1})   #{value} "
-        }
-        if options
-          puts("   #{list.length+1})   Enter your own path")
-        end
-        newline
-      end
+      private
 
-      def register_all(list)
-        list.each{ |item|
-          create_log_entry(item)
+      def standard_messages
+        {
+          all_done: ALL_DONE_MESSAGE_TEMPLATE %
+                                       [redshift_console_url, dashboard_url],
+          initial_sync: INITIAL_SYNC_MESSAGE_TEMPLATE,
+          no_de_cancel: NO_DE_CANCEL_MESSAGE_TEMPLATE % [dashboard_url],
         }
-        list.reject!{|x| x}
-      end
-
-      def choose_log_selection(list)
-        path = nil
-        list = build_recommended_entries unless list
-        path_options = list
-
-        loop do
-          if path_options.empty?
-            ask_and_create_data_entry
-            return
-          end
-          print_recommended(path_options, true)
-          choice = Readline.readline("Here are some common logs, enter the number next to the logs to add the log. Input nothing to cancel.")
-          return if choice.empty?
-          if choice.to_i==path_options.length+1
-            ask_and_create_data_entry
-            return
-          elsif (path_options[choice.to_i-1] != nil )
-            path = path_options[choice.to_i-1]
-            path_options.delete_at(choice.to_i-1)
-            break
-          else
-            puts("Not a valid entry, please try again"); 
-          end
-        end
-        create_log_entry(path)
-      end
-
-      def ask_log_deletion(path)
-        unless File.writable?(path)
-          say("Skip log deletion setting...")
-          say("  This path is readonly for current user.")
-          say("  Change user or permission, if you want to set log deletion option.")
-          newline
-          return
-        end
-        say("** Log deletion setting **")
-        say("Flydata has a log deletion feature that flydata will delete old log archives uploaded by flydata automatically.")
-        say("Flydata will delete logs whose last modified timestamp is 7 days ago.")
-        say("For more details - http://docs.hapyrus.com/faq/how-log-deletion-works/")
-        ask_yes_no("Set auto log deletion mode?")
-      end
-
-      def create_log_entry(path, log_deletion=false)
-        data_port = flydata.data_port.get
-        flydata.data_entry.create(data_port_id: data_port['id'], log_path: path, log_deletion: log_deletion)
-        say("Process added successfuly!") if flydata.response.code == 200
-      end
-
-      def more_entry?
-        ask_yes_no("Do you want to add another log path?")
-      end
-
-      def ask_enter_table_name
-        input = nil
-        loop do
-          say("Enter a table name for Redshift to store your logs. This table will be created automatically.")
-          say("Input format: [table-name] or [schema-name].[table-name]")
-          say(">> ")
-          input = gets.strip
-          if input =~ /^([a-zA-Z0-9_][a-zA-Z0-9_$]*\.)?[a-zA-Z0-9_][a-zA-Z0-9_$]*$/
-            break
-          else
-            puts "!Please enter the valid table name."
-          end
-        end
-        if development?
-          input = input.gsub(/_dev$/, '')
-        end
-        input
-      end
-
-      def ask_log_path(message = nil)
-        path = nil
-        message = "Enter the absolute path of your log (return to cancel):  " unless message
-        loop do
-          path = Readline.readline(message)
-          return if path.empty?
-          if not (FileTest.file?(path) and FileTest.readable?(path))
-            say(" ! #{path} is not a readable file!")
-          elsif @last_fetched_entries and @last_fetched_entries.any?{|e| e['log_path'] == path}
-            say(" ! #{path} has been registered already.")
-          else
-            break
-          end
-          newline
-        end
-        path
       end
     end
   end