flydata 0.1.8 → 0.1.9

data/lib/flydata/command/start.rb

@@ -1,8 +1,11 @@
 module Flydata
   module Command
     class Start < Base
+      def self.slop
+        Flydata::Command::Sender.slop_start # Needs options for Sender#start
+      end
       def run
-        sender = Flydata::Command::Sender.new
+        sender = Flydata::Command::Sender.new(opts)
         sender.start
       end
     end

data/lib/flydata/command/sync.rb

@@ -18,6 +18,17 @@ module Flydata
       STATUS_COMPLETE = 'COMPLETE'
 
       def run(*tables)
+        sender = Flydata::Command::Sender.new
+        if (sender.process_exist?)
+          if tables.empty?
+            # full sync
+            puts "FlyData Agent is already running.  If you'd like to restart FlyData Sync from scratch, run 'flydata sync:reset' first."
+          else
+            # per-table sync
+            puts "Flydata Agent is already running.  If you'd like to Sync the table(s), run 'flydata flush' first."
+          end
+          exit 1
+        end
         de = retrieve_data_entries.first
         raise "There are no data entry." unless de
         case de['type']
@@ -36,6 +47,12 @@ module Flydata
       end
 
       def reset
+        return unless ask_yes_no("This resets the current Sync.  Are you sure?")
+        sender = Flydata::Command::Sender.new
+        sender.flush_client_buffer # TODO We should rather delete buffer files
+        # TODO Reset server when API becomes available
+        sender.stop
+
         de = retrieve_data_entries.first
         sync_fm = Flydata::FileUtil::SyncFileManager.new(de)
         [
@@ -51,16 +68,36 @@ module Flydata
         end
       end
 
+      def wait_for_server_data_processing
+        state = :PROCESS
+        puts "Processing data..."
+        sleep 10
+        status = nil
+        while (status = check)
+          if state == :PROCESS && status['state'] == 'uploading'
+            puts "  -> Done"
+            state = :UPLOAD
+            puts "Uploading data to Redshift..."
+          end
+          print_progress(status)
+          sleep 10
+        end
+        if (state == :PROCESS)
+          # :UPLOAD state was skipped due to no data
+          puts "  -> Done"
+          puts "Uploading data to Redshift..."
+        end
+        puts "  -> Done"
+      end
+
       def check
         de = retrieve_data_entries.first
         retry_on(RestClient::Exception) do
-          ret = do_check(de)
-          if ret['complete']
-            puts "No buffer data on FlyData. #{ret.inspect}"
-            true
+          status = do_check(de)
+          if status['complete']
+            nil
           else
-            puts "Now processing data on FlyData. #{ret.inspect}"
-            false
+            status
           end
         end
       end
@@ -93,6 +130,11 @@ module Flydata
         flydata.data_entry.buffer_stat(de['id'], env_mode)
       end
 
+      def print_progress(buffer_stat)
+        message = buffer_stat['message']
+        puts message unless message.nil? || message.empty?
+      end
+
       DDL_DUMP_CMD_TEMPLATE = "mysqldump --protocol=tcp -d -h %s -P %s -u %s %s %s %s"
       def do_generate_table_ddl(de)
         if `which mysqldump`.empty?
@@ -110,10 +152,11 @@ module Flydata
 
         command = DDL_DUMP_CMD_TEMPLATE % params
 
-        IO.popen(command, 'r') do |io|
+        Open3.popen3(command) do |stdin, stdout, stderr|
+          stdin.close
           create_flydata_ctl_table = mp['initial_sync']
-          while !io.eof?
-            mysql_tabledef = Flydata::TableDef::MysqlTableDef.create(io)
+          while !stdout.eof?
+            mysql_tabledef = Flydata::TableDef::MysqlTableDef.create(stdout)
             if mysql_tabledef.nil?
               # stream had no more create table definition
               break
@@ -122,6 +165,10 @@ module Flydata
             puts Flydata::TableDef::RedshiftTableDef.from_flydata_tabledef(flydata_tabledef, flydata_ctl_table: create_flydata_ctl_table)
             create_flydata_ctl_table = false
           end
+          while !stderr.eof?
+            line = stderr.gets
+            $stderr.print line unless /Warning: Using a password on the command line interface can be insecure./ === line
+          end
         end
       end
 
@@ -131,7 +178,7 @@ module Flydata
 
         # Check client condition
         if File.exists?(sync_fm.binlog_path) and de['mysql_data_entry_preference']['initial_sync']
-          raise "Already synchronized. If you want to do initial sync, delete #{sync_fm.binlog_path}."
+          raise "Already synchronized. If you want to do initial sync, run 'flydata sync:reset'"
         end
 
         # Copy template if not exists
@@ -141,7 +188,7 @@ module Flydata
 
         if generate_mysqldump(de, sync_fm)
           sync_fm.save_sync_info(de['mysql_data_entry_preference']['initial_sync'], de['mysql_data_entry_preference']['tables'])
-          parse_mysqldump(dp, de, sync_fm)
+          parse_mysqldump_and_send(dp, de, sync_fm)
           complete
         end
       end
@@ -156,32 +203,40 @@ module Flydata
           end
         end
 
-        puts "Running mysqldump... host:#{de['mysql_data_entry_preference']['host']} " +
-             "username:#{de['mysql_data_entry_preference']['username']} " +
-             "database:#{de['mysql_data_entry_preference']['database']}"
-        if de['mysql_data_entry_preference']['data_servers']
-          puts "Send to Custom Data Servers: #{de['mysql_data_entry_preference']['data_servers']}"
-        end
-
-        if de['mysql_data_entry_preference']['tables']
-          puts "  target tables: #{de['mysql_data_entry_preference']['tables']}"
-        else
-          puts "  target tables: <all-tables>"
-        end
-
         fp = sync_fm.dump_file_path
         if File.exists?(fp) and File.size(fp) > 0 and not overwrite
           puts "  -> Skip"
           return fp
         end
 
-        puts "[Confirm] mysqldump path: #{fp}"
-        if ask_yes_no('OK?')
+        tables = de['mysql_data_entry_preference']['tables']
+        tables ||= '<all tables>'
+        data_servers = de['mysql_data_entry_preference']['data_servers'] ? "\n  data servers: #{de['mysql_data_entry_preference']['data_servers']}" : ""
+
+        confirmation_text = <<-EOM
+
+FlyData Sync will start synchonizing the following database tables
+  host:         #{de['mysql_data_entry_preference']['host']}
+  port:         #{de['mysql_data_entry_preference']['port']}
+  username:     #{de['mysql_data_entry_preference']['username']}
+  database:     #{de['mysql_data_entry_preference']['database']}
+  tables:       #{tables}#{data_servers}
+  dump file:    #{fp}
+
+Dump file saves contents of your tables temporarily.  Make sure you have enough disk space.
+        EOM
+
+        print confirmation_text
+
+        if ask_yes_no('Start Sync?')
+          puts "Exporting data from database..."
           Flydata::Mysql::MysqlDumpGeneratorNoMasterData.new(de['mysql_data_entry_preference']).dump(fp)
         else
           newline
-          puts "You can change the mysqldump path with 'mysqldump_path' in the conf file."
-          puts "Edit '#{Flydata::Preference::DataEntryPreference.conf_path(de)}'"
+          puts "You can change the dump file path with 'mysqldump_path' property in the following conf file."
+          puts
+          puts "    #{Flydata::Preference::DataEntryPreference.conf_path(de)}"
+          puts
           return nil
         end
         puts "  -> Done"
@@ -208,8 +263,8 @@ module Flydata
       #   <- checkpoint
       #...
       #CREATE TABLE ...
-      def parse_mysqldump(dp, de, sync_fm)
-        puts "Parsing mysqldump file..."
+      def parse_mysqldump_and_send(dp, de, sync_fm)
+        puts "Sending data to FlyData Server..."
 
         # Prepare forwarder
         de_tag_name = de["tag_name#{env_suffix}"]
@@ -250,15 +305,13 @@ module Flydata
               ret = flydata.redshift_cluster.run_query(sql)
               if ret['message'].index('ERROR:')
                 if ret['message'].index('already exists')
-                  puts " -> Skip"
+                  puts "  -> Skip"
                 else
                   raise "Failed to create table. error=#{ret['message']}"
                 end
               else
-                puts " -> OK"
+                puts "  -> OK"
               end
-            else
-              puts "- Parsing table: #{mysql_table.table_name}"
             end
 
             # dump mysql_table for resume
@@ -273,19 +326,17 @@ module Flydata
             end
             ret = forwarder.emit(records)
             tmp_num_inserted_record += 1
-            print '.'
             ret
           },
           # checkpoint
           Proc.new { |mysql_table, last_pos, binlog_pos, state, substate|
             # flush if buffer records exist
             if tmp_num_inserted_record > 0 && forwarder.buffer_record_count > 0
-              puts
               forwarder.flush # send buffer data to the server before checkpoint
             end
 
             # show the current progress
-            puts "     #{(last_pos.to_f/dump_file_size * 100).round(1)}% (#{last_pos}/#{dump_file_size}) #{Time.now.to_i - bench_start_time.to_i}sec"
+            puts "  -> #{(last_pos.to_f/dump_file_size * 100).round(1)}% (#{last_pos}/#{dump_file_size}) completed..."
 
             # save check point
             table_name = mysql_table.nil? ? '' : mysql_table.table_name
@@ -294,37 +345,57 @@ module Flydata
         )
         forwarder.close
 
+        puts "  -> Done"
+
         if ENV['FLYDATA_BENCHMARK']
-          puts "Done!"
           bench_end_time = Time.now
           elapsed_time = bench_end_time.to_i - bench_start_time.to_i
           puts "Elapsed:#{elapsed_time}sec start:#{bench_start_time} end:#{bench_end_time}"
           return true
         end
         # wait until finish
-        puts "Start waiting until all data is processed on FlyData..."
-        sleep 10
-        until check
-          sleep 10
-        end
+        wait_for_server_data_processing
 
         sync_fm.save_dump_pos(STATUS_COMPLETE, '', dump_file_size, binlog_pos)
         tables = de['mysql_data_entry_preference']['tables'].split(',').join(' ')
         sync_fm.save_table_binlog_pos(tables, binlog_pos)
       end
 
+      ALL_DONE_MESSAGE_TEMPLATE =  <<-EOM
+
+Congratulations! FlyData has started synchronizing your database tables.
+
+What's next?
+
+    - Check data on Redshift (%s)
+    - Check your FlyData usage on the FlyData Dashboard (%s)
+    - To manage the FlyData Agent, use the 'flydata' command (type 'flydata' for help)
+    - If you encounter an issue,
+        please check our documentation (https://www.flydata.com/docs/) or
+        contact our customer support team (support@flydata.com)
+
+Thank you for using FlyData!
+      EOM
       def complete
         de = load_sync_info(retrieve_data_entries.first)
         sync_fm = Flydata::FileUtil::SyncFileManager.new(de)
         info = sync_fm.load_dump_pos
         if info[:status] == STATUS_COMPLETE
+          puts "Starting FlyData Agent..."
           if de['mysql_data_entry_preference']['initial_sync']
             sync_fm.save_binlog(info[:binlog_pos])
           end
           sync_fm.move_table_binlog_files(de['mysql_data_entry_preference']['tables'].split(','))
           sync_fm.reset_table_position_files(de['mysql_data_entry_preference']['tables'].split(','))
           sync_fm.backup_dump_dir
-          Flydata::Command::Sender.new.start
+          Flydata::Command::Sender.new.start(quiet: true)
+          puts "  -> Done"
+
+          data_port = flydata.data_port.get
+          dashboard_url = "#{flydata.flydata_api_host}/data_ports/#{data_port['id']}"
+          redshift_console_url = "#{flydata.flydata_api_host}/redshift_clusters/query/new"
+          last_message = ALL_DONE_MESSAGE_TEMPLATE % [redshift_console_url, dashboard_url]
+          puts last_message
         else
           raise "Initial sync status is not complete. Try running 'flydata sync'."
         end
@@ -359,10 +430,7 @@ module Flydata
       def flush_buffer_and_stop
         sender = Flydata::Command::Sender.new
         sender.flush_client_buffer
-        puts "Checking the server."
-        until check
-          sleep 10
-        end
+        wait_for_server_data_processing
         sender.stop(quiet: true)
       end
     end
@@ -374,10 +442,10 @@ module Flydata
       def self.create(forwarder_key, tag, servers, options = {})
         case forwarder_key
         when nil, "tcpforwarder"
-          puts "Creating TCP connection"
+          puts "Creating TCP connection" if FLYDATA_DEBUG
           forward = TcpForwarder.new(tag, servers, options)
         when "sslforwarder"
-          puts "Creating SSL connection"
+          puts "Creating SSL connection" if FLYDATA_DEBUG
           forward = SslForwarder.new(tag, servers, options)
         else
           raise "Unsupported Forwarding type #{forwarder_key}"
@@ -432,7 +500,6 @@ module Flydata
       #TODO retry logic
       def send
         if @buffer_size > 0
-          puts "  -> Sending #{@buffer_record_count}records #{@buffer_size}byte"
         else
           return false
         end
@@ -1139,12 +1206,12 @@ EOS
               else
                 if chars.end_with?(')')
                   chars = chars[0..-2]
-                  @values << (chars == 'NULL' ? nil : chars)
+                  @values << (chars == 'NULL' ? nil : remove_leading_zeros(chars))
                   @values_set << @values
                   @values = []
                   cur_state = State::NEXT_VALUES
                 else
-                  @values << (chars == 'NULL' ? nil : chars)
+                  @values << (chars == 'NULL' ? nil : remove_leading_zeros(chars))
                 end
               end
             else
@@ -1154,10 +1221,10 @@ EOS
           return @values_set
         end
 
-        ESCAPE_HASH_TABLE = {"\\\\" => "\\", "\\'" => "'", "\\n" => "\n",  "\\r" => "\r"}
+        ESCAPE_HASH_TABLE = {"\\\\" => "\\", "\\'" => "'", "\\\"" => "\"", "\\n" => "\n", "\\r" => "\r"}
 
         def replace_escape_char(original)
-          original.gsub(/\\\\|\\'|\\n|\\r/, ESCAPE_HASH_TABLE)
+          original.gsub(/\\\\|\\'|\\"|\\n|\\r/, ESCAPE_HASH_TABLE)
         end
 
         # This method assume that the last character is '(single quotation)
@@ -1175,6 +1242,14 @@ EOS
           end
           flag
         end
+
+        def remove_leading_zeros(number_string)
+          if number_string.start_with?('0')
+            number_string.sub(/^0*([1-9][0-9]*(\.\d*)?|0(\.\d*)?)$/,'\1')
+          else
+            number_string
+          end
+        end
       end
     end
   end

data/lib/flydata/helpers.rb

@@ -13,8 +13,14 @@ module Flydata
     end
 
     def print_usage
-      puts '-' * 64
-      puts <<-EOM
+      puts usage_text
+    end
+
+    def usage_text
+      text = ""
+      text += '-' * 64
+      text += "\n"
+      text += <<-EOM
 Usage: flydata COMMAND
   setup                    # setup initially
   start                    # start flydata process
@@ -32,7 +38,10 @@ please setup flydata again by following commands.
 You can check the logs of sender(flydata) process.
    Log path: #{File.join(FLYDATA_HOME, 'flydata.log')}
       EOM
-      puts '-' * 64
+      text += "\n"
+      text += '-' * 64
+      text += "\n"
+      text
     end
 
     def to_command_class(class_name)

data/lib/flydata/table_def/mysql_table_def.rb

@@ -5,7 +5,7 @@ class MysqlTableDef
   TYPE_MAP_M2F = {
     'bigint' => 'int8',
     'binary' => 'binary',
-    'blob' => 'varbinary',
+    'blob' => 'varbinary(65535)',
     'bool' => 'int1',
     'boolean' => 'int1',
     'char' => 'varchar',
@@ -20,9 +20,9 @@ class MysqlTableDef
     'float' => 'float4',
     'int' => 'int4',
     'integer' => 'int4',
-    'longblob' => 'varbinary',
+    'longblob' => 'varbinary(4294967295)',
     'longtext' => 'text',
-    'mediumblob' => 'varbinary',
+    'mediumblob' => 'varbinary(16777215)',
     'mediumint' => 'int3',
     'mediumtext' => 'text',
     'numeric' => 'numeric',
@@ -30,7 +30,7 @@ class MysqlTableDef
     'text' => 'text',
     'time' => 'time',
     'timestamp' => 'datetime',
-    'tinyblob' => 'varbinary',
+    'tinyblob' => 'varbinary(255)',
     'tinyint' => 'int1',
     'tinytext' => 'text',
     'varbinary' => 'varbinary',
@@ -113,6 +113,8 @@ class MysqlTableDef
           # index creation.  No action required.
         elsif stripped_line.start_with?("CONSTRAINT")
           # constraint definition.  No acction required.
+        elsif stripped_line.start_with?("UNIQUE KEY")
+          # constraint definition.  No acction required.
         else
           $stderr.puts "Unknown table definition. Skip. (#{line})"
         end
@@ -174,6 +176,7 @@ class MysqlTableDef
 
         cond = :options
       when :options
+        column[:type] += ' unsigned' if line =~ /unsigned/i
         column[:auto_increment] = true if line =~ /AUTO_INCREMENT/i
         column[:not_null] = true if line =~ /NOT NULL/i
         if /DEFAULT\s+((?:[^'\s]+\b)|(?:'(?:\\'|[^'])*'))/i.match(line)