flydata 0.1.7 → 0.1.8

data/VERSION

@@ -1 +1 @@
-0.1.7
+0.1.8

data/flydata.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "flydata"
-  s.version = "0.1.7"
+  s.version = "0.1.8"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Koichi Fujikawa"]
-  s.date = "2014-06-01"
+  s.date = "2014-06-23"
   s.description = "FlyData Command Line Interface"
   s.email = "sysadmin@flydata.co"
   s.executables = ["fdmysqldump", "flydata"]
@@ -50,6 +50,15 @@ Gem::Specification.new do |s|
     "lib/flydata/credentials.rb",
     "lib/flydata/cron.rb",
     "lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb",
+    "lib/flydata/fluent-plugins/mysql/alter_table_query_handler.rb",
+    "lib/flydata/fluent-plugins/mysql/binlog_position.rb",
+    "lib/flydata/fluent-plugins/mysql/binlog_query_dispatcher.rb",
+    "lib/flydata/fluent-plugins/mysql/binlog_query_handler.rb",
+    "lib/flydata/fluent-plugins/mysql/binlog_record_dispatcher.rb",
+    "lib/flydata/fluent-plugins/mysql/binlog_record_handler.rb",
+    "lib/flydata/fluent-plugins/mysql/context.rb",
+    "lib/flydata/fluent-plugins/mysql/dml_record_handler.rb",
+    "lib/flydata/fluent-plugins/mysql/query_parser.rb",
     "lib/flydata/fluent-plugins/out_forward_ssl.rb",
     "lib/flydata/fluent-plugins/preference.rb",
     "lib/flydata/flydata_crontab.sh",
@@ -71,6 +80,8 @@ Gem::Specification.new do |s|
     "spec/flydata/command/sender_spec.rb",
     "spec/flydata/command/sync_spec.rb",
     "spec/flydata/fluent-plugins/in_mysql_binlog_flydata_spec.rb",
+    "spec/flydata/fluent-plugins/mysql/binlog_position_spec.rb",
+    "spec/flydata/fluent-plugins/mysql/query_parser_spec.rb",
     "spec/flydata/heroku_spec.rb",
     "spec/flydata/table_def/mysql_table_def_spec.rb",
     "spec/flydata/table_def/mysqldump_test_foreign_key.dump",
@@ -79,6 +90,7 @@ Gem::Specification.new do |s|
     "spec/flydata/table_def/mysqldump_test_table_enum.dump",
     "spec/flydata/table_def/mysqldump_test_table_multi_pk.dump",
     "spec/flydata/table_def/mysqldump_test_table_no_pk.dump",
+    "spec/flydata/table_def/redshift_table_def_spec.rb",
     "spec/flydata/util/encryptor_spec.rb",
     "spec/flydata_spec.rb",
     "spec/spec_helper.rb",
@@ -87,7 +99,7 @@ Gem::Specification.new do |s|
   s.homepage = "http://flydata.co/"
   s.licenses = ["All right reserved."]
   s.require_paths = ["lib"]
-  s.rubygems_version = "1.8.24"
+  s.rubygems_version = "1.8.23"
   s.summary = "FlyData CLI"
 
   if s.respond_to? :specification_version then

data/lib/flydata/cli.rb

@@ -16,7 +16,7 @@ module Flydata
           cmd, sub_cmd = parse_command(first_arg)
           cmd_cls = "Flydata::Command::#{cmd.capitalize}".constantize
           cmd_obj = cmd_cls.new
-          sub_cmd ? cmd_obj.send(sub_cmd) : cmd_obj.run
+          sub_cmd ? cmd_obj.send(sub_cmd,*@args) : cmd_obj.run(*@args)
         else
           raise 'no command given'
         end

data/lib/flydata/command/sender.rb

@@ -1,7 +1,7 @@
 module Flydata
   module Command
     class Sender < Base
-      def start
+      def start(show_final_message = true)
         # Check if process exist
         if process_exist?
           say("Process is still running. Please stop process first.")
@@ -19,26 +19,46 @@ module Flydata
 
         wait_until_client_ready
         #wait_until_logs_uploaded
-
-        data_port = flydata.data_port.get
-        say("Go to your Dashboard! #{flydata.flydata_api_host}/data_ports/#{data_port['id']}")
-        say <<EOF
+        if show_final_message
+          data_port = flydata.data_port.get
+          say("Go to your Dashboard! #{flydata.flydata_api_host}/data_ports/#{data_port['id']}")
+          say <<EOF
 Please Note: Records and Total Size are updated every 10-20 minutes.
 EOF
+        end
       end
-      def stop
+      def stop(options = {})
         unless process_exist?
-          say("Process doesn't exist.")
+          say("Process doesn't exist.") unless options[:quiet]
           return true
         end
 
-        say('Stopping sender process.')
+        say('Stopping sender process.') unless options[:quiet]
         if system("kill `cat #{FLYDATA_HOME}/flydata.pid`") and wait_until_client_stop
-          say('Done.')
+          say('Done.') unless options[:quiet]
           return true
         end
         raise 'Something has gone wrong..'
       end
+      def flush_client_buffer
+        unless process_exist?
+          return true if client_buffer_empty?
+          say("Process doesn't exist. But, the client buffer is not empty!!")
+          start false
+        end
+
+        say('Stopping input plugins and flushing the client buffer.')
+        system("kill -USR1 `cat #{FLYDATA_HOME}/flydata.pid`")
+
+        retry_count = 12
+        1.upto(retry_count) do |i|
+          return true if client_buffer_empty?
+          say("Waiting for the buffer to get empty... (#{i}/#{retry_count})")
+          sleep 5
+        end
+
+        raise 'Something is wrong! Unable to flush client buffer'
+      end
       def restart
         if process_exist?
           say('Restarting sender process.')
@@ -98,7 +118,7 @@ EOF
         retry_count = 5
         1.upto(retry_count) do |i|
           return true unless process_exist?
-          say("Waiting for the client stopping... (#{i}/#{retry_count})")
+          say("Waiting for the client to stop... (#{i}/#{retry_count})")
           sleep 3
         end
         false
@@ -139,6 +159,11 @@ EOF
         res = flydata.get("/data_ports/#{data_port_id}/tail.json")
         res and res['logs'] and res['logs'].size > 0
       end
+      def client_buffer_empty?
+        client_buffer = File.join(FLYDATA_HOME, 'buffer')
+        say("Checking the client buffer #{client_buffer}")
+        Dir.glob("#{client_buffer}/*").empty?
+      end
     end
   end
 end

data/lib/flydata/command/sync.rb

@@ -1,5 +1,7 @@
+require 'fiber'
 require 'msgpack'
 require 'open3'
+require 'mysql2'
 require 'flydata/sync_file_manager'
 require 'flydata/table_def'
 #require 'ruby-prof'
@@ -15,21 +17,36 @@ module Flydata
       STATUS_PARSING = 'PARSING'
       STATUS_COMPLETE = 'COMPLETE'
 
-      def run
+      def run(*tables)
         de = retrieve_data_entries.first
         raise "There are no data entry." unless de
         case de['type']
         when 'RedshiftMysqlDataEntry'
+          de = load_sync_info(override_tables(de, tables))
+          flush_buffer_and_stop  unless de['mysql_data_entry_preference']['initial_sync']
           sync_mysql_to_redshift(de)
         else
           raise "No supported data entry. Only mysql-redshift sync is supported."
         end
       end
 
+      def flush
+        flush_buffer_and_stop
+        puts "Buffers have been flushed and the sender process has been stopped."
+      end
+
       def reset
         de = retrieve_data_entries.first
         sync_fm = Flydata::FileUtil::SyncFileManager.new(de)
-        [sync_fm.dump_file_path, sync_fm.dump_pos_path, sync_fm.binlog_path, sync_fm.mysql_table_marshal_dump_path, sync_fm.table_position_file_paths].flatten.each do |path|
+        [
+          sync_fm.dump_file_path,
+          sync_fm.dump_pos_path,
+          sync_fm.binlog_path,
+          sync_fm.mysql_table_marshal_dump_path,
+          sync_fm.table_position_file_paths,
+          sync_fm.sync_info_file,
+          sync_fm.table_rev_file_paths,
+        ].flatten.each do |path|
           FileUtils.rm(path) if File.exists?(path)
         end
       end
@@ -48,18 +65,6 @@ module Flydata
         end
       end
 
-      def complete
-        de = retrieve_data_entries.first
-        sync_fm = Flydata::FileUtil::SyncFileManager.new(de)
-        info = sync_fm.load_dump_pos
-        if info[:status] == STATUS_COMPLETE
-          sync_fm.save_binlog(info[:binlog_pos])
-          Flydata::Command::Sender.new.start
-        else
-          raise "Initial sync status is not complete. Try running 'flydata sync'."
-        end
-      end
-
       # skip initial sync
       def skip
         de = retrieve_data_entries.first
@@ -71,12 +76,12 @@ module Flydata
         puts "Run 'flydata start' to start continuous sync."
       end
 
-      def generate_table_ddl
+      def generate_table_ddl(*tables)
         de = retrieve_data_entries.first
         raise "There are no data entry." unless de
         case de['type']
         when 'RedshiftMysqlDataEntry'
-          do_generate_table_ddl(de)
+          do_generate_table_ddl(override_tables(de, tables))
         else
           raise "No supported data entry. Only mysql-redshift sync is supported."
         end
@@ -102,10 +107,11 @@ module Flydata
         params << (mp['password'] ? "-p#{mp['password']}" : "")
         if mp['database'] then params << mp['database'] else raise "`database` is not defined in the data entry" end
         if mp['tables'] then params << mp['tables'].gsub(/,/, ' ') else raise "`tables` is not defined in the data entry" end
+
         command = DDL_DUMP_CMD_TEMPLATE % params
 
         IO.popen(command, 'r') do |io|
-          create_flydata_ctl_table = true
+          create_flydata_ctl_table = mp['initial_sync']
           while !io.eof?
             mysql_tabledef = Flydata::TableDef::MysqlTableDef.create(io)
             if mysql_tabledef.nil?
@@ -124,7 +130,7 @@ module Flydata
         sync_fm = Flydata::FileUtil::SyncFileManager.new(de)
 
         # Check client condition
-        if File.exists?(sync_fm.binlog_path)
+        if File.exists?(sync_fm.binlog_path) and de['mysql_data_entry_preference']['initial_sync']
           raise "Already synchronized. If you want to do initial sync, delete #{sync_fm.binlog_path}."
         end
 
@@ -134,7 +140,9 @@ module Flydata
         end
 
         if generate_mysqldump(de, sync_fm)
+          sync_fm.save_sync_info(de['mysql_data_entry_preference']['initial_sync'], de['mysql_data_entry_preference']['tables'])
           parse_mysqldump(dp, de, sync_fm)
+          complete
         end
       end
 
@@ -169,7 +177,7 @@ module Flydata
 
         puts "[Confirm] mysqldump path: #{fp}"
         if ask_yes_no('OK?')
-          Flydata::Mysql::MysqlDumpGenerator.new(de['mysql_data_entry_preference']).dump(fp)
+          Flydata::Mysql::MysqlDumpGeneratorNoMasterData.new(de['mysql_data_entry_preference']).dump(fp)
         else
           newline
           puts "You can change the mysqldump path with 'mysqldump_path' in the conf file."
@@ -293,7 +301,6 @@ module Flydata
           puts "Elapsed:#{elapsed_time}sec start:#{bench_start_time} end:#{bench_end_time}"
           return true
         end
-
         # wait until finish
         puts "Start waiting until all data is processed on FlyData..."
         sleep 10
@@ -302,8 +309,25 @@ module Flydata
         end
 
         sync_fm.save_dump_pos(STATUS_COMPLETE, '', dump_file_size, binlog_pos)
-        puts "Congratulations! All data is processed on FlyData. Please check tables and data on your Redshift Cluster."
-        puts "After checking, run 'flydata sync:complete' to start continuously synchronization."
+        tables = de['mysql_data_entry_preference']['tables'].split(',').join(' ')
+        sync_fm.save_table_binlog_pos(tables, binlog_pos)
+      end
+
+      def complete
+        de = load_sync_info(retrieve_data_entries.first)
+        sync_fm = Flydata::FileUtil::SyncFileManager.new(de)
+        info = sync_fm.load_dump_pos
+        if info[:status] == STATUS_COMPLETE
+          if de['mysql_data_entry_preference']['initial_sync']
+            sync_fm.save_binlog(info[:binlog_pos])
+          end
+          sync_fm.move_table_binlog_files(de['mysql_data_entry_preference']['tables'].split(','))
+          sync_fm.reset_table_position_files(de['mysql_data_entry_preference']['tables'].split(','))
+          sync_fm.backup_dump_dir
+          Flydata::Command::Sender.new.start
+        else
+          raise "Initial sync status is not complete. Try running 'flydata sync'."
+        end
       end
 
       def generate_json(mysql_table, values)
@@ -313,6 +337,34 @@ module Flydata
         end
         h.to_json
       end
+
+      def override_tables(de, tables)
+        de['mysql_data_entry_preference']['initial_sync'] = tables.empty?
+        if ! de['mysql_data_entry_preference']['initial_sync']
+          de['mysql_data_entry_preference']['tables'] = tables.join(',')
+        end
+        de
+      end
+
+      def load_sync_info(de)
+        sync_fm = Flydata::FileUtil::SyncFileManager.new(de)
+        mp = de['mysql_data_entry_preference']
+        unless (rs = sync_fm.load_sync_info).nil?
+          mp['initial_sync'] = rs[:initial_sync] 
+          mp['tables'] = rs[:tables]
+        end
+        de
+      end
+
+      def flush_buffer_and_stop
+        sender = Flydata::Command::Sender.new
+        sender.flush_client_buffer
+        puts "Checking the server."
+        until check
+          sleep 10
+        end
+        sender.stop(quiet: true)
+      end
     end
   end
 
@@ -668,7 +720,8 @@ EOT
 
     class MysqlDumpGenerator
       # host, port, username, password, database, tables
-      MYSQL_DUMP_CMD_TEMPLATE = "mysqldump --protocol=tcp -h %s -P %s -u%s %s --skip-lock-tables --single-transaction --flush-logs --hex-blob --master-data=2 %s %s"
+      MYSQL_DUMP_CMD_TEMPLATE = "mysqldump --protocol=tcp -h %s -P %s -u%s %s --skip-lock-tables --single-transaction --hex-blob %s %s %s"
+      EXTRA_MYSQLDUMP_PARAMS = ""
       def initialize(conf)
         password = conf['password'].to_s.empty? ? "" : "-p#{conf['password']}"
         tables = if conf['tables']
@@ -677,8 +730,16 @@ EOT
                    ''
                  end
         @dump_cmd = MYSQL_DUMP_CMD_TEMPLATE %
-          [conf['host'], conf['port'], conf['username'], password, conf['database'], tables]
+          [conf['host'], conf['port'], conf['username'], password, self.class::EXTRA_MYSQLDUMP_PARAMS, conf['database'], tables]
+        @db_opts = [:host, :port, :username, :password, :database].inject({}) {|h, sym| h[sym] = conf[sym.to_s]; h}
       end
+      def dump(file_path)
+        raise "subclass must implement the method"
+      end
+    end
+
+    class MysqlDumpGeneratorMasterData < MysqlDumpGenerator
+      EXTRA_MYSQLDUMP_PARAMS = "--flush-logs --master-data=2"
       def dump(file_path)
         cmd = "#{@dump_cmd} > #{file_path}"
         o, e, s = Open3.capture3(cmd)
@@ -700,6 +761,109 @@ EOT
       end
     end
 
+    class MysqlDumpGeneratorNoMasterData < MysqlDumpGenerator
+      EXTRA_MYSQLDUMP_PARAMS = ""
+      CHANGE_MASTER_TEMPLATE = <<EOS
+--
+-- Position to start replication or point-in-time recovery from
+--
+
+-- CHANGE MASTER TO MASTER_LOG_FILE='%s', MASTER_LOG_POS=%d;
+
+EOS
+
+      def dump(file_path)
+        # RDS doesn't allow obtaining binlog position using mysqldump.  Get it separately and insert it into the dump file.
+        table_locker = Fiber.new do
+          client = Mysql2::Client.new(@db_opts)
+          # Lock tables
+          client.query "FLUSH LOCAL TABLES;"
+          q = flush_tables_with_read_lock_query(client)
+          puts "FLUSH TABLES query: #{q}" if FLYDATA_DEBUG
+          client.query q
+          begin
+            Fiber.yield  # Lock is done.  Let dump to start
+            # obtain binlog pos
+            result = client.query "SHOW MASTER STATUS;"
+            row = result.first
+            if row.nil?
+              raise "MySQL DB has no replication master status.  Check if the DB is set up as a replication master.  In case of RDS, make sure that Backup Retention Period is set to more than 0."
+            end
+          ensure
+            # unlock tables
+            client.query "UNLOCK TABLES;"
+            client.close
+          end
+
+          [row["File"], row['Position']]
+        end
+
+        table_locker.resume # Lock tables
+        begin
+          # start dump
+          Open3.popen3 @dump_cmd do |cmd_in, cmd_out, cmd_err|
+            cmd_in.close_write
+            File.open(file_path, "w") do |f|
+              find_insert_pos = :not_started
+              cmd_out.each_line do |line|
+                if find_insert_pos == :not_started && /^-- Server version/ === line
+                  find_insert_pos = :finding
+                elsif find_insert_pos == :finding && /^--/ === line
+                  # wait before writing the first database queries
+                  file, pos = table_locker.resume # Get binlog pos
+                  # insert binlog pos
+                  change_master =  CHANGE_MASTER_TEMPLATE % [file, pos]
+                  f.print change_master
+
+                  find_insert_pos = :found
+                  # resume dump
+                end
+                f.print line
+              end
+            end
+            cmd_err.each_line do |line|
+              $stderr.print line unless /^Warning:/ === line
+            end
+          end
+        rescue
+          # Cleanup
+          FileUtils.rm(file_path) if File.exists?(file_path)
+          raise
+        ensure
+          # Let table_locker finish its task even if an exception happened
+          table_locker.resume if table_locker.alive?
+        end
+      end
+
+      private
+      # This query generates a query which flushes user tables with read lock
+      FLUSH_TABLES_QUERY_TEMPLATE = "FLUSH TABLES %s WITH READ LOCK;"
+      USER_TABLES_QUERY = <<EOS
+SELECT CONCAT('`',
+REPLACE(TABLE_SCHEMA, '`', '``'), '`.`',
+REPLACE(TABLE_NAME, '`', '``'), '` ')
+AS tables
+FROM INFORMATION_SCHEMA.TABLES
+WHERE TABLE_TYPE = 'BASE TABLE'
+AND ENGINE NOT IN ('MEMORY', 'CSV', 'PERFORMANCE_SCHEMA');
+EOS
+      def flush_tables_with_read_lock_query(client)
+        tables = ""
+        if mysql_server_version(client) >= "5.5"
+          # FLUSH TABLES table_names,... WITH READ LOCK syntax is supported from MySQL 5.5
+          result = client.query(USER_TABLES_QUERY)
+          tables = result.collect{|r| r['tables']}.join(", ")
+        end
+        FLUSH_TABLES_QUERY_TEMPLATE % [tables]
+      end
+
+      VERSION_QUERY = "SHOW VARIABLES LIKE 'version'"
+      def mysql_server_version(client)
+        result = client.query(VERSION_QUERY)
+        result.first['Value']
+      end
+    end
+
     class MysqlDumpParser
 
       module State