flydata 0.5.12 → 0.5.13

data/flydata-core/spec/mysql/command_generator_spec.rb

@@ -26,12 +26,12 @@ module FlydataCore
 
         context 'when option is empty' do
           let(:option) { {} }
-          it { is_expected.to eq("mysql --default-character-set=utf8 --protocol=tcp") }
+          it { is_expected.to eq("mysql --default-character-set=utf8 --protocol=tcp --skip-auto-rehash") }
         end
 
         context 'when all option is specified' do
           it { is_expected.to eq(
-            'mysqldump -h test-host -P 3306 -utestuser -p"testpassword" --default-character-set=utf8 --protocol=tcp testdb'
+            'mysqldump -h test-host -P 3306 -utestuser -ptestpassword --default-character-set=utf8 --protocol=tcp testdb'
           ) }
         end
 
@@ -41,7 +41,7 @@ module FlydataCore
             option['database'] = 'another_db'
           }
           it { is_expected.to eq(
-            'mysqldump -h test-host -P 3306 -utestuser -p"testpassword" --default-character-set=utf8 --protocol=tcp another_db'
+            'mysqldump -h test-host -P 3306 -utestuser -ptestpassword --default-character-set=utf8 --protocol=tcp another_db'
           ) }
         end
 
@@ -54,54 +54,54 @@ module FlydataCore
 
         context 'when password is not empty' do
           before { option[:password] = "abc`def" }
-          it { is_expected.to eq(cmd_pswd_check % '"abc\\`def"' ) }
+          it { is_expected.to eq(cmd_pswd_check % 'abc\\`def' ) }
         end
 
         context 'when password includes double quate' do
           before { option[:password] = "abc\"def" }
-          it { is_expected.to eq(cmd_pswd_check % '"abc\\"def"') }
+          it { is_expected.to eq(cmd_pswd_check % 'abc\\"def') }
         end
 
         context 'when password includes dollars sign' do
           before { option[:password] = "abc$def" }
-          it { is_expected.to eq(cmd_pswd_check % '"abc\\$def"' ) }
+          it { is_expected.to eq(cmd_pswd_check % 'abc\\$def' ) }
         end
 
         context 'when password includes multiple special characters' do
           before { option[:password] = "ab\"\"c$$def" }
-          it { is_expected.to eq(cmd_pswd_check % '"ab\\"\\"c\\$\\$def"' ) }
+          it { is_expected.to eq(cmd_pswd_check % 'ab\\"\\"c\\$\\$def' ) }
         end
 
         context 'when password does not includes special characters' do
           before { option[:password] = "abcdef" }
-          it { is_expected.to eq(cmd_pswd_check % '"abcdef"' ) }
+          it { is_expected.to eq(cmd_pswd_check % 'abcdef' ) }
         end
 
         context 'when tables are specified' do
           before { option[:tables] = %w(table_a table_b table_c) }
           it { is_expected.to eq(
-            'mysqldump -h test-host -P 3306 -utestuser -p"testpassword" --default-character-set=utf8 --protocol=tcp testdb table_a table_b table_c'
+            'mysqldump -h test-host -P 3306 -utestuser -ptestpassword --default-character-set=utf8 --protocol=tcp testdb table_a table_b table_c'
           ) }
         end
 
         context 'when ssl_ca is specified' do
           before { option[:ssl_ca] = 'mysql_sync.ssl_ca.pem' }
           it { is_expected.to eq(
-            'mysqldump -h test-host -P 3306 -utestuser -p"testpassword" --default-character-set=utf8 --protocol=tcp --ssl-ca=mysql_sync.ssl_ca.pem testdb'
+            'mysqldump -h test-host -P 3306 -utestuser -ptestpassword --default-character-set=utf8 --protocol=tcp --ssl-ca mysql_sync.ssl_ca.pem testdb'
           ) }
         end
 
         context 'when ssl_cipher is specified' do
           before { option[:ssl_cipher] = 'AES256-GCM-SHA384:AES256-SHA' }
           it { is_expected.to eq(
-            'mysqldump -h test-host -P 3306 -utestuser -p"testpassword" --default-character-set=utf8 --protocol=tcp --ssl-cipher=AES256-GCM-SHA384:AES256-SHA testdb'
+            'mysqldump -h test-host -P 3306 -utestuser -ptestpassword --default-character-set=utf8 --protocol=tcp --ssl-cipher=AES256-GCM-SHA384:AES256-SHA testdb'
           ) }
         end
 
         context 'when no_default_option is turned on' do
           before { option[:no_default_option] = true }
           it { is_expected.to eq(
-            'mysqldump -h test-host -P 3306 -utestuser -p"testpassword" testdb'
+            'mysqldump -h test-host -P 3306 -utestuser -ptestpassword testdb'
           ) }
         end
 
@@ -111,7 +111,7 @@ module FlydataCore
             option[:custom_option] = '-e "SHOW GRANTS;"'
           }
           it { is_expected.to eq(
-            'mysql -h test-host -P 3306 -utestuser -p"testpassword" --default-character-set=utf8 --protocol=tcp -e "SHOW GRANTS;" testdb'
+            'mysql -h test-host -P 3306 -utestuser -ptestpassword --default-character-set=utf8 --protocol=tcp --skip-auto-rehash -e "SHOW GRANTS;" testdb'
           ) }
         end
       end
@@ -120,7 +120,7 @@ module FlydataCore
         subject { described_class.generate_mysql_ddl_dump_cmd(option) }
 
         it { is_expected.to eq(
-          'mysqldump -h test-host -P 3306 -utestuser -p"testpassword" --default-character-set=utf8 --protocol=tcp -d testdb'
+          'mysqldump -h test-host -P 3306 -utestuser -ptestpassword --default-character-set=utf8 --protocol=tcp -d testdb'
         ) }
       end
 
@@ -128,7 +128,7 @@ module FlydataCore
         subject { described_class.generate_mysqldump_with_master_data_cmd(option) }
         before { option[:tables] = %w(table_a table_b) }
         it { is_expected.to eq(
-          'mysqldump -h test-host -P 3306 -utestuser -p"testpassword" --default-character-set=utf8 --protocol=tcp --skip-lock-tables --single-transaction --hex-blob --flush-logs --master-data=2 testdb table_a table_b'
+          'mysqldump -h test-host -P 3306 -utestuser -ptestpassword --default-character-set=utf8 --protocol=tcp --skip-lock-tables --single-transaction --hex-blob --flush-logs --master-data=2 testdb table_a table_b'
         ) }
       end
 
@@ -136,14 +136,14 @@ module FlydataCore
         subject { described_class.generate_mysqldump_without_master_data_cmd(option) }
         before { option[:tables] = %w(table_a table_b) }
         it { is_expected.to eq(
-          'mysqldump -h test-host -P 3306 -utestuser -p"testpassword" --default-character-set=utf8 --protocol=tcp --skip-lock-tables --single-transaction --hex-blob testdb table_a table_b'
+          'mysqldump -h test-host -P 3306 -utestuser -ptestpassword --default-character-set=utf8 --protocol=tcp --skip-lock-tables --single-transaction --hex-blob testdb table_a table_b'
         ) }
       end
 
       describe '.generate_mysql_show_grants_cmd' do
         subject { described_class.generate_mysql_show_grants_cmd(option) }
         it { is_expected.to eq(
-          'mysql -h test-host -P 3306 -utestuser -p"testpassword" --default-character-set=utf8 --protocol=tcp -e "SHOW GRANTS;" testdb'
+          'mysql -h test-host -P 3306 -utestuser -ptestpassword --default-character-set=utf8 --protocol=tcp --skip-auto-rehash -e "SHOW GRANTS;" testdb'
         ) }
       end
 

data/lib/flydata/command/sync.rb

@@ -9,6 +9,7 @@ require 'flydata/command/sender'
 require 'flydata/compatibility_check'
 require 'flydata/errors'
 require 'flydata/helpers'
+require 'flydata/json'
 require 'flydata/output/forwarder'
 require 'flydata/parser/mysql/dump_parser'
 require 'flydata/preference/data_entry_preference'
@@ -63,45 +64,7 @@ module Flydata
       # Command: flydata sync
       #  - Entry method
       def run(*tables)
-        # Process check
-        sender = Flydata::Command::Sender.new
-        if (sender.process_exist?)
-          if tables.empty?
-            # full sync
-            log_warn_stderr("FlyData Agent is already running.  If you'd like to restart FlyData Sync from scratch, run 'flydata sync:reset' first.")
-          else
-            # per-table sync
-            log_warn_stderr("Flydata Agent is already running.  If you'd like to Sync the table(s), run 'flydata sync:flush' first.")
-          end
-          exit 1
-        end
-
-        fluentd_started = false
-
-        quiet_option = false
-
-        start_fluentd = Proc.new do |binlog_pos|
-          # Start continuous sync by starting fluentd process
-          unless opts.no_flydata_start?
-            log_info_stdout("Starting FlyData Agent...") unless quiet_option
-            Flydata::Command::Sender.new.start(quiet: true)
-            log_info_stdout("  -> Done") unless quiet_option
-          end
-          fluentd_started = true
-        end
-
-        quiet_option = true
-        # Start initial sync with check
-        handle_mysql_sync(tables, binlog_ready_callback: start_fluentd)
-        quiet_option = false
-
-        start_fluentd.call unless fluentd_started
-
-        # Show message
-        dashboard_url = "#{flydata.flydata_api_host}/dashboard"
-        redshift_console_url = "#{flydata.flydata_api_host}/redshift_clusters/query/new"
-        last_message = ALL_DONE_MESSAGE_TEMPLATE % [redshift_console_url, dashboard_url]
-        log_info_stdout(last_message)
+        raise "Command 'flydata sync' has been deprecated.  Use 'flydata start' instead."
       end
       run_exclusive :run
 
@@ -459,8 +422,9 @@ EOS
         sent_binlog_pos = nil
         @full_tables.each do |table|
           binlog_str = table_status_hash[table]["src_pos"]
-          unless binlog_str
-            raise "Table `#{table}` has no 'src_pos' in its table status"
+          if binlog_str.nil?  || binlog_str == "-"
+            # the table status has no src_pos (which is rare.)  Skip the table
+            next
           end
           binlog_pos = FlydataCore::Mysql::BinlogPos.new(binlog_str)
           if tables.empty?
@@ -479,44 +443,49 @@ EOS
             end
           end
         end
-        if oldest_binlog && sent_binlog_pos < oldest_binlog
-          e = AgentError.new("Repair failed due to expired binlog")
-          e.description = <<EOS
-Repair failed because the starting binlog position `#{sent_binlog_pos} no longer exists.  Run full initial sync instead.
+        # if sent_binlog_pos is nil, it means sync has started for none of tables.  No need
+        # to repair positions nor clean buffer data.
+        if sent_binlog_pos
+          if oldest_binlog && sent_binlog_pos < oldest_binlog
+            e = AgentError.new("Repair failed due to expired binlog")
+            e.description = <<EOS
+  Repair failed because the starting binlog position `#{sent_binlog_pos} no longer exists.  Run full initial sync instead.
 EOS
-          raise e
-        end
-        master_binlog_pos = FlydataCore::Mysql::BinlogPos.new(sent_binlog_pos.filename, 4)
-
-        # Delete agent buffer
-        log_info_stdout "Deleting data in the agent buffer..."
-        files = Flydata::Agent.new(FLYDATA_HOME).delete_buffer_files
-        unless files.empty?
-          $log.debug "Deleted buffer files\n  " + files.join("\n  ")
-        end
-
-        # Delete query queue items for the tables
-        log_info_stdout "Deleting data stuck in the server buffer..."
-        cleanup_sync_server(de, tables, queue_only: true) unless tables.empty?
-
-        # Save the positions (binlog and seq)
-        log_info_stdout "Fixing table positions..."
-        @full_tables.each do |table|
-           binlog_str = table_status_hash[table]["src_pos"]
-           unless binlog_str
-             raise "Table `#{table}` has no 'src_pos' in its table status"
-           end
-           binlog_pos = FlydataCore::Mysql::BinlogPos.new(binlog_str)
-           pos = table_status_hash[table]["seq"]
-           old_binlog_pos, old_pos = save_table_positions(table, binlog_pos, pos)
-           if pos.to_i != old_pos.to_i && !tables.include?(table)
-             $log.debug "Fixed broken table position.  table:#{table} pos:#{old_pos} -> #{pos}"
-           end
+            raise e
+          end
+          master_binlog_pos = FlydataCore::Mysql::BinlogPos.new(sent_binlog_pos.filename, 4)
+  
+          # Delete agent buffer
+          log_info_stdout "Deleting data in the agent buffer..."
+          files = Flydata::Agent.new(FLYDATA_HOME).delete_buffer_files
+          unless files.empty?
+            $log.debug "Deleted buffer files\n  " + files.join("\n  ")
+          end
+  
+          # Delete query queue items for the tables
+          log_info_stdout "Deleting data stuck in the server buffer..."
+          cleanup_sync_server(de, tables, queue_only: true) unless tables.empty?
+  
+          # Save the positions (binlog and seq)
+          log_info_stdout "Fixing table positions..."
+          @full_tables.each do |table|
+             binlog_str = table_status_hash[table]["src_pos"]
+             if binlog_str.nil? || binlog_str == "-"
+               # no init sync has happened to the table.  No need to set positions.
+               next
+             end
+             binlog_pos = FlydataCore::Mysql::BinlogPos.new(binlog_str)
+             pos = table_status_hash[table]["seq"]
+             old_binlog_pos, old_pos = save_table_positions(table, binlog_pos, pos)
+             if pos.to_i != old_pos.to_i && !tables.include?(table)
+               $log.debug "Fixed broken table position.  table:#{table} pos:#{old_pos} -> #{pos}"
+             end
+          end
+  
+          log_info_stdout "Fixing the master position files..."
+          save_master_binlog_positions(master_binlog_pos, sent_binlog_pos)
         end
 
-        log_info_stdout "Fixing the master position files..."
-        save_master_binlog_positions(master_binlog_pos, sent_binlog_pos)
-
         # Remove the lock file if exists.
         File.delete(FLYDATA_LOCK) if File.exists?(FLYDATA_LOCK)
 
@@ -814,6 +783,7 @@ EOM
         dump_file_size = File.exists?(dump_fp) ? File.size(dump_fp) : 1
 
         begin
+          threads = []
           Flydata::Parser::Mysql::MysqlDumpParser.new(option).parse(
             mysqldump_io,
             # create table
@@ -828,7 +798,7 @@ EOM
               mysql_table_name = mysql_table.table_name
               records = values_set.collect do |values|
                 values = convert_to_flydata_values(mysql_table, values)
-                json = generate_json(mysql_table, values)
+                json = JSON.generate_kv_pairs(mysql_table.column_names, values)
                 {table_name: mysql_table_name, log: json}
               end
               ret = forwarder.emit(records)
@@ -859,10 +829,11 @@ EOM
               if mysql_table &&
                  state == Flydata::Parser::Mysql::MysqlDumpParser::State::CREATE_TABLE
                 # all records for `mysql_table` have been sent
-                send_record_counts(de, sync_fm, mysql_table.table_name)
+                send_record_counts(de, sync_fm, mysql_table.table_name, threads)
               end
             }
           )
+          threads.each{|t| t.join}
         rescue DumpParseError =>e
           ee = DumpParseError.new("ERROR: We encountered an error parsing this chunk:\n  #{e.message}")
           ee.description = "  Please contact support@flydata.com to report the issue."
@@ -972,16 +943,24 @@ EOM
       end
 
       NUM_TABLES_IN_CHUNK = 30
-      def send_record_counts(de, sync_fm, table = nil)
+      def send_record_counts(de, sync_fm, table = nil, thread_array = nil)
         stats = sync_fm.load_stats
         stats = { table => stats[table] } if table # single table stats
         stats.each_slice(NUM_TABLES_IN_CHUNK) do |slice|
           h = Hash[slice]
-          send_record_counts_chunk(de, h)
+          send_record_counts_chunk(de, h, thread_array)
+        end
+      end
+
+      def send_record_counts_chunk(de, stats, thread_array)
+        if thread_array
+          thread_array << Thread.new { do_send_record_counts_chunk(de, stats) }
+        else
+          do_send_record_counts_chunk(de, stats)
         end
       end
 
-      def send_record_counts_chunk(de, stats)
+      def do_send_record_counts_chunk(de, stats)
         retry_count = 0
         retry_interval = 3
         begin
@@ -1006,14 +985,24 @@ EOM
       end
 
       def convert_to_flydata_values(mysql_table, values)
-        types = mysql_table.columns.each_value.collect{|col_attrs| col_attrs[:format_type]}
-        types.size.times.collect{|i| FlydataCore::TableDef::MysqlTableDef.convert_to_flydata_value(values[i], types[i]) }
+        types = mysql_table.format_types
+        sz = types.size
+        result = []
+        i = 0
+        while i < sz
+          result << FlydataCore::TableDef::MysqlTableDef.convert_to_flydata_value(values[i], types[i])
+          i += 1
+        end
+        result
       end
 
-      def generate_json(mysql_table, values)
+      def generate_json(column_names, values)
         h = {}
-        mysql_table.columns.each_key.with_index do |k, i|
-          h[k] = values[i]
+        sz = column_names.size
+        i = 0
+        while i < sz
+          h[column_names[i]] = values[i]
+          i += 1
         end
         h.to_json
       end
@@ -1274,7 +1263,7 @@ Thank you for using FlyData!
 
           server_sequence = table_status["seq"]
           agent_sequence = sync_fm.get_table_position(table)
-          agent_sequence = agent_sequence.to_i if agent_sequence
+          agent_sequence = agent_sequence.to_i # nil is considred 0
           if server_sequence != agent_sequence
             pos_mismatch_tables << {table: table, agent_seq: agent_sequence, server_seq: server_sequence}
           end
@@ -1347,10 +1336,9 @@ Thank you for using FlyData!
           src_pos = table_status_hash[table]["src_pos"]
           if src_pos.nil? || src_pos == "-"
             init_binlog_pos =  sync_fm.get_table_binlog_pos_init(table)
-            unless init_binlog_pos
-              raise "File `#{table}.binlog.pos.init` is missing"
+            if init_binlog_pos
+              table_status_hash[table]["src_pos"] = init_binlog_pos
             end
-            table_status_hash[table]["src_pos"] = init_binlog_pos
           end
         end
       end

data/lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb

@@ -99,7 +99,7 @@ class MysqlBinlogFlydataInput < MysqlBinlogInput
     $log.info "mysql host:\"#{@host}\" port:\"#{@port}\" username:\"#{@username}\" database:\"#{@database}\" tables:\"#{@tables}\" tables_append_only:\"#{tables_append_only}\""
     $log.info "mysql client version: #{`mysql -V`}"
     server_msg = `echo 'select version();' | #{FlydataCore::Mysql::CommandGenerator.generate_mysql_cmd(@db_opts)} 2>&1`
-    if ($?.exitstatus == 0)
+    if ($? && $?.exitstatus == 0)
       $log.info "mysql server version: #{server_msg.strip}"
     else
       err_msg = "Failed to access mysql server... #{server_msg.strip}"
@@ -234,6 +234,9 @@ EOS
         current_ssl_cipher = @secondary_ssl_cipher
         $log.warn("Retry with secondary ssl cipher list due to '#{e}' - secondary_ssl_cipher: '#{@secondary_ssl_cipher}'")
         retry
+      elsif /binlog file.*does not exist/.match(e.to_s)
+        $log.error("#{e.to_s}.  Sync must be reset.  Terminating the agent.")
+        Process.kill(:TERM, Process.ppid)
       else
         raise e
       end

data/lib/flydata/json.rb

@@ -0,0 +1 @@
+require 'flydata/json/json_ext'

data/lib/flydata/json/.gitignore

@@ -0,0 +1 @@
+json_ext.so

data/lib/flydata/parser/mysql/.gitignore

@@ -0,0 +1 @@
+dump_parser_ext.so

data/lib/flydata/parser/mysql/dump_parser.rb

@@ -1,5 +1,6 @@
 require 'fiber'
 require 'io/wait'
+require 'mysql2'
 require 'flydata-core/mysql/config'
 require 'flydata-core/mysql/command_generator'
 
@@ -31,13 +32,19 @@ module Flydata
         def initialize(table_name, columns = {}, primary_keys = [])
           @table_name = table_name
           @columns = columns
+          @column_names = columns.collect{|k,v| v[:column_name]}
+          @format_types = columns.collect{|k,v| v[:format_type]}
           @primary_keys = primary_keys
         end
 
-        attr_accessor :table_name, :columns, :primary_keys
+        attr_accessor :table_name, :columns, :column_names, :format_types,
+                      :primary_keys
 
         def add_column(column)
-          @columns[column[:column_name]] = column
+          cn = column[:column_name]
+          @columns[cn] = column
+          @column_names << cn
+          @format_types << column[:format_type]
         end
       end
 
@@ -303,7 +310,7 @@ EOS
               line = buffered_line
               buffered_line = nil
             else
-              rawline = dump_io.readline.encode('utf-16', :undef => :replace, :invalid => :replace).encode('utf-8')
+              rawline = dump_io.readline
               bytesize += rawline.bytesize
               line = rawline.strip
             end
@@ -417,7 +424,14 @@ EOS
           state_parsing_insert_record = Proc.new do
             line = readline_proc.call
 
-            values_set = InsertParser.new.parse(line)
+            values_set = nil
+            begin
+              values_set = InsertParser.new.parse(line)
+            rescue => e
+              newe = e.class.new(e.message + "line:#{line}")
+              newe.set_backtrace(e.backtrace)
+              raise newe
+            end
             current_state = State::INSERT_RECORD
 
             if insert_record_block.call(current_table, values_set)
@@ -478,7 +492,7 @@ EOS
           def parse(line)
             start_ruby_prof
             bench_start_time = Time.now
-            _parse(line)
+            _parse2(line)
           ensure
             stop_ruby_prof
             if ENV['FLYDATA_BENCHMARK']
@@ -652,3 +666,5 @@ EOT
     end
   end
 end
+
+require 'flydata/parser/mysql/dump_parser_ext'