flydata 0.3.8 → 0.3.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 652af727f40585a7cb70d5b1f71fcdd9beda4a7e
-  data.tar.gz: f8e94b1dd631c553c787175910695065bcedee65
+  metadata.gz: abffe0873e3fe1143a5053d9f02b5c61eb1e90a3
+  data.tar.gz: c7f54222297282bcc2c13ffa1ca18ddc45bb10ef
 SHA512:
-  metadata.gz: 2e815f1b604d0ba454e82bbaae84ac13b6173507bc4066a138af3221fdaa3327aed5f636e9ebfc2c061b38eda6e0410e3ea6b57dcfcdca592b07f0cc64a6fc01
-  data.tar.gz: bdb92aee00edac39d02d4a374ca4dbf555c4b80672bcaade05801db0a0d3612715b0d19036c65162f8911401d9c7d31a5c9cc7c30730855cf0778eb53e808816
+  metadata.gz: 3b81782a6f4ac5bf2c28485b56ea1912e6f8490d375cb6ffd5f70e070776691fe4d20f74cfb444672192a0440a651b73835765054fc9da617e36182e4c021655
+  data.tar.gz: 7ef6babb2f76e4c1e75ff700a377c1b8a9504f022fe1bbe0fe09dc30f54e629277dafe2d55cfd905967edeac55d70d0019181fc1a90000c77172f9f7325961b8

data/VERSION

@@ -1 +1 @@
-0.3.8
+0.3.9

data/flydata.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: flydata 0.3.8 ruby lib
+# stub: flydata 0.3.9 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "flydata"
-  s.version = "0.3.8"
+  s.version = "0.3.9"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib"]
   s.authors = ["Koichi Fujikawa", "Masashi Miyazaki", "Matthew Luu", "Mak Inada", "Sriram NS"]
-  s.date = "2015-03-09"
+  s.date = "2015-03-21"
   s.description = "FlyData Agent"
   s.email = "sysadmin@flydata.com"
   s.executables = ["fdmysqldump", "flydata", "serverinfo"]
@@ -167,7 +167,7 @@ Gem::Specification.new do |s|
   ]
   s.homepage = "http://flydata.com/"
   s.licenses = ["All right reserved."]
-  s.rubygems_version = "2.4.3"
+  s.rubygems_version = "2.2.2"
   s.summary = "FlyData Agent"
 
   if s.respond_to? :specification_version then

data/lib/flydata/command/sync.rb

@@ -289,9 +289,9 @@ EOS
         unless Flydata::Preference::DataEntryPreference.conf_exists?(de)
           Flydata::Command::Conf.new.copy_templates
         end
-        generate_mysqldump(de, sync_fm, !opts.dump_stream?) do |mysqldump_io, db_bytesize|
+        generate_mysqldump(de, sync_fm, !opts.dump_stream?) do |mysqldump_io, binlog_pos, db_bytesize|
           sync_fm.save_sync_info(@full_initial_sync, de['mysql_data_entry_preference']['tables'])
-          parse_mysqldump_and_send(mysqldump_io, dp, de, sync_fm, db_bytesize)
+          parse_mysqldump_and_send(mysqldump_io, dp, de, sync_fm, binlog_pos, db_bytesize)
         end
         wait_for_mysqldump_processed(dp, de, sync_fm)
       end
@@ -362,13 +362,13 @@ EOM
           log_info_stdout("Exporting data from database.")
           log_info_stdout("This process can take hours depending on data size and load on your database.  Please be patient...")
           if file_dump
-            Flydata::Parser::Mysql::MysqlDumpGeneratorNoMasterData.
+            binlog_pos = Flydata::Parser::Mysql::MysqlDumpGeneratorNoMasterData.
               new(de['mysql_data_entry_preference'].merge('tables' => target_tables)).dump(fp)
             log_info_stdout("  -> Done")
-            call_block_or_return_io(fp, &block)
+            call_block_or_return_io(fp, binlog_pos, &block)
           else
             Flydata::Parser::Mysql::MysqlDumpGeneratorNoMasterData.
-              new(de['mysql_data_entry_preference'].merge('tables' => target_tables)).dump {|io| block.call(io, db_bytesize)}
+              new(de['mysql_data_entry_preference'].merge('tables' => target_tables)).dump {|io, binlog_pos| block.call(io, binlog_pos, db_bytesize)}
           end
         else
           exit 1
@@ -381,11 +381,11 @@ EOM
         stat.block_size * stat.blocks_available
       end
 
-      def call_block_or_return_io(fp, &block)
+      def call_block_or_return_io(fp, binlog_pos = nil, &block)
         if block
           f_io = open_file_io(fp)
           begin
-            block.call(f_io)
+            block.call(f_io, binlog_pos)
             return nil
           ensure
             f_io.close rescue nil
@@ -418,7 +418,7 @@ EOM
       #   <- checkpoint
       #...
       #CREATE TABLE ...
-      def parse_mysqldump_and_send(mysqldump_io, dp, de, sync_fm, db_bytesize = nil)
+      def parse_mysqldump_and_send(mysqldump_io, dp, de, sync_fm, binlog_pos, db_bytesize = nil)
         # Prepare forwarder
         de_tag_name = de["tag_name#{env_suffix}"]
         server_port = dp['server_port']
@@ -435,8 +435,10 @@ EOM
         dump_pos_info = sync_fm.load_dump_pos
         option = dump_pos_info || {}
         if option[:table_name] && option[:last_pos].to_i != -1
+          binlog_pos = option[:binlog_pos]
           log_info_stdout("Resuming... Last processed table: #{option[:table_name]}")
         else
+          option[:binlog_pos] = binlog_pos
           #If its a new sync, ensure server side resources are clean
           cleanup_sync_server(de, target_tables_for_api) unless opts.skip_cleanup?
         end
@@ -448,45 +450,53 @@ EOM
         tmp_num_inserted_record = 0
         dump_fp = sync_fm.dump_file_path
         dump_file_size = File.exists?(dump_fp) ? File.size(dump_fp) : 1
-        binlog_pos = Flydata::Parser::Mysql::MysqlDumpParser.new(option).parse(
-          mysqldump_io,
-          # create table
-          Proc.new { |mysql_table|
-            tmp_num_inserted_record = 0
-            # dump mysql_table for resume
-            #TODO: make it option
-            sync_fm.save_mysql_table_marshal_dump(mysql_table)
-          },
-          # insert record
-          Proc.new { |mysql_table, values_set|
-            mysql_table_name = mysql_table.table_name
-            records = values_set.collect do |values|
-              json = generate_json(mysql_table, values)
-              {table_name: mysql_table_name, log: json}
-            end
-            ret = forwarder.emit(records)
-            tmp_num_inserted_record += 1
-            ret
-          },
-          # checkpoint
-          Proc.new { |mysql_table, last_pos, bytesize, binlog_pos, state, substate|
-            # flush if buffer records exist
-            if tmp_num_inserted_record > 0 && forwarder.buffer_record_count > 0
-              forwarder.flush # send buffer data to the server before checkpoint
-            end
 
-            # show the current progress
-            if last_pos.to_i == -1   # stream dump
-              log_info_stdout("  -> #{as_size(bytesize)} (#{bytesize} byte) completed...")
-            else
-              log_info_stdout("  -> #{(last_pos.to_f/dump_file_size * 100).round(1)}% (#{last_pos}/#{dump_file_size}) completed...")
-            end
+        begin
+          Flydata::Parser::Mysql::MysqlDumpParser.new(option).parse(
+            mysqldump_io,
+            # create table
+            Proc.new { |mysql_table|
+              tmp_num_inserted_record = 0
+              # dump mysql_table for resume
+              #TODO: make it option
+              sync_fm.save_mysql_table_marshal_dump(mysql_table)
+            },
+            # insert record
+            Proc.new { |mysql_table, values_set|
+              mysql_table_name = mysql_table.table_name
+              records = values_set.collect do |values|
+                json = generate_json(mysql_table, values)
+                {table_name: mysql_table_name, log: json}
+              end
+              ret = forwarder.emit(records)
+              tmp_num_inserted_record += 1
+              ret
+            },
+            # checkpoint
+            Proc.new { |mysql_table, last_pos, bytesize, binlog_pos, state, substate|
+              # flush if buffer records exist
+              if tmp_num_inserted_record > 0 && forwarder.buffer_record_count > 0
+                forwarder.flush # send buffer data to the server before checkpoint
+              end
+
+              # show the current progress
+              if last_pos.to_i == -1   # stream dump
+                log_info_stdout("  -> #{as_size(bytesize)} (#{bytesize} byte) completed...")
+              else
+                log_info_stdout("  -> #{(last_pos.to_f/dump_file_size * 100).round(1)}% (#{last_pos}/#{dump_file_size}) completed...")
+              end
 
-            # save check point
-            table_name = mysql_table.nil? ? '' : mysql_table.table_name
-            sync_fm.save_dump_pos(STATUS_PARSING, table_name, last_pos, binlog_pos, state, substate)
-          }
-        )
+              # save check point
+              table_name = mysql_table.nil? ? '' : mysql_table.table_name
+              sync_fm.save_dump_pos(STATUS_PARSING, table_name, last_pos, binlog_pos, state, substate)
+            }
+          )
+        rescue DumpParseError =>e
+          ee = DumpParseError.new("ERROR: We encountered an error parsing this chunk:\n  #{e.message}")
+          ee.description = "  Please contact support@flydata.com to report the issue."
+          ee.set_backtrace e.backtrace
+          raise ee
+        end
         forwarder.close
         log_info_stdout("  -> Done")
         sync_fm.save_dump_pos(STATUS_WAITING, '', dump_file_size, binlog_pos)

data/lib/flydata/errors.rb

@@ -31,4 +31,7 @@ end
 class ServerDataProcessingTimeout < AgentError
 end
 
+class DumpParseError < AgentError
+end
+
 end

data/lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb

@@ -4,8 +4,8 @@ require 'fluent/plugin/in_mysql_binlog'
 require 'binlog'
 require 'kodama'
 
-# Load client library(flydata/cli/lib)
-lib = File.expand_path(File.dirname(__FILE__), '/../..')
+# Load client library(flydata-agent/lib)
+lib = File.expand_path(File.join(File.dirname(__FILE__), '../../'))
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'flydata'
 require 'flydata/sync_file_manager'

data/lib/flydata/parser/mysql/dump_parser.rb

@@ -41,14 +41,11 @@ module Flydata
 
       class MysqlDumpGenerator
         def initialize(conf)
-          @dump_cmd = generate_dump_cmd(conf)
+          @conf = conf
           @db_opts = [:host, :port, :username, :password, :database].inject({}) {|h, sym| h[sym] = conf[sym.to_s]; h}
         end
-        def dump(file_path)
-          raise "subclass must implement the method"
-        end
 
-        def generate_dump_cmd(conf)
+        def dump(file_path)
           raise "subclass must implement the method"
         end
       end
@@ -68,6 +65,8 @@ EOS
             raise ArgumentError.new("file_path or block must be given.")
           end
 
+          dump_cmd = generate_dump_cmd(@conf, file_path)
+
           # RDS doesn't allow obtaining binlog position using mysqldump.  Get it separately and insert it into the dump file.
           table_locker = create_table_locker
           table_locker.resume # Lock tables
@@ -78,14 +77,16 @@ EOS
             wr_io.sync = true
             wr_io.set_encoding("utf-8")
             rd_io.extend(DumpStreamIO)
+            binlog_pos = nil
 
             # start mysqldump
-            Open3.popen3 @dump_cmd do |cmd_in, cmd_out, cmd_err, wait_thr|
+            Open3.popen3 dump_cmd do |cmd_in, cmd_out, cmd_err, wait_thr|
               cmd_in.close_write
               cmd_out.set_encoding("utf-8") # mysqldump output must be in UTF-8
 
               first_line = cmd_out.gets # wait until first line comes
-              binlog_file, binlog_pos = table_locker.resume
+              binfile, pos = table_locker.resume
+              binlog_pos = {binfile: binfile, pos: pos}
 
               threads = []
 
@@ -93,7 +94,7 @@ EOS
               threads << Thread.new do
                 begin
                   wr_io.print(first_line) # write a first line
-                  filter_dump_stream(cmd_out, wr_io, binlog_file, binlog_pos)
+                  filter_dump_stream(cmd_out, wr_io)
                 ensure
                   wr_io.close rescue nil
                 end
@@ -109,10 +110,7 @@ EOS
 
               if block
                 # call callback function with io if block is given
-                block.call(rd_io)
-              elsif file_path
-                # store data to file
-                open_file_stream(file_path) {|f| rd_io.each_line{|l| f.print(l)}}
+                block.call(rd_io, binlog_pos)
               end
 
               threads.each(&:join)
@@ -123,7 +121,7 @@ EOS
               end
               raise errors unless errors.empty?
             end
-            true
+            binlog_pos
           rescue
             # Cleanup
             FileUtils.rm(file_path) if file_path && File.exists?(file_path)
@@ -136,8 +134,8 @@ EOS
           end
         end
 
-        def generate_dump_cmd(conf)
-          Util::MysqlUtil.generate_mysqldump_without_master_data_cmd(conf)
+        def generate_dump_cmd(conf, file_path = nil)
+          Util::MysqlUtil.generate_mysqldump_without_master_data_cmd(conf.merge(result_file: file_path))
         end
 
         private
@@ -201,25 +199,13 @@ EOS
           result.first['Value']
         end
 
-        def filter_dump_stream(cmd_out, w_io, binlog_file, binlog_pos)
-          find_insert_pos = :not_started
+        def filter_dump_stream(cmd_out, w_io)
           cmd_out.each_line do |line|
-            if find_insert_pos == :not_started && /^-- Server version/ === line
-              find_insert_pos = :finding
-            elsif find_insert_pos == :finding && /^--/ === line
-              # wait before writing the first database queries
-              # insert binlog pos
-              change_master =  CHANGE_MASTER_TEMPLATE % [binlog_file, binlog_pos]
-              w_io.print change_master
-
-              find_insert_pos = :found
-            end
             w_io.print line
             w_io.puts unless line.end_with?("\n")
             w_io.flush
           end
         end
-
       end
 
       # Custom mysql client that sets config params (eg:-read_timeout) uniformly for all
@@ -258,9 +244,10 @@ EOS
         end
 
         attr_accessor :binlog_pos
+        BINLOG_INV_ERROR_CHUNK_SIZE = 250
 
         def initialize(option = {})
-          @binlog_pos = option[:binlog_pos]
+          @binlog_pos = option[:binlog_pos] or raise ArgumentError.new("binlog position is required")
           @option = option
         end
 
@@ -291,10 +278,8 @@ EOS
           state_start = Proc.new do
             line = readline_proc.call
 
-            # -- CHANGE MASTER TO MASTER_LOG_FILE='mysql-bin.000002', MASTER_LOG_POS=120;
-            m = /^\-\- CHANGE MASTER TO MASTER_LOG_FILE='(?<binfile>[^']+)', MASTER_LOG_POS=(?<pos>\d+)/.match(line)
-            if m
-              @binlog_pos = {binfile: m[:binfile], pos: m[:pos].to_i}
+            # -- Server version 5.6.21-log
+            if line.start_with?('-- Server version')
               current_state = State::CREATE_TABLE
               check_point_block.call(nil, dump_io.pos, bytesize, @binlog_pos, current_state)
             end
@@ -478,8 +463,26 @@ EOS
             #
             # We are using the C native method that is like 'split', 'start_with?', 'regexp'
             # instead of 'String#each_char' and string comparision for the performance.
-            # 'String#each_char' is twice as slow as the current storategy.
-            items = target_line.split(',')
+            # 'String#each_char' is twice as slow as the current strategy.
+            begin
+              items = target_line.split(',')
+            rescue ArgumentError => e
+              # If there are any non-UTF8 characters, split will fail
+              # This will go through the chunk of data and cut the data up into chunks to make it
+              # displayable to user.
+              target_chunk=""
+              chunk_size = BINLOG_INV_ERROR_CHUNK_SIZE
+              chunk_repeat_count = target_line.length/chunk_size
+              for i in 0..chunk_repeat_count
+                line_chunk=target_line[(0+(i*chunk_size)..chunk_size+((i*chunk_size)-1))]
+                unless line_chunk.valid_encoding?
+                  target_chunk=line_chunk
+                  break
+                end
+              end
+              hex_string = target_chunk.bytes.map { |b| sprintf(", 0x%02X",b) }.join
+              raise DumpParseError, "raw_text: #{target_chunk}\n  hex_dump: #{hex_string}"
+            end
             index = 0
             cur_state = :next_values
 

data/lib/flydata/util/mysql_util.rb

@@ -54,6 +54,9 @@ module Flydata
         opt = option.dup
         opt[:command] = 'mysqldump'
         opt[:custom_option] = '--skip-lock-tables --single-transaction --hex-blob'
+        if opt[:result_file]
+          opt[:custom_option] << " --result-file=#{opt[:result_file]}"
+        end
         generate_mysql_cmd(opt)
       end
 

data/spec/flydata/parser/mysql/dump_parser_spec.rb

@@ -1,8 +1,9 @@
 # coding: utf-8
 require 'spec_helper'
-require 'flydata/parser/mysql/dump_parser'
 require 'open3'
 require 'mysql2'
+require 'flydata/command/sync'
+require 'flydata/parser/mysql/dump_parser'
 
 module Flydata
   module Parser
@@ -33,6 +34,7 @@ module Flydata
           let(:wait_thr) { double(:wait_thr) }
 
           let(:default_dump_generator) { MysqlDumpGeneratorNoMasterData.new(default_conf) }
+          let(:binlog_pos) { { binfile: 'mysql-bin.000451', pos: 89872 } }
           let(:mysql_client) do
             m = double(:mysql_client)
             allow(m).to receive(:query).with(/TABLES/)
@@ -44,12 +46,6 @@ module Flydata
             m
           end
 
-          describe '#initialize' do
-            subject { default_dump_generator.instance_variable_get(:@dump_cmd) }
-            it { is_expected.to eq('mysqldump -h localhost -P 3306 -uadmin -p"pass" --default-character-set=utf8 --protocol=tcp --skip-lock-tables ' +
-                                   '--single-transaction --hex-blob dev users groups') }
-          end
-
           describe '#dump' do
             before do
               expect(Mysql2::Client).to receive(:new).and_return(mysql_client)
@@ -59,8 +55,7 @@ module Flydata
               it do
                 expect(wait_thr).to receive(:value).and_return(0)
                 expect(stderr).to receive(:each_line).and_yield("")
-                expect(default_dump_generator.dump(file_path)).to be_truthy
-                expect(File.exists?(file_path)).to be_truthy
+                expect(default_dump_generator.dump(file_path)).to eq(binlog_pos)
               end
             end
             context "when mysqldump exits with status 1" do
@@ -115,7 +110,7 @@ module Flydata
       describe MysqlDumpParser do
         let(:file_path) { File.join('/tmp', "flydata_sync_spec_mysqldump_parse_#{Time.now.to_i}") }
         let(:dump_io) { File.open(file_path, 'r', encoding: "utf-8") }
-        let(:default_parser) { MysqlDumpParser.new }
+        let(:default_parser) { MysqlDumpParser.new(binlog_pos: default_binlog_pos) }
 
         def generate_dump_file(content)
           File.open(file_path, 'w') {|f| f.write(content)}
@@ -155,9 +150,9 @@ EOT
             content.index(string) + string.bytesize + 1
           end
 
-          let(:default_parser) { MysqlDumpParser.new }
+          let(:default_parser) { MysqlDumpParser.new(binlog_pos: default_binlog_pos) }
           let(:default_binlog_pos) { {binfile: 'mysql-bin.000267', pos: 120 } }
-          let(:dump_pos_after_binlog_pos) { index_after(DUMP_HEADER, 'MASTER_LOG_POS=120;') }
+          let(:dump_pos_after_binlog_pos) { index_after(DUMP_HEADER, '5.6.13-log') }
 
           let(:create_table_block) { double('create_table_block') }
           let(:insert_record_block) { double('insert_record_block') }
@@ -167,19 +162,6 @@ EOT
             generate_dump_file('')
           end
 
-          context 'when dump does not contain binlog pos' do
-            before { generate_dump_file('dummy content') }
-            it do
-              expect(create_table_block).to receive(:call).never
-              expect(insert_record_block).to receive(:call).never
-              expect(check_point_block).to receive(:call).never
-              binlog_pos = default_parser.parse(
-                dump_io, create_table_block, insert_record_block, check_point_block
-              )
-              expect(binlog_pos).to be_nil
-            end
-          end
-
           context 'when dump contains only binlog pos' do
             before { generate_dump_file(<<EOT
 #{DUMP_HEADER}

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flydata
 version: !ruby/object:Gem::Version
-  version: 0.3.8
+  version: 0.3.9
 platform: ruby
 authors:
 - Koichi Fujikawa
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-03-09 00:00:00.000000000 Z
+date: 2015-03-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -607,7 +607,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.3
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: FlyData Agent