flydata 0.1.5 → 0.1.6

data/VERSION

@@ -1 +1 @@
-0.1.5
+0.1.6

data/flydata.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "flydata"
-  s.version = "0.1.5"
+  s.version = "0.1.6"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Koichi Fujikawa"]
-  s.date = "2014-05-17"
+  s.date = "2014-05-22"
   s.description = "FlyData Command Line Interface"
   s.email = "sysadmin@flydata.co"
   s.executables = ["fdmysqldump", "flydata"]

data/lib/flydata/command/sync.rb

@@ -88,7 +88,7 @@ module Flydata
         flydata.data_entry.buffer_stat(de['id'], env_mode)
       end
 
-      DDL_DUMP_CMD_TEMPLATE = "mysqldump -d -h %s -P %s -u %s %s %s %s"
+      DDL_DUMP_CMD_TEMPLATE = "mysqldump --protocol=tcp -d -h %s -P %s -u %s %s %s %s"
       def do_generate_table_ddl(de)
         if `which mysqldump`.empty?
           raise "mysqldump is not installed.  mysqldump is required to run the command"
@@ -668,7 +668,7 @@ EOT
 
     class MysqlDumpGenerator
       # host, port, username, password, database, tables
-      MYSQL_DUMP_CMD_TEMPLATE = "mysqldump -h %s -P %s -u%s %s --skip-lock-tables --single-transaction --flush-logs --hex-blob --master-data=2 %s %s"
+      MYSQL_DUMP_CMD_TEMPLATE = "mysqldump --protocol=tcp -h %s -P %s -u%s %s --skip-lock-tables --single-transaction --flush-logs --hex-blob --master-data=2 %s %s"
       def initialize(conf)
         password = conf['password'].to_s.empty? ? "" : "-p#{conf['password']}"
         tables = if conf['tables']
@@ -868,6 +868,8 @@ EOT
         @binlog_pos
       end
 
+      # Parse the insert line containing multiple values. (max line size is 1kb)
+      #   ex) INSERT INTO `data_entries` VALUES (2,2,'access_log'), (2,3,'access_log2');
       class InsertParser
         #INSERT INTO `data_entries` VALUES (2,2,'access_log'), (2,3,'access_log2');
         module State
@@ -913,6 +915,12 @@ EOT
           target_line = target_line.strip
           start_index = target_line.index('(')
           target_line = target_line[start_index..-2]
+
+          # Split insert line text with ',' and take care of ',' inside of the values later.
+          #
+          # We are using the C native method that is like 'split', 'start_with?', 'regexp'
+          # instead of 'String#each_char' and string comparision for the performance.
+          # 'String#each_char' is twice as slow as the current storategy.
           items = target_line.split(',')
           index = 0
           cur_state = State::NEXT_VALUES

data/lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb

@@ -39,19 +39,42 @@ class MysqlBinlogFlydataInput < MysqlBinlogInput
     super
     positions_path = @sync_fm.table_positions_dir_path
     Dir.mkdir positions_path unless File.exists? positions_path
+  rescue Binlog::Error
+    if (/basic_string::_M_replace_aux/ === $!.to_s)
+      # TODO Fix the root cause in mysql-replication-listener
+      $log.error <<EOS
+a mysql-replication-listener error.  This could have been caused by one of the following reasons.
+  - Failed on connect: Your host is blocked because of many connection errors; unblock with 'mysqladmin flush-hosts'
+EOS
+    else
+      $log.error "unexpected mysql-replication-listener error. exception: #{$!.class.to_s}, error: #{$!.to_s}\n#{$!.backtrace.join("\n")}"
+    end
+    raise
+  rescue Exception
+    $log.error "unexpected fatal error. exception: #{$!.class.to_s}, error: #{$!.to_s}\n#{$!.backtrace.join("\n")}"
+    raise
+  end
+
+  def run
+    super
+  rescue
+    $log.error "unexpected error. exception: #{$!.class.to_s}, error: #{$!.to_s}\n#{$!.backtrace.join("\n")}"
+    raise
+  rescue SignalException
+    $log.debug "signal exception. exception: #{$!.class.to_s}, error: #{$!.to_s}"
+    raise
+  rescue Exception
+    $log.error "unexpected fatal error. exception: #{$!.class.to_s}, error: #{$!.to_s}\n#{$!.backtrace.join("\n")}"
+    raise
   end
 
   def event_listener(event)
-    begin
-      @record_handler.dispatch(event)
-    rescue Exception => e
-      position = File.open(@position_file) {|f| f.read }
-      $log.error "error occured while processing #{event.event_type} event at #{position}"
-      $log.error e.message
-      $log.error e.backtrace.join("\n")
-      # Not reraising a StandardError because the underlying code can't handle an error well.
-      raise unless e.kind_of?(StandardError)
-    end
+    @record_handler.dispatch(event)
+  rescue Exception
+    position = File.open(@position_file) {|f| f.read }
+    $log.error "error occured while processing #{event.event_type} event at #{position}\n#{e.message}\n#{$!.backtrace.join("\n")}"
+    # Not reraising a StandardError because the underlying code can't handle an error well.
+    raise unless e.kind_of?(StandardError)
   end
 
 end

data/spec/flydata/command/sync_spec.rb

@@ -50,7 +50,7 @@ module Flydata
         context 'with full options' do
           it 'issues mysqldump command with expected parameters' do
             expect(IO).to receive(:popen).with(
-             'mysqldump -d -h localhost -P 3306 -u masashi -pwelcome sync_test table1  table2', 'r').and_call_original
+             'mysqldump --protocol=tcp -d -h localhost -P 3306 -u masashi -pwelcome sync_test table1  table2', 'r').and_call_original
             subject.send(:do_generate_table_ddl, default_data_entry)
           end
         end
@@ -70,7 +70,7 @@ module Flydata
           end
           it "uses the default port" do
             expect(IO).to receive(:popen).with(
-             'mysqldump -d -h localhost -P 3306 -u masashi -pwelcome sync_test table1  table2', 'r').and_call_original
+             'mysqldump --protocol=tcp -d -h localhost -P 3306 -u masashi -pwelcome sync_test table1  table2', 'r').and_call_original
             subject.send(:do_generate_table_ddl, default_data_entry)
           end
         end
@@ -80,7 +80,7 @@ module Flydata
           end
           it "uses the specified port" do
             expect(IO).to receive(:popen).with(
-             'mysqldump -d -h localhost -P 1234 -u masashi -pwelcome sync_test table1  table2', 'r').and_call_original
+             'mysqldump --protocol=tcp -d -h localhost -P 1234 -u masashi -pwelcome sync_test table1  table2', 'r').and_call_original
             subject.send(:do_generate_table_ddl, default_data_entry)
           end
         end
@@ -100,7 +100,7 @@ module Flydata
           end
           it "call mysqldump without -p option" do
             expect(IO).to receive(:popen).with(
-             'mysqldump -d -h localhost -P 3306 -u masashi  sync_test table1  table2', 'r').and_call_original
+             'mysqldump --protocol=tcp -d -h localhost -P 3306 -u masashi  sync_test table1  table2', 'r').and_call_original
             subject.send(:do_generate_table_ddl, default_data_entry)
           end
         end
@@ -398,7 +398,7 @@ module Flydata
       describe '#initialize' do
         context 'with password' do
           subject { default_dump_generator.instance_variable_get(:@dump_cmd) }
-          it { should eq('mysqldump -h localhost -P 3306 -uadmin -ppass --skip-lock-tables ' +
+          it { should eq('mysqldump --protocol=tcp -h localhost -P 3306 -uadmin -ppass --skip-lock-tables ' +
                '--single-transaction --flush-logs --hex-blob --master-data=2 dev users groups') }
         end
         context 'without password' do
@@ -406,7 +406,7 @@ module Flydata
             MysqlDumpGenerator.new(default_conf.merge({'password' => ''}))
           end
           subject { dump_generator.instance_variable_get(:@dump_cmd) }
-          it { should eq('mysqldump -h localhost -P 3306 -uadmin  --skip-lock-tables ' +
+          it { should eq('mysqldump --protocol=tcp -h localhost -P 3306 -uadmin  --skip-lock-tables ' +
                '--single-transaction --flush-logs --hex-blob --master-data=2 dev users groups') }
         end
       end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flydata
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.1.6
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-05-17 00:00:00.000000000 Z
+date: 2014-05-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -391,7 +391,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 2175576432657099450
+      hash: -2211529075960565235
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements: