flydata 0.5.8 → 0.5.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4c79a1c7705cdd2112d4fd3465f311bf31583471
-  data.tar.gz: b5a6e9d153bbf84dca5d582de5bc5df460d01a96
+  metadata.gz: 66683f280e2cbf2c1c8ad081e4b9194dba9a928c
+  data.tar.gz: ff0e0497ec61502cbcb5a17d2860e3a98117cd54
 SHA512:
-  metadata.gz: 30e4e6fee11a8ba1b76f0e7c14e05f5d723a90655fc0e4cc0e42724f6edcef4d73514cf5d970becf3178a224172a9aa33d4fbe1bbbf27e4a0c0b73419b15fe34
-  data.tar.gz: 9ae07ffd9eb356a7947f5c8c1331f48181cc1ac8f76392f5c2d92122f4cfff38ef6f2d94e4ac001914bbb1d537423ff6ba060650d9776154b8b865285601cb77
+  metadata.gz: ed0e5930aa2494cc5ffb11d026c5edca78ba03fd97c72f42293864d5a835323f3a087133dec202245e6f143074fb8e8dbde3bf05d638f1bfdcd73d0725ae0bc8
+  data.tar.gz: c770737b2a978791723d46a9a8983c1677174c83f15a86ca03db552891a2d37529913ba62cbde987837bc931ac3d715d6cbd953e523126bd6cbc3742edeef09f

data/Gemfile

@@ -13,7 +13,7 @@ gem "slop", '~> 3.4', '>= 3.4.6'
 gem "treetop", '~> 1.5', '>= 1.5.3'
 gem "sys-filesystem", '~> 1.1', '>= 1.1.3'
 gem "io-console", '~> 0.4.2', '>= 0.4.2'
-gem "kodama", '~> 0.1.2', '>= 0.1.7'
+gem "kodama", '~> 0.1', '>= 0.1.8'
 gem "serverengine", '~> 1.5'
 
 group :development do

data/Gemfile.lock

@@ -61,7 +61,7 @@ GEM
       rdoc
     json (1.8.1)
     jwt (1.0.0)
-    kodama (0.1.7)
+    kodama (0.1.8)
       ruby-binlog (~> 1.0, >= 1.0.6)
     method_source (0.8.2)
     mime-types (2.3)
@@ -135,7 +135,7 @@ DEPENDENCIES
   io-console (~> 0.4.2, >= 0.4.2)
   jeweler (~> 1.8, >= 1.8.8)
   json (~> 1.8, >= 1.8.0)
-  kodama (~> 0.1.2, >= 0.1.7)
+  kodama (~> 0.1, >= 0.1.8)
   mysql2 (~> 0.3, >= 0.3.17)
   protected_attributes (~> 1.0, >= 1.0.8)
   pry

data/VERSION

@@ -1 +1 @@
-0.5.8
+0.5.9

data/flydata-core/lib/flydata-core/errors.rb

@@ -1,4 +1,5 @@
 require 'flydata-core/thread_context'
+require 'flydata-core/string_utils'
 
 module FlydataCore
 
@@ -129,8 +130,11 @@ class FlydataError < StandardError
 
   def filter_content(error_content = @error_content)
     error_content.each do |k, value|
-      if value.kind_of?(String) && value.length > MAX_CONTENT_SIZE
-        error_content[k] = value[0..MAX_CONTENT_SIZE-1]
+      if value.kind_of?(String)
+        if value.length > MAX_CONTENT_SIZE
+          value = error_content[k] = value[0..MAX_CONTENT_SIZE-1]
+        end
+        error_content[k] = FlydataCore::StringUtils.replace_invalid_utf8_char(value)
       end
     end
   end

data/flydata-core/lib/flydata-core/mysql/compatibility_checker.rb

@@ -128,7 +128,7 @@ module FlydataCore
         # parameter => {"comparison sign"=>minimum value}
         'net_read_timeout'=>{'>='=>600},
         'net_write_timeout'=>{'>='=>600},
-        'wait_timeout'=>{'>='=>60},
+        #'wait_timeout'=>{'>='=>60},
       }
 
       SYS_VAR_TO_CHECK = BASE_SYS_VAR_TO_CHECK.merge(ADDITIONAL_SYS_VAR_TO_CHECK)
@@ -260,15 +260,16 @@ EOT
     class PrimaryKeyChecker < MysqlCompatibilityChecker
       PK_CHECK_QUERY_TMPLT = <<EOT
 SELECT
-  c.table_name as table_name,
-  SUM(IF(c.column_key = 'PRI', 1, 0)) as num_pk
+  t.table_name,
+  SUM(IF(tc.constraint_type='PRIMARY KEY', 1, 0)) as num_pk
 FROM
-  information_schema.columns c
+  information_schema.tables t
+  LEFT OUTER JOIN information_schema.table_constraints tc USING (table_schema, table_name)
 WHERE
-  c.table_schema='%{database}'
-  AND c.table_name in (%{table_names})
+  t.table_schema = '%{database}'
+  AND t.table_name in (%{table_names})
 GROUP BY
-  c.table_name
+  t.table_schema, t.table_name
 HAVING
   num_pk = 0;
 EOT
@@ -306,6 +307,22 @@ EOT
       end
     end
 
+    class RdsMasterStatusChecker < MysqlCompatibilityChecker
+      BINLOG_RETENTION_HOURS = NonRdsRetentionChecker::BINLOG_RETENTION_HOURS
+
+      def create_query(option = @option)
+        "show master status;"
+      end
+
+      # "show master status;" returns nothing if backup retention period is not set.
+      def check_result(result, option = @option)
+        if result.first.nil?
+          raise FlydataCore::MysqlCompatibilityError,
+            "Backup Retention Period must be set on your RDS console."
+        end
+      end
+    end
+
     class RdsRetentionChecker < MysqlCompatibilityChecker
       BINLOG_RETENTION_HOURS = NonRdsRetentionChecker::BINLOG_RETENTION_HOURS
 

data/flydata-core/lib/flydata-core/mysql/ssl.rb

@@ -0,0 +1,8 @@
+module FlydataCore
+  module Mysql
+    module Ssl
+      # ssl cipher list not including DH algorithm
+      NON_DH_SSL_CIPHER = "AES256-GCM-SHA384:AES256-SHA:AES256-SHA256:CAMELLIA256-SHA:DES-CBC3-SHA:PSK-3DES-EDE-CBC-SHA:PSK-AES256-CBC-SHA:SRP-DSS-3DES-EDE-CBC-SHA:SRP-DSS-AES-128-CBC-SHA:SRP-DSS-AES-256-CBC-SHA:SRP-RSA-3DES-EDE-CBC-SHA:SRP-RSA-AES-128-CBC-S:SRP-RSA-AES-256-CBC-SHA"
+    end
+  end
+end

data/flydata-core/lib/flydata-core/string_utils.rb

@@ -0,0 +1,10 @@
+module FlydataCore
+  class StringUtils
+    def self.replace_invalid_utf8_char(str, replacement = '?')
+      str.encode("UTF-16BE",
+                 invalid: :replace,
+                 undef: :replace,
+                 replace: replacement).encode("UTF-8")
+    end
+  end
+end

data/flydata-core/lib/flydata-core/table_def/redshift_table_def.rb

@@ -346,6 +346,11 @@ EOS
       t = DateTime.strptime(value, "[%d/%b/%Y:%H:%M:%S %Z]")
     elsif time_match = TIME_REGEXP.match(value_str)
       t = convert_time_into_timestamp(time_match)
+    elsif /^(\d+)\.(\d+)$/ === value_str # epoch with fraction
+      epoch = $1
+      fraction = $2
+      t = DateTime.strptime(epoch, '%s')
+      t = DateTime.parse(t.strftime("%Y-%m-%dT%H:%M:%S.#{fraction}"))
     else
       t = DateTime.parse(convert_date(value_str))
     end

data/flydata-core/spec/mysql/compatibility_checker_spec.rb

@@ -63,15 +63,16 @@ EOT
           subject { subject_object.create_query }
           it { is_expected.to eq <<EOT
 SELECT
-  c.table_name as table_name,
-  SUM(IF(c.column_key = 'PRI', 1, 0)) as num_pk
+  t.table_name,
+  SUM(IF(tc.constraint_type='PRIMARY KEY', 1, 0)) as num_pk
 FROM
-  information_schema.columns c
+  information_schema.tables t
+  LEFT OUTER JOIN information_schema.table_constraints tc USING (table_schema, table_name)
 WHERE
-  c.table_schema='test_db'
-  AND c.table_name in ('table_1','table_2','table_3')
+  t.table_schema = 'test_db'
+  AND t.table_name in ('table_1','table_2','table_3')
 GROUP BY
-  c.table_name
+  t.table_schema, t.table_name
 HAVING
   num_pk = 0;
 EOT
@@ -135,13 +136,12 @@ EOT
               {'Variable_name'=>'binlog_checksum','Value'=>'NONE'},
               {'Variable_name'=>'log_bin_use_v1_row_events','Value'=>'ON'},
               {'Variable_name'=>'log_slave_updates','Value'=>'ON'},
-              {'Variable_name'=>'net_read_timeout','Value'=>600},
+              {'Variable_name'=>'net_read_timeout','Value'=>30},
               {'Variable_name'=>'net_write_timeout','Value'=>600},
-              {'Variable_name'=>'wait_timeout','Value'=>30}
             ]
           end
           it do
-            expect{subject}.to raise_error(FlydataCore::MysqlCompatibilityError, /binlog_format.*\n.*wait_timeout/)
+            expect{subject}.to raise_error(FlydataCore::MysqlCompatibilityError, /binlog_format.*\n.*net_read_timeout/)
           end
         end
 
@@ -353,7 +353,45 @@ EOT
             ]
           end
           it { expect{subject}.to raise_error(FlydataCore::MysqlCompatibilityError,
-                                              /^\[WARNING.*\n.*net_read_timeout.*\n.*wait_timeout/) }
+                                              /^\[WARNING.*\n.*net_read_timeout.*/) }
+        end
+      end
+    end
+
+    describe RdsMasterStatusChecker do
+      describe '#check_result' do
+        let(:subject_object) { described_class.new({}) }
+        let(:result) { [] }
+        subject { subject_object.check_result(result) }
+
+        context 'when result is empty' do
+          it do
+            expect{subject}.to raise_error(FlydataCore::MysqlCompatibilityError)
+          end
+        end
+
+        context 'when result is not empty' do
+=begin
+mysql> show master status;
++----------------------------+----------+--------------+------------------+-------------------+
+| File                       | Position | Binlog_Do_DB | Binlog_Ignore_DB | Executed_Gtid_Set |
++----------------------------+----------+--------------+------------------+-------------------+
+| mysql-bin-changelog.026292 |    31300 |              |                  |                   |
++----------------------------+----------+--------------+------------------+-------------------+
+1 row in set (0.00 sec)
+=end
+          let(:result) do
+            {
+              'File' => 'mysql-bin-changelog.026292',
+              'Position' => '31300',
+              'Binlog_Do_DB' => '',
+              'Binlog_Ignore_DB' => '',
+              'Executed_Gtid_Set' => '',
+            }
+          end
+          it do
+            expect{subject}.not_to raise_error
+          end
         end
       end
     end

data/flydata-core/spec/string_utils_spec.rb

@@ -0,0 +1,44 @@
+require 'spec_helper'
+require 'flydata-core/string_utils'
+
+describe FlydataCore::StringUtils do
+  let(:str) { '' }
+  let(:expected_str) { str }
+
+  subject { described_class.replace_invalid_utf8_char(str) }
+
+  shared_examples 'return_expected_str' do
+    it { is_expected.to eq(expected_str) }
+  end
+
+  context 'with non-invalid char' do
+    context 'with ascii code str' do
+      let(:str) { 'abcdefg' }
+      it_behaves_like 'return_expected_str'
+    end
+
+    context 'with empty str' do
+      let(:str) { '' }
+      it_behaves_like 'return_expected_str'
+    end
+
+    context 'with UTF-8 char' do
+      let(:str) { 'あいうえお' }
+      it_behaves_like 'return_expected_str'
+    end
+  end
+
+  context 'with invalid char' do
+    context 'with ascii and invalid str' do
+      let(:str) { "abc\255def" }
+      let(:expected_str) { 'abc?def' }
+      it_behaves_like 'return_expected_str'
+    end
+
+    context 'with UTF-8 and invalid str' do
+      let(:str) { "あいう\255えお" }
+      let(:expected_str) { "あいう?えお" }
+      it_behaves_like 'return_expected_str'
+    end
+  end
+end

data/flydata-core/spec/table_def/redshift_table_def_spec.rb

@@ -654,6 +654,11 @@ EOS
         let(:value) { 1400705967 }
         it { is_expected.to eq('2014-05-21 20:59:27.000000') }
       end
+
+      context 'with millisec' do
+        let(:value) { 1400705967.283928 }
+        it { is_expected.to eq('2014-05-21 20:59:27.283928') }
+      end
     end
 
     context 'with all zero string' do

data/flydata.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: flydata 0.5.8 ruby lib
+# stub: flydata 0.5.9 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "flydata"
-  s.version = "0.5.8"
+  s.version = "0.5.9"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib"]
   s.authors = ["Koichi Fujikawa", "Masashi Miyazaki", "Matthew Luu", "Mak Inada", "Sriram NS"]
-  s.date = "2015-09-25"
+  s.date = "2015-10-02"
   s.description = "FlyData Agent"
   s.email = "sysadmin@flydata.com"
   s.executables = ["fdmysqldump", "flydata", "serverinfo"]
@@ -52,10 +52,12 @@ Gem::Specification.new do |s|
     "flydata-core/lib/flydata-core/mysql/command_generator.rb",
     "flydata-core/lib/flydata-core/mysql/compatibility_checker.rb",
     "flydata-core/lib/flydata-core/mysql/config.rb",
+    "flydata-core/lib/flydata-core/mysql/ssl.rb",
     "flydata-core/lib/flydata-core/option_validator.rb",
     "flydata-core/lib/flydata-core/query_job.rb",
     "flydata-core/lib/flydata-core/record/record.rb",
     "flydata-core/lib/flydata-core/redshift/string.rb",
+    "flydata-core/lib/flydata-core/string_utils.rb",
     "flydata-core/lib/flydata-core/table_def.rb",
     "flydata-core/lib/flydata-core/table_def/autoload_redshift_table_def.rb",
     "flydata-core/lib/flydata-core/table_def/mysql_table_def.rb",
@@ -72,6 +74,7 @@ Gem::Specification.new do |s|
     "flydata-core/spec/option_validator_spec.rb",
     "flydata-core/spec/redshift/string_spec.rb",
     "flydata-core/spec/spec_helper.rb",
+    "flydata-core/spec/string_utils_spec.rb",
     "flydata-core/spec/table_def/autoload_redshift_table_def_spec.rb",
     "flydata-core/spec/table_def/mysql_table_def_spec.rb",
     "flydata-core/spec/table_def/mysql_to_redshift_table_def_spec.rb",
@@ -250,7 +253,7 @@ Gem::Specification.new do |s|
       s.add_runtime_dependency(%q<treetop>, [">= 1.5.3", "~> 1.5"])
       s.add_runtime_dependency(%q<sys-filesystem>, [">= 1.1.3", "~> 1.1"])
       s.add_runtime_dependency(%q<io-console>, [">= 0.4.2", "~> 0.4.2"])
-      s.add_runtime_dependency(%q<kodama>, [">= 0.1.7", "~> 0.1.2"])
+      s.add_runtime_dependency(%q<kodama>, [">= 0.1.8", "~> 0.1"])
       s.add_runtime_dependency(%q<serverengine>, ["~> 1.5"])
       s.add_development_dependency(%q<jeweler>, [">= 1.8.8", "~> 1.8"])
       s.add_development_dependency(%q<rspec>, ["~> 3.0"])
@@ -274,7 +277,7 @@ Gem::Specification.new do |s|
       s.add_dependency(%q<treetop>, [">= 1.5.3", "~> 1.5"])
       s.add_dependency(%q<sys-filesystem>, [">= 1.1.3", "~> 1.1"])
       s.add_dependency(%q<io-console>, [">= 0.4.2", "~> 0.4.2"])
-      s.add_dependency(%q<kodama>, [">= 0.1.7", "~> 0.1.2"])
+      s.add_dependency(%q<kodama>, [">= 0.1.8", "~> 0.1"])
       s.add_dependency(%q<serverengine>, ["~> 1.5"])
       s.add_dependency(%q<jeweler>, [">= 1.8.8", "~> 1.8"])
       s.add_dependency(%q<rspec>, ["~> 3.0"])
@@ -299,7 +302,7 @@ Gem::Specification.new do |s|
     s.add_dependency(%q<treetop>, [">= 1.5.3", "~> 1.5"])
     s.add_dependency(%q<sys-filesystem>, [">= 1.1.3", "~> 1.1"])
     s.add_dependency(%q<io-console>, [">= 0.4.2", "~> 0.4.2"])
-    s.add_dependency(%q<kodama>, [">= 0.1.7", "~> 0.1.2"])
+    s.add_dependency(%q<kodama>, [">= 0.1.8", "~> 0.1"])
     s.add_dependency(%q<serverengine>, ["~> 1.5"])
     s.add_dependency(%q<jeweler>, [">= 1.8.8", "~> 1.8"])
     s.add_dependency(%q<rspec>, ["~> 3.0"])

data/lib/flydata.rb

@@ -29,7 +29,7 @@ module Flydata
   FLYDATA_SERVERINFO = File.join(FLYDATA_GEM_BIN, 'serverinfo')
   FLYDATA_LOG = File.join(FLYDATA_HOME, 'flydata.log')
   FLYDATA_CONF = File.join(FLYDATA_HOME, 'flydata.conf')
-  FLYDATA_LOCK = File.join(FLYDATA_HOME, 'flydata.lock')
+  FLYDATA_LOCK = File.join(FLYDATA_HOME, 'flydata_lock.pid')
   FLYDATA_HELPER_HOME = File.join(FLYDATA_HOME, 'helper')
 
   VERSION_PATH = File.join(FLYDATA_GEM_HOME, 'VERSION')

data/lib/flydata/command/helper.rb

@@ -39,19 +39,19 @@ module Flydata
         create_helper_dirs
       end
 
-      def start
+      def start(options = {})
         if running?
-          log_info_stdout("Helper is already running.")
+          log_info_stdout("Helper is already running.", {}, options)
         else
           raw_start
         end
       end
 
-      def stop
+      def stop(options = {})
         if running?
           run_command(stop_cmd)
         else
-          log_info_stdout("Helper is not running.")
+          log_info_stdout("Helper is not running.", {}, options)
         end
       end
 
@@ -63,9 +63,9 @@ module Flydata
         end
       end
 
-      def restart
+      def restart(options = {})
         if running?
-          log_info_stdout("Restarting Helper.")
+          log_info_stdout("Restarting Helper.", {}, options)
           run_command(kill_hup_cmd)
         else
           raw_start

data/lib/flydata/command/restart.rb

@@ -16,7 +16,7 @@ module Flydata
         sender.restart
         unless opts.skip_helper?
           helper = Flydata::Command::Helper.new
-          helper.restart
+          helper.restart(quiet: true)
         end
       end
       run_exclusive :run

data/lib/flydata/command/sender.rb

@@ -141,6 +141,16 @@ EOF
         File.exists?(FLYDATA_LOCK)
       end
 
+      def wait_until_server_ready(options = {})
+        retry_count = 60 # 60 x 30(sec) = 1800(sec) = 30(min)
+        1.upto(retry_count) do |i|
+          return true if server_ready?
+          log_info_stdout("Waiting for the server side to become active... (#{i})") unless options[:quiet]
+          Kernel.sleep 30
+        end
+        false
+      end
+
       private
       # Return a list of fluentd parent processes run by the same user for the
       # same flydata.pid file but not the process managed by the file itself.
@@ -163,15 +173,6 @@ EOF
         orphan_pids
       end
 
-      def wait_until_server_ready(options = {})
-        retry_count = 10
-        1.upto(retry_count) do |i|
-          return true if server_ready?
-          log_info_stdout("Waiting for the server side to become active... (#{i}/#{retry_count})") unless options[:quiet]
-          Kernel.sleep 30
-        end
-        false
-      end
       def wait_until_client_ready(options = {})
         retry_count = 10
         1.upto(retry_count) do |i|

data/lib/flydata/command/start.rb

@@ -17,8 +17,8 @@ module Flydata
           helper_opts = Flydata::Command::Helper.slop_start
           helper_opts.parse!(args)
           helper = Flydata::Command::Helper.new(helper_opts)
-          helper.stop # kill existing helper process (if any) and start
-          helper.start
+          helper.stop(quiet: true) # kill existing helper process (if any) and start
+          helper.start(quiet: true)
         end
       end
       run_exclusive :run

data/lib/flydata/command/stop.rb

@@ -15,7 +15,7 @@ module Flydata
         sender.stop
         if opts.full?
           helper = Flydata::Command::Helper.new
-          helper.stop
+          helper.stop(quiet: true)
         end
       end
       run_exclusive :run

data/lib/flydata/command/sync.rb

@@ -526,6 +526,10 @@ EOS
       # Initial sync
 
       def handle_mysql_sync(tables = nil, options = {})
+        unless Flydata::Command::Sender.new.wait_until_server_ready
+          raise Timeout::Error, "Timed out to wait for the server side to become active. Pleae try again later."
+        end
+
         de = data_entry
 
         # Setup instance variables

data/lib/flydata/command_loggable.rb

@@ -11,16 +11,16 @@ module Flydata
       $stderr.puts raw_msg if options[:stderr]
     end
 
-    def log_info_stdout(message, log_params = {})
-      log_info(message, log_params, {stdout: true})
+    def log_info_stdout(message, log_params = {}, options = {})
+      log_info(message, log_params, {stdout: !options[:quiet]})
     end
 
-    def log_warn_stderr(message, log_params = {})
-      log_warn(message, log_params, {stderr: true})
+    def log_warn_stderr(message, log_params = {}, options = {})
+      log_warn(message, log_params, {stderr: !options[:quiet]})
     end
 
-    def log_error_stderr(message, log_params = {})
-      log_error(message, log_params, {stderr: true})
+    def log_error_stderr(message, log_params = {}, options = {})
+      log_error(message, log_params, {stderr: !options[:quiet]})
     end
   end
 end

data/lib/flydata/compatibility_check.rb

@@ -119,8 +119,14 @@ module Flydata
       FlydataCore::Mysql::RequiredBinlogParameterChecker.new(@db_opts).do_check
     end
 
+    def check_rds_master_status
+      if is_rds?
+        FlydataCore::Mysql::RdsMasterStatusChecker.new(@db_opts).do_check
+      end
+    end
+
     def check_mysql_binlog_retention
-      if is_rds?(@db_opts[:host])
+      if is_rds?
         run_rds_retention_check
       else
         run_mysql_retention_check
@@ -171,7 +177,7 @@ module Flydata
       end
     end
 
-    def is_rds?(hostname)
+    def is_rds?(hostname = @db_opts[:host])
       hostname.match(/rds.amazonaws.com$/) != nil
     end
   end

data/lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb

@@ -18,6 +18,7 @@ require 'flydata/fluent-plugins/idle_event_detector'
 require 'flydata/fluent-plugins/mysql/table_meta'
 require 'flydata/mysql/table_ddl'
 require 'flydata-core/fluent/config_helper'
+require 'flydata-core/mysql/ssl'
 
 #Monkey-patch fluentd class (EngineClass) to support shutdown for input plugin.
 #This will be called when USR1 signal is received
@@ -72,8 +73,7 @@ class MysqlBinlogFlydataInput < MysqlBinlogInput
   # Supported ssl cipher list for mysql(openssl)
   # https://dev.mysql.com/doc/refman/5.6/en/ssl-options.html
 
-  NON_DH_SSL_CIPHER = "AES256-GCM-SHA384:AES256-SHA:AES256-SHA256:CAMELLIA256-SHA:DES-CBC3-SHA:PSK-3DES-EDE-CBC-SHA:PSK-AES256-CBC-SHA:SRP-DSS-3DES-EDE-CBC-SHA:SRP-DSS-AES-128-CBC-SHA:SRP-DSS-AES-256-CBC-SHA:SRP-RSA-3DES-EDE-CBC-SHA:SRP-RSA-AES-128-CBC-S:SRP-RSA-AES-256-CBC-SHA"
-  config_param :secondary_ssl_cipher, :string, :default => NON_DH_SSL_CIPHER
+  config_param :secondary_ssl_cipher, :string, :default => FlydataCore::Mysql::Ssl::NON_DH_SSL_CIPHER
 
   def configure(conf)
     super
@@ -164,7 +164,6 @@ class MysqlBinlogFlydataInput < MysqlBinlogInput
     positions_path = @context.sync_fm.table_positions_dir_path
     Dir.mkdir positions_path unless File.exists? positions_path
 
-    File.open(@lock_file, "w") {|f| f.write(Process.pid)}
   rescue Binlog::Error
     if (/basic_string::_M_replace_aux/ === $!.to_s)
       # TODO Fix the root cause in mysql-replication-listener
@@ -193,33 +192,41 @@ EOS
 
     current_ssl_cipher = @ssl_cipher
     retried = false
+    transaction_broken = false
+    File.open(@lock_file, "w") {|f| f.write(Process.pid)}
     begin
-      start_kodama(mysql_url) do |c|
-        c.binlog_position_file = @position_file
-        if @ssl_ca_path.to_s != '' && c.respond_to?(:ssl_ca=)
-          $log.info "SSL is enabled. (ssl_ca: #{@ssl_ca_path})"
-          c.ssl_ca = @ssl_ca_path
-          unless current_ssl_cipher.to_s.empty?
-            $log.info "SSL cipher is set. (ssl_cipher: #{current_ssl_cipher})"
-            c.ssl_cipher = current_ssl_cipher
+      begin
+        start_kodama(mysql_url) do |c|
+          c.binlog_position_file = @position_file
+          if @ssl_ca_path.to_s != '' && c.respond_to?(:ssl_ca=)
+            $log.info "SSL is enabled. (ssl_ca: #{@ssl_ca_path})"
+            c.ssl_ca = @ssl_ca_path
+            unless current_ssl_cipher.to_s.empty?
+              $log.info "SSL cipher is set. (ssl_cipher: #{current_ssl_cipher})"
+              c.ssl_cipher = current_ssl_cipher
+            end
           end
-        end
 
-        if c.respond_to?(:sent_binlog_position_file=)
-          $log.info "Sent position feature is enabled. sent_position_file:#{@sent_position_file_path}"
-          c.sent_binlog_position_file = @sent_position_file_path
-        end
+          if c.respond_to?(:sent_binlog_position_file=)
+            $log.info "Sent position feature is enabled. sent_position_file:#{@sent_position_file_path}"
+            c.sent_binlog_position_file = @sent_position_file_path
+          end
 
-        $log.info("Binlog position - resume_pos:'#{IO.read(@position_file) rescue IOError}' " +
-                                    "sent_pos:'#{IO.read(@sent_position_file_path) rescue IOError}'")
+          $log.info("Binlog position - resume_pos:'#{IO.read(@position_file) rescue IOError}' " +
+                                      "sent_pos:'#{IO.read(@sent_position_file_path) rescue IOError}'")
 
-        c.connection_retry_limit = @retry_limit
-        c.connection_retry_wait = @retry_wait
-        c.log_level = @log_level.to_sym
-        @listen_events.each do |event_type|
-          $log.trace { "registered binlog event listener '#{event_type}'" }
-          c.send("on_#{event_type}", &method(:event_listener))
+          c.connection_retry_limit = @retry_limit
+          c.connection_retry_wait = @retry_wait
+          c.log_level = @log_level.to_sym
+          @listen_events.each do |event_type|
+            $log.trace { "registered binlog event listener '#{event_type}'" }
+            c.send("on_#{event_type}", &method(:event_listener))
+          end
         end
+      rescue Kodama::TransactionError
+        $log.debug "TransactionError"
+        transaction_broken = true
+        raise
       end
     rescue Binlog::Error => e
       if /dh key too small/.match(e.to_s) && !retried && !@secondary_ssl_cipher.to_s.empty?
@@ -230,6 +237,12 @@ EOS
       else
         raise e
       end
+    ensure
+      if !transaction_broken && # leave the lock file when a transaction is broken
+         File.exists?(@lock_file) &&
+         Process.pid == File.open(@lock_file, "r") {|f| f.read}.to_i
+        File.delete(@lock_file)
+      end
     end
 
   rescue => e
@@ -259,18 +272,12 @@ EOS
   rescue Exception => e
     position = @binlog_position_file.read
     $log.error "error occured while processing #{event.event_type} event at #{position}\n#{e.message}\n#{$!.backtrace.join("\n")}"
-    # Not reraising a StandardError because the underlying code can't handle an error well.
-    raise unless e.kind_of?(StandardError)
+    raise
   end
 
   def shutdown
     return if instance_variable_defined? :@abort
 
-    if File.exists?(@lock_file) &&
-       Process.pid == File.open(@lock_file, "r") {|f| f.read}.to_i
-      File.delete(@lock_file)
-    end
-
     @idle_event_detector.stop
     if @thread and @thread.alive?
       $log.info "Requesting stop Kodama"

data/lib/flydata/sync_file_manager.rb

@@ -123,17 +123,13 @@ module Flydata
     end
 
     def sent_binlog_path(master_binlog_path = binlog_path)
-      unless master_binlog_path && master_binlog_path.end_with?('binlog.pos')
-        raise ArgumentError.new("Invalid binlog path. binlog path needs to end with 'binlog.pos'")
-      end
+      validate_master_binlog_path(master_binlog_path)
       "#{master_binlog_path[0..-5]}.sent.pos"
     end
 
     # ssl_ca file path
     def ssl_ca_path(master_binlog_path = binlog_path)
-      unless master_binlog_path && master_binlog_path.end_with?('binlog.pos')
-        raise ArgumentError.new("Invalid binlog path. binlog path needs to end with 'binlog.pos'")
-      end
+      validate_master_binlog_path(master_binlog_path)
       # <data-entry-name>.ssl_ca.pem
       "#{master_binlog_path[0..-12]}.ssl_ca.pem"
     end
@@ -144,6 +140,19 @@ module Flydata
       end
     end
 
+    # ssl_cipher file path
+    def ssl_cipher_path(master_binlog_path = binlog_path)
+      validate_master_binlog_path(master_binlog_path)
+      # <data-entry-name>.ssl_cipher
+      "#{master_binlog_path[0..-12]}.ssl_cipher"
+    end
+
+    def save_ssl_cipher(ssl_cipher_content, path = ssl_cipher_path)
+      File.open(path, 'w') do |f|
+        f.write(ssl_cipher_content)
+      end
+    end
+
     # table files
     def reset_table_position_files(tables)
       tables.each do |table_name|
@@ -340,7 +349,9 @@ module Flydata
       FileUtils.mkdir_p(backup_dir) unless Dir.exists?(backup_dir)
       dest_dir = File.join(backup_dir, Time.now.strftime("%Y%m%d%H%M%S"))
       FileUtils.mkdir(dest_dir)
-      FileUtils.mv(Dir.glob("#{dump_dir}/*"), dest_dir)
+      %w(info pos stats mysql_table).each do |ext|
+        FileUtils.mv(Dir.glob("#{dump_dir}/*.#{ext}"), dest_dir)
+      end
     end
 
     def backup_dir
@@ -405,5 +416,11 @@ module Flydata
       FileUtils.mkdir_p(dump_dir) unless Dir.exists?(dump_dir)
       dump_dir
     end
+
+    def validate_master_binlog_path(master_binlog_path)
+      unless master_binlog_path && master_binlog_path.end_with?('binlog.pos')
+        raise ArgumentError.new("Invalid binlog path. binlog path needs to end with 'binlog.pos'")
+      end
+    end
   end
 end

data/spec/flydata/compatibility_check_spec.rb

@@ -272,5 +272,42 @@ module Flydata
         it { expect{subject}.to_not raise_error }
       end
     end
+
+    describe "#check_rds_master_status" do
+      let(:client) { double('client') }
+      let(:subject_object) { MysqlCompatibilityCheck.new(default_data_port, default_mysql_cred) }
+      subject { subject_object.check_rds_master_status }
+      let(:master_status) { [] }
+
+      before do
+        allow(Mysql2::Client).to receive(:new).and_return(client)
+        allow(client).to receive(:query).and_return(master_status)
+        allow(client).to receive(:close)
+      end
+
+      context 'when host is rds' do
+        before do
+          default_mysql_cred['host'] = 'rdrss.xxyyzz.rds.amazonaws.com'
+        end
+
+        context "where backup retention period is not set" do
+          let(:master_status) { [] }
+          it { expect{subject}.to raise_error(FlydataCore::MysqlCompatibilityError, /Backup Retention Period/) }
+        end
+
+        context "where backup retention period is set" do
+          let(:master_status) do
+            [{
+              'File' => 'mysql-bin-changelog.026292',
+              'Position' => '31300',
+              'Binlog_Do_DB' => '',
+              'Binlog_Ignore_DB' => '',
+              'Executed_Gtid_Set' => '',
+            }]
+          end
+          it { expect{subject}.not_to raise_error }
+        end
+      end
+    end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flydata
 version: !ruby/object:Gem::Version
-  version: 0.5.8
+  version: 0.5.9
 platform: ruby
 authors:
 - Koichi Fujikawa
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-09-25 00:00:00.000000000 Z
+date: 2015-10-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -248,20 +248,20 @@ dependencies:
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 0.1.7
+        version: 0.1.8
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.1.2
+        version: '0.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 0.1.7
+        version: 0.1.8
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.1.2
+        version: '0.1'
 - !ruby/object:Gem::Dependency
   name: serverengine
   requirement: !ruby/object:Gem::Requirement
@@ -489,10 +489,12 @@ files:
 - flydata-core/lib/flydata-core/mysql/command_generator.rb
 - flydata-core/lib/flydata-core/mysql/compatibility_checker.rb
 - flydata-core/lib/flydata-core/mysql/config.rb
+- flydata-core/lib/flydata-core/mysql/ssl.rb
 - flydata-core/lib/flydata-core/option_validator.rb
 - flydata-core/lib/flydata-core/query_job.rb
 - flydata-core/lib/flydata-core/record/record.rb
 - flydata-core/lib/flydata-core/redshift/string.rb
+- flydata-core/lib/flydata-core/string_utils.rb
 - flydata-core/lib/flydata-core/table_def.rb
 - flydata-core/lib/flydata-core/table_def/autoload_redshift_table_def.rb
 - flydata-core/lib/flydata-core/table_def/mysql_table_def.rb
@@ -509,6 +511,7 @@ files:
 - flydata-core/spec/option_validator_spec.rb
 - flydata-core/spec/redshift/string_spec.rb
 - flydata-core/spec/spec_helper.rb
+- flydata-core/spec/string_utils_spec.rb
 - flydata-core/spec/table_def/autoload_redshift_table_def_spec.rb
 - flydata-core/spec/table_def/mysql_table_def_spec.rb
 - flydata-core/spec/table_def/mysql_to_redshift_table_def_spec.rb