flydata 0.8.8 → 0.8.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f5fc3a906ed3837e16d5cedecaf192d488c140eb
-  data.tar.gz: b5ddcf8ea4abd89688e6ed096837bf53cfff14ee
+  metadata.gz: 18f14f5aa7cb056405bcddd04145128861746e81
+  data.tar.gz: 7137aa3faffbc50cdadf71a54e22f50c59f7beb4
 SHA512:
-  metadata.gz: 14bd2021e7c507a6ab59e29e6e210776fd6a2a7bf2eaeb52d36b1da39839d1e575203066c2b14e6bb06628f8464b1b9b9510c13c64b7ada5d727b81d46ce230e
-  data.tar.gz: 83740aa8b34dd758656071446c34a8fd2ed0486a464eca577b9ea9ff0731ece4e190ba36292210cd66953adf761e2ff5abaeb6e484582b491d93d24928eab396
+  metadata.gz: 515cc0f4a0520fc9c6f2573eb457d903dd83c43c6ff8ef114659b8e6050d67c3984e2ecc4acbfeba8307b12e4db383b1039df4647ede956c0d52799a465b6a6c
+  data.tar.gz: e2ea17f5471e190d8bcc718db2da8136bff4677337f60fb5e1f7da4011fb41de5081c9efad491bd8ebe1043296e26a969a55845a6b3446764e573d022cfad3f8

data/VERSION

@@ -1 +1 @@
-0.8.8
+0.8.9

data/flydata-core/lib/flydata-core/mysql/command_generator.rb

@@ -1,13 +1,18 @@
 require 'open3'
 require 'shellwords'
 require 'flydata-core/table_def/mysql_table_def'
+require 'flydata-core/mysql/compatibility_checker'
 
 module FlydataCore
   module Mysql
     class CommandGenerator
       DEFAULT_OPTION = "--default-character-set=utf8 --protocol=tcp"
       DEFAULT_MYSQL_OPTION = "#{DEFAULT_OPTION} --skip-auto-rehash"
+      DEFAULT_MYSQLDUMP57_OPTION = "#{DEFAULT_OPTION} --set-gtid-purged=OFF" # If you face GTID issue when start initial sync, try this option for mysqldump
 
+      # Get default command options
+      # Arguments:
+      #   command - String ('mysql' or 'mysqldump')
       def self.default_cmd_option(command)
         case command
         when 'mysql'
@@ -48,6 +53,15 @@ module FlydataCore
         default_option = option[:no_default_option] ? "" : default_cmd_option(command)
         default_option += " --ssl-ca #{ssl_ca}" if ssl_ca
         default_option += " --ssl-cipher=#{ssl_cipher}" unless ssl_cipher.to_s.empty?
+
+        if command == 'mysqldump'
+          version = MysqlCompatibilityChecker.new(option).mysql_server_version
+
+          if !(version =~ /Maria/) && MysqlCompatibilityChecker.server_version_eq_or_higher?(version, "5.6.9")
+            default_option += " --set-gtid-purged=OFF"
+          end
+        end
+
         default_option = nil if default_option == ''
 
         custom_option = option[:custom_option]

data/flydata-core/lib/flydata-core/mysql/compatibility_checker.rb

@@ -29,6 +29,33 @@ module FlydataCore
         "database"
       end
 
+      # Utility methods
+
+      VERSION_QUERY = "SHOW VARIABLES LIKE 'version'"
+      def mysql_server_version
+        result = exec_query(VERSION_QUERY)
+        result.first['Value']
+      end
+
+      # Check if the version of MySQL server is equal or higher than the given version
+      # Argument:
+      #   version: (String)
+      # Example:
+      # - MySQL server version: "5.6.10-log"
+      #   server_version_eq_or_higher?("5.6.10-log", "5.6.9") #=> true
+      # - MySQL server version: "5.6.24-2+deb.sury.org~precise+2-log"
+      #   server_version_eq_or_higher?("5.6.24-2+deb.sury.org~precise+2-log", "5.6.24") #=> true
+      def self.server_version_eq_or_higher?(server_version, version_to_compare)
+        (version_to_array(server_version) <=> version_to_array(version_to_compare)) >= 0
+      end
+
+      # Converting version string to Array of Integers
+      # Example: version_to_array("5.6.9-log") #=> [5, 6, 9]
+      #          version_to_array("5.6.24-2+deb.sury.org~precise+2-log") #=> [5, 6, 24, 0, 0]
+      def self.version_to_array(version)
+        version.split('.').map(&:to_i)
+      end
+
       # This command works only on RDS MySQL
       RDS_CHECK_QUERY = "call mysql.rds_show_configuration;"
       def rds?(hostname = @option[:host])
@@ -324,7 +351,7 @@ EOT
     class NonRdsRetentionChecker < MysqlCompatibilityChecker
       include MysqlVariablesHandling
 
-      BINLOG_RETENTION_HOURS = 24
+      BINLOG_RETENTION_HOURS = 96
       EXPIRE_LOGS_DAYS_LIMIT = BINLOG_RETENTION_HOURS / 24
 
       def check_result(result, option = @option)

data/flydata-core/spec/mysql/command_generator_spec.rb

@@ -15,6 +15,12 @@ module FlydataCore
       } }
       let(:option) { default_option }
       let(:cmd_pswd_check) { 'mysqldump -h test-host -P 3306 -utestuser -p%s --default-character-set=utf8 --protocol=tcp testdb' }
+      let(:mysql_compatibility_checker) { instance_double(MysqlCompatibilityChecker) }
+
+      before do
+        allow(MysqlCompatibilityChecker).to receive(:new).and_return(mysql_compatibility_checker)
+        allow(mysql_compatibility_checker).to receive(:mysql_server_version).and_return('5.6.8')
+      end
 
       describe '.generate_mysql_cmd' do
         subject { described_class.generate_mysql_cmd(option) }
@@ -35,7 +41,7 @@ module FlydataCore
           ) }
         end
 
-        context 'when option conatins string key' do
+        context 'when option contains string key' do
           before {
             option[:database] = nil
             option['database'] = 'another_db'
@@ -121,6 +127,34 @@ module FlydataCore
             'mysqldump -h test-host -P 3306 -utestuser -ptestpassword --default-character-set=utf8 --protocol=tcp testdb test_table'
           ) }
         end
+
+        context 'when mysql version is 5.6.9' do
+          before do
+            allow(mysql_compatibility_checker).to receive(:mysql_server_version).and_return('5.6.9')
+          end
+
+          context 'when command is not \'mysqldump\'' do
+            before { option[:command] = nil }
+
+            it { is_expected.to eq(
+                'mysql -h test-host -P 3306 -utestuser -ptestpassword --default-character-set=utf8 --protocol=tcp --skip-auto-rehash testdb'
+            ) }
+          end
+
+          context 'when server is MariaDB' do
+            before do
+              allow(mysql_compatibility_checker).to receive(:mysql_server_version).and_return('5.6.9-MariaDB')
+            end
+
+            it { is_expected.to eq(
+                'mysqldump -h test-host -P 3306 -utestuser -ptestpassword --default-character-set=utf8 --protocol=tcp testdb'
+            ) }
+          end
+
+          it { is_expected.to eq(
+              'mysqldump -h test-host -P 3306 -utestuser -ptestpassword --default-character-set=utf8 --protocol=tcp --set-gtid-purged=OFF testdb'
+          ) }
+        end
       end
 
       describe '.generate_mysql_ddl_dump_cmd' do

data/flydata-core/spec/mysql/compatibility_checker_spec.rb

@@ -19,6 +19,36 @@ module FlydataCore
       describe MysqlCompatibilityChecker do
         let(:subject_object) { described_class.new(option) }
 
+        describe '#mysql_server_version' do
+          before do
+            version = [{"Variable_name"=>"version", "Value"=>"5.5.57-log"}]
+            allow(subject_object).to receive(:exec_query).with("SHOW VARIABLES LIKE 'version'").and_return(version)
+          end
+
+          it 'returns the mysql version' do
+            expect(subject_object.mysql_server_version).to eq('5.5.57-log')
+          end
+        end
+
+        describe '.server_version_eq_or_higher' do
+          it 'returns true if versions are equal' do
+            expect(described_class.server_version_eq_or_higher?("5.6.9-log", "5.6.9")).to be_truthy
+          end
+          it 'returns true if version is higher' do
+            expect(described_class.server_version_eq_or_higher?("5.6.10-log", "5.6.9")).to be_truthy
+          end
+          it 'returns false if version is lower' do
+            expect(described_class.server_version_eq_or_higher?("5.6.8-log", "5.6.9")).to be_falsey
+          end
+        end
+
+        describe '.version_to_array' do
+          it 'converts string to array' do
+            expect(described_class.version_to_array('5.6.9-log')).to eq([5, 6, 9])
+            expect(described_class.version_to_array('5.6.24-2+deb.sury.org~precise+2-log')).to eq([5, 6, 24, 0, 0])
+          end
+        end
+
         describe '#rds?' do
           subject { subject_object.rds? }
 

data/spec/flydata/source_mysql/mysql_compatibility_check_spec.rb

@@ -170,15 +170,23 @@ module SourceMysql
         end
         context "where retention period is too low" do
           before do
-            allow(client).to receive(:query).with("call mysql.rds_show_configuration;").and_return([{"name"=>"binlog retention hours", "value"=>4}])
+            allow(client).to receive(:query).with("call mysql.rds_show_configuration;").and_return([{"name"=>"binlog retention hours", "value"=>95}])
           end
           it do
             expect{subject}.to raise_error(FlydataCore::MysqlCompatibilityError, /rds_set_config/)
           end
         end
+        context "where retention period is equal to the recommended limit" do
+          before do
+            allow(client).to receive(:query).with("call mysql.rds_show_configuration;").and_return([{"name"=>"binlog retention hours", "value"=>96}])
+          end
+          it do
+            expect{subject}.to_not raise_error
+          end
+        end
         context "where retention period is over recommended limit" do
           before do
-            allow(client).to receive(:query).with("call mysql.rds_show_configuration;").and_return([{"name"=>"binlog retention hours", "value"=>120}])
+            allow(client).to receive(:query).with("call mysql.rds_show_configuration;").and_return([{"name"=>"binlog retention hours", "value"=>97}])
           end
           it do
             expect{subject}.to_not raise_error
@@ -191,6 +199,11 @@ module SourceMysql
           it do
             expect{subject}.to_not raise_error
           end
+          it "logs the correct message" do
+            message = "[WARNING]Cannot verify RDS retention period on current MySQL user account.\nTo see retention period, please run this on your RDS:\n  $> call mysql.rds_show_configuration;\nPlease verify that the hours is not nil and is at least 96 hours\nTo set binlog retention hours, you can run this on your RDS:\n  $> call mysql.rds_set_configuration('binlog retention hours', 96);\n"
+            expect(subject_object).to receive(:log_warn_stderr).with(message)
+            subject
+          end
         end
       end
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flydata
 version: !ruby/object:Gem::Version
-  version: 0.8.8
+  version: 0.8.9
 platform: ruby
 authors:
 - Koichi Fujikawa
@@ -13,7 +13,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-03-25 00:00:00.000000000 Z
+date: 2018-02-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -950,7 +950,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.3
+rubygems_version: 2.4.8
 signing_key: 
 specification_version: 4
 summary: FlyData Agent