flydata 0.5.2 → 0.5.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ebe9adcd4a61f2b5867fa924d77c19d363a13d60
-  data.tar.gz: 5b08edfb063b96ce328457e164a8ac2ba0613642
+  metadata.gz: 550d6891d0b0315d762ded08f079e915888c9b52
+  data.tar.gz: 477cbc3a4977c15b17b9dd3a7ecedfa3fe35595d
 SHA512:
-  metadata.gz: 7ab6b64aecf376f7f0bb3a8a6e58ab5be27aab47e357f52f1904f420b9c85a7ad4808488e89a4983694b94708789dc316627a9b79ef2b92b7d28a17d7a50f3cc
-  data.tar.gz: 9291d7b54299bef2fc32ff4b725f4982b57265a0bb2656dde863b7fb413d4c18fc074125a4e86c3d60deef31a3ac23dd9706d822f0025d5d8bcd747fd50c27a6
+  metadata.gz: d9eaccb595b37a199a92244e2e27d6b8c8cf79740a8dd69a9add01cce9a68bcd1f94428c2749fdf28d35784d03ff5c0f02c13ccd6d71b324d8b129d93a9f46c5
+  data.tar.gz: bcfa91ee81945016d24f1f686cb329f7c940ab5c36db7baad3886de2fbd34f12d5be6eae80811fccc74aa0f4a2dd31477be98f96045717e486a399d33c86e8e4

data/VERSION

@@ -1 +1 @@
-0.5.2
+0.5.3

data/circle.yml

@@ -6,6 +6,7 @@ dependencies:
   pre:
     - sudo dpkg -l |grep libboost|grep 1.48|awk '{print $2}'|xargs sudo apt-get purge -y
     - sudo apt-get autoremove
+    - sudo apt-get update
     - sudo apt-get install -y libboost-system1.54.0
     - sudo apt-get install -y libboost-system1.54-dev
     - sudo apt-get install -y libboost-thread1.54.0
@@ -21,4 +22,4 @@ database:
 
 test:
   override:
-    - bundle exec rspec
+    - ./test-suite.sh

data/flydata-core/lib/flydata-core/mysql/compatibility_checker.rb

@@ -5,6 +5,10 @@ module FlydataCore
   module Mysql
     class CompatibilityChecker
       def initialize(option = {})
+        if option && option.has_key?(:host) && option[:host] == 'localhost'
+          option = option.dup
+          option[:host] = '127.0.0.1' # avoid socket access which Agent never uses
+        end
         @option = option || {}
       end
 
@@ -113,62 +117,113 @@ module FlydataCore
     class BinlogParameterChecker < MysqlCompatibilityChecker
       include MysqlVariablesHandling
 
-      SYS_VAR_TO_CHECK = {
+
+      BASE_SYS_VAR_TO_CHECK = {
         # parameter => expected value
-        'binlog_format'=>'ROW',
-        'binlog_checksum'=>'NONE',
-        'log_bin_use_v1_row_events'=>'ON',
-        'log_slave_updates'=>'ON',
+        'binlog_format' => 'ROW',
+        #'binlog_checksum' => 'NONE', # no longer necessary with mysql-replication-listener 0.2.0
+        'log_bin_use_v1_row_events' => 'ON',
+        'log_slave_updates' => 'ON',
+      }
+
+      ADDITIONAL_SYS_VAR_TO_CHECK = {
         # parameter => {"comparison sign"=>minimum value}
-        # TODO: Handled the error message of following items as warning
-        #'net_read_timeout'=>{'>'=>600},
-        #'net_write_timeout'=>{'>'=>600},
-        #'wait_timeout'=>{'>'=>60}
+        'net_read_timeout'=>{'>='=>600},
+        'net_write_timeout'=>{'>='=>600},
+        'wait_timeout'=>{'>='=>60},
       }
-      OPTIONAL_SYS_VAR = %w(binlog_checksum log_bin_use_v1_row_events)
+
+      SYS_VAR_TO_CHECK = BASE_SYS_VAR_TO_CHECK.merge(ADDITIONAL_SYS_VAR_TO_CHECK)
 
       def check_result(result, option = @option)
-        errors = {}
         param_hash = convert_result_to_hash(result)
-
-        compare_sys_var_values(param_hash, SYS_VAR_TO_CHECK, errors, OPTIONAL_SYS_VAR)
+        errors = compare_sys_var_values(param_hash, self.class::SYS_VAR_TO_CHECK)
 
         unless errors.empty?
-          error_explanation = ""
-          errors.each_key do |err_key|
-            if SYS_VAR_TO_CHECK[err_key].is_a?(Hash)
-              error_explanation << "\n    * #{err_key} is #{errors[err_key]} but should be at least #{SYS_VAR_TO_CHECK[err_key].values.first}"
-            else
-              error_explanation << "\n    * #{err_key} is #{errors[err_key]} but should be #{SYS_VAR_TO_CHECK[err_key]}"
-            end
-          end
-          raise FlydataCore::MysqlCompatibilityError,
-            "These system variable(s) are not the correct value: #{error_explanation}\n" +
-            "   Please change these system variables for FlyData Sync to run"
+          raise FlydataCore::MysqlCompatibilityError, build_error_message(errors)
         end
       end
 
+      def build_error_message(errors)
+        err_msg = errors.inject('') {|m, (k, v)| m << "\n    * #{v[:err_reason]}"}
+        "These system variable(s) are not the correct value: #{err_msg}\n" +
+        "   Please change these system variables for FlyData Sync to run"
+      end
+
       private
-      def compare_sys_var_values(input_value_hash, expected_value_hash, errors, exception_arr)
-        expected_value_hash.each_key do |exp_val|
-          if input_value_hash.has_key?(exp_val)
-            actual_val = input_value_hash[exp_val]
-            compare_sym = expected_value_hash[exp_val].is_a?(Hash) ? expected_value_hash[exp_val].keys.first : "="
-            case compare_sym
-            when "="
-              expected_val = expected_value_hash[exp_val]
-              unless actual_val == expected_val
-                errors[exp_val] = actual_val
-              end
-            when ">"
-              expected_val = expected_value_hash[exp_val].values.first
-              unless actual_val.to_i >= expected_val.to_i
-                errors[exp_val] = actual_val
+
+      def compare_sys_var_values(input_value_hash, expected_value_hash)
+        # { '<param-name>' => {actual_val: '<actual-val>', err_reason: '<err-reason>'} }
+        errors = {}
+
+        expected_value_hash.each do |param_name, exp_val_info|
+          # Ignore parameters if not returned
+          # binlog_checksum and log_bin_use_v1_row_events are supported only for Mysql v5.6
+          next unless input_value_hash.has_key?(param_name)
+          actual_val = input_value_hash[param_name]
+          err_reason = nil
+
+          if exp_val_info.kind_of?(Hash)
+            exp_val_info.each do |compare_sym, exp_val|
+              case compare_sym
+              when '='
+                unless actual_val.to_s != exp_val.to_s
+                  err_reason = "#{param_name} is '#{actual_val}' but should be '#{exp_val}'"
+                end
+              when '>='
+                unless actual_val.to_i >= exp_val.to_i
+                  err_reason = "#{param_name} is '#{actual_val}' but should be at least '#{exp_val}'"
+                end
+              else
+                raise "Unsupported compare symbol - #{compare_sym}"
               end
+              break # support just one key-value for now
             end
-          elsif not exception_arr.include?(exp_val)
-            errors[exp_val] = false
+          elsif exp_val_info.to_s != actual_val.to_s
+            err_reason = "#{param_name} is '#{actual_val}' but should be '#{exp_val_info}'"
           end
+          errors[param_name] = {actual_val: actual_val, err_reason: err_reason} if err_reason
+        end
+        errors
+      end
+    end
+
+    class RequiredBinlogParameterChecker < BinlogParameterChecker
+      SYS_VAR_TO_CHECK = BASE_SYS_VAR_TO_CHECK
+    end
+
+    class OptionalBinlogParameterChecker < BinlogParameterChecker
+      SYS_VAR_TO_CHECK = ADDITIONAL_SYS_VAR_TO_CHECK
+
+      def build_error_message(errors)
+        err_msg = errors.inject('') {|m, (k, v)| m << "\n    * #{v[:err_reason]}"}
+        "[WARNING] These system variable(s) are not the recommended values: #{err_msg}\n" +
+        "   These settings are not required but highly recommended for the FlyData Agent to run as efficiently as possible"
+      end
+    end
+
+    class TableExistenceChecker < MysqlCompatibilityChecker
+      TABLE_EXISTENCE_CHECK_QUERY_TMPLT = <<EOT
+SELECT
+  t.table_name as table_name
+FROM
+  information_schema.tables t
+WHERE
+  t.table_schema='%{database}'
+  AND t.table_name in (%{table_names});
+EOT
+      def create_query(option = @option)
+        TABLE_EXISTENCE_CHECK_QUERY_TMPLT % {database: option[:database], table_names: option[:tables].collect{|tn| "'#{tn}'"}.join(',')}
+      end
+
+      def check_result(result, option = @option)
+        existing_tables = []
+        result.each {|r| existing_tables << r['table_name']}
+        missing_tables = option[:tables] - existing_tables
+
+        unless missing_tables.empty?
+          raise FlydataCore::MysqlCompatibilityError,
+            "These tables are missing. Create these tables on your database or remove them from the data entry : #{missing_tables.join(", ")}"
         end
       end
     end
@@ -206,23 +261,26 @@ module FlydataCore
 
     class PrimaryKeyChecker < MysqlCompatibilityChecker
       PK_CHECK_QUERY_TMPLT = <<EOT
-SELECT k.table_name as table_name
-FROM information_schema.table_constraints t
-JOIN information_schema.key_column_usage k
-USING(constraint_name,table_schema,table_name)
-WHERE t.constraint_type='PRIMARY KEY'
-  AND t.table_schema='%{database}'
-  AND t.table_name in (%{table_names})
-GROUP BY k.table_name;
+SELECT
+  c.table_name as table_name,
+  SUM(IF(c.column_key = 'PRI', 1, 0)) as num_pk
+FROM
+  information_schema.columns c
+WHERE
+  c.table_schema='%{database}'
+  AND c.table_name in (%{table_names})
+GROUP BY
+  c.table_name
+HAVING
+  num_pk = 0;
 EOT
       def create_query(option = @option)
         PK_CHECK_QUERY_TMPLT % {database: option[:database], table_names: option[:tables].collect{|tn| "'#{tn}'"}.join(',')}
       end
 
       def check_result(result, option = @option)
-        pk_tables = []
-        result.each {|r| pk_tables << r['table_name']}
-        missing_pk_tables = option[:tables] - pk_tables
+        missing_pk_tables = []
+        result.each {|r| missing_pk_tables << r['table_name']}
 
         unless missing_pk_tables.empty?
           raise FlydataCore::MysqlCompatibilityError,

data/flydata-core/spec/mysql/compatibility_checker_spec.rb

@@ -0,0 +1,361 @@
+require 'spec_helper'
+require 'flydata-core/mysql/compatibility_checker'
+
+module FlydataCore
+  module Mysql
+    describe CompatibilityChecker do
+      let(:database) { 'test_db' }
+      let(:tables) { %w(table_1 table_2 table_3) }
+      let(:option) do
+        { database: database, tables: tables }
+      end
+
+      describe TableExistenceChecker do
+        let(:subject_object) { described_class.new(option) }
+
+        describe '#create_query' do
+          subject { subject_object.create_query }
+          it { is_expected.to eq <<EOT
+SELECT
+  t.table_name as table_name
+FROM
+  information_schema.tables t
+WHERE
+  t.table_schema='test_db'
+  AND t.table_name in ('table_1','table_2','table_3');
+EOT
+          }
+        end
+
+        describe '#check_reesult' do
+          let(:result) { [] }
+          subject { subject_object.check_result(result) }
+
+          context 'when all tables exist' do
+            let(:result) do
+              [ {'table_name' => 'table_1'},
+                {'table_name' => 'table_2'},
+                {'table_name' => 'table_3'}, ]
+            end
+            it { expect{subject}.not_to raise_error }
+          end
+
+          context 'when some tables does not exist' do
+            let(:result) do
+              [{"table_name" => "table_2"}]
+            end
+            it do
+              expect{subject}.to raise_error {|e|
+                expect(e).to be_kind_of(FlydataCore::MysqlCompatibilityError)
+                expect(e.to_s.include?("table_1")).to be_truthy
+                expect(e.to_s.include?("table_2")).to be_falsey
+                expect(e.to_s.include?("table_3")).to be_truthy
+              }
+            end
+          end
+        end
+      end
+
+      describe PrimaryKeyChecker do
+        let(:subject_object) { described_class.new(option) }
+
+        describe '#create_query' do
+          subject { subject_object.create_query }
+          it { is_expected.to eq <<EOT
+SELECT
+  c.table_name as table_name,
+  SUM(IF(c.column_key = 'PRI', 1, 0)) as num_pk
+FROM
+  information_schema.columns c
+WHERE
+  c.table_schema='test_db'
+  AND c.table_name in ('table_1','table_2','table_3')
+GROUP BY
+  c.table_name
+HAVING
+  num_pk = 0;
+EOT
+          }
+        end
+
+        describe '#check_reesult' do
+          let(:result) { [] }
+          subject { subject_object.check_result(result) }
+
+          context 'when all tables have pk' do
+            let(:result) do
+              []
+            end
+            it { expect{subject}.not_to raise_error }
+          end
+
+          context 'when some tables does not have pk' do
+            let(:result) do
+              [{"table_name" => "table_1"}, {"table_name" => "table_3"}]
+            end
+            it do
+              expect{subject}.to raise_error {|e|
+                expect(e).to be_kind_of(FlydataCore::MysqlCompatibilityError)
+                expect(e.to_s.include?("table_1")).to be_truthy
+                expect(e.to_s.include?("table_2")).to be_falsey
+                expect(e.to_s.include?("table_3")).to be_truthy
+              }
+            end
+          end
+        end
+      end
+    end
+
+    describe BinlogParameterChecker do
+      let(:subject_object) { described_class.new({}) }
+
+      describe "#check_result" do
+        let(:sys_var) do
+          [
+            {'Variable_name'=>'binlog_format','Value'=>'ROW'},
+            {'Variable_name'=>'binlog_checksum','Value'=>'NONE'},
+            {'Variable_name'=>'log_bin_use_v1_row_events','Value'=>'ON'},
+            {'Variable_name'=>'log_slave_updates','Value'=>'ON'},
+            {'Variable_name'=>'net_read_timeout','Value'=>600},
+            {'Variable_name'=>'net_write_timeout','Value'=>600},
+            {'Variable_name'=>'wait_timeout','Value'=>60}
+          ]
+        end
+        subject { subject_object.check_result(sys_var, {}) }
+
+        context "where parameters are valid" do
+          it do
+            expect{subject}.to_not raise_error
+          end
+        end
+        context "where parameters are invalid" do
+          let(:sys_var) do
+            [
+              {'Variable_name'=>'binlog_format','Value'=>'MIXED'},
+              {'Variable_name'=>'binlog_checksum','Value'=>'NONE'},
+              {'Variable_name'=>'log_bin_use_v1_row_events','Value'=>'ON'},
+              {'Variable_name'=>'log_slave_updates','Value'=>'ON'},
+              {'Variable_name'=>'net_read_timeout','Value'=>600},
+              {'Variable_name'=>'net_write_timeout','Value'=>600},
+              {'Variable_name'=>'wait_timeout','Value'=>30}
+            ]
+          end
+          it do
+            expect{subject}.to raise_error(FlydataCore::MysqlCompatibilityError, /binlog_format.*\n.*wait_timeout/)
+          end
+        end
+
+        context "where parameters are not returned" do
+          let(:sys_var) do
+            [
+              {'Variable_name'=>'binlog_format','Value'=>'ROW'},
+              {'Variable_name'=>'log_slave_updates','Value'=>'ON'},
+              {'Variable_name'=>'net_read_timeout','Value'=>600},
+              {'Variable_name'=>'net_write_timeout','Value'=>600},
+              {'Variable_name'=>'wait_timeout','Value'=>60},
+            ]
+          end
+          it 'ignores missing parameters' do
+            expect{subject}.not_to raise_error
+          end
+        end
+      end
+      describe "#compare_sys_var_values" do
+        let(:input_val) { nil }
+        let(:expected_val) do
+          {
+            'value_1' => 'foo',
+            'value_2' => 50,
+            'value_3' => {">="=>100}
+          }
+        end
+        let(:errors) { {} }
+
+        subject { subject_object.send(:compare_sys_var_values, input_val, expected_val) }
+
+        shared_examples "expect correct errors from values" do
+          it { is_expected.to eq(expecting_errors) }
+        end
+
+        context "when getting valid parameters" do
+          let(:input_val) do
+            {
+              'value_1' => 'foo',
+              'value_2' => 50,
+              'value_3' => 200
+            }
+          end
+          let(:expecting_errors) { {} }
+          it_behaves_like "expect correct errors from values"
+        end
+        context "when getting one invalid exact parameters for int" do
+          let(:input_val) do
+            {
+              'value_1' => 'foo',
+              'value_2' => 20,
+              'value_3' => 100
+            }
+          end
+          let(:expecting_errors) do
+            {
+              'value_2' => { actual_val: 20, err_reason: "value_2 is '20' but should be '50'" }
+            }
+          end
+          it_behaves_like "expect correct errors from values"
+        end
+        context "when getting one invalid exact parameter for string" do
+          let(:input_val) do
+            {
+              'value_1' => 'fo',
+              'value_2' => 50,
+              'value_3' => 100
+            }
+          end
+          let(:expecting_errors) do
+            {
+              'value_1' => { actual_val: 'fo', err_reason: "value_1 is 'fo' but should be 'foo'" }
+            }
+          end
+          it_behaves_like "expect correct errors from values"
+        end
+        context "when getting one invalid comparitive parameter that's lower than required" do
+          let(:input_val) do
+            {
+              'value_1' => 'foo',
+              'value_2' => 50,
+              'value_3' => 1
+            }
+          end
+          let(:expecting_errors) do
+            {
+              'value_3' => { actual_val: 1, err_reason: "value_3 is '1' but should be at least '100'" }
+            }
+          end
+          it_behaves_like "expect correct errors from values"
+        end
+        context "when getting a valid comparitive parameter that's higher than required" do
+          let(:input_val) do
+            {
+              'value_1' => 'foo',
+              'value_2' => 50,
+              'value_3' => 101
+            }
+          end
+          let(:expecting_errors) { {} }
+          it_behaves_like "expect correct errors from values"
+        end
+        context "when getting muliple invalid parameters" do
+          let(:input_val) do
+            {
+              'value_1' => 'fo',
+              'value_2' => 50,
+              'value_3' => 15
+            }
+          end
+          let(:expecting_errors) { {value_1:'fo', value_3:15} }
+          let(:expecting_errors) do
+            {
+              'value_1' => { actual_val: 'fo', err_reason: "value_1 is 'fo' but should be 'foo'" },
+              'value_3' => { actual_val: 15, err_reason: "value_3 is '15' but should be at least '100'" },
+            }
+          end
+          it_behaves_like "expect correct errors from values"
+        end
+      end
+    end
+
+    describe RequiredBinlogParameterChecker do
+      let(:subject_object) { described_class.new({}) }
+
+      describe "#check_result" do
+        let(:sys_var) do
+          [
+            {'Variable_name'=>'binlog_format','Value'=>'ROW'},
+            {'Variable_name'=>'binlog_checksum','Value'=>'NONE'},
+            {'Variable_name'=>'log_bin_use_v1_row_events','Value'=>'ON'},
+            {'Variable_name'=>'log_slave_updates','Value'=>'ON'},
+          ]
+        end
+        subject { subject_object.check_result(sys_var) }
+
+        context 'where parameters are valid' do
+          it { expect{subject}.not_to raise_error }
+        end
+
+        context 'where parameters contain optional paramters' do
+          let(:sys_var) do
+            [
+              {'Variable_name'=>'binlog_format','Value'=>'ROW'},
+              {'Variable_name'=>'binlog_checksum','Value'=>'NONE'},
+              {'Variable_name'=>'log_bin_use_v1_row_events','Value'=>'ON'},
+              {'Variable_name'=>'log_slave_updates','Value'=>'ON'},
+              {'Variable_name'=>'net_read_timeout','Value'=>1},
+              {'Variable_name'=>'net_write_timeout','Value'=>1},
+              {'Variable_name'=>'wait_timeout','Value'=>1}
+            ]
+          end
+          it { expect{subject}.not_to raise_error }
+        end
+
+        context 'where parameters contain invalid parameters' do
+          let(:sys_var) do
+            [
+              {'Variable_name'=>'binlog_format','Value'=>'MIXED'},
+              {'Variable_name'=>'binlog_checksum','Value'=>'NONE'},
+              {'Variable_name'=>'log_bin_use_v1_row_events','Value'=>'ON'},
+              {'Variable_name'=>'log_slave_updates','Value'=>'OFF'},
+            ]
+          end
+          it { expect{subject}.to raise_error(FlydataCore::MysqlCompatibilityError,
+                                              /binlog_format.*\n.*log_slave_updates/) }
+        end
+      end
+    end
+
+    describe OptionalBinlogParameterChecker do
+      let(:subject_object) { described_class.new({}) }
+
+      describe "#check_result" do
+        let(:sys_var) do
+          [
+            {'Variable_name'=>'net_read_timeout','Value'=>600},
+            {'Variable_name'=>'net_write_timeout','Value'=>600},
+            {'Variable_name'=>'wait_timeout','Value'=>60}
+          ]
+        end
+        subject { subject_object.check_result(sys_var) }
+
+        context 'where parameters are valid' do
+          it { expect{subject}.not_to raise_error }
+        end
+
+        context 'where parameters contain required paramters' do
+          let(:sys_var) do
+            [
+              {'Variable_name'=>'binlog_format','Value'=>'ROW'},
+              {'Variable_name'=>'binlog_checksum','Value'=>'NONE'},
+              {'Variable_name'=>'log_bin_use_v1_row_events','Value'=>'ON'},
+              {'Variable_name'=>'log_slave_updates','Value'=>'ON'},
+              {'Variable_name'=>'net_read_timeout','Value'=>600},
+              {'Variable_name'=>'net_write_timeout','Value'=>600},
+              {'Variable_name'=>'wait_timeout','Value'=>60},
+            ]
+          end
+          it { expect{subject}.not_to raise_error }
+        end
+
+        context 'where parameters contain invalid parameters' do
+          let(:sys_var) do
+            [
+              {'Variable_name'=>'net_read_timeout','Value'=>2},
+              {'Variable_name'=>'net_write_timeout','Value'=>600},
+              {'Variable_name'=>'wait_timeout','Value'=>1}
+            ]
+          end
+          it { expect{subject}.to raise_error(FlydataCore::MysqlCompatibilityError,
+                                              /^\[WARNING.*\n.*net_read_timeout.*\n.*wait_timeout/) }
+        end
+      end
+    end
+  end
+end

data/flydata.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: flydata 0.5.2 ruby lib
+# stub: flydata 0.5.3 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "flydata"
-  s.version = "0.5.2"
+  s.version = "0.5.3"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib"]
   s.authors = ["Koichi Fujikawa", "Masashi Miyazaki", "Matthew Luu", "Mak Inada", "Sriram NS"]
-  s.date = "2015-08-14"
+  s.date = "2015-09-03"
   s.description = "FlyData Agent"
   s.email = "sysadmin@flydata.com"
   s.executables = ["fdmysqldump", "flydata", "serverinfo"]
@@ -64,7 +64,7 @@ Gem::Specification.new do |s|
     "flydata-core/spec/fluent/config_helper_spec.rb",
     "flydata-core/spec/logger_spec.rb",
     "flydata-core/spec/mysql/command_generator_spec.rb",
-    "flydata-core/spec/mysql/compatibility_checker.rb",
+    "flydata-core/spec/mysql/compatibility_checker_spec.rb",
     "flydata-core/spec/option_validator_spec.rb",
     "flydata-core/spec/redshift/string_spec.rb",
     "flydata-core/spec/spec_helper.rb",
@@ -219,6 +219,7 @@ Gem::Specification.new do |s|
     "spec/flydata/util/shell_spec.rb",
     "spec/flydata_spec.rb",
     "spec/spec_helper.rb",
+    "test-suite.sh",
     "tmpl/redshift_mysql_data_entry.conf.tmpl"
   ]
   s.homepage = "http://flydata.com/"

data/lib/flydata/compatibility_check.rb

@@ -111,7 +111,12 @@ module Flydata
     end
 
     def check_mysql_parameters_compat
-      FlydataCore::Mysql::BinlogParameterChecker.new(@db_opts).do_check
+      begin
+        FlydataCore::Mysql::OptionalBinlogParameterChecker.new(@db_opts).do_check
+      rescue FlydataCore::MysqlCompatibilityError => e
+        log_warn_stderr(e.to_s)
+      end
+      FlydataCore::Mysql::RequiredBinlogParameterChecker.new(@db_opts).do_check
     end
 
     def check_mysql_binlog_retention

data/test-suite.sh

@@ -0,0 +1,56 @@
+#!/bin/bash
+
+# Usage: $0 [Options]
+#   Specify project
+#     [-m]  # main project
+#     [-c]  # core project
+#   Test skip option
+#     [--skip-main]
+#     [--skip-core]
+
+set -e
+BASE_DIR=$(cd $(dirname $0); pwd)
+
+#### Parse option
+
+TEST_MODE="skip" # skip or specify
+for OPT in $*
+do
+    case $OPT in
+        -m) TEST_MODE='specify'; SPECIFY_M=1; shift;;
+        -c) TEST_MODE='specify'; SPECIFY_C=1; shift;;
+        --skip-main) TEST_MODE='skip'; SKIP_M=1; shift;;
+        --skip-core) TEST_MODE='skip'; SKIP_C=1; shift;;
+    esac
+done
+
+if [ "$TEST_MODE" = 'skip' ]; then
+    if [ "$SKIP_M" = "" ]; then SPECIFY_M=1; fi
+    if [ "$SKIP_C" = "" ]; then SPECIFY_C=1; fi
+fi
+
+#### Test functions
+
+# Test flydata-agent library (flydata-agent/lib, flydata-agent/spec)
+test_main() {
+    cd $BASE_DIR
+    bundle install
+    bundle exec rspec
+}
+
+# Test flydata-core (flydata-core/lib, flydata-core/spec)
+test_core() {
+    cd $BASE_DIR/flydata-core
+    bundle install
+    bundle exec rspec
+}
+
+#### Main
+
+if [ "$SPECIFY_C" = '1' ]; then
+    test_core
+fi
+
+if [ "$SPECIFY_M" = '1' ]; then
+    test_main
+fi

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flydata
 version: !ruby/object:Gem::Version
-  version: 0.5.2
+  version: 0.5.3
 platform: ruby
 authors:
 - Koichi Fujikawa
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-08-14 00:00:00.000000000 Z
+date: 2015-09-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -501,7 +501,7 @@ files:
 - flydata-core/spec/fluent/config_helper_spec.rb
 - flydata-core/spec/logger_spec.rb
 - flydata-core/spec/mysql/command_generator_spec.rb
-- flydata-core/spec/mysql/compatibility_checker.rb
+- flydata-core/spec/mysql/compatibility_checker_spec.rb
 - flydata-core/spec/option_validator_spec.rb
 - flydata-core/spec/redshift/string_spec.rb
 - flydata-core/spec/spec_helper.rb
@@ -656,6 +656,7 @@ files:
 - spec/flydata/util/shell_spec.rb
 - spec/flydata_spec.rb
 - spec/spec_helper.rb
+- test-suite.sh
 - tmpl/redshift_mysql_data_entry.conf.tmpl
 homepage: http://flydata.com/
 licenses:

data/flydata-core/spec/mysql/compatibility_checker.rb

@@ -1,186 +0,0 @@
-require 'spec_helper'
-require 'flydata-core/mysql/compatibility_checker'
-
-module FlydataCore
-  module Mysql
-    describe CompatibilityChecker do
-      let(:database) { 'test_db' }
-      let(:tables) { %w(table_1 table_2 table_3) }
-      let(:option) do
-        { database: database, tables: tables }
-      end
-
-      describe PrimaryKeyChecker do
-        let(:subject_object) { described_class.new(option) }
-
-        describe '#create_query' do
-          subject { subject_object.create_query }
-          it { is_expected.to eq <<EOT
-SELECT k.table_name as table_name
-FROM information_schema.table_constraints t
-JOIN information_schema.key_column_usage k
-USING(constraint_name,table_schema,table_name)
-WHERE t.constraint_type='PRIMARY KEY'
-  AND t.table_schema='test_db'
-  AND t.table_name in ('table_1','table_2','table_3')
-GROUP BY k.table_name;
-EOT
-          }
-        end
-
-        describe '#check_reesult' do
-          let(:result) { [] }
-          subject { subject_object.check_result(result) }
-
-          context 'when all tables have pk' do
-            let(:result) do
-              [ {"table_name" => "table_1"},
-                {"table_name" => "table_2"},
-                {"table_name" => "table_3"}, ]
-            end
-            it { expect{subject}.not_to raise_error }
-          end
-
-          context 'when some tables does not have pk' do
-            let(:result) { [{"table_name" => "table_2"} ] }
-            it do
-              expect{subject}.to raise_error {|e|
-                expect(e).to be_kind_of(FlydataCore::MysqlCompatibilityError)
-                expect(e.to_s.include?("table_1")).to be_truthy
-                expect(e.to_s.include?("table_2")).to be_falsey
-                expect(e.to_s.include?("table_3")).to be_truthy
-              }
-            end
-          end
-        end
-      end
-    end
-
-    describe BinlogParameterChecker do
-      let(:subject_object) { BinlogParameterChecker.new({}) }
-
-      describe "#check_result" do
-        let(:sys_var) do
-          [
-            {'Variable_name'=>'binlog_format','Value'=>'ROW'},
-            {'Variable_name'=>'binlog_checksum','Value'=>'NONE'},
-            {'Variable_name'=>'log_bin_use_v1_row_events','Value'=>'ON'},
-            {'Variable_name'=>'log_slave_updates','Value'=>'ON'},
-            {'Variable_name'=>'net_read_timeout','Value'=>600},
-            {'Variable_name'=>'net_write_timeout','Value'=>600},
-            {'Variable_name'=>'wait_timeout','Value'=>60}
-          ]
-        end
-        subject { subject_object.check_result(sys_var, {}) }
-
-        context "where parameters are valid" do
-          it do
-            expect{subject}.to_not raise_error
-          end
-        end
-        context "where parameters are invalid" do
-          let(:sys_var) do
-            [
-              {'Variable_name'=>'binlog_format','Value'=>'MIXED'},
-              {'Variable_name'=>'binlog_checksum','Value'=>'NONE'},
-              {'Variable_name'=>'log_bin_use_v1_row_events','Value'=>'ON'},
-              {'Variable_name'=>'log_slave_updates','Value'=>'ON'},
-              {'Variable_name'=>'net_read_timeout','Value'=>600},
-              {'Variable_name'=>'net_write_timeout','Value'=>600},
-              {'Variable_name'=>'wait_timeout','Value'=>30}
-            ]
-          end
-          it do
-            expect{subject}.to raise_error(FlydataCore::MysqlCompatibilityError, /binlog_format.*\n.*wait_timeout/)
-          end
-        end
-      end
-      describe "#compare_sys_var_values" do
-        let(:expected_val) do
-          {
-            :value_1 => 'foo',
-            :value_2 => 50,
-            :value_3 => {">"=>100}
-          }
-        end
-        let(:errors) { {} }
-        let(:exception_arr) { [] }
-        subject { subject_object.send(:compare_sys_var_values, input_val, expected_val, errors, exception_arr) }
-
-        shared_examples "expect correct errors from values" do
-          it do
-            subject
-            expect(errors).to eq( expecting_errors )
-          end
-        end
-
-        context "when getting valid parameters" do
-          let(:input_val) do
-            {
-              :value_1 => 'foo',
-              :value_2 => 50,
-              :value_3 => 200
-            }
-          end
-          let(:expecting_errors) { {} }
-          it_behaves_like "expect correct errors from values"
-        end
-        context "when getting one invalid exact parameters for int" do
-          let(:input_val) do
-            {
-              :value_1 => 'foo',
-              :value_2 => 20,
-              :value_3 => 100
-            }
-          end
-          let(:expecting_errors) { {value_2:20} }
-          it_behaves_like "expect correct errors from values"
-        end
-        context "when getting one invalid exact parameter for string" do
-          let(:input_val) do
-            {
-              :value_1 => 'fo',
-              :value_2 => 50,
-              :value_3 => 100
-            }
-          end
-          let(:expecting_errors) { {value_1:'fo'} }
-          it_behaves_like "expect correct errors from values"
-        end
-        context "when getting one invalid comparitive parameter that's lower than required" do
-          let(:input_val) do
-            {
-              :value_1 => 'foo',
-              :value_2 => 50,
-              :value_3 => 1
-            }
-          end
-          let(:expecting_errors) { {value_3:1} }
-          it_behaves_like "expect correct errors from values"
-        end
-        context "when getting a valid comparitive parameter that's higher than required" do
-          let(:input_val) do
-            {
-              :value_1 => 'foo',
-              :value_2 => 50,
-              :value_3 => 101
-            }
-          end
-          let(:expecting_errors) { {} }
-          it_behaves_like "expect correct errors from values"
-        end
-        context "when getting muliple invalid parameters" do
-          let(:input_val) do
-            {
-              :value_1 => 'fo',
-              :value_2 => 50,
-              :value_3 => 15
-            }
-          end
-          let(:expecting_errors) { {value_1:'fo', value_3:15} }
-          it_behaves_like "expect correct errors from values"
-        end
-      end
-    end
-  end
-end