flydata 0.7.9 → 0.7.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b6544ac842d41d5b8482f1d8996e8a853efda2b9
-  data.tar.gz: 63e29a5dc775e88134129621791733b2a5ef301c
+  metadata.gz: 2dde125415d768ce91d31ba571250fcc82211a6c
+  data.tar.gz: 47de949761414acc3786397da03180e4f862cab8
 SHA512:
-  metadata.gz: 714ba6763f79f026d4dce4cd19b0fc6d9b83114292b1bde90bc30426d8f4133c86e1e9e2a36d7e4200efd3bed12bc12cc472fa8e806f45b6bc5fe0a0e70e9065
-  data.tar.gz: 1fe8b84fe9a71066df153046685ed8bc543f99fc6b32e9d8a36474e4b5e703a33c4647180ddc5a17213f6b15c97f2788ee04de97fec0da81464ca0d8c45a28c7
+  metadata.gz: 6f21358a59fb60e600a0a823a80574bd923b269093dc1d91618f9dc3e6ef7813ab385171666f62e8c86ad84a94e6db1d4c4dbad561acf9276b4ff4f94af12a32
+  data.tar.gz: 51daad9b9152816d384b367df2dbdffc3f3a0f3e9d07cf103759a8c2835d7b1bc71356961286f86da447a1dc0b2a218a9d9a669a6e4616f6da746ad8a4fb361d

data/VERSION

@@ -1 +1 @@
-0.7.9
+0.7.10

data/flydata-core/lib/flydata-core/compatibility_check/base.rb

@@ -0,0 +1,24 @@
+require 'flydata-core/errors'
+
+module FlydataCore
+  module CompatibilityCheck
+    class Base
+      def initialize(option = {})
+        @option = option || {}
+      end
+
+      def do_check(option = @option, &block)
+        result = block.call create_query(option)
+        check_result(result, option)
+      end
+
+      # Override
+      #def create_query(option = @option)
+      #end
+
+      # Override
+      #def validate_result(result, option = @option)
+      #end
+    end
+  end
+end

data/flydata-core/lib/flydata-core/compatibility_check/db_compatibility_checker.rb

@@ -0,0 +1,131 @@
+require 'flydata-core/compatibility_check/base'
+
+module FlydataCore
+  module CompatibilityCheck
+    class DbCompatibilityChecker < Base
+      def do_check(option = @option, &block)
+        query = create_query(option)
+        result = if block
+                   block.call query
+                 else
+                   exec_query(query)
+                 end
+        check_result(result, option)
+      end
+
+      def exec_query(query)
+        begin
+          client = build_db_client(@option)
+          client.query(query)
+        ensure
+          client.close rescue nil if client
+        end
+      end
+
+      # Override
+      #def build_db_client(option)
+      #end
+
+      # Utility for building query
+      # input: Array ['table_1', 'table_2', 'table_3', ...]
+      # output: String "'table_1','table_2'"
+      def table_names_in_query(tables)
+        tables.collect{|tn| "'#{tn}'"}.join(',')
+      end
+
+      def word_for_table_schema
+        "schema"
+      end
+
+      def get_schema_and_tables(result)
+        table_schema = nil
+        tables = []
+        result.each do |r|
+          tbl = r['table_name']
+          tables << tbl unless tbl.to_s.empty?
+          table_schema = r['table_schema'] if table_schema.nil?
+        end
+        [table_schema, tables]
+      end
+    end
+
+    module TableExistenceCheck
+      # Override
+      # TABLE_EXISTENCE_CHECK_QUERY_TMPLT = SELECT...
+      # COMPATIBILITY_ERROR_CLASS = MysqlCompatibilityError
+
+      def create_query(option = @option)
+        self.class::TABLE_EXISTENCE_CHECK_QUERY_TMPLT % schema_and_table_in_query(option)
+      end
+
+      def check_result(result, option = @option)
+        table_schema, existing_tables = get_schema_and_tables(result)
+        missing_tables = option[:tables] - existing_tables
+
+        unless missing_tables.empty?
+          raise self.class::COMPATIBILITY_ERROR_CLASS,
+            "These tables are missing in \"#{table_schema}\" #{word_for_table_schema}. Create these tables on your database or remove them from the data entry: #{missing_tables.join(", ")}"
+        end
+      end
+    end
+
+    module PrimaryKeyCheck
+      # Override
+      # PK_CHECK_QUERY_TMPLT = SELECT...
+      # COMPATIBILITY_ERROR_CLASS = MysqlCompatibilityError
+
+      def create_query(option = @option)
+        pk_override = option[:pk_override] || {}
+        tables = option[:tables].select{|tbl| pk_override[tbl.to_s].nil? } # exclude tables having pk_override
+        return nil if tables.empty?
+        (self.class::PK_CHECK_QUERY_TMPLT) % schema_and_table_in_query(option.merge(tables: tables))
+      end
+
+      def check_result(result, option = @option)
+        table_schema, missing_pk_tables = get_schema_and_tables(result)
+        unless missing_pk_tables.empty?
+          raise self.class::COMPATIBILITY_ERROR_CLASS,
+            "Primary key is required for tables to sync. " +
+            "Add primary key or remove following tables in \"#{table_schema}\" #{word_for_table_schema} from data entry: #{missing_pk_tables.join(", ")}"
+        end
+      end
+    end
+
+    module PkOverrideCheck
+      # Override
+      # PK_OVERRIDE_QUERY_TMPLT = SELECT...
+      # COMPATIBILITY_ERROR_CLASS = MysqlCompatibilityError
+
+      def create_query(option = @option)
+        pk_override = option[:pk_override] || {}
+        return nil if pk_override.empty?
+        self.class::PK_OVERRIDE_QUERY_TMPLT % schema_and_table_in_query(option.merge(tables: pk_override.keys))
+      end
+
+      def check_result(result, option = @option)
+        tbl_cols = Hash.new{|h, k| h[k] = []}  # key: table-name, value: array of column-name
+        table_schema = nil
+        result.each do |r|
+          tbl_cols[r['table_name']] << r['column_name']
+          table_schema = r['table_schema'] if table_schema.nil?
+        end
+
+        pk_override = option[:pk_override] || {}
+        missing_pk_override = {}
+        pk_override.each do |tbl, pk_cols|
+          missing_cols = pk_cols - tbl_cols[tbl]
+          missing_pk_override[tbl] = missing_cols unless missing_cols.empty?
+        end
+
+        unless missing_pk_override.empty?
+          msg = missing_pk_override.collect {|tbl, cols|
+            "#{tbl} (#{cols.join(',')})"
+          }.join(', ')
+          raise self.class::COMPATIBILITY_ERROR_CLASS,
+            "Columns set as primary key don't exist. " +
+            "Set correct column names to these tables in \"#{table_schema}\" #{word_for_table_schema}: #{msg}"
+        end
+      end
+    end
+  end
+end

data/flydata-core/lib/flydata-core/mysql/compatibility_checker.rb

@@ -1,47 +1,32 @@
 require 'flydata-core/errors'
 require 'flydata-core/mysql/config'
+require 'flydata-core/compatibility_check/db_compatibility_checker'
 require 'mysql2'
 
 module FlydataCore
   module Mysql
-    class CompatibilityChecker
-      def initialize(option = {})
-        option ||= {}
-        @option = option.merge FlydataCore::Mysql::Config.build_mysql_db_opts(option)
-      end
+    class MysqlCompatibilityChecker < FlydataCore::CompatibilityCheck::DbCompatibilityChecker
+      COMPATIBILITY_ERROR_CLASS = MysqlCompatibilityError
 
-      def do_check(option = @option, &block)
-        result = block.call create_query(option)
-        check_result(result, option)
+      def initialize(option = {})
+        super(option.merge FlydataCore::Mysql::Config.build_mysql_db_opts(option))
       end
 
       # Override
-      #def create_query(option = @option)
-      #end
-
-      # Override
-      #def validate_result(result, option = @option)
-      #end
-    end
+      def build_db_client(option)
+        Mysql2::Client.new(@option)
+      end
 
-    class MysqlCompatibilityChecker < CompatibilityChecker
-      def do_check(option = @option, &block)
-        query = create_query(option)
-        result = if block
-                   block.call query
-                 else
-                   exec_query(query)
-                 end
-        check_result(result, option)
+      def schema_and_table_in_query(option = @option)
+        {
+          schema_name: "'#{option[:database]}'",
+          table_names: table_names_in_query(option[:tables]),
+        }
       end
 
-      def exec_query(query)
-        begin
-          client = Mysql2::Client.new(@option)
-          client.query(query)
-        ensure
-          client.close rescue nil if client
-        end
+      # "table_schema" is equivalent to "database" on MySQL. No schema notion inside database.
+      def word_for_table_schema
+        "database"
       end
 
       # This command works only on RDS MySQL
@@ -221,29 +206,20 @@ module FlydataCore
     end
 
     class TableExistenceChecker < MysqlCompatibilityChecker
+      include FlydataCore::CompatibilityCheck::TableExistenceCheck
       TABLE_EXISTENCE_CHECK_QUERY_TMPLT = <<EOT
 SELECT
-  t.table_name as table_name
+  t.table_schema,
+  t.table_name
 FROM
   information_schema.tables t
 WHERE
-  t.table_schema='%{database}'
-  AND t.table_name in (%{table_names});
+  t.table_schema = %{schema_name}
+  AND
+  t.table_name in (%{table_names})
+UNION
+  SELECT %{schema_name}, '';
 EOT
-      def create_query(option = @option)
-        TABLE_EXISTENCE_CHECK_QUERY_TMPLT % {database: option[:database], table_names: option[:tables].collect{|tn| "'#{tn}'"}.join(',')}
-      end
-
-      def check_result(result, option = @option)
-        existing_tables = []
-        result.each {|r| existing_tables << r['table_name']}
-        missing_tables = option[:tables] - existing_tables
-
-        unless missing_tables.empty?
-          raise FlydataCore::MysqlCompatibilityError,
-            "These tables are missing. Create these tables on your database or remove them from the data entry : #{missing_tables.join(", ")}"
-        end
-      end
     end
 
     class MySqlVersion57CompabilityChecker < MysqlCompatibilityChecker
@@ -298,45 +274,46 @@ EOT
     end
 
     class PrimaryKeyChecker < MysqlCompatibilityChecker
+      include FlydataCore::CompatibilityCheck::PrimaryKeyCheck
       PK_CHECK_QUERY_TMPLT = <<EOT
 SELECT
+  t.table_schema,
   t.table_name,
   SUM(IF(tc.constraint_type='PRIMARY KEY' AND col.is_nullable='NO', 1, 0)) as num_pk,
   SUM(IF(tc.constraint_type='UNIQUE' AND col.is_nullable='NO', 1, 0)) as num_uk
 FROM
-  (select * from information_schema.tables where table_schema = '%{database}' AND table_name in (%{table_names})) t
+  (select * from information_schema.tables where table_schema = %{schema_name} AND table_name in (%{table_names})) t
   LEFT OUTER JOIN
-  (select * from information_schema.table_constraints where table_schema = '%{database}' AND table_name in (%{table_names})) tc
+  (select * from information_schema.table_constraints where table_schema = %{schema_name} AND table_name in (%{table_names})) tc
   USING (table_schema, table_name)
   LEFT JOIN
-  (select * from information_schema.key_column_usage where constraint_schema = '%{database}') kc
+  (select * from information_schema.key_column_usage where constraint_schema = %{schema_name}) kc
   USING (table_schema, table_name, constraint_name)
   LEFT JOIN
-  (select * from information_schema.columns where table_schema = '%{database}' AND table_name in (%{table_names})) col
+  (select * from information_schema.columns where table_schema = %{schema_name} AND table_name in (%{table_names})) col
   ON t.table_schema = col.table_schema AND t.table_name = col.table_name AND kc.column_name = col.column_name
 GROUP BY
   t.table_schema, t.table_name
 HAVING
   num_pk = 0 and num_uk = 0;
 EOT
+    end
 
-      def create_query(option = @option)
-        pk_override = option[:pk_override] || {}
-        tables = option[:tables].select{|tbl| pk_override[tbl.to_s].nil? } # exclude tables having pk_override
-        return nil if tables.empty?
-        PK_CHECK_QUERY_TMPLT % {database: option[:database], table_names: tables.collect{|tn| "'#{tn}'"}.join(',')}
-      end
-
-      def check_result(result, option = @option)
-        missing_pk_tables = []
-        result.each {|r| missing_pk_tables << r['table_name']}
-
-        unless missing_pk_tables.empty?
-          raise FlydataCore::MysqlCompatibilityError,
-            "Primary key is required for tables to sync. " +
-            "Add primary key or remove following tables from data entry: #{missing_pk_tables.join(", ")}"
-        end
-      end
+    class PkOverrideChecker < MysqlCompatibilityChecker
+      include FlydataCore::CompatibilityCheck::PkOverrideCheck
+      PK_OVERRIDE_QUERY_TMPLT = <<EOT
+SELECT
+  t.table_schema,
+  t.table_name,
+  col.column_name
+FROM
+  (select * from information_schema.tables where table_schema = %{schema_name} AND table_name in (%{table_names})) t
+  LEFT JOIN
+  (select * from information_schema.columns where table_schema = %{schema_name} AND table_name in (%{table_names})) col
+  ON t.table_schema = col.table_schema AND t.table_name = col.table_name
+ORDER BY
+  t.table_name;
+EOT
     end
 
     class NonRdsRetentionChecker < MysqlCompatibilityChecker

data/flydata-core/lib/flydata-core/postgresql/compatibility_checker.rb

@@ -1,114 +1,105 @@
 require 'flydata-core/errors'
+require 'flydata-core/postgresql/config'
+require 'flydata-core/compatibility_check/db_compatibility_checker'
 require 'flydata-core/postgresql/pg_client'
 
 module FlydataCore
   module Postgresql
-    class CompatibilityChecker
-      def initialize(option = {})
-        option ||= {}
-        @option = option.merge FlydataCore::Postgresql::Config.build_db_opts(option)
-      end
+    class PostgresqlCompatibilityChecker < FlydataCore::CompatibilityCheck::DbCompatibilityChecker
+      COMPATIBILITY_ERROR_CLASS = PostgresqlCompatibilityError
 
-      def do_check(option = @option, &block)
-        result = block.call create_query(option)
-        check_result(result, option)
+      def initialize(option = {})
+        super(option.merge FlydataCore::Postgresql::Config.build_db_opts(option))
       end
 
       # Override
-      #def create_query(option = @option)
-      #end
-
-      # Override
-      #def validate_result(result, option = @option)
-      #end
-    end
-
-    class PostgresqlCompatibilityChecker < CompatibilityChecker
-      def do_check(option = @option, &block)
-        query = create_query(option)
-        result = if block
-                   block.call query
-                 else
-                   exec_query(query)
-                 end
-        check_result(result, option)
-      end
-
-      def exec_query(query)
-        begin
-          client = PGClient.new(@option)
-          client.query(query)
-        ensure
-          client.close rescue nil if client
-        end
+      def build_db_client(option)
+        PGClient.new(@option)
       end
 
       def schema_and_table_in_query(option = @option)
         schema = FlydataCore::Postgresql::QueryHelper.schema_as_value(option[:schema])
         {
           schema_name: schema,
-          table_names: option[:tables].collect{|tn| "'#{tn}'"}.join(',')
+          table_names: table_names_in_query(option[:tables]),
         }
       end
     end
 
-    class TableExistenceChecker < PostgresqlCompatibilityChecker
-      TABLE_EXISTENCE_CHECK_QUERY_TMPLT = <<EOT
+    class SnapshotFunctionChecker < PostgresqlCompatibilityChecker
+      TXID_CURRENT_SNAPSHOT_CHECK_SQL = <<EOS
 SELECT
-  table_name
+  p.proname
 FROM
-  information_schema.tables
+  pg_proc p
+  JOIN pg_namespace n ON p.pronamespace = n.oid
 WHERE
-  table_schema in (%{schema_name})
-  AND
-  table_name in (%{table_names});
-EOT
+  n.nspname = 'pg_catalog'
+  AND p.proname = 'txid_current_snapshot';
+EOS
 
       def create_query(option = @option)
-        TABLE_EXISTENCE_CHECK_QUERY_TMPLT % schema_and_table_in_query(option)
+        TXID_CURRENT_SNAPSHOT_CHECK_SQL
       end
 
       def check_result(result, option = @option)
-        existing_tables = []
-        result.each {|r| existing_tables << r['table_name']}
-        missing_tables = option[:tables] - existing_tables
-
-        unless missing_tables.empty?
+        unless result.first
           raise FlydataCore::PostgresqlCompatibilityError,
-            "These tables are missing. Create these tables on your database or remove them from the data entry : #{missing_tables.join(", ")}"
+            "Unsupported PostgreSQL version. The version must be 8.3 or later."
         end
       end
     end
 
+    class TableExistenceChecker < PostgresqlCompatibilityChecker
+      include FlydataCore::CompatibilityCheck::TableExistenceCheck
+      TABLE_EXISTENCE_CHECK_QUERY_TMPLT = <<EOT
+SELECT
+  table_schema,
+  table_name
+FROM
+  information_schema.tables
+WHERE
+  table_schema = %{schema_name}
+  AND
+  table_name in (%{table_names})
+UNION
+  SELECT %{schema_name}, '';
+EOT
+    end
+
     class PrimaryKeyChecker < PostgresqlCompatibilityChecker
+      include FlydataCore::CompatibilityCheck::PrimaryKeyCheck
       PK_CHECK_QUERY_TMPLT = <<EOT
 SELECT
+  t.table_schema,
   t.table_name
 FROM
-  (select * from information_schema.tables where table_schema in (%{schema_name}) AND table_name in (%{table_names})) t
+  (select * from information_schema.tables where table_schema = %{schema_name} AND table_name in (%{table_names})) t
   LEFT OUTER JOIN
-  (select * from information_schema.table_constraints where table_schema in (%{schema_name}) AND table_name in (%{table_names})) tc
+  (select * from information_schema.table_constraints where table_schema = %{schema_name} AND table_name in (%{table_names})) tc
   USING (table_schema, table_name)
 GROUP BY
   t.table_schema, t.table_name
 HAVING
   SUM(CASE WHEN tc.constraint_type='PRIMARY KEY' THEN 1 ELSE 0 END) = 0;
 EOT
+    end
 
-      def create_query(option = @option)
-        PK_CHECK_QUERY_TMPLT % schema_and_table_in_query(option)
-      end
-
-      def check_result(result, option = @option)
-        missing_pk_tables = []
-        result.each {|r| missing_pk_tables << r['table_name']}
-
-        unless missing_pk_tables.empty?
-          raise FlydataCore::PostgresqlCompatibilityError,
-            "Primary key is required for tables to sync. " +
-            "Add primary key or remove following tables from data entry: #{missing_pk_tables.join(", ")}"
-        end
-      end
+    class PkOverrideChecker < PostgresqlCompatibilityChecker
+      include FlydataCore::CompatibilityCheck::PkOverrideCheck
+      PK_OVERRIDE_QUERY_TMPLT = <<EOT
+SELECT
+  t.table_schema,
+  t.table_name,
+  col.column_name
+FROM
+  (select * from information_schema.tables where table_schema = %{schema_name} AND table_name in (%{table_names})) t
+  LEFT JOIN
+  (select * from information_schema.columns where table_schema = %{schema_name} AND table_name in (%{table_names})) col
+  ON t.table_schema = col.table_schema AND t.table_name = col.table_name
+ORDER BY
+  t.table_name;
+EOT
     end
   end
 end

data/flydata-core/spec/mysql/compatibility_checker_spec.rb

@@ -3,7 +3,7 @@ require 'flydata-core/mysql/compatibility_checker'
 
 module FlydataCore
   module Mysql
-    describe CompatibilityChecker do
+    describe 'CompatibilityChecker' do
       let(:database) { 'test_db' }
       let(:mysql_host) { hostname + '.' + domain_name }
       let(:hostname) { 'test-hostname' }
@@ -87,12 +87,16 @@ module FlydataCore
           subject { subject_object.create_query }
           it { is_expected.to eq <<EOT
 SELECT
-  t.table_name as table_name
+  t.table_schema,
+  t.table_name
 FROM
   information_schema.tables t
 WHERE
-  t.table_schema='test_db'
-  AND t.table_name in ('table_1','table_2','table_3');
+  t.table_schema = 'test_db'
+  AND
+  t.table_name in ('table_1','table_2','table_3')
+UNION
+  SELECT 'test_db', '';
 EOT
           }
         end
@@ -133,6 +137,7 @@ EOT
           subject { subject_object.create_query }
           it { is_expected.to eq <<EOT
 SELECT
+  t.table_schema,
   t.table_name,
   SUM(IF(tc.constraint_type='PRIMARY KEY' AND col.is_nullable='NO', 1, 0)) as num_pk,
   SUM(IF(tc.constraint_type='UNIQUE' AND col.is_nullable='NO', 1, 0)) as num_uk
@@ -162,6 +167,7 @@ EOT
             end
             it { is_expected.to eq <<EOT
 SELECT
+  t.table_schema,
   t.table_name,
   SUM(IF(tc.constraint_type='PRIMARY KEY' AND col.is_nullable='NO', 1, 0)) as num_pk,
   SUM(IF(tc.constraint_type='UNIQUE' AND col.is_nullable='NO', 1, 0)) as num_uk
@@ -221,6 +227,81 @@ EOT
           end
         end
       end
+
+      describe PkOverrideChecker do
+        let(:subject_object) { described_class.new(option) }
+
+        describe '#create_query' do
+          subject { subject_object.create_query }
+
+          context 'when pk_override is given' do
+            before do
+              option[:pk_override] = {
+                'table_1' => ['id'],
+                'table_2' => ['name'],
+              }
+            end
+            it { is_expected.to eq <<EOT
+SELECT
+  t.table_schema,
+  t.table_name,
+  col.column_name
+FROM
+  (select * from information_schema.tables where table_schema = 'test_db' AND table_name in ('table_1','table_2')) t
+  LEFT JOIN
+  (select * from information_schema.columns where table_schema = 'test_db' AND table_name in ('table_1','table_2')) col
+  ON t.table_schema = col.table_schema AND t.table_name = col.table_name
+ORDER BY
+  t.table_name;
+EOT
+            }
+          end
+
+          context 'when no pk override' do
+            let(:tables) { %w(table_2) }
+            before do
+              option.delete(:pk_override)
+            end
+            it { is_expected.to be_nil }
+          end
+        end
+
+        describe '#check_reesult' do
+          subject { subject_object.check_result(result) }
+          let(:result) { [
+            {"table_name" => "table_1", "column_name" => "id"},
+            {"table_name" => "table_1", "column_name" => "value"},
+            {"table_name" => "table_2", "column_name" => "name"},
+            {"table_name" => "table_2", "column_name" => "age"},
+          ] }
+
+          before do
+            option[:pk_override] = pk_override
+          end
+
+          context 'when given pk_override exist' do
+            let(:pk_override) { {
+              'table_1' => ['id'],
+              'table_2' => ['name'],
+            } }
+            it { expect{subject}.not_to raise_error }
+          end
+
+          context 'when some columns do not exist' do
+            let(:pk_override) { {
+              'table_1' => ['id'],
+              'table_2' => ['id'],
+            } }
+            it do
+              expect{subject}.to raise_error {|e|
+                expect(e).to be_kind_of(FlydataCore::MysqlCompatibilityError)
+                expect(e.to_s.include?("table_1")).to be_falsey
+                expect(e.to_s.include?("table_2")).to be_truthy
+              }
+            end
+          end
+        end
+      end
     end
 
     describe BinlogParameterChecker do

data/flydata-core/spec/postgresql/compatibility_checker_spec.rb

@@ -3,13 +3,43 @@ require 'flydata-core/postgresql/compatibility_checker'
 
 module FlydataCore
   module Postgresql
-    describe CompatibilityChecker do
+    describe 'CompatibilityChecker' do
       let(:database) { 'test_db' }
       let(:tables) { %w(table_1 table_2 table_3) }
       let(:option) do
         { database: database, tables: tables }
       end
 
+      describe SnapshotFunctionChecker do
+        let(:subject_object) { described_class.new(option) }
+
+        describe '#check_reesult' do
+          let(:result) { [] }
+          subject { subject_object.check_result(result) }
+
+          context 'when the function exists' do
+            let(:result) do
+              [ {'proname' => 'txid_current_snapshot'} ]
+            end
+            it { expect{subject}.not_to raise_error }
+          end
+
+          context 'when the function does not exist' do
+            let(:result) do
+              []
+            end
+            it do
+              expect{subject}.to raise_error(
+                FlydataCore::PostgresqlCompatibilityError,
+                /Unsupported PostgreSQL version/
+              )
+            end
+          end
+        end
+      end
+
+
+
       describe TableExistenceChecker do
         let(:subject_object) { described_class.new(option) }
 
@@ -19,13 +49,16 @@ module FlydataCore
           context 'when schema is not set' do
             it { is_expected.to eq <<EOT
 SELECT
+  table_schema,
   table_name
 FROM
   information_schema.tables
 WHERE
-  table_schema in ((select current_schema))
+  table_schema = (select current_schema)
   AND
-  table_name in ('table_1','table_2','table_3');
+  table_name in ('table_1','table_2','table_3')
+UNION
+  SELECT (select current_schema), '';
 EOT
             }
           end
@@ -33,13 +66,16 @@ EOT
             before { option.merge!(schema: 'test_schema') }
             it { is_expected.to eq <<EOT
 SELECT
+  table_schema,
   table_name
 FROM
   information_schema.tables
 WHERE
-  table_schema in ('test_schema')
+  table_schema = 'test_schema'
   AND
-  table_name in ('table_1','table_2','table_3');
+  table_name in ('table_1','table_2','table_3')
+UNION
+  SELECT 'test_schema', '';
 EOT
             }
           end
@@ -84,11 +120,12 @@ EOT
           context 'when schema is not set' do
             it { is_expected.to eq <<EOT
 SELECT
+  t.table_schema,
   t.table_name
 FROM
-  (select * from information_schema.tables where table_schema in ((select current_schema)) AND table_name in ('table_1','table_2','table_3')) t
+  (select * from information_schema.tables where table_schema = (select current_schema) AND table_name in ('table_1','table_2','table_3')) t
   LEFT OUTER JOIN
-  (select * from information_schema.table_constraints where table_schema in ((select current_schema)) AND table_name in ('table_1','table_2','table_3')) tc
+  (select * from information_schema.table_constraints where table_schema = (select current_schema) AND table_name in ('table_1','table_2','table_3')) tc
   USING (table_schema, table_name)
 GROUP BY
   t.table_schema, t.table_name
@@ -102,11 +139,12 @@ EOT
             before { option.merge!(schema: 'test_schema') }
             it { is_expected.to eq <<EOT
 SELECT
+  t.table_schema,
   t.table_name
 FROM
-  (select * from information_schema.tables where table_schema in ('test_schema') AND table_name in ('table_1','table_2','table_3')) t
+  (select * from information_schema.tables where table_schema = 'test_schema' AND table_name in ('table_1','table_2','table_3')) t
   LEFT OUTER JOIN
-  (select * from information_schema.table_constraints where table_schema in ('test_schema') AND table_name in ('table_1','table_2','table_3')) tc
+  (select * from information_schema.table_constraints where table_schema = 'test_schema' AND table_name in ('table_1','table_2','table_3')) tc
   USING (table_schema, table_name)
 GROUP BY
   t.table_schema, t.table_name
@@ -117,7 +155,7 @@ EOT
           end
         end
 
-        describe '#check_reesult' do
+        describe '#check_result' do
           let(:result) { [] }
           subject { subject_object.check_result(result) }
 
@@ -133,6 +171,7 @@ EOT
               [{"table_name" => "table_1"}, {"table_name" => "table_3"}]
             end
             it do
+              skip "Primary key check for postgresql currenlty not raises error due to false alarm"
               expect{subject}.to raise_error {|e|
                 expect(e).to be_kind_of(FlydataCore::PostgresqlCompatibilityError)
                 expect(e.to_s.include?("table_1")).to be_truthy
@@ -143,6 +182,81 @@ EOT
           end
         end
       end
+
+      describe PkOverrideChecker do
+        let(:subject_object) { described_class.new(option) }
+
+        describe '#create_query' do
+          subject { subject_object.create_query }
+
+          context 'when pk_override is given' do
+            before do
+              option[:pk_override] = {
+                'table_1' => ['id'],
+                'table_2' => ['name'],
+              }
+            end
+            it { is_expected.to eq <<EOT
+SELECT
+  t.table_schema,
+  t.table_name,
+  col.column_name
+FROM
+  (select * from information_schema.tables where table_schema = (select current_schema) AND table_name in ('table_1','table_2')) t
+  LEFT JOIN
+  (select * from information_schema.columns where table_schema = (select current_schema) AND table_name in ('table_1','table_2')) col
+  ON t.table_schema = col.table_schema AND t.table_name = col.table_name
+ORDER BY
+  t.table_name;
+EOT
+            }
+          end
+
+          context 'when no pk override' do
+            let(:tables) { %w(table_2) }
+            before do
+              option.delete(:pk_override)
+            end
+            it { is_expected.to be_nil }
+          end
+        end
+
+        describe '#check_reesult' do
+          subject { subject_object.check_result(result) }
+          let(:result) { [
+            {"table_name" => "table_1", "column_name" => "id"},
+            {"table_name" => "table_1", "column_name" => "value"},
+            {"table_name" => "table_2", "column_name" => "name"},
+            {"table_name" => "table_2", "column_name" => "age"},
+          ] }
+
+          before do
+            option[:pk_override] = pk_override
+          end
+
+          context 'when given pk_override exist' do
+            let(:pk_override) { {
+              'table_1' => ['id'],
+              'table_2' => ['name'],
+            } }
+            it { expect{subject}.not_to raise_error }
+          end
+
+          context 'when some columns do not exist' do
+            let(:pk_override) { {
+              'table_1' => ['id'],
+              'table_2' => ['id'],
+            } }
+            it do
+              expect{subject}.to raise_error {|e|
+                expect(e).to be_kind_of(FlydataCore::PostgresqlCompatibilityError)
+                expect(e.to_s.include?("table_1")).to be_falsey
+                expect(e.to_s.include?("table_2")).to be_truthy
+              }
+            end
+          end
+        end
+      end
     end
   end
 end

data/lib/flydata/source_mysql/plugin_support/binlog_record_handler.rb

@@ -84,6 +84,10 @@ module PluginSupport
         @first_empty_binlog = true
       end
     end
+
+    def reset_table_rev(table)
+      @context.table_revs[table] = 1
+    end
   end
 end
 

data/lib/flydata/source_mysql/plugin_support/create_table_query_handler.rb

@@ -17,9 +17,14 @@ module PluginSupport
 
     def process(record)
       if acceptable_db?(record)
-        $log.error "CREATE TABLE detected. Please reset and resync the following table:'#{table_info(record)[:table_name]}'. - query:'#{record["query"]}' binlog_pos:'#{binlog_pos(record)}'"
+        emit_record(:create_table, record) do |opt|
+          table_name = table_info(record)[:table_name]
+          $log.info "type:create_table table_name:'#{table_name}' query:'#{record["query"]}' binlog_pos:'#{binlog_pos(record)}'"
+
+          opt[:increment_table_rev] = true
+          { table_name: table_name, query: record["query"] }
+        end
       end
-      #NOTE: No emit_record here because this record should not be sent to data servers for now
     end
   end
 end

data/lib/flydata/source_mysql/plugin_support/drop_table_query_handler.rb

@@ -17,9 +17,14 @@ module PluginSupport
 
     def process(record)
       if acceptable_db?(record)
-        $log.error "DROP TABLE detected for table:'#{table_info(record)[:table_name]}'. - query:'#{record["query"]}' binlog_pos:'#{binlog_pos(record)}'"
+        emit_record(:drop_table, record) do |opt|
+          table_name = table_info(record)[:table_name]
+          $log.info "type:drop_table table_name:'#{table_name}' query:'#{record["query"]}' binlog_pos:'#{binlog_pos(record)}'"
+
+          opt[:increment_table_rev] = true
+          { table_name: table_name, query: record["query"] }
+        end
       end
-      #NOTE: No emit_record here because this record should not be sent to data servers for now
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flydata
 version: !ruby/object:Gem::Version
-  version: 0.7.9
+  version: 0.7.10
 platform: ruby
 authors:
 - Koichi Fujikawa
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-08-05 00:00:00.000000000 Z
+date: 2016-08-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -522,6 +522,8 @@ files:
 - flydata-core/circle.yml
 - flydata-core/lib/flydata-core.rb
 - flydata-core/lib/flydata-core/agent.rb
+- flydata-core/lib/flydata-core/compatibility_check/base.rb
+- flydata-core/lib/flydata-core/compatibility_check/db_compatibility_checker.rb
 - flydata-core/lib/flydata-core/config/user_maintenance.rb
 - flydata-core/lib/flydata-core/core_ext.rb
 - flydata-core/lib/flydata-core/core_ext/module.rb
@@ -880,7 +882,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.4.3
 signing_key: 
 specification_version: 4
 summary: FlyData Agent