flydata 0.7.8 → 0.7.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3d9438eb32d8d567ad63149916961bde54cdf52c
-  data.tar.gz: dd2f138fd9f823e1cbee1d2d091262a6a008f336
+  metadata.gz: b6544ac842d41d5b8482f1d8996e8a853efda2b9
+  data.tar.gz: 63e29a5dc775e88134129621791733b2a5ef301c
 SHA512:
-  metadata.gz: af5eb23ccfa7691ff8a28c4767d256f988cb72377073b994bcaa7d45f94e38223253151e6702a10b5184ed28c37e22feac3c8e1eebd6792680341cddb24457e4
-  data.tar.gz: 5385e752f2531d6bea7ad8c739c1b0a485e20e1e600872319902238e4cef0b96096288192ddc9463b53afc363617e0c23a02fe0814cb33b84d01f0d8c5db891c
+  metadata.gz: 714ba6763f79f026d4dce4cd19b0fc6d9b83114292b1bde90bc30426d8f4133c86e1e9e2a36d7e4200efd3bed12bc12cc472fa8e806f45b6bc5fe0a0e70e9065
+  data.tar.gz: 1fe8b84fe9a71066df153046685ed8bc543f99fc6b32e9d8a36474e4b5e703a33c4647180ddc5a17213f6b15c97f2788ee04de97fec0da81464ca0d8c45a28c7

data/VERSION

@@ -1 +1 @@
-0.7.8
+0.7.9

data/bin/split_sync_ddl.rb

@@ -0,0 +1,53 @@
+#!/usr/bin/env ruby
+
+def usage
+  puts <<EOT
+Split sync DDL file into multiple files
+
+  Usage: split_sync_ddl.rb <ddl-file-path> [<num-tables-per-file(default:50)>]
+
+ex) split_sync_ddl ct20160722_01.sql 10
+  This command generates
+    ct20160722_01.sql_0
+    ct20160722_01.sql_1
+    ct20160722_01.sql_2
+    ....
+EOT
+  exit 1
+end
+
+@original_ddl = ARGV[0]
+unless @original_ddl
+  usage
+end
+unless File.exists?(@original_ddl)
+  $stderr.puts "Error! File doesn't exist - #{@original_ddl}"
+  exit 1
+end
+@num_tables_per_file = ARGV[1] || 50
+
+@file_no = 0
+
+def out_file_name(file_no)
+  "#{@original_ddl}_#{file_no}"
+end
+
+def save_file(lines)
+  fn = out_file_name(@file_no)
+  content = lines.kind_of?(Array) ? lines.join : lines.to_s
+  IO.write(fn, content)
+  @file_no += 1
+end
+
+SEPARATOR = "BEGIN;\n"
+def main
+  all_blocks = IO.read(@original_ddl).split(SEPARATOR)
+  first_block = all_blocks.shift
+  save_file(first_block)
+  all_blocks.each_slice(@num_tables_per_file) do |blocks|
+    content = blocks.collect{|block| "#{SEPARATOR}#{block}"}.join
+    save_file(content)
+  end
+end
+
+main

data/flydata-core/lib/flydata-core/errors.rb

@@ -32,6 +32,7 @@ module ErrorCode
   S3_ACCESS_ERROR = 10008
   REDSHIFT_ACCESS_ERROR = 10009
   BREAKING_ALTER_TABLE_ERROR = 10010
+  BREAKING_INFORMATION_SCHEMA_ERROR = 10011
   INTERNAL_ERROR = 1200
 end
 
@@ -66,6 +67,10 @@ end
 class UserMaintenanceError < StandardError
 end
 
+# Errors usded by including
+module BreakingSyncError
+end
+
 # An error that indicates that sync record was received for a table whose sync is broken
 # These records cannot be retried, should be ignored
 class BrokenTableSyncError < StandardError
@@ -203,9 +208,21 @@ end
 
 # Unsupported ALTER TABLE which breaks the tables sync consistency.
 class BreakingAlterTableError < RecordDeliveryError
+  include BreakingSyncError
   def err_code; ErrorCode::BREAKING_ALTER_TABLE_ERROR; end
 end
 
+# Unrecoverable information schema error which requires re-synchronize a table
+class BreakingInformationSchemaError < InformationSchemaError
+  include BreakingSyncError
+  def initialize(*args)
+    super
+    @error_content[:err_reason] = "Sync is broken. The table needs to be re-synchronized. (#{@error_content[:err_reason]})"
+  end
+  def err_code; ErrorCode::BREAKING_INFORMATION_SCHEMA_ERROR; end
+end
+
+
 #   + BadValueError
 # data_node_id, data_entry_id, chunk_identifier, table_name, err_code, err_reason, err_level, record_no, raw_record, colname, type, raw_value, modified_value
 

data/flydata-core/lib/flydata-core/mysql/compatibility_checker.rb

@@ -43,6 +43,26 @@ module FlydataCore
           client.close rescue nil if client
         end
       end
+
+      # This command works only on RDS MySQL
+      RDS_CHECK_QUERY = "call mysql.rds_show_configuration;"
+      def rds?(hostname = @option[:host])
+        return true if hostname.match(/rds.amazonaws.com$/) != nil
+
+        # To distinguish RDS customers using Secure Tunnel
+        begin
+          exec_query(RDS_CHECK_QUERY)
+          return true
+        rescue Mysql2::Error => e
+          if e.message =~ /command denied to user/
+            return true
+          elsif e.message =~ /PROCEDURE mysql.rds_show_configuration does not exist/
+            return false
+          else
+            raise e
+          end
+        end
+      end
     end
 
     class SyncPermissionChecker < MysqlCompatibilityChecker
@@ -281,19 +301,30 @@ EOT
       PK_CHECK_QUERY_TMPLT = <<EOT
 SELECT
   t.table_name,
-  SUM(IF(tc.constraint_type='PRIMARY KEY', 1, 0)) as num_pk
+  SUM(IF(tc.constraint_type='PRIMARY KEY' AND col.is_nullable='NO', 1, 0)) as num_pk,
+  SUM(IF(tc.constraint_type='UNIQUE' AND col.is_nullable='NO', 1, 0)) as num_uk
 FROM
   (select * from information_schema.tables where table_schema = '%{database}' AND table_name in (%{table_names})) t
   LEFT OUTER JOIN
   (select * from information_schema.table_constraints where table_schema = '%{database}' AND table_name in (%{table_names})) tc
   USING (table_schema, table_name)
+  LEFT JOIN
+  (select * from information_schema.key_column_usage where constraint_schema = '%{database}') kc
+  USING (table_schema, table_name, constraint_name)
+  LEFT JOIN
+  (select * from information_schema.columns where table_schema = '%{database}' AND table_name in (%{table_names})) col
+  ON t.table_schema = col.table_schema AND t.table_name = col.table_name AND kc.column_name = col.column_name
 GROUP BY
   t.table_schema, t.table_name
 HAVING
-  num_pk = 0;
+  num_pk = 0 and num_uk = 0;
 EOT
+
       def create_query(option = @option)
-        PK_CHECK_QUERY_TMPLT % {database: option[:database], table_names: option[:tables].collect{|tn| "'#{tn}'"}.join(',')}
+        pk_override = option[:pk_override] || {}
+        tables = option[:tables].select{|tbl| pk_override[tbl.to_s].nil? } # exclude tables having pk_override
+        return nil if tables.empty?
+        PK_CHECK_QUERY_TMPLT % {database: option[:database], table_names: tables.collect{|tn| "'#{tn}'"}.join(',')}
       end
 
       def check_result(result, option = @option)

data/flydata-core/spec/mysql/compatibility_checker_spec.rb

@@ -5,9 +5,79 @@ module FlydataCore
   module Mysql
     describe CompatibilityChecker do
       let(:database) { 'test_db' }
+      let(:mysql_host) { hostname + '.' + domain_name }
+      let(:hostname) { 'test-hostname' }
+      let(:domain_name) { 'flydata.com' }
       let(:tables) { %w(table_1 table_2 table_3) }
       let(:option) do
-        { database: database, tables: tables }
+        { database: database,
+          host: mysql_host,
+          tables: tables
+        }
+      end
+
+      describe MysqlCompatibilityChecker do
+        let(:subject_object) { described_class.new(option) }
+
+        describe '#rds?' do
+          subject { subject_object.rds? }
+
+          context 'When called for non-rds host' do
+            let(:mysql2_error_msg) { "PROCEDURE mysql.rds_show_configuration does not exist" }
+            before do
+              allow(subject_object).to receive(:exec_query).with(MysqlCompatibilityChecker::RDS_CHECK_QUERY).and_raise(Mysql2::Error, mysql2_error_msg)
+            end
+
+            context 'When called for on-premise host' do
+              let(:hostname) { 'test-hostname' }
+              let(:domain_name) { 'flydata.com' }
+              it { is_expected.to be_falsy }
+            end
+
+            context 'When called for ec2 host' do
+              let(:hostname) { 'ec2-12-345-678-90.us-west-1' }
+              let(:domain_name) { 'compute.amazonaws.com' }
+              it { is_expected.to be_falsy }
+            end
+          end
+
+          context 'When called for RDS host' do
+            let(:hostname) { 'test-read-replica.stuvwxyz.us-west-1' }
+            let(:domain_name) { 'rds.amazonaws.com' }
+            it { is_expected.to be_truthy }
+          end
+
+          context 'when called for secure tunnel user' do
+            let(:hostname) { 'mysql.dse-9999-zzz8xy765' }
+            let(:domain_name) { 'flydata.com' }
+
+            context 'When called for non-rds host' do
+              let(:mysql2_error_msg) { "PROCEDURE mysql.rds_show_configuration does not exist" }
+              it do
+                expect(subject_object).to receive(:exec_query).with(MysqlCompatibilityChecker::RDS_CHECK_QUERY).and_raise(Mysql2::Error, mysql2_error_msg)
+                is_expected.to be_falsy
+              end
+            end
+
+            context 'When called for RDS host' do
+               let(:result) { double 'rds query result' }
+               it do
+                expect(subject_object).to receive(:exec_query).with(MysqlCompatibilityChecker::RDS_CHECK_QUERY).and_return(result)
+                is_expected.to be_truthy
+              end
+            end
+          end
+
+          context 'When unexpected error returned' do
+            let(:mysql2_error_msg) { "Unknown MySQL server host '#{mysql_host}'" }
+            before do
+              allow(subject_object).to receive(:exec_query).with(MysqlCompatibilityChecker::RDS_CHECK_QUERY).and_raise(Mysql2::Error, mysql2_error_msg)
+            end
+            it do
+              expect { subject }.to raise_error(Mysql2::Error, mysql2_error_msg)
+            end
+          end
+        end
       end
 
       describe TableExistenceChecker do
@@ -64,18 +134,65 @@ EOT
           it { is_expected.to eq <<EOT
 SELECT
   t.table_name,
-  SUM(IF(tc.constraint_type='PRIMARY KEY', 1, 0)) as num_pk
+  SUM(IF(tc.constraint_type='PRIMARY KEY' AND col.is_nullable='NO', 1, 0)) as num_pk,
+  SUM(IF(tc.constraint_type='UNIQUE' AND col.is_nullable='NO', 1, 0)) as num_uk
 FROM
   (select * from information_schema.tables where table_schema = 'test_db' AND table_name in ('table_1','table_2','table_3')) t
   LEFT OUTER JOIN
   (select * from information_schema.table_constraints where table_schema = 'test_db' AND table_name in ('table_1','table_2','table_3')) tc
   USING (table_schema, table_name)
+  LEFT JOIN
+  (select * from information_schema.key_column_usage where constraint_schema = 'test_db') kc
+  USING (table_schema, table_name, constraint_name)
+  LEFT JOIN
+  (select * from information_schema.columns where table_schema = 'test_db' AND table_name in ('table_1','table_2','table_3')) col
+  ON t.table_schema = col.table_schema AND t.table_name = col.table_name AND kc.column_name = col.column_name
 GROUP BY
   t.table_schema, t.table_name
 HAVING
-  num_pk = 0;
+  num_pk = 0 and num_uk = 0;
 EOT
           }
+
+          context 'when pk_override is set' do
+            before do
+              option[:pk_override] = {
+                'table_2' => ['id']
+              }
+            end
+            it { is_expected.to eq <<EOT
+SELECT
+  t.table_name,
+  SUM(IF(tc.constraint_type='PRIMARY KEY' AND col.is_nullable='NO', 1, 0)) as num_pk,
+  SUM(IF(tc.constraint_type='UNIQUE' AND col.is_nullable='NO', 1, 0)) as num_uk
+FROM
+  (select * from information_schema.tables where table_schema = 'test_db' AND table_name in ('table_1','table_3')) t
+  LEFT OUTER JOIN
+  (select * from information_schema.table_constraints where table_schema = 'test_db' AND table_name in ('table_1','table_3')) tc
+  USING (table_schema, table_name)
+  LEFT JOIN
+  (select * from information_schema.key_column_usage where constraint_schema = 'test_db') kc
+  USING (table_schema, table_name, constraint_name)
+  LEFT JOIN
+  (select * from information_schema.columns where table_schema = 'test_db' AND table_name in ('table_1','table_3')) col
+  ON t.table_schema = col.table_schema AND t.table_name = col.table_name AND kc.column_name = col.column_name
+GROUP BY
+  t.table_schema, t.table_name
+HAVING
+  num_pk = 0 and num_uk = 0;
+EOT
+            }
+          end
+
+          context 'when no tables need to be validated' do
+            let(:tables) { %w(table_2) }
+            before do
+              option[:pk_override] = {
+                'table_2' => ['id']
+              }
+            end
+            it { is_expected.to be_nil }
+          end
         end
 
         describe '#check_reesult' do

data/lib/flydata/api/data_entry.rb

@@ -42,7 +42,7 @@ module Flydata
 #   "success": true
 # }
 
-      NUM_TABLES_PER_REQUEST = 150
+      NUM_TABLES_PER_REQUEST = 110
 
       def buffer_stat(data_entry_id, options = {})
         table_array = options[:tables] || ['']

data/lib/flydata/command/sync.rb

@@ -74,7 +74,7 @@ module Flydata
       # Public method
       #  - Called from Sender#start/restart
       def try_initial_sync(options)
-        handle_initial_sync(nil, options) if source.sync.supported?
+        handle_initial_sync(options) if source.sync.supported?
       rescue Source::UnsupportedSourceError
         return
       end
@@ -123,21 +123,10 @@ EOS
       # Command: flydata sync:reset
       #  - Entry method
       def reset(*tables)
-
-        if tables.empty? && !opts[:all] && !opts[:init]
-          log_info_stdout <<EOS
-ERROR!  Argument or option must be specified.
-
-  To reset all tables:              flydata sync:reset --all
-  To reset specific tables:         flydata sync:reset table1 table2 ...
-  To reset unfinished initial sync: flydata sync:reset --init
-EOS
-          return
-        end
-
         # Set instance variables
         reset_init = opts[:init]
         if opts[:all]
+          # Reset all tables regardless of sync_resumed
           tables = []
           reset_init = false
         end
@@ -148,7 +137,30 @@ EOS
 
         return if !sync_resumed && reset_init
 
-        message = ''
+        # Suggest to add option/arguments
+        if tables.empty? && !opts[:all] && !opts[:init]
+          if sync_resumed
+            log_info_stdout <<EOS
+ERROR!  Argument or option must be specified.
+
+  To reset unfinished initial sync, run the following command:
+
+    flydata sync:reset --init
+
+EOS
+          else
+            log_info_stdout <<EOS
+ERROR!  Argument or option must be specified.
+
+  To reset all tables:              flydata sync:reset --all
+  To reset specific tables:         flydata sync:reset table1 table2 ...
+
+EOS
+          end
+          return
+        end
+
+        # Suggest to use --init
         if sync_resumed && !tables.empty?
           log_info_stdout <<EOS
 ERROR!  You cannot reset tables because the previous initial sync has not been completed.  Reset the unfinished initial sync first with the following command:
@@ -159,17 +171,18 @@ EOS
           return
         end
 
+        # Suggest to use --force
         if target_append_only_tables.size > 0 && !opts[:force]
           log_info_stdout <<EOS
 ERROR!  Reset failed because it includes append only table(s).  Sync can no longer continue if you reset an append only table.
 
   Append only table(s): #{target_append_only_tables.join(", ")}
 
-If you really want to reset append-only tables, use '--force' option.
+If you really want to reset append-only tables, run the command with '--force' option.
 
 EOS
 
-          log_info_stdout <<EOS unless target_full_sync_tables.empty?
+          log_info_stdout <<EOS if !target_full_sync_tables.empty? && !sync_resumed
 To reset all tables except for append only tables, run the following command.
 
   flydata sync:reset #{target_full_sync_tables.join(" ")}
@@ -178,11 +191,23 @@ EOS
           return
         end
 
-        # Flush client buffer
         msg_tables = @input_tables.empty? ? '' : " for these tables : #{@input_tables.join(" ")}"
         msg_sync_type = sync_resumed ? "the current initial sync" : "the current sync"
         show_purpose_name
         return unless ask_yes_no("This resets #{msg_sync_type}#{msg_tables}.  Are you sure?")
+
+        recover_cmd = "flydata sync:reset #{tables.empty? ? '' : tables.join(" ")}"
+
+        _reset(recover_cmd, reset_client_only: opts.client?, delete_tbl_ddl: true)
+
+      end
+      run_exclusive :reset
+
+      def _reset(recover_cmd, options)
+        reset_client_only = options[:reset_client_only]
+        delete_tbl_ddl = options[:delete_tbl_ddl]
+
+        # Flush client buffer
         sender = Flydata::Command::Sender.new
         sender.flush_client_buffer # TODO We should rather delete buffer files
         sender.stop
@@ -193,9 +218,9 @@ EOS
           ee = ServerDataProcessingTimeout.new("Delayed Data Processing")
           ee.description = <<EOS
   Data processing is taking more than expected.  Please contact support@flydata.com to check the system status.
-  Once checked, you can continue sync reset with the following command
+  Once checked, you can continue your operation with the following command
 
-    flydata sync:reset #{tables.empty? ? '' : tables.join(" ")}
+    #{recover_cmd}
 
 EOS
           ee.set_backtrace e.backtrace
@@ -204,21 +229,23 @@ EOS
 
         # Cleanup tables on server
         de = data_entry
-        cleanup_sync_server(de, @input_tables) unless opts.client?
+        cleanup_sync_server(de, @input_tables) unless reset_client_only
         sync_fm = create_sync_file_manager(de)
 
         # Delete local files
         sync_fm.delete_dump_files
-        sync_fm.delete_table_control_files(*@input_tables)
+        sync_fm.delete_table_position_files(*@input_tables)
+        sync_fm.delete_table_rev_files(*@input_tables)
+        sync_fm.delete_table_ddl_files(*@input_tables) if delete_tbl_ddl
 
         new_tables_after_reset = @unsynced_tables + @input_tables
         if @input_tables.empty? or @full_tables.empty? or @full_tables.all?{|ft| new_tables_after_reset.include?(ft)}
           sync_fm.delete_master_position_files
         end
         sync_fm.close
+
         log_info_stdout("Reset completed successfully.")
       end
-      run_exclusive :reset
 
       # Depricated Command: flydata sync:skip
       # skip initial sync
@@ -554,13 +581,13 @@ EOS
 
       # Initial sync
 
-      def handle_initial_sync(tables = nil, options = {})
+      def handle_initial_sync(options = {})
         unless Flydata::Command::Sender.new.wait_until_server_ready
           raise Timeout::Error, "Timed out to wait for the server side to become active. Pleae try again later."
         end
 
         # Setup instance variables
-        sync_resumed = set_current_tables(tables, resume: true)
+        sync_resumed = set_current_tables(nil, resume: true)
 
         if sync_resumed
           # skip confirmation prompts and resume sync right away.
@@ -595,7 +622,22 @@ EOS
       def initial_sync(opt)
         de = data_entry
         # Load sync information from file
-        validate_initial_sync_status
+        aborted_during_dump = validate_initial_sync_status
+        if aborted_during_dump
+          # The previous init-sync failed during dump.
+          # - Discard remainng continuous-sync data by automatically running `sync:reset --init`
+          #  and run init-sync from scratch.
+          # - Will keep ./positions/*.generated_ddl files.
+          #  No need to re-run sync:generate_table_ddl to drop tables on Redshift because
+          #  no data was written to Redshift during the previous init-sync.
+          $log.info "Resetting the initial sync..."
+          recover_cmd = "flydata restart"
+          _reset(recover_cmd, reset_client_only: false, delete_tbl_ddl: false)
+
+          # Setup instance variables again
+          sync_resumed = set_current_tables(nil, resume: true)
+        end
+
         begin
           if opt[:sync_resumed]
             # parallel cont sync has sent buffer data by now so server buffer
@@ -732,12 +774,7 @@ EOM
               #Catch all exceptions including SystemExit and Interrupt.
               log_info_stdout "Quit while running dump, deleting dump file..."
               sync_fm.delete_dump_file
-              log_info_stdout "Dump file deleted. To restart the FlyData Agent, Following tables may need reset : #{target_tables}"
-              if (target_tables.nil? or target_tables.empty?)
-                log_info_stdout "To do full reset sync use command: flydata sync:reset"
-              else
-                log_info_stdout "To reset these tables, use command: flydata sync:reset #{target_tables.join(' ')}"
-              end
+              log_info_stdout "Dump file deleted. To restart the FlyData Agent, run the following command: flydata restart"
               raise e
             end
             call_block_or_return_io(fp, source_pos, &dump_ready_callback)
@@ -1294,9 +1331,10 @@ Thank you for using FlyData!
 
         @input_tables = sync_resumed ? sync_info[:tables] : input_tables
         @input_tables ||= []
+
+        # true if all tables in data entry hasn't init-synced
         @full_initial_sync = sync_resumed ? sync_info[:initial_sync] :
                                             (@unsynced_tables == @full_tables)
-
         sync_fm.close
 
         verify_input_tables(@input_tables, @full_tables)
@@ -1307,13 +1345,17 @@ Thank you for using FlyData!
       def validate_initial_sync_status
         sync_fm = create_sync_file_manager
         dump_pos_info = sync_fm.load_dump_pos
-        fp = sync_fm.dump_file_path
+        sync_info_exists = !!sync_fm.load_sync_info
+        dump_file_deleted = !File.exists?(sync_fm.dump_file_path)
         sync_fm.close
 
         # status is parsing but dumpfile doesn't exist due to streaming -> raise error
-        if dump_pos_info[:status] == STATUS_PARSING && !File.exists?(fp)
+        if dump_pos_info[:status] == STATUS_PARSING && dump_file_deleted
           raise "FlyData Sync was interrupted with invalid state. Run 'flydata sync:reset#{@input_tables.join(',')}' first."
         end
+
+        # check if the previous initial sync was aborted during dump.
+        sync_info_exists && dump_file_deleted
       end
 
       def target_tables

data/lib/flydata/source_mysql/mysql_compatibility_check.rb

@@ -17,6 +17,7 @@ module SourceMysql
       @dump_dir = options[:dump_dir] || nil
       @backup_dir = options[:backup_dir] || nil
       @tables = de_hash['tables']
+      @rds = FlydataCore::Mysql::MysqlCompatibilityChecker.new(@db_opts).rds?
     end
 
     def print_errors
@@ -99,8 +100,8 @@ module SourceMysql
       end
     end
 
-    def is_rds?(hostname = @db_opts[:host])
-      hostname.match(/rds.amazonaws.com$/) != nil
+    def is_rds?
+      @rds
     end
   end
 

data/lib/flydata/source_mysql/plugin_support/binlog_query_dispatcher.rb

@@ -1,4 +1,6 @@
 require 'flydata/source_mysql/plugin_support/alter_table_query_handler'
+require 'flydata/source_mysql/plugin_support/create_table_query_handler'
+require 'flydata/source_mysql/plugin_support/drop_table_query_handler'
 require 'flydata/source_mysql/plugin_support/truncate_table_query_handler'
 require 'flydata/source_mysql/plugin_support/drop_database_query_handler'
 require 'flydata/source_mysql/plugin_support/unsupported_query_handler'
@@ -63,6 +65,8 @@ module PluginSupport
       @handlers = [
         AlterTableQueryHandler.new(context),
         TruncateTableQueryHandler.new(context),
+        CreateTableQueryHandler.new(context),
+        DropTableQueryHandler.new(context),
         DropDatabaseQueryHandler.new(context),
         UnsupportedQueryHandler.new(context), # This must be a last element
       ]

data/lib/flydata/source_mysql/plugin_support/create_table_query_handler.rb

@@ -0,0 +1,28 @@
+require 'flydata/source_mysql/plugin_support/ddl_query_handler'
+
+module Flydata
+module SourceMysql
+
+module PluginSupport
+  class CreateTableQueryHandler < TableDdlQueryHandler
+    PATTERN = /^CREATE TABLE/i
+
+    def initialize(context)
+      super
+    end
+
+    def pattern
+      PATTERN
+    end
+
+    def process(record)
+      if acceptable_db?(record)
+        $log.error "CREATE TABLE detected. Please reset and resync the following table:'#{table_info(record)[:table_name]}'. - query:'#{record["query"]}' binlog_pos:'#{binlog_pos(record)}'"
+      end
+      #NOTE: No emit_record here because this record should not be sent to data servers for now
+    end
+  end
+end
+
+end
+end

data/lib/flydata/source_mysql/plugin_support/drop_table_query_handler.rb

@@ -0,0 +1,28 @@
+require 'flydata/source_mysql/plugin_support/ddl_query_handler'
+
+module Flydata
+module SourceMysql
+
+module PluginSupport
+  class DropTableQueryHandler < TableDdlQueryHandler
+    PATTERN = /^DROP TABLE/i
+
+    def initialize(context)
+      super
+    end
+
+    def pattern
+      PATTERN
+    end
+
+    def process(record)
+      if acceptable_db?(record)
+        $log.error "DROP TABLE detected for table:'#{table_info(record)[:table_name]}'. - query:'#{record["query"]}' binlog_pos:'#{binlog_pos(record)}'"
+      end
+      #NOTE: No emit_record here because this record should not be sent to data servers for now
+    end
+  end
+end
+
+end
+end

data/lib/flydata/source_postgresql/table_meta.rb

@@ -60,9 +60,13 @@ FROM
   JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = ANY(i.indkey)
   RIGHT JOIN
     (SELECT
-      (table_catalog ||'.'|| table_schema ||'.'|| table_name)::regclass AS regid,
+      (table_schema ||'.'|| table_name)::regclass AS regid,
       *
      FROM information_schema.columns
+     WHERE
+       table_catalog = '%{database}'
+       AND table_schema IN (%{schema})
+       AND table_name IN (%{tables})
     ) c
   ON i.indrelid = c.regid AND a.attname = c.column_name
 WHERE

data/lib/flydata/sync_file_manager.rb

@@ -324,19 +324,31 @@ module Flydata
       new_rev
     end
 
-    def delete_table_control_files(*tables)
+    def delete_table_position_files(*tables)
       files_to_delete = [
           table_position_file_paths(*tables),
           table_source_pos_paths(*tables),
           table_source_pos_init_paths(*tables),
-          table_rev_file_paths(*tables),
-          table_ddl_file_paths(*tables)
       ]
       files_to_delete.flatten.each do |path|
         FileUtils.rm(path) if File.exists?(path)
       end
     end
 
+    def delete_table_rev_files(*tables)
+      files_to_delete = table_rev_file_paths(*tables)
+      files_to_delete.each do |path|
+        FileUtils.rm(path) if File.exists?(path)
+      end
+    end
+
+    def delete_table_ddl_files(*tables)
+      files_to_delete = table_ddl_file_paths(*tables)
+      files_to_delete.each do |path|
+        FileUtils.rm(path) if File.exists?(path)
+      end
+    end
+
     def tables_from_positions_dir
       all_table_control_files = Dir.glob(File.join(table_positions_dir_path, '*.{pos,generated_ddl,init,rev}'))
       tables = Set.new

data/spec/flydata/api/data_entry_spec.rb

@@ -177,8 +177,8 @@ describe DataEntry do
     end
     context "with more than 100 tables" do
       let(:tables) { 180.times.collect{|i| "tbl#{i}"} }
-      let(:expected_tables1) { tables[0...150].join(",") }
-      let(:expected_tables2) { tables[150..-1].join(",") }
+      let(:expected_tables1) { tables[0...110].join(",") }
+      let(:expected_tables2) { tables[110..-1].join(",") }
 
       let(:expected_message) { double('expected_message') }
 
@@ -320,8 +320,8 @@ describe DataEntry do
 
     context "with more than 100 tables" do
       let(:tables) { 180.times.collect {|i| "tbl#{i}"} }
-      let(:expected_tables1) { tables[0...150].join(",") }
-      let(:expected_tables2) { tables[150..-1].join(",") }
+      let(:expected_tables1) { tables[0...110].join(",") }
+      let(:expected_tables2) { tables[110..-1].join(",") }
 
       it do
         expect(api_client).to receive(:post).

data/spec/flydata/command/sync_spec.rb

@@ -33,7 +33,8 @@ module Flydata
         "data_port_key"=>"a458c641",
         "mysql_data_entry_preference" =>
           { "host"=>"localhost", "port"=>3306, "username"=>"masashi",
-            "password"=>"welcome", "database"=>"sync_test", "tables"=>["table1", "table2", "table4"],
+            "password"=>"welcome", "database"=>"sync_test",
+            "tables"=>["table1", "table2", "table4"],
             "invalid_tables"=>["table3"],
             "new_tables"=>["table4"],
             "dump_dir"=>default_dump_dir, "forwarder" => "tcpforwarder",
@@ -419,6 +420,112 @@ module Flydata
           end
         end
       end
+
+      describe '#reset' do
+        let(:subject_object) { described_class.new(reset_opts) }
+        subject { subject_object.reset(*arg_tables) }
+
+        let(:command_options) { ["-y"] }
+        let(:reset_opts) { Sync.slop_reset }
+        let(:timeout_value) { Sync::SERVER_DATA_PROCESSING_TIMEOUT } #3600
+
+        let(:test_data_entry) { default_data_entry }
+        let(:valid_tables_append_only) { nil }
+        let(:invalid_tables_append_only) { nil }
+        let(:full_tables) { ["table1", "table2"] } #not include invalid_tables and real_new_tables
+        let(:new_tables) { ["table4"] }
+
+        before do
+          command_options.each {|opt_str| reset_opts.parse!([opt_str]) }
+          allow(subject_object).to receive(:data_entry).and_return(test_data_entry)
+          allow(subject_object).to receive(:check_server_status).and_return( { "complete" => true, "state" => "complete" } )
+          allow_any_instance_of(Api::DataEntry).to receive(:cleanup_sync)
+
+          allow_any_instance_of(SyncFileManager).to receive(:load_sync_info).and_return(sync_info)
+          allow_any_instance_of(SyncFileManager).to receive(:get_new_table_list).with(full_tables, "generated_ddl").and_return(full_tables)
+          allow_any_instance_of(SyncFileManager).to receive(:get_new_table_list).with(new_tables, "pos").and_return(new_tables)
+          allow_any_instance_of(SyncFileManager).to receive(:get_new_table_list).with(full_tables, "pos").and_return([])
+        end
+
+        context 'when called with --all' do
+          let(:arg_tables) { [] }
+          let(:command_options) { ["-y", "--all"] }
+          let(:sync_info) { nil }
+          it do
+            expect(subject_object).to receive(:set_current_tables).with([], resume: false).and_call_original
+            expect(subject_object).to receive(:wait_for_server_buffer).with( {:timeout=>timeout_value, :tables=>arg_tables} )
+            expect(subject_object).to receive(:cleanup_sync_server).with( default_data_entry, arg_tables )
+            subject
+          end
+        end
+
+        context 'when called with --init' do
+          let(:arg_tables) { [] }
+          let(:command_options) { ["-y", "--init"] }
+          let(:sync_info) { {initial_sync: is_full_init_sync, tables: sync_info_tables} }
+          let(:full_tables) { ["table1", "table2", "table4"] }
+          let(:new_tables) { [] }
+          before do
+            # no new tables.
+            test_data_entry["mysql_data_entry_preference"]["new_tables"] = []
+            # continuous sync has started behind the aborted init-sync and all tables have .pos file
+            allow_any_instance_of(SyncFileManager).to receive(:load_sync_info).and_return(sync_info)
+            allow_any_instance_of(SyncFileManager).to receive(:get_new_table_list).with(full_tables, "generated_ddl").and_return([])
+            allow_any_instance_of(SyncFileManager).to receive(:get_new_table_list).with(new_tables, "pos").and_return([])
+            allow_any_instance_of(SyncFileManager).to receive(:get_new_table_list).with(full_tables, "pos").and_return([])
+          end
+
+          context 'when the aborted init-sync was for all tables in the application' do
+            let(:is_full_init_sync) { true }
+            let(:sync_info_tables) { ["table1", "table2", "table4"] }
+            let(:target_tables_for_api) { [] }
+            it do
+              expect(subject_object).to receive(:set_current_tables).with([], resume: true).and_call_original
+              expect(subject_object).to receive(:wait_for_server_buffer).with(
+                                          {:timeout=>timeout_value, :tables=>target_tables_for_api} )
+              expect(subject_object).to receive(:cleanup_sync_server).with(
+                                          default_data_entry, sync_info_tables )
+              subject
+            end
+          end
+
+          context 'when the aborted init-sync was for a part of tables in the application' do
+            let(:is_full_init_sync) { false }
+            let(:sync_info_tables) { ["table2"] }
+            let(:target_tables_for_api) { sync_info_tables }
+            it do
+              expect(subject_object).to receive(:set_current_tables).with([], resume: true).and_call_original
+              expect(subject_object).to receive(:wait_for_server_buffer).with(
+                                          {:timeout=>timeout_value, :tables=>target_tables_for_api} )
+              expect(subject_object).to receive(:cleanup_sync_server).with(
+                                          default_data_entry, sync_info_tables )
+              subject
+            end
+          end
+        end
+
+        context 'when called with a table as an argument' do
+          let(:arg_tables) { ["table2"] }
+          let(:command_options) { ["-y"] }
+          let(:sync_info) { nil }
+          let(:target_tables_for_api) { arg_tables }
+          before do
+            allow_any_instance_of(SyncFileManager).to receive(:get_new_table_list).with(
+                                                        full_tables, "pos").and_return([])
+            allow_any_instance_of(SyncFileManager).to receive(:get_new_table_list).with(full_tables, "pos").and_return([])
+            allow_any_instance_of(SyncFileManager).to receive(:get_new_table_list).with(new_tables, "pos").and_return(new_tables)
+            allow_any_instance_of(SyncFileManager).to receive(:get_new_table_list).with(full_tables, "generated_ddl").and_return([])
+          end
+          it do
+            expect(subject_object).to receive(:set_current_tables).with(arg_tables, resume: true).and_call_original
+            expect(subject_object).to receive(:wait_for_server_buffer).with(
+                                                {:timeout=>timeout_value, :tables=>arg_tables} )
+            expect(subject_object).to receive(:cleanup_sync_server).with(
+                                                default_data_entry, arg_tables )
+            subject
+          end
+        end
+      end
     end
   end
 end

data/spec/flydata/source_mysql/mysql_compatibility_check_spec.rb

@@ -32,6 +32,7 @@ module SourceMysql
     let(:client) { double('client') }
     before do
       allow(Mysql2::Client).to receive(:new).and_return(client)
+      allow(client).to receive(:query).with("call mysql.rds_show_configuration;")
       allow(client).to receive(:close)
     end
 

data/spec/flydata/source_mysql/plugin_support/binlog_query_dispatcher_spec.rb

@@ -59,6 +59,16 @@ EOT
       allow(r).to receive(:pattern).and_return(Flydata::SourceMysql::PluginSupport::TruncateTableQueryHandler::PATTERN)
       r
     end
+    let(:create_table_query_handler) do
+      r = double('create_table_query_handler')
+      allow(r).to receive(:pattern).and_return(Flydata::SourceMysql::PluginSupport::CreateTableQueryHandler::PATTERN)
+      r
+    end
+    let(:drop_table_query_handler) do
+      r = double('drop_table_query_handler')
+      allow(r).to receive(:pattern).and_return(Flydata::SourceMysql::PluginSupport::DropTableQueryHandler::PATTERN)
+      r
+    end
     let(:drop_database_query_handler) do
       r = double('drop_database_query_handler')
       allow(r).to receive(:pattern).and_return(Flydata::SourceMysql::PluginSupport::DropDatabaseQueryHandler::PATTERN)
@@ -83,6 +93,8 @@ EOT
     before do
       expect(AlterTableQueryHandler).to receive(:new).with(context).and_return(alter_query_handler)
       expect(TruncateTableQueryHandler).to receive(:new).with(context).and_return(truncate_query_handler)
+      expect(CreateTableQueryHandler).to receive(:new).with(context).and_return(create_table_query_handler)
+      expect(DropTableQueryHandler).to receive(:new).with(context).and_return(drop_table_query_handler)
       expect(DropDatabaseQueryHandler).to receive(:new).with(context).and_return(drop_database_query_handler)
       allow(UnsupportedQueryHandler).to receive(:new).with(context).and_return(unsupported_query_handler)
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flydata
 version: !ruby/object:Gem::Version
-  version: 0.7.8
+  version: 0.7.9
 platform: ruby
 authors:
 - Koichi Fujikawa
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-07-05 00:00:00.000000000 Z
+date: 2016-08-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -484,6 +484,7 @@ executables:
 - fdredshift
 - flydata
 - serverinfo
+- split_sync_ddl.rb
 extensions:
 - ext/flydata/json/extconf.rb
 - ext/flydata/source_mysql/parser/extconf.rb
@@ -502,6 +503,7 @@ files:
 - bin/fdredshift
 - bin/flydata
 - bin/serverinfo
+- bin/split_sync_ddl.rb
 - circle.yml
 - ext/flydata/flydata.h
 - ext/flydata/json/extconf.rb
@@ -732,9 +734,11 @@ files:
 - lib/flydata/source_mysql/plugin_support/binlog_record_dispatcher.rb
 - lib/flydata/source_mysql/plugin_support/binlog_record_handler.rb
 - lib/flydata/source_mysql/plugin_support/context.rb
+- lib/flydata/source_mysql/plugin_support/create_table_query_handler.rb
 - lib/flydata/source_mysql/plugin_support/ddl_query_handler.rb
 - lib/flydata/source_mysql/plugin_support/dml_record_handler.rb
 - lib/flydata/source_mysql/plugin_support/drop_database_query_handler.rb
+- lib/flydata/source_mysql/plugin_support/drop_table_query_handler.rb
 - lib/flydata/source_mysql/plugin_support/source_position_file.rb
 - lib/flydata/source_mysql/plugin_support/truncate_table_query_handler.rb
 - lib/flydata/source_mysql/plugin_support/unsupported_query_handler.rb