flydata 0.1.9 → 0.1.10

data/Gemfile

@@ -15,7 +15,7 @@ gem "slop"
 group :development do
   gem "bundler"
   gem "jeweler"
-  gem "rspec"
+  gem "rspec", '~> 3.0'
   #gem "autotest"
   #gem "autotest-standalone"
   #gem "autotest-notification"

data/Gemfile.lock

@@ -22,7 +22,7 @@ GEM
     atomic (1.1.14)
     builder (3.1.4)
     cool.io (1.2.3)
-    diff-lcs (1.2.4)
+    diff-lcs (1.2.5)
     faraday (0.8.8)
       multipart-post (~> 1.2.0)
     fluent-plugin-mysql-binlog (0.0.2)
@@ -86,14 +86,18 @@ GEM
       json (~> 1.4)
     rest-client (1.6.7)
       mime-types (>= 1.16)
-    rspec (2.14.1)
-      rspec-core (~> 2.14.0)
-      rspec-expectations (~> 2.14.0)
-      rspec-mocks (~> 2.14.0)
-    rspec-core (2.14.5)
-    rspec-expectations (2.14.3)
-      diff-lcs (>= 1.1.3, < 2.0)
-    rspec-mocks (2.14.3)
+    rspec (3.0.0)
+      rspec-core (~> 3.0.0)
+      rspec-expectations (~> 3.0.0)
+      rspec-mocks (~> 3.0.0)
+    rspec-core (3.0.2)
+      rspec-support (~> 3.0.0)
+    rspec-expectations (3.0.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.0.0)
+    rspec-mocks (3.0.2)
+      rspec-support (~> 3.0.0)
+    rspec-support (3.0.2)
     ruby-binlog (1.0.1)
     ruby-prof (0.14.2)
     sigdump (0.2.2)
@@ -122,7 +126,7 @@ DEPENDENCIES
   mysql2 (~> 0.3.11)
   protected_attributes
   rest-client (~> 1.6.7)
-  rspec
+  rspec (~> 3.0)
   ruby-binlog (>= 1.0.1)
   ruby-prof
   slop

data/VERSION

@@ -1 +1 @@
-0.1.9
+0.1.10

data/flydata.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "flydata"
-  s.version = "0.1.9"
+  s.version = "0.1.10"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Koichi Fujikawa"]
-  s.date = "2014-07-01"
+  s.date = "2014-07-15"
   s.description = "FlyData Command Line Interface"
   s.email = "sysadmin@flydata.co"
   s.executables = ["fdmysqldump", "flydata"]
@@ -58,7 +58,10 @@ Gem::Specification.new do |s|
     "lib/flydata/fluent-plugins/mysql/binlog_record_handler.rb",
     "lib/flydata/fluent-plugins/mysql/context.rb",
     "lib/flydata/fluent-plugins/mysql/dml_record_handler.rb",
-    "lib/flydata/fluent-plugins/mysql/query_parser.rb",
+    "lib/flydata/fluent-plugins/mysql/parser.rb",
+    "lib/flydata/fluent-plugins/mysql/parser/alter_table_add_column_parser.rb",
+    "lib/flydata/fluent-plugins/mysql/parser/alter_table_drop_column_parser.rb",
+    "lib/flydata/fluent-plugins/mysql/parser/query_parser.rb",
     "lib/flydata/fluent-plugins/out_forward_ssl.rb",
     "lib/flydata/fluent-plugins/preference.rb",
     "lib/flydata/flydata_crontab.sh",
@@ -82,7 +85,8 @@ Gem::Specification.new do |s|
     "spec/flydata/command/sync_spec.rb",
     "spec/flydata/fluent-plugins/in_mysql_binlog_flydata_spec.rb",
     "spec/flydata/fluent-plugins/mysql/binlog_position_spec.rb",
-    "spec/flydata/fluent-plugins/mysql/query_parser_spec.rb",
+    "spec/flydata/fluent-plugins/mysql/parser/alter_table_add_column_parser_spec.rb",
+    "spec/flydata/fluent-plugins/mysql/parser/alter_table_drop_column_parser_spec.rb",
     "spec/flydata/heroku_spec.rb",
     "spec/flydata/table_def/mysql_table_def_spec.rb",
     "spec/flydata/table_def/mysqldump_test_foreign_key.dump",
@@ -120,7 +124,7 @@ Gem::Specification.new do |s|
       s.add_runtime_dependency(%q<slop>, [">= 0"])
       s.add_development_dependency(%q<bundler>, [">= 0"])
       s.add_development_dependency(%q<jeweler>, [">= 0"])
-      s.add_development_dependency(%q<rspec>, [">= 0"])
+      s.add_development_dependency(%q<rspec>, ["~> 3.0"])
       s.add_development_dependency(%q<timecop>, [">= 0"])
       s.add_development_dependency(%q<sqlite3>, [">= 0"])
       s.add_development_dependency(%q<ruby-prof>, [">= 0"])
@@ -140,7 +144,7 @@ Gem::Specification.new do |s|
       s.add_dependency(%q<slop>, [">= 0"])
       s.add_dependency(%q<bundler>, [">= 0"])
       s.add_dependency(%q<jeweler>, [">= 0"])
-      s.add_dependency(%q<rspec>, [">= 0"])
+      s.add_dependency(%q<rspec>, ["~> 3.0"])
       s.add_dependency(%q<timecop>, [">= 0"])
       s.add_dependency(%q<sqlite3>, [">= 0"])
       s.add_dependency(%q<ruby-prof>, [">= 0"])
@@ -161,7 +165,7 @@ Gem::Specification.new do |s|
     s.add_dependency(%q<slop>, [">= 0"])
     s.add_dependency(%q<bundler>, [">= 0"])
     s.add_dependency(%q<jeweler>, [">= 0"])
-    s.add_dependency(%q<rspec>, [">= 0"])
+    s.add_dependency(%q<rspec>, ["~> 3.0"])
     s.add_dependency(%q<timecop>, [">= 0"])
     s.add_dependency(%q<sqlite3>, [">= 0"])
     s.add_dependency(%q<ruby-prof>, [">= 0"])

data/lib/flydata/api/data_entry.rb

@@ -10,6 +10,10 @@ module Flydata
       def buffer_stat(data_entry_id, mode = nil)
         @client.get("/#{@model_name.pluralize}/#{data_entry_id}/buffer_stat/#{mode}")
       end
+      
+      def cleanup_sync(data_entry_id, tables)
+        @client.post("/#{@model_name.pluralize}/#{data_entry_id}/cleanup_sync", nil, {tables: tables.join(',')})      
+      end
     end
   end
 end

data/lib/flydata/command/sync.rb

@@ -25,7 +25,7 @@ module Flydata
             puts "FlyData Agent is already running.  If you'd like to restart FlyData Sync from scratch, run 'flydata sync:reset' first."
           else
             # per-table sync
-            puts "Flydata Agent is already running.  If you'd like to Sync the table(s), run 'flydata flush' first."
+            puts "Flydata Agent is already running.  If you'd like to Sync the table(s), run 'flydata sync:flush' first."
           end
           exit 1
         end
@@ -46,14 +46,20 @@ module Flydata
         puts "Buffers have been flushed and the sender process has been stopped."
       end
 
+      def self.slop_reset
+        Slop.new do
+          on 'c', 'client', 'Resets client only.'
+        end
+      end
+
       def reset
         return unless ask_yes_no("This resets the current Sync.  Are you sure?")
         sender = Flydata::Command::Sender.new
         sender.flush_client_buffer # TODO We should rather delete buffer files
-        # TODO Reset server when API becomes available
         sender.stop
 
         de = retrieve_data_entries.first
+        cleanup_sync_server(de) unless opts.client?
         sync_fm = Flydata::FileUtil::SyncFileManager.new(de)
         [
           sync_fm.dump_file_path,
@@ -70,14 +76,14 @@ module Flydata
 
       def wait_for_server_data_processing
         state = :PROCESS
-        puts "Processing data..."
+        puts "Uploading data to Redshift..."
         sleep 10
         status = nil
         while (status = check)
           if state == :PROCESS && status['state'] == 'uploading'
             puts "  -> Done"
             state = :UPLOAD
-            puts "Uploading data to Redshift..."
+            puts "Finishing data upload..."
           end
           print_progress(status)
           sleep 10
@@ -85,7 +91,7 @@ module Flydata
         if (state == :PROCESS)
           # :UPLOAD state was skipped due to no data
           puts "  -> Done"
-          puts "Uploading data to Redshift..."
+          puts "Finishing data upload..."
         end
         puts "  -> Done"
       end
@@ -126,6 +132,11 @@ module Flydata
 
       private
 
+      def cleanup_sync_server(de, tables = [])
+        puts "Checking the server and cleaning up"
+        flydata.data_entry.cleanup_sync(de['id'], tables)
+      end
+      
       def do_check(de)
         flydata.data_entry.buffer_stat(de['id'], env_mode)
       end
@@ -264,8 +275,6 @@ Dump file saves contents of your tables temporarily.  Make sure you have enough
       #...
       #CREATE TABLE ...
       def parse_mysqldump_and_send(dp, de, sync_fm)
-        puts "Sending data to FlyData Server..."
-
         # Prepare forwarder
         de_tag_name = de["tag_name#{env_suffix}"]
         server_port = dp['server_port']
@@ -283,8 +292,12 @@ Dump file saves contents of your tables temporarily.  Make sure you have enough
         option = dump_pos_info || {}
         if option[:table_name]
           puts "Resuming... Last processed table: #{option[:table_name]}"
+        else
+          #If its a new sync, ensure server side resources are clean          
+          cleanup_sync_server(de, de['mysql_data_entry_preference']['tables'].split(','))  
         end
-
+        puts "Sending data to FlyData Server..."
+        
         bench_start_time = Time.now
 
         # Start parsing dump file

data/lib/flydata/fluent-plugins/mysql/alter_table_query_handler.rb

@@ -1,4 +1,5 @@
 require_relative 'binlog_query_handler'
+require_relative 'parser'
 
 module Mysql
   class AlterTableQueryHandler < BinlogQueryHandler
@@ -16,6 +17,8 @@ module Mysql
       case normalized_query
       when /^alter table [^\s]+ add column/i
         on_add_column(record, normalized_query)
+      when /^alter table [^\s]+ drop/i
+        on_drop_column(record, normalized_query)
       else
         $log.debug("not supported alter table query:'#{normalized_query}'")
       end
@@ -27,7 +30,15 @@ module Mysql
       $log.debug("on_add_column query:'#{record['query']}'")
       #TODO: Uncomment following lines after supporting alter table on the server side
       #emit_record(:alter_table, record, increment_table_rev: true) do
-      #  AlterTableAddColumnParser.new.parse(record['query'])
+      #  Parser::AlterTableAddColumnParser.new.parse(record['query'])
+      #end
+    end
+
+    def on_drop_column(record, query)
+      $log.debug("on_drop_column query:'#{record['query']}'")
+      #TODO: Uncomment following lines after supporting alter table on the server side
+      #emit_record(:alter_table, record, increment_table_rev: true) do
+      #  Parser::AlterTableDropColumnParser.new.parse(record['query'])
       #end
     end
   end

data/lib/flydata/fluent-plugins/mysql/binlog_record_handler.rb

@@ -1,7 +1,6 @@
 require 'fluent/plugin/in_mysql_binlog'
 require 'binlog'
 require_relative 'binlog_position'
-require_relative 'query_parser'
 
 module Mysql
   class BinlogRecordHandler
@@ -54,6 +53,7 @@ module Mysql
       check_empty_binlog
 
       records = yield
+      return if records.nil? # skip
       records = [records] unless records.kind_of?(Array)
 
       table = records.first[TABLE_NAME] || record['table_name']

data/lib/flydata/fluent-plugins/mysql/parser.rb

@@ -0,0 +1,2 @@
+require_relative 'parser/alter_table_add_column_parser'
+require_relative 'parser/alter_table_drop_column_parser'

data/lib/flydata/fluent-plugins/mysql/{query_parser.rb → parser/alter_table_add_column_parser.rb}

@@ -1,11 +1,6 @@
-require_relative '../../table_def'
+require_relative 'query_parser'
 
-module Mysql
-  class QueryParser
-    # Return hash object
-    def parse(query)
-    end
-  end
+module Mysql::Parser
 
   class AlterTableAddColumnParser < QueryParser
     # Return hash object or array
@@ -67,3 +62,4 @@ module Mysql
     end
   end
 end
+

data/lib/flydata/fluent-plugins/mysql/parser/alter_table_drop_column_parser.rb

@@ -0,0 +1,49 @@
+require_relative 'query_parser'
+
+module Mysql::Parser
+
+  class AlterTableDropColumnParser < QueryParser
+    # Return hash object or array
+    # {
+    #   alter_action : "drop_column",
+    #   table_name : table_name,
+    #   column : {
+    #     name: column_name,
+    #   }
+    # }
+    def parse(query)
+      do_parse(query)
+    rescue
+      $log.error("Failed to parse query. query:'#{query}' error:#{$!}")
+      raise
+    end
+
+    private
+
+    def do_parse(query)
+      m = /^\s*alter\s+table\s+`?(?<table_name>[^\s`]+)`?\s+drop\s+/i.match(query)
+      table_name = m['table_name']
+      query = query[m[0].length..-1]
+
+      column_name = extract_column_name(query)
+      return nil unless column_name
+
+      { subtype: :drop_column,
+        table_name: table_name,
+        column: { name: column_name } }
+    end
+
+    # Parse following drop column
+    #
+    # DROP [COLUMN] col_name
+    # DROP PRIMARY KEY             # ignore
+    # DROP {INDEX|KEY} index_name  # ignore
+    # DROP FOREIGN KEY fk_symbol   # ignore
+    def extract_column_name(query)
+      return nil if /primary\s+key|(foreign\s+key|key|index)\s+[^\s]+/i.match(query)
+      m = /(column\s+)?`?(?<column_name>[^\s`]+)`?/i.match(query)
+      m['column_name']
+    end
+  end
+end
+

data/lib/flydata/fluent-plugins/mysql/parser/query_parser.rb

@@ -0,0 +1,9 @@
+require_relative '../../../table_def'
+
+module Mysql::Parser
+  class QueryParser
+    # Return hash object
+    def parse(query)
+    end
+  end
+end

data/spec/flydata/api/data_entry_spec.rb

@@ -6,7 +6,7 @@ module Flydata
       describe '#create' do
         it 'creates data entry' do
           api_client = double('api_client')
-          api_client.stub(:post)
+          allow(api_client).to receive(:post)
 
           api = DataEntry.new(api_client)
           api.create(data_port_id: 1, log_path: 'log-path')

data/spec/flydata/command/sender_spec.rb

@@ -42,8 +42,8 @@ module Flydata
           end
         end
       end
-      pending '#stop'
-      pending '#restart'
+      skip '#stop'
+      skip '#restart'
     end
   end
 end

data/spec/flydata/command/sync_spec.rb

@@ -300,21 +300,21 @@ module Flydata
         context 'with nil forwarder_key' do
           it 'should return TcpForwarder object' do
             forwarder = ForwarderFactory.create(nil, 'test_tag', ['localhost:3456', 'localhost:4567'])
-            expect(forwarder.kind_of?(TcpForwarder)).to be_true
+            expect(forwarder.kind_of?(TcpForwarder)).to be_truthy
           end
         end
 
         context 'with tcpforwarder forwarder_key' do
           it 'should return TcpForwarder object' do
             forwarder = ForwarderFactory.create('tcpforwarder', 'test_tag', ['localhost:3456', 'localhost:4567'])
-            expect(forwarder.kind_of?(TcpForwarder)).to be_true
+            expect(forwarder.kind_of?(TcpForwarder)).to be_truthy
           end
         end
 
         context 'with sslforwarder forwarder_key' do
           it 'should return SslForwarder object' do
             forwarder = ForwarderFactory.create('sslforwarder', 'test_tag', ['localhost:3456', 'localhost:4567'])
-            expect(forwarder.kind_of?(SslForwarder)).to be_true
+            expect(forwarder.kind_of?(SslForwarder)).to be_truthy
           end
         end
       end
@@ -397,7 +397,7 @@ module Flydata
       describe '#initialize' do
         context 'with password' do
           subject { default_dump_generator.instance_variable_get(:@dump_cmd) }
-          it { should eq('mysqldump --protocol=tcp -h localhost -P 3306 -uadmin -ppass --skip-lock-tables ' +
+          it { is_expected.to eq('mysqldump --protocol=tcp -h localhost -P 3306 -uadmin -ppass --skip-lock-tables ' +
                '--single-transaction --hex-blob --flush-logs --master-data=2 dev users groups') }
         end
         context 'without password' do
@@ -405,7 +405,7 @@ module Flydata
             MysqlDumpGeneratorMasterData.new(default_conf.merge({'password' => ''}))
           end
           subject { dump_generator.instance_variable_get(:@dump_cmd) }
-          it { should eq('mysqldump --protocol=tcp -h localhost -P 3306 -uadmin  --skip-lock-tables ' +
+          it { is_expected.to eq('mysqldump --protocol=tcp -h localhost -P 3306 -uadmin  --skip-lock-tables ' +
                '--single-transaction --hex-blob --flush-logs --master-data=2 dev users groups') }
         end
       end
@@ -421,7 +421,7 @@ module Flydata
           end
           it do
             expect{ default_dump_generator.dump(file_path) }.to raise_error
-            expect(File.exists?(file_path)).to be_false
+            expect(File.exists?(file_path)).to be_falsey
           end
         end
         context 'when exit status is 0 but no file' do
@@ -433,7 +433,7 @@ module Flydata
           end
           it do
             expect{ default_dump_generator.dump(file_path) }.to raise_error
-            expect(File.exists?(file_path)).to be_false
+            expect(File.exists?(file_path)).to be_falsey
           end
         end
         context 'when exit status is 0 but file size is 0' do
@@ -446,7 +446,7 @@ module Flydata
           end
           it do
             expect{ default_dump_generator.dump(file_path) }.to raise_error
-            expect(File.exists?(file_path)).to be_true
+            expect(File.exists?(file_path)).to be_truthy
           end
         end
         context 'when exit status is 0' do
@@ -458,8 +458,8 @@ module Flydata
             )
           end
           it do
-            expect(default_dump_generator.dump(file_path)).to be_true
-            expect(File.exists?(file_path)).to be_true
+            expect(default_dump_generator.dump(file_path)).to be_truthy
+            expect(File.exists?(file_path)).to be_truthy
           end
         end
         after :each do

data/spec/flydata/fluent-plugins/in_mysql_binlog_flydata_spec.rb

@@ -155,6 +155,7 @@ EOT
     let(:update_two_byte_event) { create_event(TEST_EVENT_TWO_BYTE_UPDATE) }
     let(:update_three_byte_event) { create_event(TEST_EVENT_THREE_BYTE_UPDATE) }
     let(:alter_table_add_column_event) { create_event(TEST_EVENT_QUERY_ALTER_TABLE_ADD_COLUMN) }
+    let(:alter_table_drop_column_event) { create_event(TEST_EVENT_QUERY_ALTER_TABLE_DROP_COLUMN) }
 
     let(:query_event) { create_event(TEST_EVENT_QUERY_CREATE_DATABSE) }
     let(:table_map_event) { create_event(TEST_EVENT_TABLE_MAP) }
@@ -166,9 +167,9 @@ EOT
 
     let(:table_seq_file) {
       f = double('table_seq_file')
-      f.stub(:read).and_return('1')
-      f.stub(:rewind)
-      f.stub(:truncate)
+      allow(f).to receive(:read).and_return('1')
+      allow(f).to receive(:rewind)
+      allow(f).to receive(:truncate)
       f
     }
 
@@ -197,7 +198,7 @@ EOT
 
         context 'when received insert event' do
           it do
-            table_seq_file.should_receive(:write).exactly(3).with(2)
+            expect(table_seq_file).to receive(:write).exactly(3).with(2)
             expect_emitted_records_with_rows(insert_event, :insert, TEST_TABLE, 628, "mysql-bin.000048", 
               [{"1"=>"0SL00000001", "2"=>"foo"}, {"1"=>"0SL00000002", "2"=>"var"}, {"1"=>"0SL00000003", "2"=>"hoge"}])
           end
@@ -205,7 +206,7 @@ EOT
 
         context 'when received insert event containing two byte UTF8 chars' do
           it do
-            table_seq_file.should_receive(:write).exactly(3).with(2)
+            expect(table_seq_file).to receive(:write).exactly(3).with(2)
             expect_emitted_records_with_rows(insert_two_byte_event, :insert, TEST_TABLE, 628, "mysql-bin.000048",
               [{"1"=>"0SL00000001", "2"=>"føø"}, {"1"=>"0SL00000002", "2"=>"vår"}, {"1"=>"0SL00000003", "2"=>"høgé"}])
           end
@@ -213,7 +214,7 @@ EOT
 
         context 'when received insert event containing three byte UTF8 chars' do
           it do
-            table_seq_file.should_receive(:write).exactly(3).with(2)
+            expect(table_seq_file).to receive(:write).exactly(3).with(2)
             expect_emitted_records_with_rows(insert_three_byte_event, :insert, TEST_TABLE, 628, "mysql-bin.000048",
               [{"1"=>"0SL00000001", "2"=>"富无无"}, {"1"=>"0SL00000002", "2"=>"易变的"}, {"1"=>"0SL00000003", "2"=>"切实切实"}])
           end
@@ -221,7 +222,7 @@ EOT
 
         context 'when received delete event' do
           it do
-            table_seq_file.should_receive(:write).twice.with(2)
+            expect(table_seq_file).to receive(:write).twice.with(2)
             expect_emitted_records_with_rows(delete_event, :delete, TEST_TABLE, 5324, "mysql-bin.000048",
               [{"1"=>"0SL00000002", "2"=>"var"}, {"1"=>"0SL00000003", "2"=>"hoge"}])
           end
@@ -229,7 +230,7 @@ EOT
 
         context 'when received delete event containing two byte UTF8 chars' do
           it do
-            table_seq_file.should_receive(:write).twice.with(2)
+            expect(table_seq_file).to receive(:write).twice.with(2)
             expect_emitted_records_with_rows(delete_two_byte_event, :delete, TEST_TABLE, 5324, "mysql-bin.000048",
               [{"1"=>"0SL00000002", "2"=>"vår"}, {"1"=>"0SL00000003", "2"=>"høgé"}])
           end
@@ -237,7 +238,7 @@ EOT
 
         context 'when received delete event with containing byte UTF8 chars' do
           it do
-            table_seq_file.should_receive(:write).twice.with(2)
+            expect(table_seq_file).to receive(:write).twice.with(2)
             expect_emitted_records_with_rows(delete_three_byte_event, :delete, TEST_TABLE, 5324, "mysql-bin.000048", 
               [{"1"=>"0SL00000002", "2"=>"易变的"}, {"1"=>"0SL00000003", "2"=>"切实切实"}])
           end
@@ -245,7 +246,7 @@ EOT
 
         context 'when received update event' do
           it do
-            table_seq_file.should_receive(:write).twice.with(2)
+            expect(table_seq_file).to receive(:write).twice.with(2)
             expect_emitted_records_with_rows(update_event, :update, TEST_TABLE, 2528, "mysql-bin.000048", 
               [{"1"=>"0SL00000001", "2"=>"wow"}, {"1"=>"0SL00000003", "2"=>"fuga"}])
           end
@@ -253,7 +254,7 @@ EOT
 
         context 'when received update event with two byte utf8 chars' do
           it do
-            table_seq_file.should_receive(:write).twice.with(2)
+            expect(table_seq_file).to receive(:write).twice.with(2)
             expect_emitted_records_with_rows(update_two_byte_event, :update, TEST_TABLE, 2528, "mysql-bin.000048",
               [{"1"=>"0SL00000001", "2"=>"∑ø∑"}, {"1"=>"0SL00000003", "2"=>"fügå"}])
           end
@@ -261,7 +262,7 @@ EOT
 
         context 'when received update event with three byte utf8 chars' do
           it do
-            table_seq_file.should_receive(:write).twice.with(2)
+            expect(table_seq_file).to receive(:write).twice.with(2)
             expect_emitted_records_with_rows(update_three_byte_event, :update, TEST_TABLE, 2528, "mysql-bin.000048",
               [{"1"=>"0SL00000001", "2"=>"很兴奋"}, {"1"=>"0SL00000003", "2"=>"興奮虎"}])
           end
@@ -278,12 +279,12 @@ EOT
         end
 
         #TODO: Uncomment the following test and delete the above one after supporting alter table
-=begin
-        context 'when received alter table event' do
+        context 'when received alter table add column event' do
           before do
             allow(File).to receive(:open).with(/test_table\.rev/, 'w')
           end
           it do
+            skip "Now alter table add column is not supported"
             expect(table_seq_file).to receive(:write).with(2).once
             expect_emitted_records(alter_table_add_column_event, {
               subtype: :add_column,
@@ -297,7 +298,26 @@ EOT
             })
           end
         end
-=end
+
+        context 'when received alter table drop column event' do
+          before do
+            allow(File).to receive(:open).with(/test_table\.rev/, 'w')
+          end
+          it do
+            skip "Now alter table drop column is not supported"
+            expect(table_seq_file).to receive(:write).with(2).once
+            expect_emitted_records(alter_table_drop_column_event, {
+              subtype: :drop_column,
+              table_name: "test_table",
+              type: :alter_table,
+              respect_order: true,
+              src_pos: "mysql-bin.000048\t800",
+              table_rev: 2,  # increment revision
+              seq: 2,
+              column: {:name=>"sum"},
+            })
+          end
+        end
 
         context 'when received event with another database name' do
           it do
@@ -331,28 +351,28 @@ EOT
         end
 
         it 'logs a warning and emits FET with a blank binlog file name, when it receives an insert event' do
-          table_seq_file.should_receive(:write).exactly(3).with(2)
+          expect(table_seq_file).to receive(:write).exactly(3).with(2)
           expect($log).to receive(:warn).with("Binlog file name is empty. Rotate event not received!").once
           expect_emitted_records_with_rows(insert_event, :insert, TEST_TABLE, 628, "", 
             [{"1"=>"0SL00000001", "2"=>"foo"}, {"1"=>"0SL00000002", "2"=>"var"}, {"1"=>"0SL00000003", "2"=>"hoge"}])
         end
 
         it 'logs a warning and emits FET with a blank binlog file name, when it receives an update event'  do
-          table_seq_file.should_receive(:write).twice.with(2)
+          expect(table_seq_file).to receive(:write).twice.with(2)
           expect($log).to receive(:warn).with("Binlog file name is empty. Rotate event not received!").once
           expect_emitted_records_with_rows(update_event, :update, TEST_TABLE, 2528, "", 
             [{"1"=>"0SL00000001", "2"=>"wow"}, {"1"=>"0SL00000003", "2"=>"fuga"}])
         end
 
         it 'logs a warning emits FET with a blank binlog file name, when it receives a delete event' do
-          table_seq_file.should_receive(:write).twice.with(2)
+          expect(table_seq_file).to receive(:write).twice.with(2)
           expect($log).to receive(:warn).with("Binlog file name is empty. Rotate event not received!").once     
           expect_emitted_records_with_rows(delete_event, :delete, TEST_TABLE, 5324, "",
             [{"1"=>"0SL00000002", "2"=>"var"}, {"1"=>"0SL00000003", "2"=>"hoge"}])
         end
 
         it 'logs warning once when it receives consecutive events' do
-          table_seq_file.should_receive(:write).exactly(10).with(2)
+          expect(table_seq_file).to receive(:write).exactly(10).with(2)
           expect($log).to receive(:warn).with("Binlog file name is empty. Rotate event not received!").once
           expect_emitted_records_with_rows(insert_event, :insert, TEST_TABLE, 628, "", 
             [{"1"=>"0SL00000001", "2"=>"foo"}, {"1"=>"0SL00000002", "2"=>"var"}, {"1"=>"0SL00000003", "2"=>"hoge"}])
@@ -373,7 +393,7 @@ EOT
         end
 
         it 'logs a warning and emits FET with a blank binlog file name, when it receives an insert event' do
-          table_seq_file.should_receive(:write).exactly(3).with(2)
+          expect(table_seq_file).to receive(:write).exactly(3).with(2)
           expect($log).to receive(:warn).with("Binlog file name is empty. Rotate event not received!").once
           expect_emitted_records_with_rows(insert_event, :insert, TEST_TABLE, 628, "", 
             [{"1"=>"0SL00000001", "2"=>"foo"}, {"1"=>"0SL00000002", "2"=>"var"}, {"1"=>"0SL00000003", "2"=>"hoge"}])
@@ -398,7 +418,7 @@ EOT
         it 'deletes the per-table-binlog file (if there is one) as soon as a record with bigger binlog position is received' do
           event = insert_event
           event['next_position'] = 1250
-          table_seq_file.should_receive(:write).exactly(3).with(2)  
+          expect(table_seq_file).to receive(:write).exactly(3).with(2)  
           expect(FileUtils).to receive(:rm).with(/test_table.binlog.pos/, :force => true)
           expect_emitted_records_with_rows(event, :insert, TEST_TABLE, 1211, "mysql-bin.000048", 
             [{"1"=>"0SL00000001", "2"=>"foo"}, {"1"=>"0SL00000002", "2"=>"var"}, {"1"=>"0SL00000003", "2"=>"hoge"}])

data/spec/flydata/fluent-plugins/mysql/binlog_position_spec.rb

@@ -9,24 +9,24 @@ module Mysql
     let(:pos5) { BinLogPosition.new('mysql-bin.000064	1107') }
 
     it 'should respond to greater than or equal to operator' do
-      pos1.respond_to?('>=').should be_true
+      expect(pos1.respond_to?('>=')).to be_truthy
     end
 
     context 'when testing greater than or equal to operator' do
       it 'should return true when it is compared with another object with smaller position value' do
-        (pos2 >= pos1).should be_true
+        expect(pos2 >= pos1).to be_truthy
       end
 
       it 'should return false when it is compared with another object with bigger binlogfile value' do
-        (pos2 >= pos3).should be_false
+        expect(pos2 >= pos3).to be_falsey
       end 
 
       it 'should compare the position as an integer (and not string)' do
-        (pos2 >= pos4).should be_false
+        expect(pos2 >= pos4).to be_falsey
       end
 
       it 'should return true when it is compared with another object with equal value' do
-        (pos4 >= pos5).should be_true
+        expect(pos4 >= pos5).to be_truthy
       end
     end
   end

data/spec/flydata/fluent-plugins/mysql/parser/alter_table_add_column_parser_spec.rb

@@ -0,0 +1,52 @@
+require_relative '../../../../spec_helper'
+
+describe Mysql::Parser::AlterTableAddColumnParser do
+  describe '#parse' do
+    let(:query) { "" }
+    subject { described_class.new.parse(query) }
+
+    context 'with normal alter table query' do
+      let(:query) { "alter table test_table add column value varchar(26)" }
+      it do
+        expect(subject).to eq([{
+          :subtype => :add_column,
+          :table_name => "test_table",
+          :column => {
+            :name => "value",
+            :type => "varchar(78)"
+          }
+        }])
+      end
+    end
+
+    context 'with after option' do
+      let(:query) { "alter table test_table add column value varchar(26) after id" }
+      it do
+        expect(subject).to eq([{
+          :subtype => :add_column,
+          :table_name => "test_table",
+          :column => {
+            :name => "value",
+            :type => "varchar(78)",
+            :after => "id"
+          }
+        }])
+      end
+    end
+
+    context 'with first option' do
+      let(:query) { "alter table test_table add column value varchar(26) first" }
+      it do
+        expect(subject).to eq([{
+          :subtype => :add_column,
+          :table_name => "test_table",
+          :column => {
+            :name => "value",
+            :type => "varchar(78)",
+            :position => :first
+          }
+        }])
+      end
+    end
+  end
+end

data/spec/flydata/fluent-plugins/mysql/parser/alter_table_drop_column_parser_spec.rb

@@ -0,0 +1,68 @@
+require_relative '../../../../spec_helper'
+
+describe Mysql::Parser::AlterTableDropColumnParser do
+  describe '#parse' do
+    let(:query) { "" }
+    subject { described_class.new.parse(query) }
+
+    context 'with normal alter table query' do
+      let(:query) { "alter table test_table drop column value" }
+      it do
+        expect(subject).to eq(
+          :subtype => :drop_column,
+          :table_name => "test_table",
+          :column => { :name => "value" })
+      end
+    end
+
+    context 'without column keyword' do
+      let(:query) { "alter table test_table drop value" }
+      it do
+        expect(subject).to eq(
+          :subtype => :drop_column,
+          :table_name => "test_table",
+          :column => { :name => "value" })
+      end
+    end
+
+    context 'when column_name is wrapped with backquote' do
+      let(:query) { "alter table test_table drop `value`" }
+      it do
+        expect(subject).to eq(
+          :subtype => :drop_column,
+          :table_name => "test_table",
+          :column => { :name => "value" })
+      end
+    end
+
+    context 'when table_name is wrapped with backquote' do
+      let(:query) { "alter table `test_table` drop value" }
+      it do
+        expect(subject).to eq(
+          :subtype => :drop_column,
+          :table_name => "test_table",
+          :column => { :name => "value" })
+      end
+    end
+
+    context 'with primary key drop' do
+      let(:query) { "alter table test_table drop PRIMARY KEY" }
+      it { expect(subject).to be_nil }
+    end
+
+    context 'with index drop' do
+      let(:query) { "alter table test_table drop Index index_name" }
+      it { expect(subject).to be_nil }
+    end
+
+    context 'with key drop' do
+      let(:query) { "alter table test_table drop  key  index_name" }
+      it { expect(subject).to be_nil }
+    end
+
+    context 'with foreign key drop' do
+      let(:query) { "alter table test_table drop  foreign key foreign_key_name" }
+      it { expect(subject).to be_nil }
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flydata
 version: !ruby/object:Gem::Version
-  version: 0.1.9
+  version: 0.1.10
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-07-01 00:00:00.000000000 Z
+date: 2014-07-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -208,17 +208,17 @@ dependencies:
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.0'
 - !ruby/object:Gem::Dependency
   name: timecop
   requirement: !ruby/object:Gem::Requirement
@@ -367,7 +367,10 @@ files:
 - lib/flydata/fluent-plugins/mysql/binlog_record_handler.rb
 - lib/flydata/fluent-plugins/mysql/context.rb
 - lib/flydata/fluent-plugins/mysql/dml_record_handler.rb
-- lib/flydata/fluent-plugins/mysql/query_parser.rb
+- lib/flydata/fluent-plugins/mysql/parser.rb
+- lib/flydata/fluent-plugins/mysql/parser/alter_table_add_column_parser.rb
+- lib/flydata/fluent-plugins/mysql/parser/alter_table_drop_column_parser.rb
+- lib/flydata/fluent-plugins/mysql/parser/query_parser.rb
 - lib/flydata/fluent-plugins/out_forward_ssl.rb
 - lib/flydata/fluent-plugins/preference.rb
 - lib/flydata/flydata_crontab.sh
@@ -391,7 +394,8 @@ files:
 - spec/flydata/command/sync_spec.rb
 - spec/flydata/fluent-plugins/in_mysql_binlog_flydata_spec.rb
 - spec/flydata/fluent-plugins/mysql/binlog_position_spec.rb
-- spec/flydata/fluent-plugins/mysql/query_parser_spec.rb
+- spec/flydata/fluent-plugins/mysql/parser/alter_table_add_column_parser_spec.rb
+- spec/flydata/fluent-plugins/mysql/parser/alter_table_drop_column_parser_spec.rb
 - spec/flydata/heroku_spec.rb
 - spec/flydata/table_def/mysql_table_def_spec.rb
 - spec/flydata/table_def/mysqldump_test_foreign_key.dump
@@ -421,7 +425,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 4366584009775605873
+      hash: 1050886461710559963
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:

data/spec/flydata/fluent-plugins/mysql/query_parser_spec.rb

@@ -1,54 +0,0 @@
-require_relative '../../../spec_helper'
-
-module Mysql
-  describe AlterTableAddColumnParser do
-    describe '#parse' do
-      let(:query) { "" }
-      subject { AlterTableAddColumnParser.new.parse(query) }
-
-      context 'with normal alter table query' do
-        let(:query) { "alter table test_table add column value varchar(26)" }
-        it do
-          expect(subject).to eq([{
-            :subtype => :add_column,
-            :table_name => "test_table",
-            :column => {
-              :name => "value",
-              :type => "varchar(78)"
-            }
-          }])
-        end
-      end
-
-      context 'with after option' do
-        let(:query) { "alter table test_table add column value varchar(26) after id" }
-        it do
-          expect(subject).to eq([{
-            :subtype => :add_column,
-            :table_name => "test_table",
-            :column => {
-              :name => "value",
-              :type => "varchar(78)",
-              :after => "id"
-            }
-          }])
-        end
-      end
-
-      context 'with first option' do
-        let(:query) { "alter table test_table add column value varchar(26) first" }
-        it do
-          expect(subject).to eq([{
-            :subtype => :add_column,
-            :table_name => "test_table",
-            :column => {
-              :name => "value",
-              :type => "varchar(78)",
-              :position => :first
-            }
-          }])
-        end
-      end
-    end
-  end
-end