flydata 0.1.7 → 0.1.8

data/lib/flydata/table_def/mysql_table_def.rb

@@ -19,6 +19,7 @@ class MysqlTableDef
     'fixed' => 'numeric',
     'float' => 'float4',
     'int' => 'int4',
+    'integer' => 'int4',
     'longblob' => 'varbinary',
     'longtext' => 'text',
     'mediumblob' => 'varbinary',
@@ -36,6 +37,31 @@ class MysqlTableDef
     'varchar' => 'varchar',
   }
 
+  def self.convert_to_flydata_type(type)
+    TYPE_MAP_M2F.each do |mysql_type, flydata_type|
+      if /^#{mysql_type}\(|^#{mysql_type}$/.match(type)
+        ret_type = type.gsub(/^#{mysql_type}/, flydata_type)
+        ret_type = check_and_set_varchar_length(ret_type, mysql_type, flydata_type)
+        return ret_type
+      end
+    end
+    nil
+  end
+
+  # Check and set the varchar(char) size which is converted from
+  # length to byte size.
+  # On Mysql the record size of varchar(char) is a length of characters.
+  # ex) varchar(6) on mysql -> varchar(18) on flydata
+  def self.check_and_set_varchar_length(type, mysql_type, flydata_type)
+    return type unless %w(char varchar).include?(mysql_type)
+    if type =~ /\((\d+)\)/
+      # expect 3 byte UTF-8 character
+      "#{flydata_type}(#{$1.to_i * 3})"
+    else
+      raise "Invalid varchar type. It must be a bug... type:#{type}"
+    end
+  end
+
   def self.create(io)
     params = _create(io)
     params ? self.new(*params) : nil
@@ -74,7 +100,7 @@ class MysqlTableDef
         stripped_line = line.strip
         # `col_smallint` smallint(6) DEFAULT NULL,
         if stripped_line.start_with?('`')
-          columns << parse_column_line(line)
+          columns << parse_one_column_def(line)
         # PRIMARY KEY (`id`)
         elsif stripped_line.start_with?("PRIMARY KEY")
           parse_primary_key(line, columns)
@@ -112,49 +138,56 @@ class MysqlTableDef
     tabledef
   end
 
-  private
-
-  def self.parse_column_line(line)
-    line = line.strip
+  def self.parse_one_column_def(query)
+    line = query.strip
     line = line[0..-2] if line.end_with?(',')
     pos = 0
     cond = :column_name
     column = {}
 
     while pos < line.length
-      #cur_char = line[pos]
       case cond
       when :column_name  #`column_name` ...
-        pos = line.index('`', 1)
-        column[:name] = line[1..pos-1]
+        pos = line.index(' ', 1)
+        column[:name] = if line[0] == '`'
+                          line[1..pos-2]
+                        else
+                          line[0..pos-1]
+                        end
         cond = :column_type
         pos += 1
       when :column_type  #... formattype(,,,) ...
         pos += 1 until line[pos] != ' '
         start_pos = pos
         pos += 1 until line[pos].nil? || line[pos] =~ /\s|\(/
+
+        # meta
         if line[pos] == '('
-          pos = parse_column_type_values(line, pos)
+          #TODO: implement better parser
+          pos = line.index(')', pos)
           pos += 1
         end
+
+        # type
         type = line[start_pos..pos-1]
-        TYPE_MAP_M2F.each do |mysql_type, flydata_type|
-          type.gsub!(/\b#{mysql_type}\b/, flydata_type)
-        end
-        column[:type] = type
+        column[:type] = convert_to_flydata_type(type)
+
         cond = :options
       when :options
-        column[:auto_increment] = true if line =~ /AUTO_INCREMENT/
-        column[:not_null] = true if line =~ /NOT NULL/
-        if /DEFAULT\s+((?:[^'\s]+\b)|(?:'(?:\\'|[^'])*'))/.match(line)
+        column[:auto_increment] = true if line =~ /AUTO_INCREMENT/i
+        column[:not_null] = true if line =~ /NOT NULL/i
+        if /DEFAULT\s+((?:[^'\s]+\b)|(?:'(?:\\'|[^'])*'))/i.match(line)
           val = $1
           val = val.slice(1..-1) if val.start_with?("'")
           val = val.slice(0..-2) if val.end_with?("'")
           column[:default] = val == "NULL" ? nil : val
         end
-        if /COMMENT\s+'(((?:\\'|[^'])*))'/.match(line)
+        if /COMMENT\s+'(((?:\\'|[^'])*))'/i.match(line)
           column[:comment] = $1
         end
+        if block_given?
+          column = yield(column, query, pos)
+        end
         break
       else
         raise "Invalid condition. It must be a bug..."
@@ -163,6 +196,8 @@ class MysqlTableDef
     column
   end
 
+  private
+
   def self.parse_primary_key(line, columns)
     primary_keys = line.scan(/`(.*?)`/).collect{|item| item[0]}
     primary_keys.each do |primary_key|
@@ -173,11 +208,6 @@ class MysqlTableDef
       column[:primary_key] = true
     end
   end
-
-  def self.parse_column_type_values(line, pos)
-    #TODO: more better parse
-    line.index(')', pos)
-  end
 end
 
 end

data/lib/flydata/table_def/redshift_table_def.rb

@@ -27,7 +27,7 @@ class RedshiftTableDef
     'text' => {type: 'varchar(max)'},
     'time' => {type: 'timestamp'},
     'varbinary' => {type: 'varchar', use_params: true},
-    'varchar' => {type: 'varchar', use_params: true},
+    'varchar' => {type: 'varchar', use_params: true, max_size: 65535},
   }
   def self.from_flydata_tabledef(flydata_tabledef, options = {})
     options[:flydata_ctl_table] = true unless options.has_key?(:flydata_ctl_table)
@@ -40,11 +40,14 @@ class RedshiftTableDef
   end
 
   CREATE_FLYDATA_CTL_TABLE_SQL = <<EOS
+DROP TABLE flydata_ctl_columns;
 CREATE TABLE flydata_ctl_columns (
   id integer NOT NULL IDENTITY(1,1),
   table_name varchar(128) NOT NULL,
   column_name varchar(128) NOT NULL,
   src_data_type varchar(1024) NOT NULL,
+  revision int NOT NULL DEFAULT 1,
+  ordinal_position int NOT NULL,
   PRIMARY KEY(id)
 ) DISTKEY(table_name) SORTKEY(table_name);
 EOS
@@ -73,17 +76,22 @@ EOS
 
   def self.column_def_sql(column)
     type = column[:type]
-    if type =~ /\(.*?\)/
+    if type =~ /\((.*?)\)/
       type = $` + $'
-      params = $&
+      params = $1
     end
 
     type_info = TYPE_MAP_F2R[type]
     raise "Unsupported type '#{column[:type]}'" if type_info.nil?
 
-    rs_type = (type_info[:use_params] && params && !params.nil?) ?
-                 type_info[:type] + "#{params}"
-               : type_info[:type]
+    rs_type = if type_info[:use_params] && params && !params.nil?
+                if type_info[:max_size] && /\d+/.match(params) && params.to_i > type_info[:max_size]
+                  params = type_info[:max_size]
+                end
+                type_info[:type] + "(#{params})"
+              else
+                type_info[:type]
+              end
     line = %Q|  "#{column[:name]}" #{rs_type}|
     line += " NOT NULL" if column[:not_null]
     if (column.has_key?(:default))
@@ -127,13 +135,13 @@ EOS
 
   FLYDATA_CTL_COLUMNS_SQL = <<EOS
 DELETE FROM flydata_ctl_columns WHERE table_name = '%s';
-INSERT INTO flydata_ctl_columns (table_name, column_name, src_data_type) VALUES
+INSERT INTO flydata_ctl_columns (table_name, column_name, src_data_type, ordinal_position) VALUES
 EOS
   def self.flydata_ctl_sql(flydata_tabledef)
     sql = FLYDATA_CTL_COLUMNS_SQL % [ flydata_tabledef[:table_name] ]
     values = []
-    flydata_tabledef[:columns].each do |col|
-      values << "('#{flydata_tabledef[:table_name]}', '#{col[:name]}', '#{escape(col[:type])}')"
+    flydata_tabledef[:columns].each.with_index(1) do |col, i|
+      values << "('#{flydata_tabledef[:table_name]}', '#{col[:name]}', '#{escape(col[:type])}', #{i})"
     end
     sql += values.join(",\n") + ';'
     sql

data/spec/flydata/command/sync_spec.rb

@@ -384,7 +384,7 @@ module Flydata
   end
 
   module Mysql
-    describe MysqlDumpGenerator do
+    describe MysqlDumpGeneratorMasterData do
       let(:stdout) { double(:stdout) }
       let(:stderr) { double(:stderr) }
       let(:status) { double(:status) }
@@ -393,21 +393,21 @@ module Flydata
         'host' => 'localhost', 'port' => 3306, 'username' => 'admin',
         'password' => 'pass', 'database' => 'dev', 'tables' => 'users,groups',
       } }
-      let(:default_dump_generator) { MysqlDumpGenerator.new(default_conf) }
+      let(:default_dump_generator) { MysqlDumpGeneratorMasterData.new(default_conf) }
 
       describe '#initialize' do
         context 'with password' do
           subject { default_dump_generator.instance_variable_get(:@dump_cmd) }
           it { should eq('mysqldump --protocol=tcp -h localhost -P 3306 -uadmin -ppass --skip-lock-tables ' +
-               '--single-transaction --flush-logs --hex-blob --master-data=2 dev users groups') }
+               '--single-transaction --hex-blob --flush-logs --master-data=2 dev users groups') }
         end
         context 'without password' do
           let (:dump_generator) do
-            MysqlDumpGenerator.new(default_conf.merge({'password' => ''}))
+            MysqlDumpGeneratorMasterData.new(default_conf.merge({'password' => ''}))
           end
           subject { dump_generator.instance_variable_get(:@dump_cmd) }
           it { should eq('mysqldump --protocol=tcp -h localhost -P 3306 -uadmin  --skip-lock-tables ' +
-               '--single-transaction --flush-logs --hex-blob --master-data=2 dev users groups') }
+               '--single-transaction --hex-blob --flush-logs --master-data=2 dev users groups') }
         end
       end
 

data/spec/flydata/fluent-plugins/in_mysql_binlog_flydata_spec.rb

@@ -12,6 +12,7 @@ module Fluent
   TEST_SEQUENCE_FILE = /positions\/#{TEST_TABLE}.pos$/
   TEST_TABLES = "#{TEST_TABLE},test_table_1,test_table_2,test_table_3"
   TEST_POSITION_FILE = "test_position.log"
+  TEST_REVISION_FILE = "#{ENV['HOME']}/.flydata/positions/#{TEST_TABLE}.rev"
   TEST_TIMESTAMP = 1389214083
   TEST_CONFIG = <<EOT
   tag #{TEST_TAG}
@@ -66,27 +67,27 @@ EOT
 EOT
   # QUERY: create table
   TEST_EVENT_QUERY_CREATE_TABLE = <<EOT
-{"marker"=>0, "timestamp"=>1389309656, "type_code"=>2, "server_id"=>1, "event_length"=>121, "next_position"=>317, "flags"=>0, "event_type"=>"Query", "thread_id"=>42, "exec_time"=>0, "error_code"=>0, "variables"=>[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3, 115, 116, 100, 4, 33, 0, 33, 0, 8, 0], "db_name"=>"test_db", "query"=>"create table test_table(id int primary key, value text)"}
+{"marker"=>0, "timestamp"=>#{TEST_TIMESTAMP}, "type_code"=>2, "server_id"=>1, "event_length"=>121, "next_position"=>317, "flags"=>0, "event_type"=>"Query", "thread_id"=>42, "exec_time"=>0, "error_code"=>0, "variables"=>[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3, 115, 116, 100, 4, 33, 0, 33, 0, 8, 0], "db_name"=>"test_db", "query"=>"create table test_table(id int primary key, value text)"}
 EOT
   # QUERY: drop table
   TEST_EVENT_QUERY_DROP_TABLE = <<EOT
-{"marker"=>0, "timestamp"=>1389309995, "type_code"=>2, "server_id"=>1, "event_length"=>115, "next_position"=>568, "flags"=>0, "event_type"=>"Query", "thread_id"=>42, "exec_time"=>0, "error_code"=>0, "variables"=>[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3, 115, 116, 100, 4, 33, 0, 33, 0, 8, 0], "db_name"=>"test_db", "query"=>"DROP TABLE `test_table` /* generated by server */"}
+{"marker"=>0, "timestamp"=>#{TEST_TIMESTAMP}, "type_code"=>2, "server_id"=>1, "event_length"=>115, "next_position"=>568, "flags"=>0, "event_type"=>"Query", "thread_id"=>42, "exec_time"=>0, "error_code"=>0, "variables"=>[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3, 115, 116, 100, 4, 33, 0, 33, 0, 8, 0], "db_name"=>"test_db", "query"=>"DROP TABLE `test_table` /* generated by server */"}
 EOT
   # QUERY: alter table add column
   TEST_EVENT_QUERY_ALTER_TABLE_ADD_COLUMN = <<EOT
-{"marker"=>0, "timestamp"=>1389310404, "type_code"=>2, "server_id"=>1, "event_length"=>111, "next_position"=>800, "flags"=>0, "event_type"=>"Query", "thread_id"=>42, "exec_time"=>0, "error_code"=>0, "variables"=>[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3, 115, 116, 100, 4, 33, 0, 33, 0, 8, 0], "db_name"=>"test_db", "query"=>"alter table test_table add column sum integer"}
+{"marker"=>0, "timestamp"=>#{TEST_TIMESTAMP}, "type_code"=>2, "server_id"=>1, "event_length"=>111, "next_position"=>800, "flags"=>0, "event_type"=>"Query", "thread_id"=>42, "exec_time"=>0, "error_code"=>0, "variables"=>[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3, 115, 116, 100, 4, 33, 0, 33, 0, 8, 0], "db_name"=>"test_db", "query"=>"alter table test_table add column sum integer"}
 EOT
   # QUERY: alter table drop column
   TEST_EVENT_QUERY_ALTER_TABLE_DROP_COLUMN = <<EOT
-{"marker"=>0, "timestamp"=>1389310533, "type_code"=>2, "server_id"=>1, "event_length"=>104, "next_position"=>904, "flags"=>0, "event_type"=>"Query", "thread_id"=>42, "exec_time"=>0, "error_code"=>0, "variables"=>[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3, 115, 116, 100, 4, 33, 0, 33, 0, 8, 0], "db_name"=>"test_db", "query"=>"alter table test_table drop column sum"}
+{"marker"=>0, "timestamp"=>#{TEST_TIMESTAMP}, "type_code"=>2, "server_id"=>1, "event_length"=>104, "next_position"=>904, "flags"=>0, "event_type"=>"Query", "thread_id"=>42, "exec_time"=>0, "error_code"=>0, "variables"=>[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3, 115, 116, 100, 4, 33, 0, 33, 0, 8, 0], "db_name"=>"test_db", "query"=>"alter table test_table drop column sum"}
 EOT
   # QUERY: alter table modify column
   TEST_EVENT_QUERY_ALTER_TABLE_MODIFY_COLUMN = <<EOT
-{"marker"=>0, "timestamp"=>1389310735, "type_code"=>2, "server_id"=>1, "event_length"=>112, "next_position"=>1352, "flags"=>0, "event_type"=>"Query", "thread_id"=>48, "exec_time"=>0, "error_code"=>0, "variables"=>[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3, 115, 116, 100, 4, 33, 0, 33, 0, 8, 0], "db_name"=>"test_db", "query"=>"alter table test_table modify column sum float"}
+{"marker"=>0, "timestamp"=>#{TEST_TIMESTAMP}, "type_code"=>2, "server_id"=>1, "event_length"=>112, "next_position"=>1352, "flags"=>0, "event_type"=>"Query", "thread_id"=>48, "exec_time"=>0, "error_code"=>0, "variables"=>[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3, 115, 116, 100, 4, 33, 0, 33, 0, 8, 0], "db_name"=>"test_db", "query"=>"alter table test_table modify column sum float"}
 EOT
   # - 03 STOP_EVENT
   TEST_EVENT_STOP = <<EOT
-{"marker"=>0, "timestamp"=>1389309286, "type_code"=>3, "server_id"=>1, "event_length"=>19, "next_position"=>2929, "flags"=>0, "event_type"=>"Stop"}
+{"marker"=>0, "timestamp"=>#{TEST_TIMESTAMP}, "type_code"=>3, "server_id"=>1, "event_length"=>19, "next_position"=>2929, "flags"=>0, "event_type"=>"Stop"}
 EOT
   # - 04 ROTATE_EVENT
   TEST_EVENT_ROTATE = <<EOT
@@ -137,7 +138,8 @@ EOT
     def expect_emitted_records_with_rows(event, type, table, position, binlog_file, rows)
       rows = [rows] unless rows.kind_of?(Array)
       records = rows.collect do |row|
-        { "type"=>type, "table_name"=>table, "respect_order"=>true, "seq"=>2, "src_pos"=>"#{binlog_file}\t#{position}", "row"=>row }
+        { :type=>type, :table_name=>table, :respect_order=>true, :seq=>2,
+          :src_pos=>"#{binlog_file}\t#{position}", :table_rev=>1, :row=>row }
       end
       expect_emitted_records(event, records)
     end
@@ -152,6 +154,7 @@ EOT
     let(:update_event) { create_event(TEST_EVENT_UPDATE) }
     let(:update_two_byte_event) { create_event(TEST_EVENT_TWO_BYTE_UPDATE) }
     let(:update_three_byte_event) { create_event(TEST_EVENT_THREE_BYTE_UPDATE) }
+    let(:alter_table_add_column_event) { create_event(TEST_EVENT_QUERY_ALTER_TABLE_ADD_COLUMN) }
 
     let(:query_event) { create_event(TEST_EVENT_QUERY_CREATE_DATABSE) }
     let(:table_map_event) { create_event(TEST_EVENT_TABLE_MAP) }
@@ -173,6 +176,7 @@ EOT
       allow(MysqlBinlogInput::BinlogUtil).to receive(:to_hash) {|e| e}
       allow(File).to receive(:open).with(TEST_POSITION_FILE).and_return('test_position')
       allow(File).to receive(:exists?).with(TEST_POSITION_FILE).and_return(true)
+      allow(File).to receive(:exists?).with(TEST_REVISION_FILE).and_return(false)
       allow(File).to receive(:open).with(TEST_SEQUENCE_FILE, "r+").and_yield(table_seq_file)
       Timecop.freeze(now)
     end
@@ -185,7 +189,9 @@ EOT
 
       context 'when received rotate event' do
         before do
-          Test.configure_plugin(plugin, TEST_CONFIG) 
+          allow(File).to receive(:exists?).with(/binlog.pos/).and_return(false)
+          allow(File).to receive(:exists?).with(/\.rev$/).and_return(false)
+          Test.configure_plugin(plugin, TEST_CONFIG)
           plugin.event_listener(rotate_event)
         end
 
@@ -261,6 +267,38 @@ EOT
           end
         end
 
+        context 'when received alter table event' do
+          before do
+            allow(File).to receive(:open).with(/test_table\.rev/, 'w')
+          end
+          it do
+            expect(table_seq_file).to receive(:write).with(2).never
+            expect_no_emitted_record(alter_table_add_column_event)
+          end
+        end
+
+        #TODO: Uncomment the following test and delete the above one after supporting alter table
+=begin
+        context 'when received alter table event' do
+          before do
+            allow(File).to receive(:open).with(/test_table\.rev/, 'w')
+          end
+          it do
+            expect(table_seq_file).to receive(:write).with(2).once
+            expect_emitted_records(alter_table_add_column_event, {
+              subtype: :add_column,
+              table_name: "test_table",
+              type: :alter_table,
+              respect_order: true,
+              src_pos: "mysql-bin.000048\t689",
+              table_rev: 2,  # increment revision
+              seq: 2,
+              column: {:name=>"sum", :type=>'int4'},
+            })
+          end
+        end
+=end
+
         context 'when received event with another database name' do
           it do
             event = insert_event
@@ -285,9 +323,10 @@ EOT
           end
         end
       end
-      
+
       context 'when rotate event is not received' do
         before do
+          allow(File).to receive(:exists?).with(/binlog.pos/).and_return(false)
           Test.configure_plugin(plugin, TEST_CONFIG) 
         end
 
@@ -311,7 +350,7 @@ EOT
           expect_emitted_records_with_rows(delete_event, :delete, TEST_TABLE, 5324, "",
             [{"1"=>"0SL00000002", "2"=>"var"}, {"1"=>"0SL00000003", "2"=>"hoge"}])
         end
-        
+
         it 'logs warning once when it receives consecutive events' do
           table_seq_file.should_receive(:write).exactly(10).with(2)
           expect($log).to receive(:warn).with("Binlog file name is empty. Rotate event not received!").once
@@ -322,24 +361,56 @@ EOT
           expect_emitted_records_with_rows(insert_event, :insert, TEST_TABLE, 628, "", 
             [{"1"=>"0SL00000001", "2"=>"foo"}, {"1"=>"0SL00000002", "2"=>"var"}, {"1"=>"0SL00000003", "2"=>"hoge"}])
           expect_emitted_records_with_rows(delete_event, :delete, TEST_TABLE, 5324, "",
-            [{"1"=>"0SL00000002", "2"=>"var"}, {"1"=>"0SL00000003", "2"=>"hoge"}])        
-        end                        
+            [{"1"=>"0SL00000002", "2"=>"var"}, {"1"=>"0SL00000003", "2"=>"hoge"}])
+        end
       end
 
       context 'when received rotate event with missing binlog file' do
         before do
+          allow(File).to receive(:exists?).with(/binlog.pos/).and_return(false)
           Test.configure_plugin(plugin, TEST_CONFIG) 
           plugin.event_listener(rotate_event_corrupt)
         end
-        
+
         it 'logs a warning and emits FET with a blank binlog file name, when it receives an insert event' do
           table_seq_file.should_receive(:write).exactly(3).with(2)
           expect($log).to receive(:warn).with("Binlog file name is empty. Rotate event not received!").once
           expect_emitted_records_with_rows(insert_event, :insert, TEST_TABLE, 628, "", 
             [{"1"=>"0SL00000001", "2"=>"foo"}, {"1"=>"0SL00000002", "2"=>"var"}, {"1"=>"0SL00000003", "2"=>"hoge"}])
-        end      
+        end
+      end
+
+      context 'when per-table binlog file is present' do
+        before do
+          f = double('per-table-binlog-file')
+          allow(f).to receive(:readline).and_return("mysql-bin.000048\t1010")
+          allow(File).to receive(:exists?).with(/binlog.pos/).and_return(false)
+          allow(File).to receive(:exists?).with(/test_table.binlog.pos/).and_return(true)
+          allow(File).to receive(:open).with(/test_table.binlog.pos/, "r").and_return(f)
+          Test.configure_plugin(plugin, TEST_CONFIG)
+          plugin.event_listener(rotate_event)
+        end
+
+        it  'emits no record when it receives an event whose binlog position is less than per-table binlog position' do
+          expect_no_emitted_record(insert_event)
+        end
+
+        it 'deletes the per-table-binlog file (if there is one) as soon as a record with bigger binlog position is received' do
+          event = insert_event
+          event['next_position'] = 1250
+          table_seq_file.should_receive(:write).exactly(3).with(2)  
+          expect(FileUtils).to receive(:rm).with(/test_table.binlog.pos/, :force => true)
+          expect_emitted_records_with_rows(event, :insert, TEST_TABLE, 1211, "mysql-bin.000048", 
+            [{"1"=>"0SL00000001", "2"=>"foo"}, {"1"=>"0SL00000002", "2"=>"var"}, {"1"=>"0SL00000003", "2"=>"hoge"}])
+        end
+
+        it  'emits no record when it receives an event whose binlog position is equal to per-table binlog position' do
+          event = insert_event
+          event['next_position'] = 1049
+          expect_no_emitted_record(event)
+        end
       end
-            
     end
   end
+
 end

data/spec/flydata/fluent-plugins/mysql/binlog_position_spec.rb

@@ -0,0 +1,33 @@
+require_relative '../../../spec_helper'
+
+module Mysql
+  describe BinLogPosition do
+    let(:pos1) { BinLogPosition.new('mysql-bin.000064	107') }
+    let(:pos2) { BinLogPosition.new('mysql-bin.000064	978') }
+    let(:pos3) { BinLogPosition.new('mysql-bin.000065	107') }
+    let(:pos4) { BinLogPosition.new('mysql-bin.000064	1107') }
+    let(:pos5) { BinLogPosition.new('mysql-bin.000064	1107') }
+
+    it 'should respond to greater than or equal to operator' do
+      pos1.respond_to?('>=').should be_true
+    end
+
+    context 'when testing greater than or equal to operator' do
+      it 'should return true when it is compared with another object with smaller position value' do
+        (pos2 >= pos1).should be_true
+      end
+
+      it 'should return false when it is compared with another object with bigger binlogfile value' do
+        (pos2 >= pos3).should be_false
+      end 
+
+      it 'should compare the position as an integer (and not string)' do
+        (pos2 >= pos4).should be_false
+      end
+
+      it 'should return true when it is compared with another object with equal value' do
+        (pos4 >= pos5).should be_true
+      end
+    end
+  end
+end