flydata 0.1.6 → 0.1.7

data/Gemfile

@@ -7,7 +7,7 @@ gem "activesupport", "~> 4.0.0"
 gem "json", "~> 1.8.0"
 gem "highline", "~> 1.6.19"
 gem "fluentd", "0.10.46"
-gem "ruby-binlog", ">= 1.0.0"
+gem "ruby-binlog", ">= 1.0.1"
 gem "fluent-plugin-mysql-binlog", "~> 0.0.2"
 gem "mysql2", "~> 0.3.11"
 

data/Gemfile.lock

@@ -94,7 +94,7 @@ GEM
     rspec-expectations (2.14.3)
       diff-lcs (>= 1.1.3, < 2.0)
     rspec-mocks (2.14.3)
-    ruby-binlog (1.0.0)
+    ruby-binlog (1.0.1)
     ruby-prof (0.14.2)
     sigdump (0.2.2)
     sqlite3 (1.3.8)
@@ -122,7 +122,7 @@ DEPENDENCIES
   protected_attributes
   rest-client (~> 1.6.7)
   rspec
-  ruby-binlog (>= 1.0.0)
+  ruby-binlog (>= 1.0.1)
   ruby-prof
   sqlite3
   timecop

data/VERSION

@@ -1 +1 @@
-0.1.6
+0.1.7

data/flydata.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "flydata"
-  s.version = "0.1.6"
+  s.version = "0.1.7"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Koichi Fujikawa"]
-  s.date = "2014-05-22"
+  s.date = "2014-06-01"
   s.description = "FlyData Command Line Interface"
   s.email = "sysadmin@flydata.co"
   s.executables = ["fdmysqldump", "flydata"]
@@ -100,7 +100,7 @@ Gem::Specification.new do |s|
       s.add_runtime_dependency(%q<json>, ["~> 1.8.0"])
       s.add_runtime_dependency(%q<highline>, ["~> 1.6.19"])
       s.add_runtime_dependency(%q<fluentd>, ["= 0.10.46"])
-      s.add_runtime_dependency(%q<ruby-binlog>, [">= 1.0.0"])
+      s.add_runtime_dependency(%q<ruby-binlog>, [">= 1.0.1"])
       s.add_runtime_dependency(%q<fluent-plugin-mysql-binlog>, ["~> 0.0.2"])
       s.add_runtime_dependency(%q<mysql2>, ["~> 0.3.11"])
       s.add_development_dependency(%q<bundler>, [">= 0"])
@@ -119,7 +119,7 @@ Gem::Specification.new do |s|
       s.add_dependency(%q<json>, ["~> 1.8.0"])
       s.add_dependency(%q<highline>, ["~> 1.6.19"])
       s.add_dependency(%q<fluentd>, ["= 0.10.46"])
-      s.add_dependency(%q<ruby-binlog>, [">= 1.0.0"])
+      s.add_dependency(%q<ruby-binlog>, [">= 1.0.1"])
       s.add_dependency(%q<fluent-plugin-mysql-binlog>, ["~> 0.0.2"])
       s.add_dependency(%q<mysql2>, ["~> 0.3.11"])
       s.add_dependency(%q<bundler>, [">= 0"])
@@ -139,7 +139,7 @@ Gem::Specification.new do |s|
     s.add_dependency(%q<json>, ["~> 1.8.0"])
     s.add_dependency(%q<highline>, ["~> 1.6.19"])
     s.add_dependency(%q<fluentd>, ["= 0.10.46"])
-    s.add_dependency(%q<ruby-binlog>, [">= 1.0.0"])
+    s.add_dependency(%q<ruby-binlog>, [">= 1.0.1"])
     s.add_dependency(%q<fluent-plugin-mysql-binlog>, ["~> 0.0.2"])
     s.add_dependency(%q<mysql2>, ["~> 0.3.11"])
     s.add_dependency(%q<bundler>, [">= 0"])

data/lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb

@@ -107,6 +107,7 @@ class FlydataMysqlBinlogRecordHandler < MysqlBinlogRecordHandler
                    'LONGLONG' => 8
                   }
   SIGNLESS_INTEGER_PREFIX = '0SL'
+  SRC_POS = 'src_pos'
 
   def initialize(opts)
     mandatory_opts = [:database, :tables, :tag, :sync_fm]
@@ -119,10 +120,15 @@ class FlydataMysqlBinlogRecordHandler < MysqlBinlogRecordHandler
     @tables = opts[:tables]
     @tag = opts[:tag]
     @sync_fm = opts[:sync_fm]
-
+    @current_binlog_file = ""
+    @first_empty_binlog = true
     @query_handler = FlydataMysqlBinlogQueryHandler.new(record_handler: self)
   end
 
+  def on_rotate(record)
+    @current_binlog_file = record["binlog_file"]
+  end
+
   def on_write_rows(record)
     emit_insert(record)
   end
@@ -167,11 +173,14 @@ class FlydataMysqlBinlogRecordHandler < MysqlBinlogRecordHandler
     return unless acceptable?(record)
 
     table = record['table_name']
-
+    position = record['next_position'] - record['event_length']
+    check_empty_binlog    
+    
     records = record["rows"].collect do |row|
                 row = yield(row) if block_given?  # Give the caller a chance to generate the correct row
                 { TYPE => type, TABLE_NAME => table,
                   RESPECT_ORDER => true, # Continuous sync needs record order to be kept
+                  SRC_POS => "#{@current_binlog_file}\t#{position}",        
                   ROW => row.each.with_index(1).inject({}) do |h, (v, i)|
                     if v.kind_of?(String)
                       v = v.encode('utf-16', :undef => :replace, :invalid => :replace).encode('utf-8')
@@ -212,6 +221,18 @@ class FlydataMysqlBinlogRecordHandler < MysqlBinlogRecordHandler
       end
     end
   end
+  
+  def check_empty_binlog
+    #Log one warning per consecutive records that have empty binlog filename
+    if @current_binlog_file.to_s.empty? 
+      if @first_empty_binlog
+        $log.warn "Binlog file name is empty. Rotate event not received!"
+        @first_empty_binlog = false
+      end  
+    else
+      @first_empty_binlog = true
+    end        
+  end
 end
 
 class MysqlBinlogQueryHandler

data/spec/flydata/fluent-plugins/in_mysql_binlog_flydata_spec.rb

@@ -108,6 +108,10 @@ EOT
   # la
   TEST_EVENT_INCIDENT = <<EOT
 {"marker"=>0, "timestamp"=>0, "type_code"=>26, "server_id"=>1, "event_length"=>40, "next_position"=>2883, "flags"=>32, "event_type"=>"Incident", "incident_type"=>175, "message"=>"Operation canceled"}
+EOT
+  # - 04 ROTATE_EVENT
+  TEST_EVENT_ROTATE_MISSING_BINLOG_FILE = <<EOT
+{"marker"=>0, "timestamp"=>0, "type_code"=>4, "server_id"=>1, "event_length"=>43, "next_position"=>0, "flags"=>32, "event_type"=>"Rotate", "binlog_pos"=>2883}
 EOT
 
   describe MysqlBinlogFlydataInput do
@@ -130,10 +134,10 @@ EOT
       plugin.event_listener(event)
     end
 
-    def expect_emitted_records_with_rows(event, type, table, rows)
+    def expect_emitted_records_with_rows(event, type, table, position, binlog_file, rows)
       rows = [rows] unless rows.kind_of?(Array)
       records = rows.collect do |row|
-        { "type"=>type, "table_name"=>table, "respect_order"=>true, "seq"=>2, "row"=>row }
+        { "type"=>type, "table_name"=>table, "respect_order"=>true, "seq"=>2, "src_pos"=>"#{binlog_file}\t#{position}", "row"=>row }
       end
       expect_emitted_records(event, records)
     end
@@ -152,7 +156,9 @@ EOT
     let(:query_event) { create_event(TEST_EVENT_QUERY_CREATE_DATABSE) }
     let(:table_map_event) { create_event(TEST_EVENT_TABLE_MAP) }
     let(:xid_event) { create_event(TEST_EVENT_XID) }
-
+    let(:rotate_event){ create_event(TEST_EVENT_ROTATE) }  
+    let(:rotate_event_corrupt){ create_event(TEST_EVENT_ROTATE_MISSING_BINLOG_FILE) } 
+    
     let(:now) { Time.now }
 
     let(:table_seq_file) {
@@ -176,103 +182,164 @@ EOT
     end
 
     describe '#event_listener' do
-      before { Test.configure_plugin(plugin, TEST_CONFIG) }
 
-      context 'when received insert event' do
-        it do
-          table_seq_file.should_receive(:write).exactly(3).with(2)
-          expect_emitted_records_with_rows(insert_event, :insert, TEST_TABLE,
-            [{"1"=>"0SL00000001", "2"=>"foo"}, {"1"=>"0SL00000002", "2"=>"var"}, {"1"=>"0SL00000003", "2"=>"hoge"}])
+      context 'when received rotate event' do
+        before do
+          Test.configure_plugin(plugin, TEST_CONFIG) 
+          plugin.event_listener(rotate_event)
         end
-      end
 
-      context 'when received insert event containing two byte UTF8 chars' do
-        it do
-          table_seq_file.should_receive(:write).exactly(3).with(2)
-          expect_emitted_records_with_rows(insert_two_byte_event, :insert, TEST_TABLE,
-            [{"1"=>"0SL00000001", "2"=>"føø"}, {"1"=>"0SL00000002", "2"=>"vår"}, {"1"=>"0SL00000003", "2"=>"høgé"}])
+        context 'when received insert event' do
+          it do
+            table_seq_file.should_receive(:write).exactly(3).with(2)
+            expect_emitted_records_with_rows(insert_event, :insert, TEST_TABLE, 628, "mysql-bin.000048", 
+              [{"1"=>"0SL00000001", "2"=>"foo"}, {"1"=>"0SL00000002", "2"=>"var"}, {"1"=>"0SL00000003", "2"=>"hoge"}])
+          end
         end
-      end
 
-      context 'when received insert event containing three byte UTF8 chars' do
-        it do
-          table_seq_file.should_receive(:write).exactly(3).with(2)
-          expect_emitted_records_with_rows(insert_three_byte_event, :insert, TEST_TABLE,
-            [{"1"=>"0SL00000001", "2"=>"富无无"}, {"1"=>"0SL00000002", "2"=>"易变的"}, {"1"=>"0SL00000003", "2"=>"切实切实"}])
+        context 'when received insert event containing two byte UTF8 chars' do
+          it do
+            table_seq_file.should_receive(:write).exactly(3).with(2)
+            expect_emitted_records_with_rows(insert_two_byte_event, :insert, TEST_TABLE, 628, "mysql-bin.000048",
+              [{"1"=>"0SL00000001", "2"=>"føø"}, {"1"=>"0SL00000002", "2"=>"vår"}, {"1"=>"0SL00000003", "2"=>"høgé"}])
+          end
         end
-      end
 
-      context 'when received delete event' do
-        it do
-          table_seq_file.should_receive(:write).twice.with(2)
-          expect_emitted_records_with_rows(delete_event, :delete, TEST_TABLE,
-            [{"1"=>"0SL00000002", "2"=>"var"}, {"1"=>"0SL00000003", "2"=>"hoge"}])
+        context 'when received insert event containing three byte UTF8 chars' do
+          it do
+            table_seq_file.should_receive(:write).exactly(3).with(2)
+            expect_emitted_records_with_rows(insert_three_byte_event, :insert, TEST_TABLE, 628, "mysql-bin.000048",
+              [{"1"=>"0SL00000001", "2"=>"富无无"}, {"1"=>"0SL00000002", "2"=>"易变的"}, {"1"=>"0SL00000003", "2"=>"切实切实"}])
+          end
         end
-      end
 
-      context 'when received delete event containing two byte UTF8 chars' do
-        it do
-          table_seq_file.should_receive(:write).twice.with(2)
-          expect_emitted_records_with_rows(delete_two_byte_event, :delete, TEST_TABLE,
-            [{"1"=>"0SL00000002", "2"=>"vår"}, {"1"=>"0SL00000003", "2"=>"høgé"}])
+        context 'when received delete event' do
+          it do
+            table_seq_file.should_receive(:write).twice.with(2)
+            expect_emitted_records_with_rows(delete_event, :delete, TEST_TABLE, 5324, "mysql-bin.000048",
+              [{"1"=>"0SL00000002", "2"=>"var"}, {"1"=>"0SL00000003", "2"=>"hoge"}])
+          end
         end
-      end
 
-      context 'when received delete event with containing byte UTF8 chars' do
-        it do
-          table_seq_file.should_receive(:write).twice.with(2)
-          expect_emitted_records_with_rows(delete_three_byte_event, :delete, TEST_TABLE,
-            [{"1"=>"0SL00000002", "2"=>"易变的"}, {"1"=>"0SL00000003", "2"=>"切实切实"}])
+        context 'when received delete event containing two byte UTF8 chars' do
+          it do
+            table_seq_file.should_receive(:write).twice.with(2)
+            expect_emitted_records_with_rows(delete_two_byte_event, :delete, TEST_TABLE, 5324, "mysql-bin.000048",
+              [{"1"=>"0SL00000002", "2"=>"vår"}, {"1"=>"0SL00000003", "2"=>"høgé"}])
+          end
         end
-      end
 
-      context 'when received update event' do
-        it do
-          table_seq_file.should_receive(:write).twice.with(2)
-          expect_emitted_records_with_rows(update_event, :update, TEST_TABLE,
-            [{"1"=>"0SL00000001", "2"=>"wow"}, {"1"=>"0SL00000003", "2"=>"fuga"}])
+        context 'when received delete event with containing byte UTF8 chars' do
+          it do
+            table_seq_file.should_receive(:write).twice.with(2)
+            expect_emitted_records_with_rows(delete_three_byte_event, :delete, TEST_TABLE, 5324, "mysql-bin.000048", 
+              [{"1"=>"0SL00000002", "2"=>"易变的"}, {"1"=>"0SL00000003", "2"=>"切实切实"}])
+          end
         end
-      end
 
-      context 'when received update event with two byte utf8 chars' do
-        it do
-          table_seq_file.should_receive(:write).twice.with(2)
-          expect_emitted_records_with_rows(update_two_byte_event, :update, TEST_TABLE,
-            [{"1"=>"0SL00000001", "2"=>"∑ø∑"}, {"1"=>"0SL00000003", "2"=>"fügå"}])
+        context 'when received update event' do
+          it do
+            table_seq_file.should_receive(:write).twice.with(2)
+            expect_emitted_records_with_rows(update_event, :update, TEST_TABLE, 2528, "mysql-bin.000048", 
+              [{"1"=>"0SL00000001", "2"=>"wow"}, {"1"=>"0SL00000003", "2"=>"fuga"}])
+          end
         end
-      end
 
-      context 'when received update event with three byte utf8 chars' do
-        it do
-          table_seq_file.should_receive(:write).twice.with(2)
-          expect_emitted_records_with_rows(update_three_byte_event, :update, TEST_TABLE,
-            [{"1"=>"0SL00000001", "2"=>"很兴奋"}, {"1"=>"0SL00000003", "2"=>"興奮虎"}])
+        context 'when received update event with two byte utf8 chars' do
+          it do
+            table_seq_file.should_receive(:write).twice.with(2)
+            expect_emitted_records_with_rows(update_two_byte_event, :update, TEST_TABLE, 2528, "mysql-bin.000048",
+              [{"1"=>"0SL00000001", "2"=>"∑ø∑"}, {"1"=>"0SL00000003", "2"=>"fügå"}])
+          end
+        end
+
+        context 'when received update event with three byte utf8 chars' do
+          it do
+            table_seq_file.should_receive(:write).twice.with(2)
+            expect_emitted_records_with_rows(update_three_byte_event, :update, TEST_TABLE, 2528, "mysql-bin.000048",
+              [{"1"=>"0SL00000001", "2"=>"很兴奋"}, {"1"=>"0SL00000003", "2"=>"興奮虎"}])
+          end
+        end
+
+        context 'when received event with another database name' do
+          it do
+            event = insert_event
+            event['db_name'] = 'another_db'
+            expect_no_emitted_record(event)
+          end
+        end
+
+        context 'when received event with unsupported table name' do
+          it do
+            event = insert_event
+            event['table_name'] = 'another_table'
+            expect_no_emitted_record(event)
+          end
         end
-      end
 
-      context 'when received event with another database name' do
-        it do
-          event = insert_event
-          event['db_name'] = 'another_db'
-          expect_no_emitted_record(event)
+        context 'when received unsupported event' do
+          it do
+            expect_no_emitted_record(query_event)
+            expect_no_emitted_record(table_map_event)
+            expect_no_emitted_record(xid_event)
+          end
         end
       end
+      
+      context 'when rotate event is not received' do
+        before do
+          Test.configure_plugin(plugin, TEST_CONFIG) 
+        end
 
-      context 'when received event with unsupported table name' do
-        it do
-          event = insert_event
-          event['table_name'] = 'another_table'
-          expect_no_emitted_record(event)
+        it 'logs a warning and emits FET with a blank binlog file name, when it receives an insert event' do
+          table_seq_file.should_receive(:write).exactly(3).with(2)
+          expect($log).to receive(:warn).with("Binlog file name is empty. Rotate event not received!").once
+          expect_emitted_records_with_rows(insert_event, :insert, TEST_TABLE, 628, "", 
+            [{"1"=>"0SL00000001", "2"=>"foo"}, {"1"=>"0SL00000002", "2"=>"var"}, {"1"=>"0SL00000003", "2"=>"hoge"}])
         end
+
+        it 'logs a warning and emits FET with a blank binlog file name, when it receives an update event'  do
+          table_seq_file.should_receive(:write).twice.with(2)
+          expect($log).to receive(:warn).with("Binlog file name is empty. Rotate event not received!").once
+          expect_emitted_records_with_rows(update_event, :update, TEST_TABLE, 2528, "", 
+            [{"1"=>"0SL00000001", "2"=>"wow"}, {"1"=>"0SL00000003", "2"=>"fuga"}])
+        end
+
+        it 'logs a warning emits FET with a blank binlog file name, when it receives a delete event' do
+          table_seq_file.should_receive(:write).twice.with(2)
+          expect($log).to receive(:warn).with("Binlog file name is empty. Rotate event not received!").once     
+          expect_emitted_records_with_rows(delete_event, :delete, TEST_TABLE, 5324, "",
+            [{"1"=>"0SL00000002", "2"=>"var"}, {"1"=>"0SL00000003", "2"=>"hoge"}])
+        end
+        
+        it 'logs warning once when it receives consecutive events' do
+          table_seq_file.should_receive(:write).exactly(10).with(2)
+          expect($log).to receive(:warn).with("Binlog file name is empty. Rotate event not received!").once
+          expect_emitted_records_with_rows(insert_event, :insert, TEST_TABLE, 628, "", 
+            [{"1"=>"0SL00000001", "2"=>"foo"}, {"1"=>"0SL00000002", "2"=>"var"}, {"1"=>"0SL00000003", "2"=>"hoge"}])
+          expect_emitted_records_with_rows(update_event, :update, TEST_TABLE, 2528, "", 
+            [{"1"=>"0SL00000001", "2"=>"wow"}, {"1"=>"0SL00000003", "2"=>"fuga"}])
+          expect_emitted_records_with_rows(insert_event, :insert, TEST_TABLE, 628, "", 
+            [{"1"=>"0SL00000001", "2"=>"foo"}, {"1"=>"0SL00000002", "2"=>"var"}, {"1"=>"0SL00000003", "2"=>"hoge"}])
+          expect_emitted_records_with_rows(delete_event, :delete, TEST_TABLE, 5324, "",
+            [{"1"=>"0SL00000002", "2"=>"var"}, {"1"=>"0SL00000003", "2"=>"hoge"}])        
+        end                        
       end
 
-      context 'when received unsupported event' do
-        it do
-          expect_no_emitted_record(query_event)
-          expect_no_emitted_record(table_map_event)
-          expect_no_emitted_record(xid_event)
+      context 'when received rotate event with missing binlog file' do
+        before do
+          Test.configure_plugin(plugin, TEST_CONFIG) 
+          plugin.event_listener(rotate_event_corrupt)
         end
+        
+        it 'logs a warning and emits FET with a blank binlog file name, when it receives an insert event' do
+          table_seq_file.should_receive(:write).exactly(3).with(2)
+          expect($log).to receive(:warn).with("Binlog file name is empty. Rotate event not received!").once
+          expect_emitted_records_with_rows(insert_event, :insert, TEST_TABLE, 628, "", 
+            [{"1"=>"0SL00000001", "2"=>"foo"}, {"1"=>"0SL00000002", "2"=>"var"}, {"1"=>"0SL00000003", "2"=>"hoge"}])
+        end      
       end
+            
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flydata
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 0.1.7
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-05-22 00:00:00.000000000 Z
+date: 2014-06-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -114,7 +114,7 @@ dependencies:
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: 1.0.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -122,7 +122,7 @@ dependencies:
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: 1.0.1
 - !ruby/object:Gem::Dependency
   name: fluent-plugin-mysql-binlog
   requirement: !ruby/object:Gem::Requirement
@@ -391,7 +391,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -2211529075960565235
+      hash: 4300353101989721136
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements: