flydata 0.5.10 → 0.5.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9ae9067dda053aa56404eb3260e3518679d787cd
-  data.tar.gz: f4af787def23a4860a8bbe5ed4619e61bb8abb7e
+  metadata.gz: 48cf9129bf088b6ab52c5b1933c344f772cc208d
+  data.tar.gz: 577ead8b2fd3962be51b610542ae52030ecd88d0
 SHA512:
-  metadata.gz: e6160a696088969741b7fb49cc44c1a7e28cf8af0d839a8fec431b8549ec9ebfc5e54cb4845bfb3ccb789585e5fd9953e75d0fc16136d443c0b251f269aa7af4
-  data.tar.gz: 6df8c0c9a3bd151d157ba2ac01e1c74757abfb1abc86cc23774b33ddebcc06331f4f7af798a49432af9a6cf0bf750363dc039d8f40f119494cdc0da102eb17ac
+  metadata.gz: e6ed2e5faf4aab3e8347b5d56ce7c5fc08e9213e16629fbdd5ab46084ec6a213f1a3917ae308b46d58701c10b2be926d32268416a4a38341198f014700f61349
+  data.tar.gz: 38e9f620d6b617cb3461918de1159adfb7a2084f6a3667bb2d7f8f68533d159d13be0ff85b2d08016fe8ef4bbcdb925b212c9ae122b3efd400c1bd6350ea52f8

data/Gemfile.lock

@@ -21,7 +21,7 @@ GEM
     arel (4.0.2)
     builder (3.1.4)
     coderay (1.1.0)
-    cool.io (1.2.4)
+    cool.io (1.4.1)
     diff-lcs (1.2.5)
     faraday (0.8.9)
       multipart-post (~> 1.2.0)

data/VERSION

@@ -1 +1 @@
-0.5.10
+0.5.11

data/flydata.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: flydata 0.5.10 ruby lib
+# stub: flydata 0.5.11 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "flydata"
-  s.version = "0.5.10"
+  s.version = "0.5.11"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib"]
   s.authors = ["Koichi Fujikawa", "Masashi Miyazaki", "Matthew Luu", "Mak Inada", "Sriram NS"]
-  s.date = "2015-10-04"
+  s.date = "2015-10-06"
   s.description = "FlyData Agent"
   s.email = "sysadmin@flydata.com"
   s.executables = ["fdmysqldump", "flydata", "serverinfo"]
@@ -140,6 +140,7 @@ Gem::Specification.new do |s|
     "lib/flydata/fluent-plugins/mysql/context.rb",
     "lib/flydata/fluent-plugins/mysql/ddl_query_handler.rb",
     "lib/flydata/fluent-plugins/mysql/dml_record_handler.rb",
+    "lib/flydata/fluent-plugins/mysql/drop_database_query_handler.rb",
     "lib/flydata/fluent-plugins/mysql/table_meta.rb",
     "lib/flydata/fluent-plugins/mysql/truncate_table_query_handler.rb",
     "lib/flydata/fluent-plugins/out_forward_ssl.rb",
@@ -205,6 +206,7 @@ Gem::Specification.new do |s|
     "spec/flydata/fluent-plugins/mysql/binlog_query_dispatcher_spec.rb",
     "spec/flydata/fluent-plugins/mysql/ddl_query_handler_spec.rb",
     "spec/flydata/fluent-plugins/mysql/dml_record_handler_spec.rb",
+    "spec/flydata/fluent-plugins/mysql/drop_database_query_handler_spec.rb",
     "spec/flydata/fluent-plugins/mysql/shared_query_handler_context.rb",
     "spec/flydata/fluent-plugins/mysql/table_meta_spec.rb",
     "spec/flydata/fluent-plugins/mysql/truncate_query_handler_spec.rb",
@@ -236,7 +238,7 @@ Gem::Specification.new do |s|
   ]
   s.homepage = "http://flydata.com/"
   s.licenses = ["All right reserved."]
-  s.rubygems_version = "2.4.3"
+  s.rubygems_version = "2.4.6"
   s.summary = "FlyData Agent"
 
   if s.respond_to? :specification_version then

data/lib/flydata/command/sync.rb

@@ -1348,7 +1348,8 @@ Thank you for using FlyData!
       # from 'table.binlog.pos.init' file.
       def populate_initial_binlog_positions(table_status_hash, sync_fm)
         table_status_hash.keys.each do |table|
-          if table_status_hash[table]["src_pos"] == "-"
+          src_pos = table_status_hash[table]["src_pos"]
+          if src_pos.nil? || src_pos == "-"
             init_binlog_pos =  sync_fm.get_table_binlog_pos_init(table)
             unless init_binlog_pos
               raise "File `#{table}.binlog.pos.init` is missing"

data/lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb

@@ -281,12 +281,16 @@ EOS
     @idle_event_detector.stop
     if @thread and @thread.alive?
       $log.info "Requesting stop Kodama"
-      @kodama_client.stop_request
-      if wait_till_safe_to_stop
-        $log.info "Killing Kodama client"
-        Thread.kill(@thread)
-      else
-        $log.error "Unable to stop Kodama"
+      begin
+        @kodama_client.stop_request
+        if wait_till_safe_to_stop
+          $log.info "Killing Kodama client"
+          Thread.kill(@thread)
+        else
+          $log.error "Unable to stop Kodama"
+        end
+      rescue => e
+         $log.warn "an error occurred during Kodama shutdown.  error:'#{e.to_s}'\n#{e.backtrace.join("\n")}"
       end
     end
     @sync_fm.close

data/lib/flydata/fluent-plugins/mysql/binlog_query_dispatcher.rb

@@ -1,5 +1,6 @@
 require 'flydata/fluent-plugins/mysql/alter_table_query_handler'
 require 'flydata/fluent-plugins/mysql/truncate_table_query_handler'
+require 'flydata/fluent-plugins/mysql/drop_database_query_handler'
 
 module Mysql
   class BinlogQueryDispatcher
@@ -58,6 +59,7 @@ module Mysql
       @handlers = [
         AlterTableQueryHandler.new(context),
         TruncateTableQueryHandler.new(context),
+        DropDatabaseQueryHandler.new(context),
       ]
     end
   end

data/lib/flydata/fluent-plugins/mysql/binlog_record_handler.rb

@@ -4,14 +4,8 @@ require 'flydata/mysql/binlog_position'
 require 'flydata-core/record/record'
 
 module Mysql
+
   class BinlogRecordHandler
-    TABLE_NAME = :table_name # A Flydata JSON tag to specify a table name
-    TYPE = :type
-    SEQ = :seq
-    RESPECT_ORDER = :respect_order
-    SRC_POS = :src_pos
-    TABLE_REV = :table_rev
-    V = :v  # FlyData record format version
 
     def initialize(context)
       @context = context
@@ -59,7 +53,7 @@ module Mysql
 
     def emit_record(type, record)
       return unless acceptable_db?(record)
-      return unless record["table_name"].nil? or acceptable_table?(record, record["table_name"])
+      return if record["table_name"] and not acceptable_table?(record, record["table_name"])
 
       check_empty_binlog
 
@@ -68,47 +62,135 @@ module Mysql
       return if records.nil? # skip
       records = [records] unless records.kind_of?(Array)
 
-      table = records.first[TABLE_NAME] || record['table_name']
-      raise "Missing table name. #{record}" if table.to_s.empty?
-      return unless acceptable_table?(record, table) && acceptable_event?(type, table)
+      FlydataJsonHandlerFactory.create(records, record, @context, type, opt[:increment_table_rev])
+    end
 
-      table_rev = @context.table_revs[table]
-      position = record['next_position'] - record['event_length']
+    def check_empty_binlog
+      #Log one warning per consecutive records that have empty binlog filename
+      if @context.current_binlog_file.to_s.empty?
+        if @first_empty_binlog
+          $log.warn "Binlog file name is empty. Rotate event not received!"
+          @first_empty_binlog = false
+        end
+      else
+        @first_empty_binlog = true
+      end
+    end
+  end
+  class FlydataJsonHandlerFactory
+
+    def self.create(records, record, context, type, increment_table_rev)
+      table = records.first[:table_name] || record['table_name']
+      database = records.first[:db_name] || record['db_name']
+
+      case table
+      when nil
+        DatabaseFlydataJsonHandler.new(records, record, context, type)
+      else
+        TableFlydataJsonHandler.new(records, record, context, type, increment_table_rev)
+      end
+    end
 
-      # Add common information to each record
-      records.each do |r|
-        if opt[:increment_table_rev]
-          table_rev = @context.sync_fm.increment_table_rev(table, table_rev)
-          @context.table_revs[table] = table_rev
+    class FlydataJsonHandler
+      TABLE_NAME = :table_name # A Flydata JSON tag to specify a table name
+      TYPE = :type
+      SEQ = :seq
+      RESPECT_ORDER = :respect_order
+      SRC_POS = :src_pos
+      TABLE_REV = :table_rev
+      V = :v  # FlyData record format version
+      DB_NAME = :db_name
+
+      def initialize(records, record, context, type, increment_table_rev=nil)
+        @context = context
+      end
+
+      def emit(timestamp, row)
+      end
+    end
+
+    class TableFlydataJsonHandler < FlydataJsonHandler
+      def initialize(records, record, context, type, increment_table_rev=nil)
+        super
+        table = records.first[TABLE_NAME] || record['table_name']
+        raise "Missing table name. #{record}" if table.to_s.empty?
+
+        return unless acceptable_event?(type, table)
+
+        table_rev = @context.table_revs[table]
+        position = record['next_position'] - record['event_length']
+
+        # Add common information to each record
+        records.each do |r|
+          if increment_table_rev
+            table_rev = @context.sync_fm.increment_table_rev(table, table_rev)
+            @context.table_revs[table] = table_rev
+          end
+          r[TYPE] = type
+          r[RESPECT_ORDER] = true
+          r[TABLE_NAME] = table
+          r[SRC_POS] = "#{@context.current_binlog_file}\t#{position}"
+          r[TABLE_REV] = table_rev
+          r[V] = FlydataCore::Record::V2
+        end
+
+        # Use binlog's timestamp
+        timestamp = record["timestamp"].to_i
+        records.each do |row|
+          emit(timestamp, row)
         end
-        r[TYPE] = type
-        r[RESPECT_ORDER] = true
-        r[TABLE_NAME] = table
-        r[SRC_POS] = "#{@context.current_binlog_file}\t#{position}"
-        r[TABLE_REV] = table_rev
-        r[V] = FlydataCore::Record::V2
       end
 
-      # Use binlog's timestamp
-      timestamp = record["timestamp"].to_i
-      records.each do |row|
+      def emit(timestamp, row)
         @context.sync_fm.increment_and_save_table_position(row[TABLE_NAME]) do |seq|
           row[SEQ] = seq
           Fluent::Engine.emit(@context.tag, timestamp, row)
         end
       end
+
+      private
+      def acceptable_event?(type, table)
+        @context.omit_events[table].nil? || !@context.omit_events[table].include?(type)
+      end
+
     end
 
-    def check_empty_binlog
-      #Log one warning per consecutive records that have empty binlog filename
-      if @context.current_binlog_file.to_s.empty?
-        if @first_empty_binlog
-          $log.warn "Binlog file name is empty. Rotate event not received!"
-          @first_empty_binlog = false
+    class DatabaseFlydataJsonHandler < FlydataJsonHandler
+      def initialize(records, record, context, type, increment_table_rev=nil)
+        super
+        database = records.first[DB_NAME] || record['db_name']
+
+        return unless acceptable_event?(type)
+
+        position = record['next_position'] - record['event_length']
+
+        # Add common information to each record
+        records.each do |r|
+          r[TYPE] = type
+          r[RESPECT_ORDER] = true
+          r[SRC_POS] = "#{@context.current_binlog_file}\t#{position}"
+          r[V] = FlydataCore::Record::V2
         end
-      else
-        @first_empty_binlog = true
+
+        # Use binlog's timestamp
+        timestamp = record["timestamp"].to_i
+        records.each do |row|
+          emit(timestamp, row)
+        end
+      end
+
+      def emit(timestamp, row)
+        Fluent::Engine.emit(@context.tag, timestamp, row)
+      end
+
+      private
+      #TODO: No support on DS for database records currently so no record should be emitted
+      def acceptable_event?(type)
+        false
       end
+
     end
+
   end
+
 end

data/lib/flydata/fluent-plugins/mysql/drop_database_query_handler.rb

@@ -0,0 +1,22 @@
+require 'flydata/parser_provider'
+require 'flydata/fluent-plugins/mysql/ddl_query_handler'
+
+module Mysql
+  class DropDatabaseQueryHandler < DdlQueryHandler
+    # For MySQL, database and schema are exchangable
+    PATTERN = /^DROP DATABASE|SCHEMA/i
+
+    def initialize(context)
+      super
+    end
+
+    def pattern
+      PATTERN
+    end
+
+    def process(record)
+      $log.warn("DROP DATABASE detected. A full re-sync is required to provide sync consistency")
+      #NOTE: No emit_record here because this record should not be sent to data servers for now
+    end
+  end
+end

data/lib/flydata/helper/action/check_remote_actions.rb

@@ -6,9 +6,14 @@ module Flydata
   module Helper
     module Action
       class CheckRemoteActions < BaseAction
+        include FlydataCore::Logger
+
+        MAX_ERROR_RETRY = 20
+
         def initialize(config)
           super
           @api_client = ApiClient.instance
+          @retry_count = 0
         end
 
         def execute(opts = {})
@@ -16,7 +21,20 @@ module Flydata
 
           # Get actions from flydata web server
           last_id = action_position.load
-          actions = @api_client.agent.actions(last_id)
+          actions = nil
+          begin
+            actions = @api_client.agent.actions(last_id)
+            @retry_count = 0
+          rescue => e
+            if @retry_count >= MAX_ERROR_RETRY
+              @retry_count = 0
+              raise
+            else
+               @retry_count += 1
+               log_warn "Failed to get actions with error '#{e.to_s}'.  Wait for the next turn."
+               actions = {'actions' => []}
+            end
+          end
 
           actions['actions'].each do |action|
             action_name = action['name']

data/spec/flydata/fluent-plugins/mysql/binlog_query_dispatcher_spec.rb

@@ -59,6 +59,11 @@ EOT
       allow(r).to receive(:pattern).and_return(Mysql::TruncateTableQueryHandler::PATTERN)
       r
     end
+    let(:drop_database_query_handler) do
+      r = double('drop_database_query_handler')
+      allow(r).to receive(:pattern).and_return(Mysql::DropDatabaseQueryHandler::PATTERN)
+      r
+    end
     let(:context) { double('context') }
     let(:subject_object) { described_class.new(context) }
     subject { subject_object }
@@ -73,6 +78,7 @@ EOT
     before do
       expect(AlterTableQueryHandler).to receive(:new).with(context).and_return(alter_query_handler)
       expect(TruncateTableQueryHandler).to receive(:new).with(context).and_return(truncate_query_handler)
+      expect(DropDatabaseQueryHandler).to receive(:new).with(context).and_return(drop_database_query_handler)
     end
 
     describe '#dispatch' do
@@ -198,6 +204,30 @@ EOS
           it_behaves_like "a dispatcher that calls query handler with correct query"
         end
       end
+      context "drop database queries" do
+        let(:correct_query_handler) { drop_database_query_handler }
+
+        shared_examples "drop database tests" do
+          context "drop database query" do
+            let(:query) { "DROP DATABASE #{db_name};" }
+            let(:normalized_query) { "DROP DATABASE #{db_name};" }
+            it_behaves_like "a dispatcher that calls query handler with correct query"
+          end
+
+          context "drop schema query" do
+            let(:query) { "DROP SCHEMA #{db_name};" }
+            let(:normalized_query) { "DROP SCHEMA #{db_name};" }
+            it_behaves_like "a dispatcher that calls query handler with correct query"
+          end
+        end
+
+        context "without table key word" do
+          let(:db_name) { "foo" }
+          include_examples "drop database tests"
+        end
+
+      end
+
       context "truncate table queries" do
         let(:correct_query_handler) { truncate_query_handler }
 

data/spec/flydata/fluent-plugins/mysql/drop_database_query_handler_spec.rb

@@ -0,0 +1,56 @@
+require 'fluent_plugins_spec_helper'
+require 'flydata/fluent-plugins/mysql/drop_database_query_handler'
+require 'flydata/fluent-plugins/mysql/shared_query_handler_context'
+
+module Mysql
+  describe DropDatabaseQueryHandler do
+    include_context "query handler context"
+
+    describe '#process' do
+      let(:drop_database_query) { "DROP database foo" }
+
+      before do
+        allow(record).to receive(:[]).with("query").and_return(drop_database_query)
+        allow(record).to receive(:[]).with("normalized_query").and_return(drop_database_query)
+      end
+
+      shared_examples "process drop database queries correctly" do
+        let(:expected_record) do
+          {
+            query: drop_database_query,
+            type: :drop_database,
+            respect_order: true,
+            src_pos: "#{current_binlog_file}\t#{next_position - event_length}",
+            v: flydata_record_version
+          }
+        end
+
+        context "for the registered database" do
+          it "should call Fluent's emit with appropriate params" do
+            # Should mock up the method to return true
+            #expect(Fluent::Engine).to receive(:emit).with(tag, timestamp, expected_record)
+            expect(Fluent::Engine).to receive(:emit).never
+            expect($log).to receive(:warn)
+            expect(subject.process(record))
+          end
+        end
+      end
+
+      shared_examples "skip processing queries" do
+        it "should not call Fluent's emit" do
+          expect(Fluent::Engine).to receive(:emit).never
+          subject.process(record)
+        end
+      end
+
+      context "drop database queries with database keyword" do
+        let(:drop_database_query) { "DROP DATABASE foo" }
+        include_examples "process drop database queries correctly"
+      end
+      context "drop database queries with schema keyword" do
+        let(:drop_database_query) { "DROP SCHEMA foo" }
+        include_examples "process drop database queries correctly"
+      end
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flydata
 version: !ruby/object:Gem::Version
-  version: 0.5.10
+  version: 0.5.11
 platform: ruby
 authors:
 - Koichi Fujikawa
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-10-04 00:00:00.000000000 Z
+date: 2015-10-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -577,6 +577,7 @@ files:
 - lib/flydata/fluent-plugins/mysql/context.rb
 - lib/flydata/fluent-plugins/mysql/ddl_query_handler.rb
 - lib/flydata/fluent-plugins/mysql/dml_record_handler.rb
+- lib/flydata/fluent-plugins/mysql/drop_database_query_handler.rb
 - lib/flydata/fluent-plugins/mysql/table_meta.rb
 - lib/flydata/fluent-plugins/mysql/truncate_table_query_handler.rb
 - lib/flydata/fluent-plugins/out_forward_ssl.rb
@@ -642,6 +643,7 @@ files:
 - spec/flydata/fluent-plugins/mysql/binlog_query_dispatcher_spec.rb
 - spec/flydata/fluent-plugins/mysql/ddl_query_handler_spec.rb
 - spec/flydata/fluent-plugins/mysql/dml_record_handler_spec.rb
+- spec/flydata/fluent-plugins/mysql/drop_database_query_handler_spec.rb
 - spec/flydata/fluent-plugins/mysql/shared_query_handler_context.rb
 - spec/flydata/fluent-plugins/mysql/table_meta_spec.rb
 - spec/flydata/fluent-plugins/mysql/truncate_query_handler_spec.rb
@@ -690,7 +692,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.3
+rubygems_version: 2.4.6
 signing_key: 
 specification_version: 4
 summary: FlyData Agent