flydata 0.2.21 → 0.2.22

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b2bb40962e8bd93a3daf20cee2de578c4abc1788
-  data.tar.gz: 266ca371867bd725c6721bae7d23fbea9fd4a0e6
+  metadata.gz: 99feeb76fa109f4255e1d76c881d1631e82fdfbe
+  data.tar.gz: 8c080c31ac049d2ad3e30bef154507a6ec15aec7
 SHA512:
-  metadata.gz: daaa7f5468f4f334ef0b7d0cea6cfd672c0a816fd119ca4b54c4f6fa5619987ff37c5df587b11c734d4a29fd2518072277c98a810755d0f0db8008fc719a0ec1
-  data.tar.gz: 3140aea09a5fad9977f1a0bf9d60f317f71637129e0b7ac423198e43e84159534f1715d49d28db28ddc135e522bcbdc32f3ed1415904d2f6cffeffc1f46066c3
+  metadata.gz: f51151ca084d61e1fdf9bea821151f86f54c773c53223daa35a0ac5e173125906cc2cee2ab53ef691bcebcd7f1ae59b96476977786e6d076b3542021dea982cb
+  data.tar.gz: fbca06f1f714479abcf848e16cd9905579ed857ffbcf7a1b9b3d42895f17fb5446f7fc5e17ffa103595e25c87ac1c06e011c03449a16b9c1f4c5afc6f52968ed

data/VERSION

@@ -1 +1 @@
-0.2.21
+0.2.22

data/flydata.gemspec

@@ -2,16 +2,14 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: flydata 0.2.21 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "flydata"
-  s.version = "0.2.21"
+  s.version = "0.2.22"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
-  s.require_paths = ["lib"]
   s.authors = ["Koichi Fujikawa", "Masashi Miyazaki", "Matthew Luu", "Mak Inada", "Sriram NS"]
-  s.date = "2014-12-12"
+  s.date = "2014-12-15"
   s.description = "FlyData Agent"
   s.email = "sysadmin@flydata.com"
   s.executables = ["fdmysqldump", "flydata", "serverinfo"]
@@ -91,6 +89,7 @@ Gem::Specification.new do |s|
     "lib/flydata/compatibility_check.rb",
     "lib/flydata/credentials.rb",
     "lib/flydata/cron.rb",
+    "lib/flydata/fluent-plugins/idle_event_detector.rb",
     "lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb",
     "lib/flydata/fluent-plugins/mysql/alter_table_query_handler.rb",
     "lib/flydata/fluent-plugins/mysql/binlog_position.rb",
@@ -126,6 +125,7 @@ Gem::Specification.new do |s|
     "spec/flydata/command/sender_spec.rb",
     "spec/flydata/command/sync_spec.rb",
     "spec/flydata/compatibility_check_spec.rb",
+    "spec/flydata/fluent-plugins/idle_event_detector_spec.rb",
     "spec/flydata/fluent-plugins/in_mysql_binlog_flydata_spec.rb",
     "spec/flydata/fluent-plugins/mysql/alter_table_query_handler_spec.rb",
     "spec/flydata/fluent-plugins/mysql/binlog_position_spec.rb",
@@ -143,7 +143,8 @@ Gem::Specification.new do |s|
   ]
   s.homepage = "http://flydata.com/"
   s.licenses = ["All right reserved."]
-  s.rubygems_version = "2.2.2"
+  s.require_paths = ["lib"]
+  s.rubygems_version = "2.0.14"
   s.summary = "FlyData Agent"
 
   if s.respond_to? :specification_version then

data/lib/flydata/fluent-plugins/idle_event_detector.rb

@@ -0,0 +1,68 @@
+module Fluent
+
+class IdleEventDetector
+  def initialize(initial_interval, continuous_interval, check_interval = 5)
+    @mutex = Mutex.new
+    @initial_interval = initial_interval
+    @continuous_interval = continuous_interval
+    @check_interval = check_interval
+  end
+
+  def start(&block)
+    raise "block must be given" unless block_given?
+    @callback = block
+    stop if @thread
+    @last_event_at = @timestamp = Time.current
+    @is_absent = false
+
+    @thread = Thread.new do
+      loop do
+        check_state
+        sleep @check_interval
+      end
+    end
+  end
+
+  def stop
+    @thread.exit if @thread
+    @thread = nil
+    @is_absent = false
+    @last_event_at = @timestamp = nil
+    @callback = nil
+  end
+
+  def notify
+    update_last_event_time
+  end
+
+  private
+
+  def update_last_event_time
+    @mutex.synchronize do
+      @last_event_at = @timestamp = Time.current
+      if @is_absent
+        @callback.call(:event_arrived_finally, @last_event_at) if @callback
+        @is_absent = false
+      end
+    end
+  end
+
+  def check_state
+    @mutex.synchronize do
+      if @is_absent
+        if Time.current - @timestamp >= @continuous_interval
+          @callback.call(:event_still_not_coming, @last_event_at) if @callback
+          @timestamp = Time.current
+        end
+      else
+        if Time.current - @timestamp >= @initial_interval
+          @callback.call(:event_not_coming, @last_event_at) if @callback
+          @is_absent = true
+          @timestamp = Time.current
+        end
+      end
+    end
+  end
+end
+
+end

data/lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb

@@ -14,6 +14,7 @@ require_relative 'preference'
 require_relative 'mysql/binlog_position_file'
 require_relative 'mysql/binlog_record_dispatcher'
 require_relative 'mysql/context'
+require_relative 'idle_event_detector'
 require_relative 'mysql/table_meta'
 
 #Monkey-patch fluentd class (EngineClass) to support shutdown for input plugin.
@@ -49,6 +50,9 @@ class MysqlBinlogFlydataInput < MysqlBinlogInput
   config_param :database, :string
   config_param :tables, :string
   config_param :tables_append_only, :string
+  config_param :initial_idle_interval, :integer, :default => 30
+  config_param :continuous_idle_interval, :integer, :default => 600
+  config_param :check_interval, :integer, :default => 5
 
   def configure(conf)
     super
@@ -80,10 +84,22 @@ class MysqlBinlogFlydataInput < MysqlBinlogInput
       table_meta: table_meta,
     )
     @record_dispatcher = Mysql::FlydataBinlogRecordDispatcher.new(@context)
+    @idle_event_detector = IdleEventDetector.new(@initial_idle_interval, @continuous_idle_interval, @check_interval)
   end
 
   def start
     super
+    @idle_event_detector.start do |reason, timestamp|
+      case reason
+      when :event_not_coming
+        $log.warn "No binary log event since #{timestamp}"
+      when :event_still_not_coming
+        $log.warn "No binary log event since #{timestamp}"
+      when :event_arrived_finally
+        $log.info "Binary log event has come at #{timestamp}"
+      end
+    end
+
     positions_path = @context.sync_fm.table_positions_dir_path
     Dir.mkdir positions_path unless File.exists? positions_path
   rescue Binlog::Error
@@ -132,6 +148,7 @@ EOS
   end
 
   def event_listener(event)
+    @idle_event_detector.notify
     @record_dispatcher.dispatch(event)
   rescue Exception => e
     position = @binlog_position_file.read
@@ -141,6 +158,7 @@ EOS
   end
 
   def shutdown
+    @idle_event_detector.stop
     if @thread and @thread.alive?
       $log.info "Requesting stop Kodama"
       @kodama_client.stop_request

data/lib/flydata/output/forwarder.rb

@@ -23,6 +23,8 @@ module Flydata
     end
 
     class TcpForwarder
+      include CommandLoggable
+
       FORWARD_HEADER = [0x92].pack('C')
       BUFFER_SIZE = 1024 * 1024 * 32 # 32M
       DEFUALT_SEND_TIMEOUT = 60 # 1 minute
@@ -95,7 +97,7 @@ module Flydata
             log_error_stderr("! Error: Failed to send data. Exceeded the retry limit. retry_count:#{retry_count}")
             raise e
           end
-          log_warn_stderr("! Warn: Retring to send data. retry_count:#{retry_count} error=#{e.to_s}")
+          log_warn_stderr("! Warn: Retrying to send data. retry_count:#{retry_count} error=#{e.to_s}")
           wait_time = RETRY_INTERVAL ** retry_count
           log_warn_stderr("  Now waiting for next retry. time=#{wait_time}sec")
           sleep wait_time

data/spec/flydata/fluent-plugins/idle_event_detector_spec.rb

@@ -0,0 +1,403 @@
+# encoding: utf-8
+
+require 'fluent_plugins_spec_helper'
+require 'idle_event_detector'
+require 'timecop'
+
+module Fluent
+
+describe IdleEventDetector do
+  let(:time) { Time.local(2014, 12, 1, 5, 0, 0) }
+  let(:initial_interval) { 0.1 }
+  let(:continuous_interval) { 0.18 }
+  let(:check_interval) { 0.1 }
+
+  subject { described_class.new(initial_interval, continuous_interval, check_interval) }
+
+  describe '#start' do
+    context 'without block' do
+      it 'errors out' do
+        expect{ subject.start }.to raise_error
+      end
+    end
+
+    context 'no event within initial_interval' do
+      before do
+        Timecop.freeze(time)
+        @callback_called = false
+      end
+      let(:callback) { Proc.new { @callback_called = true } }
+      it 'fires :event_not_coming callback' do
+
+        subject.start(&callback)
+
+        Timecop.freeze(time + initial_interval + continuous_interval * 0.2)
+
+        sleep check_interval * 1.1
+
+        expect(@callback_called).to eq true
+      end
+      after do
+        Timecop.return
+        @callback_called = false
+      end
+    end
+    context 'no event during test which completes before initial_interval' do
+      before do
+        Timecop.freeze(time)
+        @callback_called = false
+      end
+      let(:callback) { Proc.new { @callback_called = true } }
+      it 'does not fire :event_not_coming callback' do
+
+        subject.start(&callback)
+
+        Timecop.freeze(time + initial_interval * 0.2)
+
+        sleep check_interval * 1.1
+
+        expect(@callback_called).to eq false
+
+        subject.stop
+      end
+      after do
+        Timecop.return
+        @callback_called = false
+      end
+    end
+    context 'an event during test which completes after initial_interval' do
+      before do
+        Timecop.freeze(time)
+        @callback_called = false
+      end
+      let(:callback) { Proc.new { @callback_called = true } }
+      it 'does not fire :event_not_coming callback' do
+
+        subject.start(&callback)
+
+        Timecop.freeze(time + initial_interval * 0.2)
+
+        sleep check_interval * 1.1
+
+        subject.notify
+
+        Timecop.freeze(time + initial_interval + initial_interval * 0.1)
+
+        sleep check_interval * 1.1
+
+        expect(@callback_called).to eq false
+
+        subject.stop
+      end
+      after do
+        Timecop.return
+        @callback_called = false
+      end
+    end
+    context 'no event during test which completes after initial_interval + continuous_interval' do
+      let(:callback) {
+        Proc.new do |reason|
+          case reason
+          when :event_not_coming
+            @initial_callback_called = true
+          when :event_still_not_coming
+            @continuous_callback_called = true
+          end
+        end
+      }
+      before do
+        Timecop.freeze(time)
+        @initial_callback_called = false
+        @continuous_callback_called = false
+      end
+      it 'fires an :event_not_coming callback and an :event_still_not_coming' do
+        subject.start(&callback)
+
+        expect(@initial_callback_called).to eq false
+        expect(@continuous_callback_called).to eq false
+
+        Timecop.freeze(time + initial_interval + continuous_interval * 0.2)
+        sleep check_interval * 1.1
+
+        expect(@initial_callback_called).to eq true
+        expect(@continuous_callback_called).to eq false
+
+        Timecop.freeze(time + initial_interval + continuous_interval + check_interval)
+
+        sleep check_interval * 1.1
+
+        expect(@initial_callback_called).to eq true
+        expect(@continuous_callback_called).to eq true
+
+        subject.stop
+      end
+      after do
+        Timecop.return
+        @initial_callback_called = false
+        @continuous_callback_called = false
+        @arrived_callback_called = false
+      end
+    end
+    context 'an event during test after initial_interval' do
+      let(:callback) {
+        Proc.new do |reason|
+          case reason
+          when :event_not_coming
+            @initial_callback_called = true
+          when :event_still_not_coming
+            @continuous_callback_called = true
+         when :event_arrived_finally
+            @arrived_callback_called = true
+          end
+        end
+      }
+      before do
+        Timecop.freeze(time)
+        @initial_callback_called = false
+        @continuous_callback_called = false
+        @arrived_callback_called = false
+      end
+      it 'fires an :event_not_coming callback and an :event_arrived_finaly callback' do
+        subject.start(&callback)
+
+        expect(@initial_callback_called).to eq false
+        expect(@continuous_callback_called).to eq false
+        expect(@arrived_callback_called).to eq false
+
+        Timecop.freeze(time + initial_interval + initial_interval * 0.2)
+        sleep check_interval * 1.1
+
+        expect(@initial_callback_called).to eq true
+        expect(@continuous_callback_called).to eq false
+        expect(@arrived_callback_called).to eq false
+
+        Timecop.freeze(time + initial_interval + initial_interval * 0.8)
+
+        subject.notify
+
+        sleep check_interval * 1.1
+
+        expect(@initial_callback_called).to eq true
+        expect(@continuous_callback_called).to eq false
+        expect(@arrived_callback_called).to eq true
+
+        subject.stop
+      end
+      after do
+        Timecop.return
+        @initial_callback_called = false
+        @continuous_callback_called = false
+        @arrived_callback_called = false
+      end
+    end
+    context 'an event during test after initial_interval + continuous_interval' do
+      let(:callback) {
+        Proc.new do |reason|
+          case reason
+          when :event_not_coming
+            @initial_callback_called = true
+          when :event_still_not_coming
+            @continuous_callback_called = true
+         when :event_arrived_finally
+            @arrived_callback_called = true
+          end
+        end
+      }
+      before do
+        Timecop.freeze(time)
+        @initial_callback_called = false
+        @continuous_callback_called = false
+        @arrived_callback_called = false
+      end
+      it 'fires all 3 callbacks' do
+        subject.start(&callback)
+
+        expect(@initial_callback_called).to eq false
+        expect(@continuous_callback_called).to eq false
+        expect(@arrived_callback_called).to eq false
+
+        Timecop.freeze(time + initial_interval + initial_interval * 0.2)
+        sleep check_interval * 1.1
+
+        expect(@initial_callback_called).to eq true
+        expect(@continuous_callback_called).to eq false
+        expect(@arrived_callback_called).to eq false
+
+        @initial_callback_called = false
+
+        Timecop.freeze(time + initial_interval + continuous_interval + initial_interval * 0.2)
+
+        sleep check_interval * 1.1
+
+        expect(@initial_callback_called).to eq false
+        expect(@continuous_callback_called).to eq true
+        expect(@arrived_callback_called).to eq false
+
+        @continuous_callback_called = false
+
+        Timecop.freeze(time + initial_interval + continuous_interval + initial_interval * 0.4)
+
+        subject.notify
+
+        sleep check_interval * 1.1
+
+        expect(@initial_callback_called).to eq false
+        expect(@continuous_callback_called).to eq false
+        expect(@arrived_callback_called).to eq true
+
+        subject.stop
+      end
+      after do
+        Timecop.return
+        @initial_callback_called = false
+        @continuous_callback_called = false
+        @arrived_callback_called = false
+      end
+    end
+    context 'no event during test which is long enough for two event_still_not_coming callbacks' do
+      let(:callback) {
+        Proc.new do |reason|
+          case reason
+          when :event_not_coming
+            @initial_callback_called = true
+          when :event_still_not_coming
+            @continuous_callback_called = true
+         when :event_arrived_finally
+            @arrived_callback_called = true
+          end
+        end
+      }
+      before do
+        Timecop.freeze(time)
+        @initial_callback_called = false
+        @continuous_callback_called = false
+        @arrived_callback_called = false
+      end
+      it 'fires two event_still_not_coming callbacks after an event_not_coming callback' do
+        subject.start(&callback)
+
+        expect(@initial_callback_called).to eq false
+        expect(@continuous_callback_called).to eq false
+        expect(@arrived_callback_called).to eq false
+
+        Timecop.freeze(time + initial_interval + initial_interval * 0.2)
+        sleep check_interval * 1.1
+
+        expect(@initial_callback_called).to eq true
+        expect(@continuous_callback_called).to eq false
+        expect(@arrived_callback_called).to eq false
+
+        @initial_callback_called = false
+
+        Timecop.freeze(time + initial_interval + continuous_interval + initial_interval * 0.2)
+
+        sleep check_interval * 1.1
+
+        expect(@initial_callback_called).to eq false
+        expect(@continuous_callback_called).to eq true
+        expect(@arrived_callback_called).to eq false
+
+        @continuous_callback_called = false
+
+        Timecop.freeze(time + initial_interval + continuous_interval + continuous_interval + initial_interval * 0.2)
+
+        sleep check_interval * 1.1
+
+        expect(@initial_callback_called).to eq false
+        expect(@continuous_callback_called).to eq true
+        expect(@arrived_callback_called).to eq false
+
+        subject.stop
+      end
+      after do
+        Timecop.return
+        @initial_callback_called = false
+        @continuous_callback_called = false
+        @arrived_callback_called = false
+      end
+    end
+    context 'an event after initial_interval and no event for another initial_interval time' do
+      let(:callback) {
+        Proc.new do |reason|
+          case reason
+          when :event_not_coming
+            @initial_callback_called = true
+          when :event_still_not_coming
+            @continuous_callback_called = true
+         when :event_arrived_finally
+            @arrived_callback_called = true
+          end
+        end
+      }
+      before do
+        Timecop.freeze(time)
+        @initial_callback_called = false
+        @continuous_callback_called = false
+        @arrived_callback_called = false
+      end
+      it 'fires two event_still_not_coming callbacks after an event_not_coming callback' do
+        subject.start(&callback)
+
+        expect(@initial_callback_called).to eq false
+        expect(@continuous_callback_called).to eq false
+        expect(@arrived_callback_called).to eq false
+
+        Timecop.freeze(time + initial_interval + initial_interval * 0.2)
+        sleep check_interval * 1.1
+
+        expect(@initial_callback_called).to eq true
+        expect(@continuous_callback_called).to eq false
+        expect(@arrived_callback_called).to eq false
+
+        @initial_callback_called = false
+
+        Timecop.freeze(time + initial_interval + initial_interval * 0.4)
+
+        subject.notify
+
+        sleep check_interval * 1.1
+
+        expect(@initial_callback_called).to eq false
+        expect(@continuous_callback_called).to eq false
+        expect(@arrived_callback_called).to eq true
+
+        @arrived_callback_called = false
+
+        Timecop.freeze(time + initial_interval + initial_interval * 0.4 + initial_interval + initial_interval * 0.2)
+
+        sleep check_interval * 1.1
+
+        expect(@initial_callback_called).to eq true
+        expect(@continuous_callback_called).to eq false
+        expect(@arrived_callback_called).to eq false
+
+        subject.stop
+      end
+      after do
+        Timecop.return
+        @initial_callback_called = false
+        @continuous_callback_called = false
+        @arrived_callback_called = false
+      end
+    end
+  end
+  describe '#stop' do
+    context 'stop without corresponding start' do
+      it 'behaves nicely' do
+        subject.stop
+      end
+    end
+    context 'stop with corresponding start' do
+      it 'kills the thread' do
+        subject.start {}
+        thread = subject.instance_variable_get(:@thread)
+        expect(thread.alive?).to eq true
+        subject.stop
+        sleep 0.05
+        expect(thread.alive?).to eq false
+      end
+    end
+  end
+end
+
+end

data/spec/flydata/output/forwarder_spec.rb

@@ -8,8 +8,8 @@ module Flydata
       let(:forwarder) do
         ForwarderFactory.create('tcpforwarder', 'test_tag', ['localhost:3456', 'localhost:4567'])
       end
+      let(:conn) { double(TCPSocket) }
       before :each do
-        conn = double(TCPSocket)
         allow(conn).to receive(:setsockopt)
         allow(conn).to receive(:write)
         allow(conn).to receive(:close)
@@ -73,6 +73,24 @@ module Flydata
             expect(forwarder.buffer_record_count).to be(0)
           end
         end
+        context 'when the error happens during the data sending' do
+          before do
+            @never_received = true
+            allow(conn).to receive(:write) do
+              if @never_received
+                @never_received = false
+                raise "test-error"
+              else
+                nil
+              end
+            end
+          end
+          it 'retry and succeed sending data' do
+            forwarder.emit(record)
+            forwarder.emit(record)
+            expect(forwarder.emit(record)).to be(true)
+          end
+        end
       end
 
       describe '#pickup_server' do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flydata
 version: !ruby/object:Gem::Version
-  version: 0.2.21
+  version: 0.2.22
 platform: ruby
 authors:
 - Koichi Fujikawa
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-12-12 00:00:00.000000000 Z
+date: 2014-12-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -474,6 +474,7 @@ files:
 - lib/flydata/compatibility_check.rb
 - lib/flydata/credentials.rb
 - lib/flydata/cron.rb
+- lib/flydata/fluent-plugins/idle_event_detector.rb
 - lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb
 - lib/flydata/fluent-plugins/mysql/alter_table_query_handler.rb
 - lib/flydata/fluent-plugins/mysql/binlog_position.rb
@@ -509,6 +510,7 @@ files:
 - spec/flydata/command/sender_spec.rb
 - spec/flydata/command/sync_spec.rb
 - spec/flydata/compatibility_check_spec.rb
+- spec/flydata/fluent-plugins/idle_event_detector_spec.rb
 - spec/flydata/fluent-plugins/in_mysql_binlog_flydata_spec.rb
 - spec/flydata/fluent-plugins/mysql/alter_table_query_handler_spec.rb
 - spec/flydata/fluent-plugins/mysql/binlog_position_spec.rb
@@ -543,7 +545,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.0.14
 signing_key: 
 specification_version: 4
 summary: FlyData Agent