fluent-plugin-keep-forward 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ae46708b29b44194062b9985c6d5cab371785a27
-  data.tar.gz: 2038eb737536f4bacd779e0d2f40a8a2081f6f14
+  metadata.gz: 9c2af8c036f4fd144a1db429d94cc08988eec860
+  data.tar.gz: 376bb6ac2acceab104079b72c214e619e1f8afac
 SHA512:
-  metadata.gz: 8f885899158c22a334a6fe027dad4fd5c2ff48e081bb78b93c56593aeb2337411d4b64251ffb54d43f8b5f0a579b30d3cc01a7daf80030536e67006484777366
-  data.tar.gz: 74cec077f9b1bf83d62eac10a9ab6e8308a58a5b0baa1f0868ae4ace0a234e3fc5a313e9e18330aca0d2556a1194a32639b74f791bb9277d5a490b3b4809eece
+  metadata.gz: 51070129d6930e68aabad1e66c195f1355b3548302673c6bf3bf82585a4f9b3f3df31fdff45c5ce22da00b13c4c31154269f7285d066dcee1c8b19b3aaf5457a
+  data.tar.gz: 7708a5f03c1dae8c93b35525610f1cee0b3f9025aa2519b8fc58e6937dc421c779a3937262693736aa622d10cbd0db74ff4f31f1826d27059cac1d7d0f582630

data/.travis.yml

@@ -1,6 +1,6 @@
 rvm:
-  - 1.9.2
   - 1.9.3
   - 2.0.0
+  - 2.1.*
 gemfile:
   - Gemfile

data/CHANGELOG.md

@@ -1,3 +1,14 @@
+## 0.1.3 (2014/03/19)
+
+Enhancement
+
+* Enrich warn log message
+* Add `keepforward` option
+
+Fixes
+
+* Explicitly close socket the case of non-keepalive
+
 ## 0.1.2
 
 Enhancement:

data/README.md

@@ -20,6 +20,12 @@ Following parameters are additionally available:
 
     Keepalive expired time. Default is nil (which means to keep connection as long as possible).
 
+- keepforward
+
+    `one` for keep forwarding all data to the one node.
+    `tag` for keep forwarding data with the same tag to the same node.
+    Default is `one`.
+
 - prefer_recover (bool)
 
     Switch connection to a recovered node from standby nodes or less weighted nodes. Default is `true`.

data/fluent-plugin-keep-forward.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |s|
   s.name        = "fluent-plugin-keep-forward"
-  s.version     = "0.1.2"
+  s.version     = "0.1.3"
   s.authors     = ["Naotoshi Seo"]
   s.email       = ["sonots@gmail.com"]
   s.homepage    = "https://github.com/sonots/fluent-plugin-keep-forward"

data/lib/fluent/plugin/out_keep_forward.rb

@@ -11,6 +11,16 @@ class Fluent::KeepForwardOutput < Fluent::ForwardOutput
   config_param :prefer_recover, :bool, :default => true
   config_param :keepalive, :bool, :default => false
   config_param :keepalive_time, :time, :default => nil # infinite
+  config_param :keepforward, :default => :one do |val|
+    case val.downcase
+    when 'one'
+      :one
+    when 'tag'
+      :tag
+    else
+      raise ConfigError, "out_keep_forward keepforward should be 'one' or 'tag'"
+    end
+  end
 
   # for test
   attr_accessor :watcher_interval
@@ -25,6 +35,18 @@ class Fluent::KeepForwardOutput < Fluent::ForwardOutput
     @watcher_interval = 1
   end
 
+  def get_node(tag)
+    @node[keepforward(tag)]
+  end
+
+  def keepforward(tag)
+    @keepforward == :one ? :one : tag
+  end
+
+  def cache_node(tag, node)
+    @node[keepforward(tag)] = node
+  end
+
   def start
     super
     start_watcher
@@ -52,23 +74,29 @@ class Fluent::KeepForwardOutput < Fluent::ForwardOutput
   def write_objects(tag, chunk)
     return if chunk.empty?
     error = nil
-    node = @node[tag]
+    node = get_node(tag)
 
     if node and node.available? and (!@prefer_recover or @weight_array.include?(node))
       begin
         send_data(node, tag, chunk)
         return
       rescue
-        weight_send_data(tag, chunk)
+        node = weight_send_data(tag, chunk, error_node = node)
+        cache_node(tag, node)
       end
     else
-      weight_send_data(tag, chunk)
+      node = weight_send_data(tag, chunk, error_node = node)
+      cache_node(tag, node)
     end
   end
 
-  def weight_send_data(tag, chunk)
+  def weight_send_data(tag, chunk, error_node = nil)
     error = nil
 
+    if error_node
+      sock_close(error_node) if @keepalive and @keepforward == :one
+    end
+
     wlen = @weight_array.length
     wlen.times do
       @rr = (@rr + 1) % wlen
@@ -77,8 +105,7 @@ class Fluent::KeepForwardOutput < Fluent::ForwardOutput
       if node.available?
         begin
           send_data(node, tag, chunk)
-          @node[tag] = node
-          return
+          return node
         rescue
           # for load balancing during detecting crashed servers
           error = $!  # use the latest error
@@ -86,7 +113,7 @@ class Fluent::KeepForwardOutput < Fluent::ForwardOutput
       end
     end
 
-    @node[tag] = nil
+    cache_node(tag, nil)
     if error
       raise error
     else
@@ -100,15 +127,20 @@ class Fluent::KeepForwardOutput < Fluent::ForwardOutput
       sock = get_sock[node]
       unless sock
         sock = reconnect(node)
+        cache_sock(node, sock) if @keepalive
       end
 
       begin
         sock_write(sock, tag, chunk)
         node.heartbeat(false)
       rescue Errno::EPIPE, Errno::ECONNRESET, Errno::ECONNABORTED, Errno::ETIMEDOUT => e
-        log.warn "out_keep_forward: #{e.class} #{e.message}"
+        log.warn "out_keep_forward: send_data failed #{e.class} #{e.message}, try to reconnect", :host=>node.host, :port=>node.port
+        sock.close rescue IOError
         sock = reconnect(node)
+        cache_sock(node, sock) if @keepalive
         retry
+      ensure
+        sock.close if sock and !@keepalive
       end
     end
   end
@@ -121,11 +153,6 @@ class Fluent::KeepForwardOutput < Fluent::ForwardOutput
     opt = [@send_timeout.to_i, 0].pack('L!L!')  # struct timeval
     sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_SNDTIMEO, opt)
 
-    if @keepalive
-      get_sock[node] = sock
-      get_sock_expired_at[node] = Time.now + @keepalive_time if @keepalive_time
-    end
-
     sock
   end
 
@@ -172,12 +199,26 @@ class Fluent::KeepForwardOutput < Fluent::ForwardOutput
     end
   end
 
+  def sock_close(node)
+    get_mutex(node).synchronize do
+      sock = get_sock[node]
+      sock.close rescue IOError if sock
+      get_sock[node] = nil
+      get_sock_expired_at[node] = nil
+    end
+  end
+
   def get_mutex(node)
     thread_id = Thread.current.object_id
     @mutex[thread_id] ||= {}
     @mutex[thread_id][node] ||= Mutex.new
   end
 
+  def cache_sock(node, sock)
+    get_sock[node] = sock
+    get_sock_expired_at[node] = Time.now + @keepalive_time if @keepalive_time
+  end
+
   def get_sock
     @sock[Thread.current.object_id] ||= {}
   end

data/spec/out_keep_forward_spec.rb

@@ -38,7 +38,7 @@ shared_context 'keep_forward_try_once' do
   after do
     driver.stop_watcher
   end
-  let!(:keep_node) { driver.instance_variable_get(:@node)[tag] }
+  let!(:keep_node) { driver.get_node(tag) }
   let!(:another_node) { (driver.instance_variable_get(:@nodes) - [keep_node]).first }
 end
 
@@ -86,7 +86,7 @@ describe Fluent::KeepForwardOutput do
       before { driver.should_receive(:send_data).with(keep_node, tag, chunk) }
     end
     it_should_behave_like 'error_occurs' do
-      before { driver.stub(:weight_send_data).with(tag, chunk) } # re-call weight_send_data
+      before { driver.stub(:weight_send_data).with(tag, chunk, keep_node) } # re-call weight_send_data
     end
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-keep-forward
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - Naotoshi Seo
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-02-04 00:00:00.000000000 Z
+date: 2014-03-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd