fluent-plugin-hash-forward 0.2.0 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a216c4741f2a87d809deb2a2dcd6006f4315056b
-  data.tar.gz: db7b1ca95255839c9a3134eacb34e3d580d75daf
+  metadata.gz: a0afb6cf6c53fa7e63173685f40958889da2f770
+  data.tar.gz: e6a1bc25cc85e1d075cb4bc48579a77dc4a71958
 SHA512:
-  metadata.gz: bbff18a4bcb208ded9babba3f8f11992b62fa033285aa17b73f3ca0de441cfcea2b58eed8b6a4bbecf68ab2bb92250c543abf4bf0f22ac1fcc089f1f2846a285
-  data.tar.gz: afe9ed287898a602f6bacf7d1fdc18eb226587e11c6ae2d82a97957028bd0f358a2e93cc72df235b86a4720a70af06865f6d405a55a8c526a9a8faed2820116d
+  metadata.gz: 0075b38f21756454e85d18632e7d57d3994c14491b119494560142e379c6bd6e4b968ee42b147bd118d4cd53ace4cc65888f2a19e40b25418ecc59ba439fecc6
+  data.tar.gz: c8b4a6b1361a1c17a78aaf022729c37e8ea09be612fc870c062559b154182422c0e5f740646825e215a62cd31a96a7cab0037ae392e22c686f64576e3781af04

data/CHANGELOG.md

@@ -1,3 +1,15 @@
+# 0.3.1 (2013/12/11)
+
+Changes
+
+* Change default `keepalive` option to false
+
+# 0.3.0 (2013/12/11)
+
+Enhancement
+
+* Add `keepalive` and `keepalive_time` option
+
 # 0.2.0 (2013/11/02)
 
 Enhancement

data/README.md

@@ -31,6 +31,15 @@ Basically same with out\_forward plugin. See [http://docs.fluentd.org/articles/o
 
 Following parameters are additionally available:
 
+
+* keepalive (bool)
+
+    Keepalive connection. Default is `false`.
+
+* keepalive_time (time)
+
+    Keepalive expired time. Default is nil (which means to keep connection as long as possible).
+
 * hash\_key\_slice *min*..*max*
 
     Use sliced `tag` as a hash key to determine a forwarding node. Default: use entire `tag`. 

data/fluent-plugin-hash-forward.gemspec

@@ -3,10 +3,10 @@ $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |s|
   s.name        = "fluent-plugin-hash-forward"
-  s.version     = "0.2.0"
-  s.authors     = ["Ryosuke IWANAGA", "Naotoshi SEO"]
+  s.version     = "0.3.1"
+  s.authors     = ["Ryosuke IWANAGA", "Naotoshi Seo"]
   s.email       = ["riywo.jp@gmail.com", "sonots@gmail.com"]
-  s.homepage    = "https://github.com/riywo/fluent-plugin-hash-forward"
+  s.homepage    = "https://github.com/sonots/fluent-plugin-hash-forward"
   s.summary     = %q{Fluentd plugin to keep forwarding messsages of a specific tag pattern to a specific node}
   s.description = s.summary
   s.licenses    = ["MIT"]

data/lib/fluent/plugin/out_hash_forward.rb

@@ -4,6 +4,8 @@ class Fluent::HashForwardOutput < Fluent::ForwardOutput
   Fluent::Plugin.register_output('hash_forward', self)
 
   config_param :hash_key_slice, :string, :default => nil
+  config_param :keepalive, :bool, :default => false
+  config_param :keepalive_time, :time, :default => nil # infinite
 
   def configure(conf)
     super
@@ -22,6 +24,10 @@ class Fluent::HashForwardOutput < Fluent::ForwardOutput
     @standby_weight_array = build_weight_array(@standby_nodes)
 
     @cache_nodes = {}
+    @sock = {}
+    @sock_expired_at = {}
+    @mutex = {}
+    @watcher_interval = 1
   end
 
   # for test
@@ -31,13 +37,35 @@ class Fluent::HashForwardOutput < Fluent::ForwardOutput
   attr_reader :standby_weight_array
   attr_accessor :hash_key_slice_lindex
   attr_accessor :hash_key_slice_rindex
+  attr_accessor :watcher_interval
+
+  def start
+    super
+    start_watcher
+  end
+
+  def shutdown
+    super
+    stop_watcher
+  end
+
+  def start_watcher
+    if @keepalive and @keepalive_time
+      @watcher = Thread.new(&method(:watch_keepalive_time))
+    end
+  end
+
+  def stop_watcher
+    if @watcher
+      @watcher.terminate
+      @watcher.join
+    end
+  end
 
   # Override
   def write_objects(tag, chunk)
     return if chunk.empty?
-
     error = nil
-
     nodes = nodes(tag)
 
     # below is just copy from out_forward
@@ -64,8 +92,8 @@ class Fluent::HashForwardOutput < Fluent::ForwardOutput
   def rebuild_weight_array
   end
 
+  # This is just a partial copy from ForwardOuput#rebuild_weight_array
   def build_weight_array(nodes)
-    # below is just a partial copy from out_forward#rebuild_weight_array
     weight_array = []
     gcd = nodes.map {|n| n.weight }.inject(0) {|r,w| r.gcd(w) }
     nodes.each {|n|
@@ -104,4 +132,96 @@ class Fluent::HashForwardOutput < Fluent::ForwardOutput
     sliced = tags[@hash_key_slice_lindex..@hash_key_slice_rindex]
     return sliced.nil? ? "" : sliced.join('.')
   end
+
+  # Override for keepalive
+  def send_data(node, tag, chunk)
+    get_mutex(node).synchronize do
+      sock = get_sock[node]
+      unless sock
+        sock = reconnect(node)
+      end
+
+      begin
+        sock_write(sock, tag, chunk)
+        node.heartbeat(false)
+      rescue Errno::EPIPE, Errno::ECONNRESET, Errno::ECONNABORTED, Errno::ETIMEDOUT => e
+        $log.warn "out_hash_forward: #{e.class} #{e.message}"
+        sock = reconnect(node)
+        retry
+      end
+    end
+  end
+
+  def reconnect(node)
+    sock = connect(node)
+    opt = [1, @send_timeout.to_i].pack('I!I!')  # { int l_onoff; int l_linger; }
+    sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_LINGER, opt)
+
+    opt = [@send_timeout.to_i, 0].pack('L!L!')  # struct timeval
+    sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_SNDTIMEO, opt)
+
+    if @keepalive
+      get_sock[node] = sock
+      get_sock_expired_at[node] = Time.now + @keepalive_time if @keepalive_time
+    end
+
+    sock
+  end
+
+  def sock_write(sock, tag, chunk)
+    # beginArray(2)
+    sock.write FORWARD_HEADER
+
+    # writeRaw(tag)
+    sock.write tag.to_msgpack  # tag
+
+    # beginRaw(size)
+    sz = chunk.size
+    #if sz < 32
+    #  # FixRaw
+    #  sock.write [0xa0 | sz].pack('C')
+    #elsif sz < 65536
+    #  # raw 16
+    #  sock.write [0xda, sz].pack('Cn')
+    #else
+    # raw 32
+    sock.write [0xdb, sz].pack('CN')
+    #end
+
+    # writeRawBody(packed_es)
+    chunk.write_to(sock)
+  end
+
+  # watcher thread callback
+  def watch_keepalive_time
+    while true
+      sleep @watcher_interval
+      thread_ids = @sock.keys
+      thread_ids.each do |thread_id|
+        @sock[thread_id].each do |node, sock|
+          @mutex[thread_id][node].synchronize do
+            next unless sock_expired_at = @sock_expired_at[thread_id][node]
+            next unless Time.now >= sock_expired_at
+            sock.close rescue IOError if sock
+            @sock[thread_id][node] = nil
+            @sock_expired_at[thread_id][node] = nil
+          end
+        end
+      end
+    end
+  end
+
+  def get_mutex(node)
+    thread_id = Thread.current.object_id
+    @mutex[thread_id] ||= {}
+    @mutex[thread_id][node] ||= Mutex.new
+  end
+
+  def get_sock
+    @sock[Thread.current.object_id] ||= {}
+  end
+
+  def get_sock_expired_at
+    @sock_expired_at[Thread.current.object_id] ||= {}
+  end
 end

data/spec/out_hash_forward_spec.rb

@@ -117,7 +117,7 @@ describe Fluent::HashForwardOutput do
           <server>
             host 192.168.1.3
             port 24224
-            weight 1
+            weight 100
           </server>
           <server>
             host 192.168.1.4
@@ -189,10 +189,10 @@ describe Fluent::HashForwardOutput do
       let(:tag2) { 'test.tag2' }
       let(:config) { CONFIG + %[hash_key_slice 0..-2] }
       before do
-        @node1 = driver.nodes(tag1).first
+        @node = driver.nodes(tag1).first
       end
-      it 'should forward to the different node' do
-        expect(driver.nodes(tag2).first).to eq(@node1)
+      it 'should forward to the same node' do
+        expect(driver.nodes(tag2).first).to eq(@node)
       end
     end
   end

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-hash-forward
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.1
 platform: ruby
 authors:
 - Ryosuke IWANAGA
-- Naotoshi SEO
+- Naotoshi Seo
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-11-02 00:00:00.000000000 Z
+date: 2013-12-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -99,7 +99,7 @@ files:
 - lib/fluent/plugin/out_hash_forward.rb
 - spec/out_hash_forward_spec.rb
 - spec/spec_helper.rb
-homepage: https://github.com/riywo/fluent-plugin-hash-forward
+homepage: https://github.com/sonots/fluent-plugin-hash-forward
 licenses:
 - MIT
 metadata: {}