fluent-plugin-hash-forward 0.3.2 → 0.3.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: dc5f372dbd73daa73bc4a7da282a7edf386195a6
-  data.tar.gz: bff3830548afa79073a48dab264cb7a484f36cab
+  metadata.gz: d01dea2a66bc0f774fc40e36ccb1ebf6e2a01e49
+  data.tar.gz: aca6bf525e53870912ed253c7aff0dceda19826d
 SHA512:
-  metadata.gz: 513d76863848a0de51a6675eb9b2e7b472d371e2ab69a0c1c47b3769ffc9dc2c75a005a064b5c3a8fd8bf8b4aaa43ca38365310d2f344f41ddf1fb319eef2fc7
-  data.tar.gz: 5f8e77d7b1eb80719aad7b9ce1a8981dc84526b9451619074ec2920f595e25833b25652ff8a6e9dc16fa18efa9e4f1855f1c60edf4c7a4e7c92120793dce9dae
+  metadata.gz: 95dd5908ae5750478ace61a0e940d177993326645d01ea24dd948d2e54e52ec5d16b91581390265e6056dab17a902928e187043757bd998117cd78f98b5ed6d2
+  data.tar.gz: 362fd432c9a0898905154e3cf530cd689e10729556ac746283e952b31ffd22b71a5e26d974cf798959b9a88b10c8bc4086b487f71b02ad9e4aa2a7b7de88f1c1

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+# 0.3.3 (2014/03/17)
+
+Enhancement:
+
+* Support `heartbeat_type` `none`
+
 # 0.3.2 (2014/02/04)
 
 Enhancement:

data/README.md

@@ -40,6 +40,11 @@ Following parameters are additionally available:
 
     Keepalive expired time. Default is nil (which means to keep connection as long as possible).
 
+* heartbeat_type
+
+    The transport protocol to use for heartbeats. The default is “udp”, but you can select “tcp” as well.
+    Furthermore, in hash_forward, you can also select "none" to disable heartbeat. 
+
 * hash\_key\_slice *min*..*max*
 
     Use sliced `tag` as a hash key to determine a forwarding node. Default: use entire `tag`. 

data/example/fluent.conf

@@ -0,0 +1,39 @@
+<source>
+  type tail
+  path dummy.log
+  pos_file /var/tmp/_var_log_dummy.pos
+  format none
+  tag dummy.localhost
+</source>
+<match dummy.localhost>
+  type copy
+  <store>
+    type stdout
+  </store>
+  <store>
+    type hash_forward
+    flush_interval 0
+    buffer_queue_limit 312
+    buffer_chunk_limit 10m
+    num_threads 1
+    retry_wait 0.1
+    retry_limit 17
+    # max_retry_wait 131072
+    send_timeout 60s # 60s
+    # recover_wait 10s
+    heartbeat_type none # udp
+    # heartbeat_interval 1s
+    phi_threshold 70 # 8
+    hard_timeout 60s # 60s
+    hash_key_slice 0..-2
+    <server>
+      host localhost
+      port 26000
+    </server>
+    <server>
+      host localhost
+      port 27000
+      standby true
+    </server>
+  </store>
+</match>

data/fluent-plugin-hash-forward.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |s|
   s.name        = "fluent-plugin-hash-forward"
-  s.version     = "0.3.2"
+  s.version     = "0.3.3"
   s.authors     = ["Ryosuke IWANAGA", "Naotoshi Seo"]
   s.email       = ["riywo.jp@gmail.com", "sonots@gmail.com"]
   s.homepage    = "https://github.com/riywo/fluent-plugin-hash-forward"

data/lib/fluent/plugin/out_hash_forward.rb

@@ -1,4 +1,5 @@
 require 'fluent/plugin/out_forward'
+require 'forwardable'
 
 class Fluent::HashForwardOutput < Fluent::ForwardOutput
   Fluent::Plugin.register_output('hash_forward', self)
@@ -11,6 +12,18 @@ class Fluent::HashForwardOutput < Fluent::ForwardOutput
   config_param :hash_key_slice, :string, :default => nil
   config_param :keepalive, :bool, :default => false
   config_param :keepalive_time, :time, :default => nil # infinite
+  config_param :heartbeat_type, :default => :udp do |val|
+    case val.downcase
+    when 'tcp'
+      :tcp
+    when 'udp'
+      :udp
+    when 'none' # custom
+      :none
+    else
+      raise ConfigError, "forward output heartbeat type should be 'tcp' or 'udp', or 'none'"
+    end
+  end
 
   def configure(conf)
     super
@@ -24,6 +37,10 @@ class Fluent::HashForwardOutput < Fluent::ForwardOutput
       end
     end
 
+    if @heartbeat_type == :none
+      @nodes = @nodes.map {|node| NonHeartbeatNode.new(node) }
+    end
+
     @standby_nodes, @regular_nodes = @nodes.partition {|n| n.standby? }
     @regular_weight_array = build_weight_array(@regular_nodes)
     @standby_weight_array = build_weight_array(@standby_nodes)
@@ -50,7 +67,11 @@ class Fluent::HashForwardOutput < Fluent::ForwardOutput
   end
 
   def shutdown
-    super
+    @finished = true
+    @loop.watchers.each {|w| w.detach }
+    @loop.stop unless @heartbeat_type == :none # custom
+    @thread.join
+    @usock.close if @usock
     stop_watcher
   end
 
@@ -67,6 +88,37 @@ class Fluent::HashForwardOutput < Fluent::ForwardOutput
     end
   end
 
+  # Override to disable heartbeat
+  def run
+    unless @heartbeat_type == :none
+      super
+    end
+  end
+
+  # Delegate to Node instance disabling heartbeat
+  class NonHeartbeatNode
+    extend Forwardable
+    attr_reader :node
+    def_delegators :@node, :standby?, :resolved_host, :resolve_dns!, :to_msgpack,
+      :name, :host, :port, :weight, :weight=, :standby=, :available=, :sockaddr
+
+    def initialize(node)
+      @node = node
+    end
+
+    def available?
+      true
+    end
+
+    def tick
+      false
+    end
+
+    def heartbeat(detect=true)
+      true
+    end
+  end
+
   # Override
   def write_objects(tag, chunk)
     return if chunk.empty?
@@ -150,7 +202,7 @@ class Fluent::HashForwardOutput < Fluent::ForwardOutput
         sock_write(sock, tag, chunk)
         node.heartbeat(false)
       rescue Errno::EPIPE, Errno::ECONNRESET, Errno::ECONNABORTED, Errno::ETIMEDOUT => e
-        log.warn "out_hash_forward: #{e.class} #{e.message}"
+        log.warn "out_hash_forward: send_data failed #{e.class} #{e.message}, try to reconnect", :host=>node.host, :port=>node.port
         sock = reconnect(node)
         retry
       end

data/spec/out_hash_forward_spec.rb

@@ -314,5 +314,16 @@ describe Fluent::HashForwardOutput do
       end
     end
   end
+
+  describe 'test heartbeat_type :none' do
+    let(:tag) { 'test.tag' }
+    let(:es) { Array.new(1) }
+    let(:chain) { Fluent::NullOutputChain.instance }
+    let(:config) { CONFIG + %[heartbeat_type none] }
+
+    context 'NonHeartbeatNode' do
+      it { driver.nodes(tag).first.class.should == Fluent::HashForwardOutput::NonHeartbeatNode }
+    end
+  end
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-hash-forward
 version: !ruby/object:Gem::Version
-  version: 0.3.2
+  version: 0.3.3
 platform: ruby
 authors:
 - Ryosuke IWANAGA
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-02-04 00:00:00.000000000 Z
+date: 2014-03-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -95,6 +95,7 @@ files:
 - Gemfile
 - README.md
 - Rakefile
+- example/fluent.conf
 - fluent-plugin-hash-forward.gemspec
 - lib/fluent/plugin/out_hash_forward.rb
 - spec/out_hash_forward_spec.rb