fluent-plugin-secure-forward 0.2.2 → 0.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 17a1806c021cb45491bbea5c7d65ec21a1d94783
-  data.tar.gz: ea346ee1d2013beed5cd8582510e2869bbfb9be6
+  metadata.gz: 0129c03cd79fb35962e3b4106ce6065a91a7c3a0
+  data.tar.gz: f1138674a8cb6be47aab0eb9215472a8984e8b3b
 SHA512:
-  metadata.gz: 792d9fc40c27269746c796061ee68381419d271c4c8e736e855e6a023a6e2cf03bdeb99348c3b8dd2e854777671da2a5b096acf0872232ea69562d0f7ed119d5
-  data.tar.gz: f7e2c2947ebbc4b229a12e667440bee903836cb62c3f2fb02e90f878e23352b3f8be98fcdfe7a056971eb6776167c645b483f61ea565dfa3d6501cf61efdafc4
+  metadata.gz: a44b920b0e142947d94d4869efd3bf8d1d34f43ca6d545382894201cbf153ab6c3a9287633e5243b9795cb64c62072c3cc1951e6256005eefecae86861d3e26a
+  data.tar.gz: b972a083ada6760f7968b425b87322123a0f081926f6670f87a8ff4058f6e2d11be06b2657009b5cecd6ab9c1b6057b2df0139e3be04ba064e2b8fa9782597b4

data/fluent-plugin-secure-forward.gemspec

@@ -1,7 +1,7 @@
 # -*- encoding: utf-8 -*-
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-secure-forward"
-  gem.version       = "0.2.2"
+  gem.version       = "0.2.3"
   gem.authors       = ["TAGOMORI Satoshi"]
   gem.email         = ["tagomoris@gmail.com"]
   gem.summary       = %q{Fluentd input/output plugin to forward over SSL with authentications}

data/lib/fluent/plugin/out_secure_forward.rb

@@ -38,15 +38,16 @@ module Fluent
 
     attr_reader :read_interval, :socket_interval
 
+    config_section :server, param_name: :servers do
+      config_param :host, :string
+      config_param :hostlabel, :string, default: nil
+      config_param :port, :integer, default: DEFAULT_SECURE_CONNECT_PORT
+      config_param :shared_key, :string, default: nil
+      config_param :username, :string, default: ''
+      config_param :password, :string, default: ''
+      config_param :standby, :bool, default: false
+    end
     attr_reader :nodes
-    # <server>
-    #   host ipaddr/hostname
-    #   hostlabel labelname # certification common name
-    #   port 24284
-    #   shared_key .... # optional shared key
-    #   username name # if required
-    #   password pass # if required
-    # </server>
 
     attr_reader :hostname_resolver
 
@@ -73,25 +74,17 @@ module Fluent
       @read_interval = @read_interval_msec / 1000.0
       @socket_interval = @socket_interval_msec / 1000.0
 
-      # read <server> tags and set to nodes
       @nodes = []
-      conf.elements.each do |element|
-        case element.name
-        when 'server'
-          unless element['host']
-            raise Fluent::ConfigError, "host missing in <server>"
-          end
-          node_shared_key = element['shared_key'] || @shared_key
-          node = Node.new(self, node_shared_key, element)
-          node.first_session = true
-          node.keepalive = @keepalive
-          @nodes.push node
-        when 'secondary'
-          # ignore
-        else
-          raise Fluent::ConfigError, "unknown config tag name #{element.name}"
-        end
+      @servers.each do |server|
+        node = Node.new(self, server)
+        node.first_session = true
+        @nodes.push node
+      end
+
+      if @num_threads > @nodes.select{|n| not n.standby}.size
+        log.warn "Too many num_threads for secure-forward: threads should be smaller or equal to non standby servers"
       end
+
       @next_node = 0
       @mutex = Mutex.new
 
@@ -110,7 +103,10 @@ module Fluent
           @next_node += 1
           @next_node = 0 if @next_node >= nodes
 
-          return n if n && n.established? && (!n.standby || permit_standby)
+          if n && n.established? && (! n.tained?) && (! n.detached?) && (!n.standby || permit_standby)
+            n.tain!
+            return n
+          end
 
           tries += 1
         end
@@ -181,7 +177,7 @@ module Fluent
             end
 
             log.trace "old connection shutting down"
-            oldconn.shutdown if oldconn # connection object doesn't raise any exceptions
+            oldconn.detach! if oldconn # connection object doesn't raise any exceptions
             log.trace "old connection shutted down"
 
             reconnectings[i] = nil
@@ -196,7 +192,7 @@ module Fluent
           timeout_conn = reconnectings[i][:conn]
           log.debug "SSL connection is not established until timemout", :host => timeout_conn.host, :port => timeout_conn.port, :timeout => @established_timeout
           reconnectings[i] = nil
-          timeout_conn.shutdown if timeout_conn # connection object doesn't raise any exceptions
+          timeout_conn.detach! if timeout_conn # connection object doesn't raise any exceptions
         end
       end
     end
@@ -208,7 +204,7 @@ module Fluent
       @nodewatcher.join
 
       @nodes.each do |node|
-        node.detach = true
+        node.detach!
         node.join
       end
     end
@@ -222,13 +218,11 @@ module Fluent
 
       begin
         send_data(node, tag, es)
+        node.release!
       rescue Errno::EPIPE, IOError, OpenSSL::SSL::SSLError => e
         log.warn "Failed to send messages to #{node.host}, parging.", :error_class => e.class, :error => e
-        begin
-          node.shutdown
-        rescue => e2
-          # ignore all errors
-        end
+        node.release!
+        node.detach!
 
         raise # to retry #write_objects
       end

data/lib/fluent/plugin/output_node.rb

@@ -16,20 +16,23 @@ class Fluent::SecureForwardOutput::Node
 
   attr_reader :expire
 
-  def initialize(sender, shared_key, conf)
+  def initialize(sender, conf)
     @sender = sender
-    @shared_key = shared_key
+    @shared_key = conf.shared_key || sender.shared_key
 
-    @host = conf['host']
-    @port = (conf['port'] || Fluent::SecureForwardOutput::DEFAULT_SECURE_CONNECT_PORT).to_i
-    @hostlabel = conf['hostlabel'] || conf['host']
-    @username = conf['username'] || ''
-    @password = conf['password'] || ''
-    @standby = conf.has_key?('standby') && Fluent::Config.bool_value(conf['standby']) != false
+    @host = conf.host
+    @port = conf.port
+    @hostlabel = conf.hostlabel || conf.host
+    @username = conf.username
+    @password = conf.password
+    @standby = conf.standby
+
+    @keepalive = sender.keepalive
 
     @authentication = nil
 
-    @keepalive = nil
+    @writing = false
+
     @expire = nil
     @first_session = false
     @detach = false
@@ -50,10 +53,8 @@ class Fluent::SecureForwardOutput::Node
   def dup
     renewed = self.class.new(
       @sender,
-      @shared_key,
-      {'host' => @host, 'port' => @port, 'hostlabel' => @hostlabel, 'username' => @username, 'password' => @password}
+      Fluent::Config::Section.new({host: @host, port: @port, hostlabel: @hostlabel, username: @username, password: @password, shared_key: @shared_key, standby: @standby})
     )
-    renewed.keepalive = @keepalive if @keepalive
     renewed
   end
 
@@ -63,6 +64,27 @@ class Fluent::SecureForwardOutput::Node
     # @thread.abort_on_exception = true
   end
 
+  def detach!
+    @detach = true
+  end
+
+  def detached?
+    @detach
+  end
+
+  def tain!
+    raise RuntimeError, "BUG: taining detached node" if @detach
+    @writing = true
+  end
+
+  def tained?
+    @writing
+  end
+
+  def release!
+    @writing = false
+  end
+
   def shutdown
     log.debug "shutting down node #{@host}"
     @state = :closed
@@ -273,6 +295,9 @@ class Fluent::SecureForwardOutput::Node
         break
       end
     end
+    while @writing
+      sleep read_interval
+    end
     self.shutdown
   end
 end

data/test/plugin/test_out_secure_forward.rb

@@ -27,4 +27,72 @@ class SecureForwardOutputTest < Test::Unit::TestCase
 CONFIG
   end
 
+  def test_configure_standby_server
+    p1 = nil
+    assert_nothing_raised { p1 = create_driver(<<CONFIG).instance }
+  type secure_forward
+  shared_key secret_string
+  self_hostname client.fqdn.local
+  keepalive 1m
+  <server>
+    host server1.fqdn.local
+  </server>
+  <server>
+    host server2.fqdn.local
+    hostlabel server2
+  </server>
+  <server>
+    host server1.fqdn.local
+    hostlabel server1
+    port 24285
+    shared_key secret_string_more
+    standby
+  </server>
+CONFIG
+    assert_equal 3, p1.servers.size
+    assert_equal 3, p1.nodes.size
+
+    assert_equal 'server1.fqdn.local', p1.nodes[0].host
+    assert_equal 'server1.fqdn.local', p1.nodes[0].hostlabel
+    assert_equal 24284, p1.nodes[0].port
+    assert_equal false, p1.nodes[0].standby
+    assert_equal 'secret_string', p1.nodes[0].shared_key
+    assert_equal 60, p1.nodes[0].keepalive
+
+    assert_equal 'server2.fqdn.local', p1.nodes[1].host
+    assert_equal 'server2', p1.nodes[1].hostlabel
+    assert_equal 24284, p1.nodes[1].port
+    assert_equal false, p1.nodes[1].standby
+    assert_equal 'secret_string', p1.nodes[1].shared_key
+    assert_equal 60, p1.nodes[1].keepalive
+
+    assert_equal 'server1.fqdn.local', p1.nodes[2].host
+    assert_equal 'server1', p1.nodes[2].hostlabel
+    assert_equal 24285, p1.nodes[2].port
+    assert_equal true, p1.nodes[2].standby
+    assert_equal 'secret_string_more', p1.nodes[2].shared_key
+    assert_equal 60, p1.nodes[2].keepalive
+  end
+
+  def test_configure_standby_server
+    p1 = nil
+    assert_nothing_raised { p1 = create_driver(<<CONFIG).instance }
+  type secure_forward
+  shared_key secret_string
+  self_hostname client.fqdn.local
+  num_threads 3
+  <server>
+    host server1.fqdn.local
+  </server>
+  <server>
+    host server2.fqdn.local
+  </server>
+  <server>
+    host server3.fqdn.local
+    standby
+  </server>
+CONFIG
+    assert_equal 3, p1.num_threads
+    assert_equal 1, p1.log.logs.select{|line| line =~ /\[warn\]: Too many num_threads for secure-forward:/}.size
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-secure-forward
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.2.3
 platform: ruby
 authors:
 - TAGOMORI Satoshi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-10-24 00:00:00.000000000 Z
+date: 2014-10-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd