fluentd 0.12.17 → 0.12.18

data/lib/fluent/plugin/out_exec_filter.rb

@@ -25,33 +25,44 @@ module Fluent
       require 'fluent/timezone'
     end
 
+    desc 'The command (program) to execute.'
     config_param :command, :string
 
     config_param :remove_prefix, :string, :default => nil
     config_param :add_prefix, :string, :default => nil
 
+    desc "The format used to map the incoming event to the program input.(#{ExecUtil::SUPPORTED_FORMAT.keys.join(',')})"
     config_param :in_format, :default => :tsv do |val|
       f = ExecUtil::SUPPORTED_FORMAT[val]
       raise ConfigError, "Unsupported in_format '#{val}'" unless f
       f
     end
+    desc 'Specify comma-separated values for tsv format.'
     config_param :in_keys, :default => [] do |val|
       val.split(',')
     end
+    desc 'The name of the key to use as the event tag.'
     config_param :in_tag_key, :default => nil
+    desc 'The name of the key to use as the event time.'
     config_param :in_time_key, :default => nil
+    desc 'The format for event time used when the in_time_key parameter is specified.(Defauls is UNIX time)'
     config_param :in_time_format, :default => nil
 
+    desc "The format used to process the program output.(#{ExecUtil::SUPPORTED_FORMAT.keys.join(',')})"
     config_param :out_format, :default => :tsv do |val|
       f = ExecUtil::SUPPORTED_FORMAT[val]
       raise ConfigError, "Unsupported out_format '#{val}'" unless f
       f
     end
+    desc 'Specify comma-separated values for tsv format.'
     config_param :out_keys, :default => [] do |val|  # for tsv format
       val.split(',')
     end
+    desc 'The name of the key to use as the event tag.'
     config_param :out_tag_key, :default => nil
+    desc 'The name of the key to use as the event time.'
     config_param :out_time_key, :default => nil
+    desc 'The format for event time used when the in_time_key parameter is specified.(Defauls is UNIX time)'
     config_param :out_time_format, :default => nil
 
     config_param :tag, :string, :default => nil
@@ -59,10 +70,14 @@ module Fluent
     config_param :time_key, :string, :default => nil
     config_param :time_format, :string, :default => nil
 
+    desc 'If true, use localtime with in_time_format.'
     config_param :localtime, :bool, :default => true
+    desc 'If true, use timezone with in_time_format.'
     config_param :timezone, :string, :default => nil
+    desc 'The number of spawned process for command.'
     config_param :num_children, :integer, :default => 1
 
+    desc 'Respawn command when command exit.'
     # nil, 'none' or 0: no respawn, 'inf' or -1: infinite times, positive integer: try to respawn specified times only
     config_param :child_respawn, :string, :default => nil
 

data/lib/fluent/plugin/out_file.rb

@@ -23,9 +23,13 @@ module Fluent
       'gzip' => :gz,
     }
 
+    desc "The Path of the file."
     config_param :path, :string
+    desc "The format of the file content. The default is out_file."
     config_param :format, :string, :default => 'out_file'
+    desc "The flushed chunk is appended to existence file or not."
     config_param :append, :bool, :default => false
+    desc "Compress flushed file."
     config_param :compress, :default => nil do |val|
       c = SUPPORTED_COMPRESS[val]
       unless c
@@ -33,6 +37,7 @@ module Fluent
       end
       c
     end
+    desc "Create symlink to temporary buffered file when buffer_type is file."
     config_param :symlink_path, :string, :default => nil
 
     def initialize

data/lib/fluent/plugin/out_forward.rb

@@ -36,7 +36,9 @@ module Fluent
       @nodes = []  #=> [Node]
     end
 
+    desc 'The timeout time when sending event logs.'
     config_param :send_timeout, :time, :default => 60
+    desc 'The transport protocol to use for heartbeats.(udp,tcp,none)'
     config_param :heartbeat_type, :default => :udp do |val|
       case val.downcase
       when 'tcp'
@@ -49,19 +51,28 @@ module Fluent
         raise ConfigError, "forward output heartbeat type should be 'tcp', 'udp', or 'none'"
       end
     end
+    desc 'The interval of the heartbeat packer.'
     config_param :heartbeat_interval, :time, :default => 1
+    desc 'The wait time before accepting a server fault recovery.'
     config_param :recover_wait, :time, :default => 10
+    desc 'The hard timeout used to detect server failure.'
     config_param :hard_timeout, :time, :default => 60
+    desc 'Set TTL to expire DNS cache in seconds.'
     config_param :expire_dns_cache, :time, :default => nil  # 0 means disable cache
+    desc 'The threshold parameter used to detect server faults.'
     config_param :phi_threshold, :integer, :default => 16
+    desc 'Use the "Phi accrual failure detector" to detect server failure.'
     config_param :phi_failure_detector, :bool, :default => true
 
     # if any options added that requires extended forward api, fix @extend_internal_protocol
 
+    desc 'Change the protocol to at-least-once.'
     config_param :require_ack_response, :bool, :default => false  # require in_forward to respond with ack
+    desc 'This option is used when require_ack_response is true.'
     config_param :ack_response_timeout, :time, :default => 190  # 0 means do not wait for ack responses
     # Linux default tcp_syn_retries is 5 (in many environment)
     # 3 + 6 + 12 + 24 + 48 + 96 -> 189 (sec)
+    desc 'Enable client-side DNS round robin.'
     config_param :dns_round_robin, :bool, :default => false # heartbeat_type 'udp' is not available for this
 
     attr_reader :nodes
@@ -129,6 +140,10 @@ module Fluent
         end
         log.info "adding forwarding server '#{name}'", :host=>host, :port=>port, :weight=>weight, :plugin_id=>plugin_id
       }
+
+      if @nodes.empty?
+        raise ConfigError, "forward output plugin requires at least one <server> is required"
+      end
     end
 
     def start

data/lib/fluent/plugin/out_stdout.rb

@@ -18,6 +18,7 @@ module Fluent
   class StdoutOutput < Output
     Plugin.register_output('stdout', self)
 
+    desc 'Output format.(json,hash)'
     config_param :output_type, :default => 'json'
 
     def configure(conf)

data/lib/fluent/plugin/socket_util.rb

@@ -94,10 +94,15 @@ module Fluent
         require 'fluent/parser'
       end
 
+      desc 'Tag of output events.'
       config_param :tag, :string
+      desc 'The format of the payload.'
       config_param :format, :string
+      desc 'The port to listen to.'
       config_param :port, :integer, :default => 5150
+      desc 'The bind address to listen to.'
       config_param :bind, :string, :default => '0.0.0.0'
+      desc "The field name of the client's hostname."
       config_param :source_host_key, :string, :default => nil
       config_param :blocking_timeout, :time, :default => 0.5
 

data/lib/fluent/process.rb

@@ -232,26 +232,36 @@ module Fluent
         @w = w
         @interval = interval
         @buffer = {}
+        @mutex = Mutex.new
         Thread.new(&method(:run))
       end
 
       def emit(tag, es)
-        if ms = @buffer[tag]
-          ms << es.to_msgpack_stream
-        else
-          @buffer[tag] = es.to_msgpack_stream
+        stream = es.to_msgpack_stream
+        @mutex.synchronize do
+          if @buffer[tag]
+            @buffer[tag] << stream
+          else
+            @buffer[tag] = stream
+          end
         end
       end
 
       def run
         while true
           sleep @interval
-          @buffer.keys.each {|tag|
-            if ms = @buffer.delete(tag)
-              [tag, ms].to_msgpack(@w)
-              #@w.write [tag, ms].to_msgpack
+
+          pairs = []
+          @mutex.synchronize do
+            @buffer.keys.each do |tag|
+              if ms = @buffer.delete(tag)
+                pairs << [tag, ms]
+              end
             end
-          }
+          end
+          pairs.each do |pair|
+            pair.to_msgpack(@w)
+          end
         end
       rescue
         $log.error "error on forwerder thread", :error=>$!.to_s

data/lib/fluent/supervisor.rb

@@ -111,6 +111,7 @@ module Fluent
       @chgroup = opt[:chgroup]
       @chuser = opt[:chuser]
       @rpc_server = nil
+      @process_name = nil
 
       @log_level = opt[:log_level]
       @suppress_interval = opt[:suppress_interval]
@@ -193,7 +194,19 @@ module Fluent
       $log.info "Show config for #{@show_plugin_config}"
       name, type = @show_plugin_config.split(":")
       plugin = Plugin.__send__("new_#{name}", type)
-      $log.info plugin.class.dump
+      dumped_config = "\n"
+      level = 0
+      plugin.class.ancestors.reverse_each do |plugin_class|
+        if plugin_class.respond_to?(:dump)
+          $log.on_debug do
+            dumped_config << plugin_class.name
+            dumped_config << "\n"
+            level = 1
+          end
+          dumped_config << plugin_class.dump(level)
+        end
+      end
+      $log.info dumped_config
       exit 0
     rescue => e
       $log.error "show config failed: #{e}"
@@ -306,6 +319,7 @@ module Fluent
     def supervise(&block)
       start_time = Time.now
 
+      Process.setproctitle("supervisor:#{@process_name}") if @process_name
       $log.info "starting fluentd-#{Fluent::VERSION}"
       @main_pid = fork do
         main_process(&block)
@@ -332,6 +346,8 @@ module Fluent
     end
 
     def main_process(&block)
+      Process.setproctitle("worker:#{@process_name}") if @process_name
+
       begin
         block.call
 
@@ -462,6 +478,7 @@ module Fluent
       config_param :without_source, :bool, :default => nil
       config_param :rpc_endpoint, :string, :default => nil
       config_param :enable_get_dump, :bool, :default => nil
+      config_param :process_name, :default => nil
 
       def initialize(conf)
         super()
@@ -478,6 +495,7 @@ module Fluent
           @without_source = system.without_source unless system.without_source.nil?
           @rpc_endpoint = system.rpc_endpoint unless system.rpc_endpoint.nil?
           @enable_get_dump = system.enable_get_dump unless system.enable_get_dump.nil?
+          @process_name = system.process_name unless system.process_name.nil?
         }
       end
     end

data/lib/fluent/test.rb

@@ -20,5 +20,7 @@ require 'fluent/test/base'
 require 'fluent/test/input_test'
 require 'fluent/test/output_test'
 require 'fluent/test/filter_test'
+require 'fluent/test/parser_test'
+require 'fluent/test/formatter_test'
 
 $log ||= Fluent::Log.new(Fluent::Test::DummyLogDevice.new)

data/lib/fluent/test/formatter_test.rb

@@ -0,0 +1,60 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module Fluent
+  module Test
+    class FormatterTestDriver
+      def initialize(klass_or_str, proc=nil, &block)
+        if klass_or_str.is_a?(Class)
+          if block
+            # Create new class for test w/ overwritten methods
+            #   klass.dup is worse because its ancestors does NOT include original class name
+            klass_or_str = Class.new(klass_or_str)
+            klass_or_str.module_eval(&block)
+          end
+          @instance = klass_or_str.new
+        elsif klass_or_str.is_a?(String)
+          @instance = TextFormatter::TEMPLATE_REGISTRY.lookup(klass_or_str).call
+        else
+          @instance = klass_or_str
+        end
+        @config = Config.new
+      end
+
+      attr_reader :instance, :config
+
+      def configure(conf)
+        case conf
+        when Fluent::Config::Element
+          @config = conf
+        when String
+          io = StringIO.new(conf)
+          @config = Config::Parser.parse(io, 'fluent.conf')
+        when Hash
+          @config = Config::Element.new('ROOT', '', conf, [])
+        else
+          raise "Unknown type... #{conf}"
+        end
+        @instance.configure(@config)
+        self
+      end
+
+      def format(tag, time, record)
+        @instance.format(tag, time, record)
+      end
+    end
+  end
+end

data/lib/fluent/test/parser_test.rb

@@ -0,0 +1,66 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module Fluent
+  module Test
+    class ParserTestDriver
+      def initialize(klass_or_str, format=nil, conf={}, &block)
+        if klass_or_str.is_a?(Class)
+          if block
+            # Create new class for test w/ overwritten methods
+            #   klass.dup is worse because its ancestors does NOT include original class name
+            klass_or_str = Class.new(klass_or_str)
+            klass_or_str.module_eval(&block)
+          end
+          case klass_or_str.instance_method(:initialize).arity
+          when 0
+            @instance = klass_or_str.new
+          when -2
+            # for RegexpParser
+            @instance = klass_or_str.new(format, conf)
+          end
+        elsif klass_or_str.is_a?(String)
+          @instance = TextParser::TEMPLATE_REGISTRY.lookup(klass_or_str).call
+        else
+          @instance = klass_or_str
+        end
+        @config = Config.new
+      end
+
+      attr_reader :instance, :config
+
+      def configure(conf)
+        case conf
+        when Fluent::Config::Element
+          @config = conf
+        when String
+          io = StringIO.new(conf)
+          @config = Config::Parser.parse(io, 'fluent.conf')
+        when Hash
+          @config = Config::Element.new('ROOT', '', conf, [])
+        else
+          raise "Unknown type... #{conf}"
+        end
+        @instance.configure(@config)
+        self
+      end
+
+      def parse(text, &block)
+        @instance.parse(text, &block)
+      end
+    end
+  end
+end

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '0.12.17'
+  VERSION = '0.12.18'
 
 end

data/test/config/test_configure_proxy.rb

@@ -138,13 +138,12 @@ module Fluent::Config
         end
 
         test 'empty proxy' do
-          assert_equal("\n", @proxy.dump)
+          assert_equal("", @proxy.dump)
         end
 
         test 'plain proxy w/o default value' do
           @proxy.config_param(:name, :string)
           assert_equal(<<CONFIG, @proxy.dump)
-
 name: string: <nil>
 CONFIG
         end
@@ -152,7 +151,6 @@ CONFIG
         test 'plain proxy w/ default value' do
           @proxy.config_param(:name, :string, default: "name1")
           assert_equal(<<CONFIG, @proxy.dump)
-
 name: string: <"name1">
 CONFIG
         end
@@ -161,7 +159,6 @@ CONFIG
           @proxy.config_param(:name, :string)
           @proxy.config_set_default(:name, "name1")
           assert_equal(<<CONFIG, @proxy.dump)
-
 name: string: <"name1">
 CONFIG
         end
@@ -171,7 +168,6 @@ CONFIG
             config_param(:name, :string, default: "name1")
           end
           assert_equal(<<CONFIG, @proxy.dump)
-
 sub
  name: string: <"name1">
 CONFIG
@@ -187,7 +183,6 @@ CONFIG
             end
           end
           assert_equal(<<CONFIG, @proxy.dump)
-
 sub
  name1: string: <"name1">
  name2: string: <"name2">
@@ -201,7 +196,6 @@ CONFIG
           test 'single proxy' do
             @proxy.config_param(:name, :string, desc: "description for name")
           assert_equal(<<CONFIG, @proxy.dump)
-
 name: string: <nil> # description for name
 CONFIG
           end
@@ -210,7 +204,6 @@ CONFIG
             @proxy.config_param(:name, :string)
             @proxy.config_set_desc(:name, "description for name")
           assert_equal(<<CONFIG, @proxy.dump)
-
 name: string: <nil> # description for name
 CONFIG
           end
@@ -225,7 +218,6 @@ CONFIG
               end
             end
             assert_equal(<<CONFIG, @proxy.dump)
-
 sub
  name1: string: <"name1"> # desc1
  name2: string: <"name2"> # desc2
@@ -247,7 +239,6 @@ CONFIG
               end
             end
             assert_equal(<<CONFIG, @proxy.dump)
-
 sub
  name1: string: <"name1"> # desc1
  name2: string: <"name2"> # desc2