fluentd 0.12.17 → 0.12.18

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9a8a90a863a94d939296d18ce47d4a37d4568ae6
-  data.tar.gz: 4ccca67d8a418215bca28e7b2297292b6782ca7d
+  metadata.gz: 07240444ef56a97335d1f42c0153a8be4059e77a
+  data.tar.gz: 31cec22258753e9e6e51780b6a8d33e1194879c2
 SHA512:
-  metadata.gz: 95398c478a7b63b0b2f497cd4cdda0507ca9af63b6f55d9f9c76f6efe9c377c08e05864e67d9049cddb787082bc911ca04990ad2044a810ef9ec556de0fcd876
-  data.tar.gz: 58ad404181a58040ede0b7e1fd8bba4311ddae24378b3f777708092bdfe3a32d74b836602c1e003a0eb1a9ef6a98f3e0836a274ced77ccf2cf99da7adc3e62f3
+  metadata.gz: 404559093a12caba1b073e0106287975f9e3dd3ea4ae8be191e95a71abf8d49b13c81c53539830bc2672e0a4741d5088cae800b6906fe3789a7849a8f19997b0
+  data.tar.gz: d8d4411e79d2211526ca7ce69d47263c18b7d8b47606be018a926b2dbf44a25195ffc35fa4aadb23a77067e390d7f4dbe2d395b9d81e912ad25aca085683e63b

data/.travis.yml

@@ -4,7 +4,7 @@ rvm:
   - 1.9.3
   - 2.0.0
   - 2.1
-  - 2.2
+  - 2.2.3
   - ruby-head
   - rbx-2
 
@@ -30,4 +30,3 @@ matrix:
   allow_failures:
     - rvm: ruby-head
     - rvm: rbx-2
-    - rvm: 2.2

data/ChangeLog

@@ -1,5 +1,30 @@
 # v0.12
 
+## Release 0.12.18 - 2015/12/09
+
+### New features / Enhancement
+
+* in_exec: Stop in_exec immediately at shutdown
+  https://github.com/fluent/fluentd/pull/526
+* in_exec: Support built-in text parsers
+  https://github.com/fluent/fluentd/pull/704
+* out_forward: raises ZeroDivisionError when no <server> is available
+  https://github.com/fluent/fluentd/pull/707
+* Add process_name parameter in system to change fluentd's process name
+  https://github.com/fluent/fluentd/pull/708
+* test: Add ParserTestDriver
+  https://github.com/fluent/fluentd/pull/702
+* test: Add FormatterTestDriver
+  https://github.com/fluent/fluentd/pull/712
+* Add parameter description to built-in plugins
+
+### Bug fixes
+
+* fix bug not to protect in-memory-buffer for multi thread emitting / run loop
+  https://github.com/fluent/fluentd/pull/710
+* in_tail: expand_paths exclude unreadable files
+  https://github.com/fluent/fluentd/pull/734
+
 ## Release 0.12.17 - 2015/11/04
 
 ### New features / Enhancement

data/lib/fluent/buffer.rb

@@ -139,7 +139,9 @@ module Fluent
     # This configuration assumes plugins to send records to a remote server.
     # Local file based plugins which should provide more reliability and efficiency
     # should override buffer_chunk_limit with a larger size.
+    desc 'The size of each buffer chunk.'
     config_param :buffer_chunk_limit, :size, :default => 8*1024*1024
+    desc 'The length limit of the chunk queue.'
     config_param :buffer_queue_limit, :integer, :default => 256
 
     alias chunk_limit buffer_chunk_limit

data/lib/fluent/config/configure_proxy.rb

@@ -235,7 +235,7 @@ module Fluent
       end
 
       def dump(level = 0)
-        dumped_config = "\n"
+        dumped_config = ""
         indent = " " * level
         @params.each do |name, config|
           dumped_config << "#{indent}#{name}: #{config[1][:type]}: <#{@defaults[name].inspect}>"
@@ -243,7 +243,7 @@ module Fluent
           dumped_config << "\n"
         end
         @sections.each do |section_name, sub_proxy|
-          dumped_config << "#{indent}#{section_name}#{sub_proxy.dump(level + 1)}"
+          dumped_config << "#{indent}#{section_name}\n#{sub_proxy.dump(level + 1)}"
         end
         dumped_config
       end

data/lib/fluent/configurable.rb

@@ -130,8 +130,8 @@ module Fluent
         configurables.map{ |a| a.configure_proxy(a.name || a.object_id.to_s) }.reduce(:merge)
       end
 
-      def dump
-        configure_proxy_map[self.to_s].dump
+      def dump(level = 0)
+        configure_proxy_map[self.to_s].dump(level)
       end
     end
   end

data/lib/fluent/log.rb

@@ -343,6 +343,7 @@ module Fluent
   module PluginLoggerMixin
     def self.included(klass)
       klass.instance_eval {
+        desc 'Allows the user to set different levels of logging for each plugin.'
         config_param :log_level, :string, :default => nil, :alias => :@log_level
       }
     end

data/lib/fluent/plugin/buf_file.rb

@@ -82,7 +82,9 @@ module Fluent
       @uri_parser = URI::Parser.new
     end
 
+    desc 'The path where buffer chunks are stored.'
     config_param :buffer_path, :string
+    desc 'If true, queued chunks are flushed at shutdown process.'
     config_param :flush_at_shutdown, :bool, :default => false
 
     # 'symlink_path' is currently only for out_file.

data/lib/fluent/plugin/buf_memory.rb

@@ -70,6 +70,7 @@ module Fluent
       super
     end
 
+    desc 'If true, queued chunks are flushed at shutdown process. Otherwise queued chunks are discarded'
     config_param :flush_at_shutdown, :bool, :default => true
     # Overwrite default BasicBuffer#buffer_queue_limit
     # to limit total memory usage upto 512MB.

data/lib/fluent/plugin/exec_util.rb

@@ -16,6 +16,8 @@
 
 module Fluent
   module ExecUtil
+    require 'fluent/parser'
+
     SUPPORTED_FORMAT = {
       'tsv' => :tsv,
       'json' => :json,
@@ -28,6 +30,25 @@ module Fluent
       end
     end
 
+    class TextParserWrapperParser < Parser
+      def initialize(conf, on_message)
+        @parser = Plugin.new_parser(conf['format'])
+        @parser.configure(conf)
+        super(on_message)
+      end
+
+      def call(io)
+        io.each_line(&method(:each_line))
+      end
+
+      def each_line(line)
+        line.chomp!
+        @parser.parse(line) { |time, record|
+          @on_message.call(record, time)
+        }
+      end
+    end
+
     class TSVParser < Parser
       def initialize(keys, on_message)
         @keys = keys

data/lib/fluent/plugin/filter_record_transformer.rb

@@ -25,11 +25,17 @@ module Fluent
       super
     end
 
+    desc 'A comma-delimited list of keys to delete.'
     config_param :remove_keys, :string, :default => nil
+    desc 'A comma-delimited list of keys to keep.'
     config_param :keep_keys, :string, :default => nil
+    desc 'Create new Hash to transform incoming data'
     config_param :renew_record, :bool, :default => false
+    desc 'Specify field name of the record to overwrite the time of events. Its value must be unix time.'
     config_param :renew_time_key, :string, :default => nil
+    desc 'When set to true, the full Ruby syntax is enabled in the ${...} expression.'
     config_param :enable_ruby, :bool, :default => false
+    desc 'Use original value type.'
     config_param :auto_typecast, :bool, :default => false # false for lower version compatibility
 
     def configure(conf)

data/lib/fluent/plugin/filter_stdout.rb

@@ -21,6 +21,7 @@ module Fluent
     # for tests
     attr_reader :formatter
 
+    desc 'The format of the output.'
     config_param :format, :string, :default => 'stdout'
     # config_param :output_type, :string, :default => 'json' (StdoutFormatter defines this)
 

data/lib/fluent/plugin/in_exec.rb

@@ -24,25 +24,23 @@ module Fluent
       require 'fluent/timezone'
     end
 
-    SUPPORTED_FORMAT = {
-      'tsv' => :tsv,
-      'json' => :json,
-      'msgpack' => :msgpack,
-    }
-
+    desc 'The command (program) to execute.'
     config_param :command, :string
-    config_param :format, :default => :tsv do |val|
-      f = SUPPORTED_FORMAT[val]
-      raise ConfigError, "Unsupported format '#{val}'" unless f
-      f
-    end
+    desc 'The format used to map the program output to the incoming event.(tsv,json,msgpack)'
+    config_param :format, :string, :default => 'tsv'
+    desc 'Specify the comma-separated keys when using the tsv format.'
     config_param :keys, :default => [] do |val|
       val.split(',')
     end
+    desc 'Tag of the output events.'
     config_param :tag, :string, :default => nil
+    desc 'The key to use as the event tag instead of the value in the event record. '
     config_param :tag_key, :string, :default => nil
+    desc 'The key to use as the event time instead of the value in the event record.'
     config_param :time_key, :string, :default => nil
+    desc 'The format of the event time used for the time_key parameter.'
     config_param :time_format, :string, :default => nil
+    desc 'The interval time between periodic program runs.'
     config_param :run_interval, :time, :default => nil
 
     def configure(conf)
@@ -72,16 +70,22 @@ module Fluent
         end
       end
 
+      @parser = setup_parser(conf)
+    end
+
+    def setup_parser(conf)
       case @format
-      when :tsv
+      when 'tsv'
         if @keys.empty?
           raise ConfigError, "keys option is required on exec input for tsv format"
         end
-        @parser = ExecUtil::TSVParser.new(@keys, method(:on_message))
-      when :json
-        @parser = ExecUtil::JSONParser.new(method(:on_message))
-      when :msgpack
-        @parser = ExecUtil::MessagePackParser.new(method(:on_message))
+        ExecUtil::TSVParser.new(@keys, method(:on_message))
+      when 'json'
+        ExecUtil::JSONParser.new(method(:on_message))
+      when 'msgpack'
+        ExecUtil::MessagePackParser.new(method(:on_message))
+      else
+        ExecUtil::TextParserWrapperParser.new(conf, method(:on_message))
       end
     end
 
@@ -99,6 +103,8 @@ module Fluent
     def shutdown
       if @run_interval
         @finished = true
+        # call Thread#run which interupts sleep in order to stop run_periodic thread immediately.
+        @thread.run
         @thread.join
       else
         begin
@@ -122,12 +128,13 @@ module Fluent
     end
 
     def run_periodic
+      sleep @run_interval
       until @finished
         begin
-          sleep @run_interval
           io = IO.popen(@command, "r")
           @parser.call(io)
           Process.waitpid(io.pid)
+          sleep @run_interval
         rescue
           log.error "exec failed to run or shutdown child process", :error => $!.to_s, :error_class => $!.class.to_s
           log.warn_backtrace $!.backtrace
@@ -137,17 +144,21 @@ module Fluent
 
     private
 
-    def on_message(record)
+    def on_message(record, parsed_time = nil)
       if val = record.delete(@tag_key)
         tag = val
       else
         tag = @tag
       end
 
-      if val = record.delete(@time_key)
-        time = @time_parse_proc.call(val)
+      if parsed_time
+        time = parsed_time
       else
-        time = Engine.now
+        if val = record.delete(@time_key)
+          time = @time_parse_proc.call(val)
+        else
+          time = Engine.now
+        end
       end
 
       router.emit(tag, time, record)

data/lib/fluent/plugin/in_forward.rb

@@ -23,15 +23,20 @@ module Fluent
       require 'fluent/plugin/socket_util'
     end
 
+    desc 'The port to listen to.'
     config_param :port, :integer, :default => DEFAULT_LISTEN_PORT
+    desc 'The bind address to listen to.'
     config_param :bind, :string, :default => '0.0.0.0'
     config_param :backlog, :integer, :default => nil
     # SO_LINGER 0 to send RST rather than FIN to avoid lots of connections sitting in TIME_WAIT at src
+    desc 'The timeout time used to set linger option.'
     config_param :linger_timeout, :integer, :default => 0
     # This option is for Cool.io's loop wait timeout to avoid loop stuck at shutdown. Almost users don't need to change this value.
     config_param :blocking_timeout, :time, :default => 0.5
 
+    desc 'Log warning if received chunk size is larger than this value.'
     config_param :chunk_size_warn_limit, :size, :default => nil
+    desc 'Received chunk is dropped if it is larger than this value.'
     config_param :chunk_size_limit, :size, :default => nil
 
     def configure(conf)

data/lib/fluent/plugin/in_http.rb

@@ -30,16 +30,25 @@ module Fluent
 
     EMPTY_GIF_IMAGE = "GIF89a\u0001\u0000\u0001\u0000\x80\xFF\u0000\xFF\xFF\xFF\u0000\u0000\u0000,\u0000\u0000\u0000\u0000\u0001\u0000\u0001\u0000\u0000\u0002\u0002D\u0001\u0000;".force_encoding("UTF-8")
 
+    desc 'The port to listen to.'
     config_param :port, :integer, :default => 9880
+    desc 'The bind address to listen to.'
     config_param :bind, :string, :default => '0.0.0.0'
+    desc 'The size limit of the POSTed element. Default is 32MB.'
     config_param :body_size_limit, :size, :default => 32*1024*1024  # TODO default
+    desc 'The timeout limit for keeping the connection alive.'
     config_param :keepalive_timeout, :time, :default => 10   # TODO default
     config_param :backlog, :integer, :default => nil
+    desc 'Add HTTP_ prefix headers to the record.'
     config_param :add_http_headers, :bool, :default => false
+    desc 'Add REMOTE_ADDR header to the record.'
     config_param :add_remote_addr, :bool, :default => false
+    desc 'The format of the HTTP body.'
     config_param :format, :string, :default => 'default'
     config_param :blocking_timeout, :time, :default => 0.5
+    desc 'Set a white list of domains that can do CORS (Cross-Origin Resource Sharing)'
     config_param :cors_allow_origins, :array, :default => nil
+    desc 'Respond with empty gif image of 1x1 pixel.'
     config_param :respond_with_empty_img, :bool, :default => false
 
     def configure(conf)

data/lib/fluent/plugin/in_stream.rb

@@ -164,7 +164,9 @@ module Fluent
   class UnixInput < StreamInput
     Plugin.register_input('unix', self)
 
+    desc 'The path to your Unix Domain Socket.'
     config_param :path, :string, :default => DEFAULT_SOCKET_PATH
+    desc 'The backlog of Unix Domain Socket.'
     config_param :backlog, :integer, :default => nil
 
     def configure(conf)

data/lib/fluent/plugin/in_syslog.rb

@@ -64,9 +64,13 @@ module Fluent
       require 'fluent/plugin/socket_util'
     end
 
+    desc 'The port to listen to.'
     config_param :port, :integer, :default => 5140
+    desc 'The bind address to listen to.'
     config_param :bind, :string, :default => '0.0.0.0'
+    desc 'The prefix of the tag. The tag itself is generated by the tag prefix, facility level, and priority.'
     config_param :tag, :string
+    desc 'The transport protocol used to receive logs.(udp, tcp)'
     config_param :protocol_type, :default => :udp do |val|
       case val.downcase
       when 'tcp'
@@ -77,7 +81,9 @@ module Fluent
         raise ConfigError, "syslog input protocol type should be 'tcp' or 'udp'"
       end
     end
+    desc 'If true, add source host to event record.'
     config_param :include_source_host, :bool, :default => false
+    desc 'Specify key of source host when include_source_host is true.'
     config_param :source_host_key, :string, :default => 'source_host'.freeze
     config_param :blocking_timeout, :time, :default => 0.5
 

data/lib/fluent/plugin/in_tail.rb

@@ -24,13 +24,21 @@ module Fluent
       @tails = {}
     end
 
+    desc 'The paths to read. Multiple paths can be specified, separated by comma.'
     config_param :path, :string
+    desc 'The tag of the event.'
     config_param :tag, :string
+    desc 'The paths to exclude the files from watcher list.'
     config_param :exclude_path, :array, :default => []
+    desc 'Specify interval to keep reference to old file when rotate a file.'
     config_param :rotate_wait, :time, :default => 5
+    desc 'Fluentd will record the position it last read into this file.'
     config_param :pos_file, :string, :default => nil
+    desc 'Start to read the logs from the head of file, not bottom.'
     config_param :read_from_head, :bool, :default => false
+    desc 'The interval of refreshing the list of watch file.'
     config_param :refresh_interval, :time, :default => 60
+    desc 'The number of reading lines at each IO.'
     config_param :read_lines_limit, :integer, :default => 1000
 
     attr_reader :paths
@@ -106,7 +114,14 @@ module Fluent
       @paths.each { |path|
         path = date.strftime(path)
         if path.include?('*')
-          paths += Dir.glob(path)
+          paths += Dir.glob(path).select { |p|
+            if File.readable?(p)
+              true
+            else
+              log.warn "#{p} unreadable. It is excluded and would be examined next time."
+              false
+            end
+          }
         else
           # When file is not created yet, Dir.glob returns an empty array. So just add when path is static.
           paths << path

data/lib/fluent/plugin/in_tcp.rb

@@ -21,6 +21,7 @@ module Fluent
     Plugin.register_input('tcp', self)
 
     config_set_default :port, 5170
+    desc 'The payload is read up to this character.'
     config_param :delimiter, :string, :default => "\n" # syslog family add "\n" to each message and this seems only way to split messages in tcp stream
 
     def listen(callback)

data/lib/fluent/plugin/out_copy.rb

@@ -18,6 +18,7 @@ module Fluent
   class CopyOutput < MultiOutput
     Plugin.register_output('copy', self)
 
+    desc 'If true, pass different record to each `store` plugin.'
     config_param :deep_copy, :bool, :default => false
 
     def initialize

data/lib/fluent/plugin/out_exec.rb

@@ -26,13 +26,19 @@ module Fluent
       @localtime = false
     end
 
+    desc 'The command (program) to execute. The exec plugin passes the path of a TSV file as the last argumen'
     config_param :command, :string
+    desc 'Specify the comma-separated keys when using the tsv format.'
     config_param :keys, :default => [] do |val|
       val.split(',')
     end
+    desc 'The name of the key to use as the event tag. This replaces the value in the event record.'
     config_param :tag_key, :string, :default => nil
+    desc 'The name of the key to use as the event time. This replaces the the value in the event record.'
     config_param :time_key, :string, :default => nil
+    desc 'The format for event time used when the time_key parameter is specified. The default is UNIX time (integer).'
     config_param :time_format, :string, :default => nil
+    desc "The format used to map the incoming events to the program input. (#{ExecUtil::SUPPORTED_FORMAT.keys.join(',')})"
     config_param :format, :default => :tsv do |val|
       f = ExecUtil::SUPPORTED_FORMAT[val]
       raise ConfigError, "Unsupported format '#{val}'" unless f