fluentd 0.14.20.rc1 → 0.14.20

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 27bc8dbc0114bfd96c3f6eea899abb46036b00b3
-  data.tar.gz: 731f9f97662048edae1476538510747a6279b52d
+  metadata.gz: 419af849ffe16bdb83d13106862b94018d71dca1
+  data.tar.gz: 6f1435c4d5f9f1645267bb88f43aa86ee838075e
 SHA512:
-  metadata.gz: 7cb206015a95afcfa03a96c455e9f9631e3b2357f63d76d67dd198a9a435d7d6f91bf1dc9f81690b66d885bc458dfcbc2611a139ef46548841d081488e818f05
-  data.tar.gz: 33c0185e918d5b78810a86226d9f17bead748ca4f24574f3b633143bbbf1d0d30cab99d8a6e950084ff77443899f145d0440562ec8b442c6c9e0d54ca8a8050c
+  metadata.gz: 96ff26acaccd6870648fd36ca991e101058296abfa8585366c0b132d94a19c79ef75214527c026b3719061c943d0d0aeaec45c3bb58655b3f08f7da88d5a6aa6
+  data.tar.gz: a83370feeeabb3f9a3200f5575ce8ba9f4fe0a57349b9fe73dfd3f6b386492993483412883f91214b8903709a86dcacccac658cf663d1734aa37d5b1e4f5fa21

data/ChangeLog

@@ -1,5 +1,25 @@
 # v0.14
 
+## Release v0.14.20 - 2017/07/31
+
+### New features / Enhancements
+
+* plugin: Add record_accessor plugin helper
+  https://github.com/fluent/fluentd/pull/1637
+* log: Add format and time_format parameters to <system> setting
+  https://github.com/fluent/fluentd/pull/1644
+
+### Bug fixes
+
+* buf_file: Improve file handling to mitigate broken meta file
+  https://github.com/fluent/fluentd/pull/1628
+* in_syslog: Fix the description of resolve_hostname parameter
+  https://github.com/fluent/fluentd/pull/1633
+* process: Fix signal handling. Send signal to all workers
+  https://github.com/fluent/fluentd/pull/1642
+* output: Fix error message typo
+  https://github.com/fluent/fluentd/pull/1643
+
 ## Release v0.14.19 - 2017/07/12
 
 ### New features / Enhancements

data/fluentd.gemspec

@@ -28,6 +28,7 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency("tzinfo", ["~> 1.0"])
   gem.add_runtime_dependency("tzinfo-data", ["~> 1.0"])
   gem.add_runtime_dependency("strptime", ["~> 0.1.7"])
+  gem.add_runtime_dependency("ruby_dig", ["~> 0.0.2"])
 
   # build gem for a certain platform. see also Rakefile
   fake_platform = ENV['GEM_BUILD_FAKE_PLATFORM'].to_s

data/lib/fluent/log.rb

@@ -96,8 +96,12 @@ module Fluent
       @level = logger.level + 1
       @debug_mode = false
       @log_event_enabled = false
-      @time_format = '%Y-%m-%d %H:%M:%S %z '
       @depth_offset = 1
+      @format = nil
+      @time_format = nil
+      @formatter = nil
+
+      self.format = :text
       enable_color out.tty?
       # TODO: This variable name is unclear so we should change to better name.
       @threads_exclude_events = []
@@ -136,12 +140,14 @@ module Fluent
       dl_opts[:log_level] = @level - 1
       logger = ServerEngine::DaemonLogger.new(@out, dl_opts)
       clone = self.class.new(logger, suppress_repeated_stacktrace: @suppress_repeated_stacktrace, process_type: @process_type, worker_id: @worker_id)
+      clone.format = @format
       clone.time_format = @time_format
       clone.log_event_enabled = @log_event_enabled
       # optional headers/attrs are not copied, because new PluginLogger should have another one of it
       clone
     end
 
+    attr_reader :format
     attr_accessor :log_event_enabled
     attr_accessor :out
     attr_accessor :level
@@ -154,6 +160,37 @@ module Fluent
       nil
     end
 
+    def format=(fmt)
+      return if @format == fmt
+
+      case fmt
+      when :text
+        @format = :text
+        @time_format = '%Y-%m-%d %H:%M:%S %z'
+        @formatter = Proc.new { |type, time, level, msg|
+          r = caller_line(type, time, @depth_offset, level)
+          r << msg
+          r
+        }
+      when :json
+        @format = :json
+        @time_format = '%Y-%m-%d %H:%M:%S %z'
+        @formatter = Proc.new { |type, time, level, msg|
+          r = {
+            'time' => time.strftime(@time_format),
+            'level' => LEVEL_TEXT[level],
+            'message' => msg
+          }
+          if wid = get_worker_id(type)
+            r['worker_id'] = wid
+          end
+          Yajl.dump(r)
+        }
+      end
+
+      nil
+    end
+
     def reopen!
       # do nothing in @logger.reopen! because it's already reopened in Supervisor.load_config
       @logger.reopen! if @logger
@@ -235,7 +272,7 @@ module Fluent
       return if skipped_type?(type)
       args << block.call if block
       time, msg = event(:trace, args)
-      puts [@color_trace, caller_line(type, time, @depth_offset, LEVEL_TRACE), msg, @color_reset].join
+      puts [@color_trace, @formatter.call(type, time, LEVEL_TRACE, msg), @color_reset].join
     rescue
       # logger should not raise an exception. This rescue prevents unexpected behaviour.
     end
@@ -256,7 +293,7 @@ module Fluent
       return if skipped_type?(type)
       args << block.call if block
       time, msg = event(:debug, args)
-      puts [@color_debug, caller_line(type, time, @depth_offset, LEVEL_DEBUG), msg, @color_reset].join
+      puts [@color_debug, @formatter.call(type, time, LEVEL_DEBUG, msg), @color_reset].join
     rescue
     end
     alias DEBUG debug
@@ -276,7 +313,7 @@ module Fluent
       return if skipped_type?(type)
       args << block.call if block
       time, msg = event(:info, args)
-      puts [@color_info, caller_line(type, time, @depth_offset, LEVEL_INFO), msg, @color_reset].join
+      puts [@color_info, @formatter.call(type, time, LEVEL_INFO, msg), @color_reset].join
     rescue
     end
     alias INFO info
@@ -296,7 +333,7 @@ module Fluent
       return if skipped_type?(type)
       args << block.call if block
       time, msg = event(:warn, args)
-      puts [@color_warn, caller_line(type, time, @depth_offset, LEVEL_WARN), msg, @color_reset].join
+      puts [@color_warn, @formatter.call(type, time, LEVEL_WARN, msg), @color_reset].join
     rescue
     end
     alias WARN warn
@@ -316,7 +353,7 @@ module Fluent
       return if skipped_type?(type)
       args << block.call if block
       time, msg = event(:error, args)
-      puts [@color_error, caller_line(type, time, @depth_offset, LEVEL_ERROR), msg, @color_reset].join
+      puts [@color_error, @formatter.call(type, time, LEVEL_ERROR, msg), @color_reset].join
     rescue
     end
     alias ERROR error
@@ -336,7 +373,7 @@ module Fluent
       return if skipped_type?(type)
       args << block.call if block
       time, msg = event(:fatal, args)
-      puts [@color_fatal, caller_line(type, time, @depth_offset, LEVEL_FATAL), msg, @color_reset].join
+      puts [@color_fatal, @formatter.call(type, time, LEVEL_FATAL, msg), @color_reset].join
     rescue
     end
     alias FATAL fatal
@@ -373,19 +410,47 @@ module Fluent
       return if @level > level
 
       time = Time.now
-      line = caller_line(type, time, 5, level)
-      if @suppress_repeated_stacktrace && (Thread.current[:last_repeated_stacktrace] == backtrace)
-        puts ["  ", line, 'suppressed same stacktrace'].join
+
+      if @format == :text
+        line = caller_line(type, time, 5, level)
+        if @suppress_repeated_stacktrace && (Thread.current[:last_repeated_stacktrace] == backtrace)
+          puts ["  ", line, 'suppressed same stacktrace'].join
+        else
+          backtrace.each { |msg|
+            puts ["  ", line, msg].join
+          }
+          Thread.current[:last_repeated_stacktrace] = backtrace if @suppress_repeated_stacktrace
+        end
       else
-        backtrace.each { |msg|
-          puts ["  ", line, msg].join
+        r = {
+          'time' => time.strftime(@time_format),
+          'level' => LEVEL_TEXT[level],
         }
-        Thread.current[:last_repeated_stacktrace] = backtrace if @suppress_repeated_stacktrace
+        if wid = get_worker_id(type)
+          r['worker_id'] = wid
+        end
+
+        if @suppress_repeated_stacktrace && (Thread.current[:last_repeated_stacktrace] == backtrace)
+          r['message'] = 'suppressed same stacktrace'
+        else
+          r['message'] = backtrace.join("\n")
+          Thread.current[:last_repeated_stacktrace] = backtrace if @suppress_repeated_stacktrace
+        end
+
+        puts Yajl.dump(r)
       end
 
       nil
     end
 
+    def get_worker_id(type)
+      if type == :default && (@process_type == :worker0 || @process_type == :workers)
+        @worker_id
+      else
+        nil
+      end
+    end
+
     def event(level, args)
       time = Time.now
       message = @optional_header ? @optional_header.dup : ''
@@ -426,7 +491,7 @@ module Fluent
                        else
                          "".freeze
                        end
-      log_msg = "#{time.strftime(@time_format)}[#{LEVEL_TEXT[level]}]: #{worker_id_part}"
+      log_msg = "#{time.strftime(@time_format)} [#{LEVEL_TEXT[level]}]: #{worker_id_part}"
       if @debug_mode
         line = caller(depth+1)[0]
         if match = /^(.+?):(\d+)(?::in `(.*)')?/.match(line)
@@ -450,11 +515,13 @@ module Fluent
     def initialize(logger)
       @logger = logger
       @level = @logger.level
+      @format = nil
       @depth_offset = 2
       if logger.instance_variable_defined?(:@suppress_repeated_stacktrace)
         @suppress_repeated_stacktrace = logger.instance_variable_get(:@suppress_repeated_stacktrace)
       end
 
+      self.format = @logger.format
       enable_color @logger.enable_color?
     end
 
@@ -462,15 +529,21 @@ module Fluent
       @level = Log.str_to_level(log_level_str)
     end
 
+    alias orig_format= format=
     alias orig_enable_color enable_color
 
+    def format=(fmt)
+      self.orig_format = fmt
+      @logger.format = fmt
+    end
+
     def enable_color(b = true)
       orig_enable_color b
       @logger.enable_color b
     end
 
     extend Forwardable
-    def_delegators '@logger', :enable_color?, :enable_debug, :enable_event,
+    def_delegators '@logger', :get_worker_id, :enable_color?, :enable_debug, :enable_event,
       :disable_events, :log_event_enabled, :log_event_enamed=, :time_format, :time_format=,
       :event, :caller_line, :puts, :write, :<<, :flush, :reset, :out, :out=,
       :optional_header, :optional_header=, :optional_attrs, :optional_attrs=

data/lib/fluent/plugin/filter_grep.rb

@@ -22,6 +22,15 @@ module Fluent::Plugin
   class GrepFilter < Filter
     Fluent::Plugin.register_filter('grep', self)
 
+    def initialize
+      super
+
+      @_regexps = {}
+      @_excludes = {}
+    end
+
+    helpers :record_accessor
+
     REGEXP_MAX_NUM = 20
 
     (1..REGEXP_MAX_NUM).each {|i| config_param :"regexp#{i}",  :string, default: nil, deprecated: "Use <regexp> section" }
@@ -52,31 +61,38 @@ module Fluent::Plugin
     def configure(conf)
       super
 
-      @_regexps = {}
+      rs = {}
       (1..REGEXP_MAX_NUM).each do |i|
         next unless conf["regexp#{i}"]
         key, regexp = conf["regexp#{i}"].split(/ /, 2)
         raise Fluent::ConfigError, "regexp#{i} does not contain 2 parameters" unless regexp
-        raise Fluent::ConfigError, "regexp#{i} contains a duplicated key, #{key}" if @_regexps[key]
-        @_regexps[key] = Regexp.compile(regexp)
+        raise Fluent::ConfigError, "regexp#{i} contains a duplicated key, #{key}" if rs[key]
+        rs[key] = Regexp.compile(regexp)
       end
 
-      @_excludes = {}
+      es = {}
       (1..REGEXP_MAX_NUM).each do |i|
         next unless conf["exclude#{i}"]
         key, exclude = conf["exclude#{i}"].split(/ /, 2)
         raise Fluent::ConfigError, "exclude#{i} does not contain 2 parameters" unless exclude
-        raise Fluent::ConfigError, "exclude#{i} contains a duplicated key, #{key}" if @_excludes[key]
-        @_excludes[key] = Regexp.compile(exclude)
+        raise Fluent::ConfigError, "exclude#{i} contains a duplicated key, #{key}" if es[key]
+        es[key] = Regexp.compile(exclude)
       end
 
       @regexps.each do |e|
-        raise Fluent::ConfigError, "Duplicate key: #{e.key}" if @_regexps.key?(e.key)
-        @_regexps[e.key] = e.pattern
+        raise Fluent::ConfigError, "Duplicate key: #{e.key}" if rs.key?(e.key)
+        rs[e.key] = e.pattern
       end
       @excludes.each do |e|
-        raise Fluent::ConfigError, "Duplicate key: #{e.key}" if @_excludes.key?(e.key)
-        @_excludes[e.key] = e.pattern
+        raise Fluent::ConfigError, "Duplicate key: #{e.key}" if es.key?(e.key)
+        es[e.key] = e.pattern
+      end
+
+      rs.each_pair do |k, v|
+        @_regexps[record_accessor_create(k)] = v
+      end
+      es.each_pair do |k, v|
+        @_excludes[record_accessor_create(k)] = v
       end
     end
 
@@ -85,10 +101,10 @@ module Fluent::Plugin
       begin
         catch(:break_loop) do
           @_regexps.each do |key, regexp|
-            throw :break_loop unless ::Fluent::StringUtil.match_regexp(regexp, record[key].to_s)
+            throw :break_loop unless ::Fluent::StringUtil.match_regexp(regexp, key.call(record).to_s)
           end
           @_excludes.each do |key, exclude|
-            throw :break_loop if ::Fluent::StringUtil.match_regexp(exclude, record[key].to_s)
+            throw :break_loop if ::Fluent::StringUtil.match_regexp(exclude, key.call(record).to_s)
           end
           result = record
         end

data/lib/fluent/plugin/in_syslog.rb

@@ -83,8 +83,8 @@ module Fluent::Plugin
 
     desc 'The field name of hostname of sender.'
     config_param :source_hostname_key, :string, default: nil
+    desc 'Try to resolve hostname from IP addresses or not.'
     config_param :resolve_hostname, :bool, default: nil
-    desc 'Connections will be disconnected right after receiving first message if this value is true.'
     desc 'The field name of source address of sender.'
     config_param :source_address_key, :string, default: nil
     desc 'The field name of the priority.'

data/lib/fluent/plugin/output.rb

@@ -700,7 +700,7 @@ module Fluent
             rvalue = rvalue.gsub(CHUNK_KEY_PLACEHOLDER_PATTERN, hash)
           end
           if rvalue =~ CHUNK_KEY_PLACEHOLDER_PATTERN
-            log.warn "chunk key placeholder '#{$1}' not replaced. templace:#{str}"
+            log.warn "chunk key placeholder '#{$1}' not replaced. template:#{str}"
           end
           rvalue
         end

data/lib/fluent/plugin_helper.rb

@@ -27,6 +27,7 @@ require 'fluent/plugin_helper/extract'
 require 'fluent/plugin_helper/socket'
 require 'fluent/plugin_helper/server'
 require 'fluent/plugin_helper/retry_state'
+require 'fluent/plugin_helper/record_accessor'
 require 'fluent/plugin_helper/compat_parameters'
 
 module Fluent

data/lib/fluent/plugin_helper/record_accessor.rb

@@ -0,0 +1,172 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/config/error'
+unless {}.respond_to?(:dig)
+  begin
+    # backport_dig is faster than ruby_dig so prefer backport_dig.
+    require 'backport_dig'
+  rescue LoadError
+    require 'ruby_dig'
+  end
+end
+
+module Fluent
+  module PluginHelper
+    module RecordAccessor
+      def record_accessor_create(param)
+        Accessor.new(param)
+      end
+
+      class Accessor
+        def initialize(param)
+          @keys = Accessor.parse_parameter(param)
+
+          # Call [] for single key to reduce dig overhead
+          m = method(@keys.is_a?(Array) ? :call_dig : :call_index)
+          (class << self; self; end).module_eval do
+            define_method(:call, m)
+          end
+        end
+
+        def call(r)
+        end
+
+        # To optimize the performance, use class_eval with pre-expanding @keys
+        # See https://gist.github.com/repeatedly/ab553ed260cd080bd01ec71da9427312
+        def call_dig(r)
+          r.dig(*@keys)
+        end
+
+        def call_index(r)
+          r[@keys]
+        end
+
+        def self.parse_parameter(param)
+          if param.start_with?('$.')
+            parse_dot_notation(param)
+          elsif param.start_with?('$[')
+            parse_bracket_notation(param)
+          else
+            param
+          end
+        end
+
+        def self.parse_dot_notation(param)
+          result = []
+          keys = param[2..-1].split('.')
+          keys.each { |key|
+            if key.include?('[')
+              result.concat(parse_dot_array_op(key, param))
+            else
+              result << key
+            end
+          }
+
+          raise Fluent::ConfigError, "empty keys in dot notation" if result.empty?
+          validate_dot_keys(result)
+
+          result
+        end
+
+        def self.validate_dot_keys(keys)
+          keys.each { |key|
+            next unless key.is_a?(String)
+            if /\s+/.match(key)
+              raise Fluent::ConfigError, "whitespace character is not allowed in dot notation. Use bracket notation: #{key}"
+            end
+          }
+        end
+
+        def self.parse_dot_array_op(key, param)
+          start = key.index('[')
+          result = if start.zero?
+                     []
+                   else
+                     [key[0..start - 1]]
+                   end
+          key = key[start + 1..-1]
+          in_bracket = true
+
+          until key.empty?
+            if in_bracket
+              if i = key.index(']')
+                index_value = key[0..i - 1]
+                raise Fluent::ConfigError, "missing array index in '[]'. Invalid syntax: #{param}" if index_value == ']'
+                result << Integer(index_value)
+                key = key[i + 1..-1]
+                in_bracket = false
+              else
+                raise Fluent::ConfigError, "'[' found but ']' not found. Invalid syntax: #{param}"
+              end
+            else
+              if i = key.index('[')
+                key = key[i + 1..-1]
+                in_bracket = true
+              else
+                raise Fluent::ConfigError, "found more characters after ']'. Invalid syntax: #{param}"
+              end
+            end
+          end
+
+          result
+        end
+
+        def self.parse_bracket_notation(param)
+          orig_param = param
+          result = []
+          param = param[1..-1]
+          in_bracket = false
+
+          until param.empty?
+            if in_bracket
+              if param[0] == "'"
+                if i = param.index("']")
+                  result << param[1..i - 1]
+                  param = param[i + 2..-1]
+                  in_bracket = false
+                else
+                  raise Fluent::ConfigError, "Incomplete bracket. Invalid syntax: #{orig_param}"
+                end
+              else
+                if i = param.index(']')
+                  index_value = param[0..i - 1]
+                  raise Fluent::ConfigError, "missing array index in '[]'. Invalid syntax: #{param}" if index_value == ']'
+                  result << Integer(index_value)
+                  param = param[i + 1..-1]
+                  in_bracket = false
+                else
+                  raise Fluent::ConfigError, "'[' found but ']' not found. Invalid syntax: #{orig_param}"
+                end
+              end
+            else
+              if i = param.index('[')
+                param = param[i + 1..-1]
+                in_bracket = true
+              else
+                raise Fluent::ConfigError, "found more characters after ']'. Invalid syntax: #{orig_param}"
+              end
+            end
+          end
+
+          raise Fluent::ConfigError, "empty keys in bracket notation" if result.empty?
+
+          result
+        end
+      end
+    end
+  end
+end

data/lib/fluent/supervisor.rb

@@ -163,18 +163,22 @@ module Fluent
         log.reopen!
       end
 
-      if pid = config[:worker_pid]
-        Process.kill(:USR1, pid)
-        # don't rescue Erro::ESRSH here (invalid status)
+      if config[:worker_pid]
+        config[:worker_pid].each do |pid|
+          Process.kill(:USR1, pid)
+          # don't rescue Erro::ESRSH here (invalid status)
+        end
       end
     end
 
     def kill_worker
-      if pid = config[:worker_pid]
-        if Fluent.windows?
-          Process.kill :KILL, pid
-        else
-          Process.kill :TERM, pid
+      if config[:worker_pid]
+        config[:worker_pid].each do |pid|
+          if Fluent.windows?
+            Process.kill :KILL, pid
+          else
+            Process.kill :TERM, pid
+          end
         end
       end
     end
@@ -198,7 +202,7 @@ module Fluent
     end
 
     def after_start
-      config[:worker_pid] = @pm.pid
+      (config[:worker_pid] ||= []).push(@pm.pid)
     end
   end
 
@@ -360,6 +364,11 @@ module Fluent
         self
       end
 
+      def apply_options(opts)
+        $log.format = opts[:format] if opts[:format]
+        $log.time_format = opts[:time_format] if opts[:time_format]
+      end
+
       def level=(level)
         @level = level
         $log.level = level
@@ -439,6 +448,9 @@ module Fluent
       show_plugin_config if @show_plugin_config
       read_config
       set_system_config
+      @log.apply_options(format: @system_config.log.format, time_format: @system_config.log.time_format)
+
+      $log.info :supervisor, "parsing config file is succeeded", path: @config_path
 
       if @workers < 1
         raise Fluent::ConfigError, "invalid number of workers (must be > 0):#{@workers}"
@@ -485,11 +497,12 @@ module Fluent
                      else :workers
                      end
       @log.init(process_type, worker_id)
-      Process.setproctitle("worker:#{@process_name}") if @process_name
-
       show_plugin_config if @show_plugin_config
       read_config
       set_system_config
+      @log.apply_options(format: @system_config.log.format, time_format: @system_config.log.time_format)
+
+      Process.setproctitle("worker:#{@process_name}") if @process_name
 
       if @standalone_worker && @workers != 1
         raise Fluent::ConfigError, "invalid number of workers (must be 1 or unspecified) with --no-supervisor: #{@workers}"
@@ -715,7 +728,6 @@ module Fluent
     end
 
     def read_config
-      $log.info :supervisor, "reading config file", path: @config_path
       @config_fname = File.basename(@config_path)
       @config_basedir = File.dirname(@config_path)
       @config_data = File.open(@config_path, "r:utf-8:utf-8") {|f| f.read }

data/lib/fluent/system_config.rb

@@ -46,6 +46,10 @@ module Fluent
     config_param :dir_permission, default: nil do |v|
       v.to_i(8)
     end
+    config_section :log, required: false, init: true, multi: false do
+      config_param :format, :enum, list: [:text, :json], default: :text
+      config_param :time_format, :string, default: '%Y-%m-%d %H:%M:%S %z'
+    end
 
     def self.create(conf)
       systems = conf.elements(name: 'system')

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '0.14.20.rc1'
+  VERSION = '0.14.20'
 
 end

data/test/config/test_system_config.rb

@@ -55,6 +55,8 @@ module Fluent::Config
       assert_nil(sc.emit_error_log_interval)
       assert_nil(sc.suppress_config_dump)
       assert_nil(sc.without_source)
+      assert_equal(:text, sc.log.format)
+      assert_equal('%Y-%m-%d %H:%M:%S %z', sc.log.time_format)
       assert_equal(1, s.instance_variable_get(:@workers))
       assert_nil(s.instance_variable_get(:@root_dir))
       assert_equal(Fluent::Log::LEVEL_INFO, s.instance_variable_get(:@log_level))
@@ -91,6 +93,22 @@ module Fluent::Config
       assert_not_nil(s.instance_variable_get("@#{key}"))
     end
 
+    test "log parameters" do
+      conf = parse_text(<<-EOS)
+          <system>
+            <log>
+              format json
+              time_format %Y
+            </log>
+          </system>
+      EOS
+      s = FakeSupervisor.new
+      sc = Fluent::SystemConfig.new(conf)
+      sc.apply(s)
+      assert_equal(:json, sc.log.format)
+      assert_equal('%Y', sc.log.time_format)
+    end
+
     data(
       'foo' => ['foo', 'bar'],
       'hoge' => ['hoge', 'fuga'],

data/test/plugin/test_filter_grep.rb

@@ -23,12 +23,16 @@ class GrepFilterTest < Test::Unit::TestCase
 
     test "regexpN can contain a space" do
       d = create_driver(%[regexp1 message  foo])
-      assert_equal(Regexp.compile(/ foo/), d.instance._regexps['message'])
+      d.instance._regexps.each_pair { |key, value|
+        assert_equal(Regexp.compile(/ foo/), value)
+      }
     end
 
     test "excludeN can contain a space" do
       d = create_driver(%[exclude1 message  foo])
-      assert_equal(Regexp.compile(/ foo/), d.instance._excludes['message'])
+      d.instance._excludes.each_pair { |key, value|
+        assert_equal(Regexp.compile(/ foo/), value)
+      }
     end
 
     sub_test_case "duplicate key" do
@@ -163,6 +167,58 @@ class GrepFilterTest < Test::Unit::TestCase
     end
   end
 
+  sub_test_case 'nested keys' do
+    def messages
+      [
+        {"nest1" => {"nest2" => "INFO"}},
+        {"nest1" => {"nest2" => "WARN"}},
+        {"nest1" => {"nest2" => "WARN"}}
+      ]
+    end
+
+    def filter(config, msgs)
+      d = create_driver(config)
+      d.run {
+        msgs.each { |msg|
+          d.feed("filter.test", @time, {'foo' => 'bar', 'message' => msg})
+        }
+      }
+      d.filtered_records
+    end
+
+    test 'regexps' do
+      conf = %[
+        <regexp>
+          key $.message.nest1.nest2
+          pattern WARN
+        </regexp>
+      ]
+      filtered_records = filter(conf, messages)
+      assert_equal(2, filtered_records.size)
+      assert_block('only 2 nested logs') do
+        filtered_records.all? { |r|
+          r['message']['nest1']['nest2'] == 'WARN'
+        }
+      end
+    end
+
+    test 'excludes' do
+      conf = %[
+        <exclude>
+          key $.message.nest1.nest2
+          pattern WARN
+        </exclude>
+      ]
+      filtered_records = filter(conf, messages)
+      assert_equal(1, filtered_records.size)
+      assert_block('only 2 nested logs') do
+        filtered_records.all? { |r|
+          r['message']['nest1']['nest2'] == 'INFO'
+        }
+      end
+    end
+  end
+
   sub_test_case 'grep non-string jsonable values' do
     def filter(msg, config = 'regexp1 message 0')
       d = create_driver(config)

data/test/plugin/test_output.rb

@@ -372,7 +372,7 @@ class OutputTest < Test::Unit::TestCase
     end
 
     sub_test_case '#placeholder_validate!' do
-      test 'raises configuration error for a templace when timestamp placeholders exist but time key is missing' do
+      test 'raises configuration error for a template when timestamp placeholders exist but time key is missing' do
         @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))
         assert_raise Fluent::ConfigError.new("Parameter 'path: /path/without/timestamp/file.%Y%m%d-%H%M.log' has timestamp placeholders, but chunk key 'time' is not configured") do
           @i.placeholder_validate!(:path, "/path/without/timestamp/file.%Y%m%d-%H%M.log")

data/test/plugin_helper/test_record_accessor.rb

@@ -0,0 +1,108 @@
+require_relative '../helper'
+require 'fluent/plugin_helper/record_accessor'
+require 'fluent/plugin/base'
+
+require 'time'
+
+class RecordAccessorHelperTest < Test::Unit::TestCase
+  class Dummy < Fluent::Plugin::TestBase
+    helpers :record_accessor
+  end
+
+  sub_test_case 'parse nested key expression' do
+    data('normal' => 'key1',
+         'space' => 'ke y2',
+         'dot key' => 'this.is.key3')
+    test 'parse single key' do |param|
+      result = Fluent::PluginHelper::RecordAccessor::Accessor.parse_parameter(param)
+      assert_equal param, result
+    end
+
+    test "nested bracket keys with dot" do
+      result = Fluent::PluginHelper::RecordAccessor::Accessor.parse_parameter("$['key1']['this.is.key3']")
+      assert_equal ['key1', 'this.is.key3'], result
+    end
+
+    data('dot' => '$.key1.key2[0]',
+         'bracket' => "$['key1']['key2'][0]")
+    test "nested keys ['key1', 'key2', 0]" do |param|
+      result = Fluent::PluginHelper::RecordAccessor::Accessor.parse_parameter(param)
+      assert_equal ['key1', 'key2', 0], result
+    end
+
+    data('bracket' => "$['key1'][0]['ke y2']")
+    test "nested keys ['key1', 0, 'ke y2']" do |param|
+      result = Fluent::PluginHelper::RecordAccessor::Accessor.parse_parameter(param)
+      assert_equal ['key1', 0, 'ke y2'], result
+    end
+
+    data('dot' => '$.[0].key1.[1].key2',
+         'bracket' => "$[0]['key1'][1]['key2']")
+    test "nested keys [0, 'key1', 1, 'key2']" do |param|
+      result = Fluent::PluginHelper::RecordAccessor::Accessor.parse_parameter(param)
+      assert_equal [0, 'key1', 1, 'key2'], result
+    end
+
+    data("missing ']'" => "$['key1'",
+         "missing array index with dot" => "$.hello[]",
+         "missing array index with braket" => "$[]",
+         "more chars" => "$.key1[0]foo",
+         "whitespace char included key in dot notation" => "$.key[0].ke y",
+         "empty keys with dot" => "$.",
+         "empty keys with bracket" => "$[")
+    test 'invalid syntax' do |param|
+      assert_raise Fluent::ConfigError do
+        Fluent::PluginHelper::RecordAccessor::Accessor.parse_parameter(param)
+      end
+    end
+  end
+
+  sub_test_case Fluent::PluginHelper::RecordAccessor::Accessor do
+    setup do
+      @d = Dummy.new
+    end
+
+    data('normal' => 'key1',
+         'space' => 'ke y2',
+         'dot key' => 'this.is.key3')
+    test 'access single key' do |param|
+      r = {'key1' => 'v1', 'ke y2' => 'v2', 'this.is.key3' => 'v3'}
+      accessor = @d.record_accessor_create(param)
+      assert_equal r[param], accessor.call(r)
+    end
+
+    test "nested bracket keys with dot" do
+      r = {'key1' => {'this.is.key3' => 'value'}}
+      accessor = @d.record_accessor_create("$['key1']['this.is.key3']")
+      assert_equal 'value', accessor.call(r)
+    end
+
+    data('dot' => '$.key1.key2[0]',
+         'bracket' => "$['key1']['key2'][0]")
+    test "nested keys ['key1', 'key2', 0]" do |param|
+      r = {'key1' => {'key2' => [1, 2, 3]}}
+      accessor = @d.record_accessor_create(param)
+      assert_equal 1, accessor.call(r)
+    end
+
+    data('bracket' => "$['key1'][0]['ke y2']")
+    test "nested keys ['key1', 0, 'ke y2']" do |param|
+      r = {'key1' => [{'ke y2' => "value"}]}
+      accessor = @d.record_accessor_create(param)
+      assert_equal 'value', accessor.call(r)
+    end
+
+    data("missing ']'" => "$['key1'",
+         "missing array index with dot" => "$.hello[]",
+         "missing array index with braket" => "$['hello'][]",
+         "whitespace char included key in dot notation" => "$.key[0].ke y",
+         "more chars" => "$.key1[0]foo",
+         "empty keys with dot" => "$.",
+         "empty keys with bracket" => "$[")
+    test 'invalid syntax' do |param|
+      assert_raise Fluent::ConfigError do
+        @d.record_accessor_create(param)
+      end
+    end
+  end
+end

data/test/test_log.rb

@@ -379,6 +379,56 @@ class LogTest < Test::Unit::TestCase
     assert_equal(Fluent::Log::LEVEL_TRACE, log2.level)
   end
 
+  def test_format_json
+    logdev = @log_device
+    logger = ServerEngine::DaemonLogger.new(logdev)
+    log = Fluent::Log.new(logger)
+    log.format = :json
+    log.level = Fluent::Log::LEVEL_TRACE
+    log.trace "trace log"
+    log.debug "debug log"
+    log.info "info log"
+    log.warn "warn log"
+    log.error "error log"
+    log.fatal "fatal log"
+    expected = [
+      "#{@timestamp_str} [trace]: trace log\n",
+      "#{@timestamp_str} [debug]: debug log\n",
+      "#{@timestamp_str} [info]: info log\n",
+      "#{@timestamp_str} [warn]: warn log\n",
+      "#{@timestamp_str} [error]: error log\n",
+      "#{@timestamp_str} [fatal]: fatal log\n"
+    ]
+    assert_equal(expected, log.out.logs.map { |l|
+                   r = JSON.parse(l)
+                   "#{r['time']} [#{r['level']}]: #{r['message']}\n"
+                 })
+  end
+
+  def test_time_format
+    logdev = @log_device
+    logger = ServerEngine::DaemonLogger.new(logdev)
+    log = Fluent::Log.new(logger)
+    log.time_format = "%Y"
+    log.level = Fluent::Log::LEVEL_TRACE
+    log.trace "trace log"
+    log.debug "debug log"
+    log.info "info log"
+    log.warn "warn log"
+    log.error "error log"
+    log.fatal "fatal log"
+    timestamp_str = @timestamp.strftime("%Y")
+    expected = [
+      "#{timestamp_str} [trace]: trace log\n",
+      "#{timestamp_str} [debug]: debug log\n",
+      "#{timestamp_str} [info]: info log\n",
+      "#{timestamp_str} [warn]: warn log\n",
+      "#{timestamp_str} [error]: error log\n",
+      "#{timestamp_str} [fatal]: fatal log\n"
+    ]
+    assert_equal(expected, log.out.logs)
+  end
+
   def test_disable_events
     dl_opts = {}
     dl_opts[:log_level] = ServerEngine::DaemonLogger::TRACE

data/test/test_supervisor.rb

@@ -130,6 +130,10 @@ class SupervisorTest < ::Test::Unit::TestCase
   process_name "process_name"
   log_level info
   root_dir #{TMP_ROOT_DIR}
+  <log>
+    format json
+    time_format %Y
+  </log>
 </system>
     EOC
     conf = Fluent::Config.parse(conf_data, "(test)", "(test_dir)", true)
@@ -145,6 +149,8 @@ class SupervisorTest < ::Test::Unit::TestCase
     assert_equal "process_name", sys_conf.process_name
     assert_equal 2, sys_conf.log_level
     assert_equal TMP_ROOT_DIR, sys_conf.root_dir
+    assert_equal :json, sys_conf.log.format
+    assert_equal '%Y', sys_conf.log.time_format
   end
 
   def test_main_process_signal_handlers

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 0.14.20.rc1
+  version: 0.14.20
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-07-21 00:00:00.000000000 Z
+date: 2017-07-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -160,6 +160,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.1.7
+- !ruby/object:Gem::Dependency
+  name: ruby_dig
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.2
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -503,6 +517,7 @@ files:
 - lib/fluent/plugin_helper/formatter.rb
 - lib/fluent/plugin_helper/inject.rb
 - lib/fluent/plugin_helper/parser.rb
+- lib/fluent/plugin_helper/record_accessor.rb
 - lib/fluent/plugin_helper/retry_state.rb
 - lib/fluent/plugin_helper/server.rb
 - lib/fluent/plugin_helper/socket.rb
@@ -665,6 +680,7 @@ files:
 - test/plugin_helper/test_formatter.rb
 - test/plugin_helper/test_inject.rb
 - test/plugin_helper/test_parser.rb
+- test/plugin_helper/test_record_accessor.rb
 - test/plugin_helper/test_retry_state.rb
 - test/plugin_helper/test_server.rb
 - test/plugin_helper/test_storage.rb
@@ -716,9 +732,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '2.1'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.6.11
@@ -830,6 +846,7 @@ test_files:
 - test/plugin_helper/test_formatter.rb
 - test/plugin_helper/test_inject.rb
 - test/plugin_helper/test_parser.rb
+- test/plugin_helper/test_record_accessor.rb
 - test/plugin_helper/test_retry_state.rb
 - test/plugin_helper/test_server.rb
 - test/plugin_helper/test_storage.rb