fluentd 0.14.20 → 0.14.21

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 419af849ffe16bdb83d13106862b94018d71dca1
-  data.tar.gz: 6f1435c4d5f9f1645267bb88f43aa86ee838075e
+  metadata.gz: e0c04447f174a283ac32276dd0dbde42bfabac9b
+  data.tar.gz: dc5da1036653db4373345239dc4486918423e479
 SHA512:
-  metadata.gz: 96ff26acaccd6870648fd36ca991e101058296abfa8585366c0b132d94a19c79ef75214527c026b3719061c943d0d0aeaec45c3bb58655b3f08f7da88d5a6aa6
-  data.tar.gz: a83370feeeabb3f9a3200f5575ce8ba9f4fe0a57349b9fe73dfd3f6b386492993483412883f91214b8903709a86dcacccac658cf663d1734aa37d5b1e4f5fa21
+  metadata.gz: 7015711b0401b911a3242e96bf90f5bd56693e1ccc62565d2cee241403354599a3c29bf487f10989297bfcf41a7d0c90c274396f0a07dc9c338fc26a66769323
+  data.tar.gz: 9cd584791a19c95166e34e325f104cb38918769c836f1779008e77e7d36367ccb786d3fa9b16381d8d9ce680ec22b903b60e9a8e0244f635954d188c6a5e2a6c

data/ChangeLog

@@ -1,5 +1,25 @@
 # v0.14
 
+## Release v0.14.21 - 2017/09/07
+
+### New features / Enhancements
+
+* filter_parser: Support record_accessor in key_name
+  https://github.com/fluent/fluentd/pull/1654
+* buffer: Support record_accessor in chunk keys
+  https://github.com/fluent/fluentd/pull/1662
+
+### Bug fixes
+
+* compat_parameters: Support all syslog parser parameters
+  https://github.com/fluent/fluentd/pull/1650
+* filter_record_transformer: Don't create new keys if the original record doesn't have `keep_keys` keys
+  https://github.com/fluent/fluentd/pull/1663
+* in_tail: Fix the error when 'tag *' is configured
+  https://github.com/fluent/fluentd/pull/1664
+* supervisor: Clear previous worker pids when receive kill signals.
+  https://github.com/fluent/fluentd/pull/1683
+
 ## Release v0.14.20 - 2017/07/31
 
 ### New features / Enhancements

data/ENTERPRISE_PROVIDERS.md

@@ -0,0 +1,43 @@
+# Fluentd Enterprise Providers
+
+[Fluentd](http://www.fluentd.org) is widely adopted in the enterprise and in most of scenarios, production environments requires special support and services. The following document describe the certified  service providers and further guidelines to apply to become one.
+
+## Fluentd Providers
+
+The following section lists the companies that provides Support, Consulting Services and Products around Fluentd for the Enterprise needs.
+
+- [Treasure Data](https://fluentd.treasuredata.com)
+
+### Treasure Data
+
+[Treasure Data](https://www.treasuredata.com) is one of the principal sponsors of Fluentd, it provides technical support, consulting services and [Fluentd Enterprise](https://fluentd.treasuredata.com/), a Fluentd-on steroids with enhanced security and connectors for enterprise backends such as [Apache Kafka](http://kafka.apache.org) and [Splunk](http://www.splunk.com) within others.
+
+For more details about Fluentd Enterprise provided by Treasure Data, click [here](https://fluentd.treasuredata.com/).
+
+## Apply to become a Fluentd Service Provider
+
+In order to keep a transparent involvement of companies in Fluentd growth, to be listed as a service provider your company must fulfil the following criteria:
+
+- The company offering the services or products must be fully incorporated.
+- At least two former employees must be contributors of Fluentd ecosystem (core, plugins, documentation, etc).
+- Company participate from Fluentd community activities: meetups, conferences (or webinars) or online communication channels (Slack)
+
+In order to be listed as a service provider, please send an email using your corporate account to the maintainers (listed at bottom) with the following information:
+
+- Company Information
+  - Company Name
+  - Company URL
+  - Country / Location
+- Employees
+  - Name, email and github handle of employees who contribute to Fluentd ecosystem
+  - List of projects or contributions where the company is involved
+- Services
+  - What kind of services around Fluentd do you provide ?
+- Community
+  - List of activities of your company within the Fluentd community in the last 12 months
+  - Are you a member of the [Cloud Native Computing Foundation](http://cncf.io) ?
+
+### Maintainers / Recipients
+
+- Kiyoto Tamura <kiyoto@treasure-data.com>
+- Eduardo Silva <eduardo@treasure-data.com>

data/MAINTAINERS.md

@@ -3,3 +3,4 @@
 - Kiyoto Tamura <me@ktamura.com>
 - Kazuki Ohta <kazuki.ohta@gmail.com>
 - Satoshi "Moris" Tagomori <tagomoris@gmail.com>
+- Eduardo Silva <eduardo@treasure-data.com>

data/README.md

@@ -2,6 +2,8 @@ Fluentd: Open-Source Log Collector
 ===================================
 
 [<img src="https://travis-ci.org/fluent/fluentd.svg" />](https://travis-ci.org/fluent/fluentd) [![Code Climate](https://codeclimate.com/github/fluent/fluentd/badges/gpa.svg)](https://codeclimate.com/github/fluent/fluentd)
+[![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/1189/badge)](https://bestpractices.coreinfrastructure.org/projects/1189)
+[![FOSSA Status](https://app.fossa.io/api/projects/git%2Bhttps%3A%2F%2Fgithub.com%2Ffluent%2Ffluentd.svg?type=shield)](https://app.fossa.io/projects/git%2Bhttps%3A%2F%2Fgithub.com%2Ffluent%2Ffluentd?ref=badge_shield)
 
 [Fluentd](http://fluentd.org/) collects events from various data sources and writes them to files, RDBMS, NoSQL, IaaS, SaaS, Hadoop and so on. Fluentd helps you unify your logging infrastructure (Learn more about the [Unified Logging Layer](http://www.fluentd.org/blog/unified-logging-layer)).
 

data/lib/fluent/config/element.rb

@@ -76,6 +76,11 @@ module Fluent
         "name:#{@name}, arg:#{@arg}, " + attrs + ", " + @elements.inspect
       end
 
+      # Used by PP and Pry
+      def pretty_print(q)
+        q.text(inspect)
+      end
+
       # This method assumes _o_ is an Element object. Should return false for nil or other object
       def ==(o)
         self.name == o.name && self.arg == o.arg &&

data/lib/fluent/plugin/filter_parser.rb

@@ -24,7 +24,7 @@ module Fluent::Plugin
   class ParserFilter < Filter
     Fluent::Plugin.register_filter('parser', self)
 
-    helpers :parser, :compat_parameters
+    helpers :parser, :record_accessor, :compat_parameters
 
     config_param :key_name, :string
     config_param :reserve_data, :bool, default: false
@@ -41,6 +41,7 @@ module Fluent::Plugin
 
       super
 
+      @accessor = record_accessor_create(@key_name)
       @parser = parser_create
     end
 
@@ -48,7 +49,7 @@ module Fluent::Plugin
     REPLACE_CHAR = '?'.freeze
 
     def filter_with_time(tag, time, record)
-      raw_value = record[@key_name]
+      raw_value = @accessor.call(record)
       if raw_value.nil?
         if @emit_invalid_record_to_error
           router.emit_error_event(tag, time, record, ArgumentError.new("#{@key_name} does not exist"))

data/lib/fluent/plugin/filter_record_transformer.rb

@@ -28,9 +28,9 @@ module Fluent::Plugin
     Fluent::Plugin.register_filter('record_transformer', self)
 
     desc 'A comma-delimited list of keys to delete.'
-    config_param :remove_keys, :string, default: nil
+    config_param :remove_keys, :array, default: nil
     desc 'A comma-delimited list of keys to keep.'
-    config_param :keep_keys, :string, default: nil
+    config_param :keep_keys, :array, default: nil
     desc 'Create new Hash to transform incoming data'
     config_param :renew_record, :bool, default: false
     desc 'Specify field name of the record to overwrite the time of events. Its value must be unix time.'
@@ -52,13 +52,8 @@ module Fluent::Plugin
         end
       end
 
-      if @remove_keys
-        @remove_keys = @remove_keys.split(',')
-      end
-
       if @keep_keys
         raise Fluent::ConfigError, "`renew_record` must be true to use `keep_keys`" unless @renew_record
-        @keep_keys = @keep_keys.split(',')
       end
 
       placeholder_expander_params = {
@@ -129,7 +124,9 @@ module Fluent::Plugin
       placeholders = @placeholder_expander.prepare_placeholders(placeholder_values)
 
       new_record = @renew_record ? {} : record.dup
-      @keep_keys.each {|k| new_record[k] = record[k]} if @keep_keys and @renew_record
+      @keep_keys.each do |k|
+        new_record[k] = record[k] if record.has_key?(k)
+      end if @keep_keys && @renew_record
       new_record.merge!(expand_placeholders(@map, placeholders))
 
       new_record
@@ -218,7 +215,7 @@ module Fluent::Plugin
       # @param [String] str
       # @param [Boolean] force_stringify the value must be string, used for hash key
       def expand(str, placeholders, force_stringify = false)
-        if @auto_typecast and !force_stringify
+        if @auto_typecast && !force_stringify
           single_placeholder_matched = str.match(/\A(\${[^}]+}|__[A-Z_]+__)\z/)
           if single_placeholder_matched
             log_if_unknown_placeholder($1, placeholders)
@@ -261,9 +258,9 @@ module Fluent::Plugin
       def preprocess_map(value, force_stringify = false)
         new_value = nil
         if value.is_a?(String)
-          if @auto_typecast and !force_stringify
+          if @auto_typecast && !force_stringify
             num_placeholders = value.scan('${').size
-            if num_placeholders == 1 and value.start_with?('${') && value.end_with?('}')
+            if num_placeholders == 1 && value.start_with?('${') && value.end_with?('}')
               new_value = value[2..-2] # ${..} => ..
             end
           end

data/lib/fluent/plugin/in_tail.rb

@@ -130,6 +130,7 @@ module Fluent::Plugin
     def configure_tag
       if @tag.index('*')
         @tag_prefix, @tag_suffix = @tag.split('*')
+        @tag_prefix ||= ''
         @tag_suffix ||= ''
       else
         @tag_prefix = nil

data/lib/fluent/plugin/output.rb

@@ -15,6 +15,7 @@
 #
 
 require 'fluent/plugin/base'
+require 'fluent/plugin_helper/record_accessor'
 require 'fluent/log'
 require 'fluent/plugin_id'
 require 'fluent/plugin_helper'
@@ -36,7 +37,7 @@ module Fluent
       helpers_internal :thread, :retry_state
 
       CHUNK_KEY_PATTERN = /^[-_.@a-zA-Z0-9]+$/
-      CHUNK_KEY_PLACEHOLDER_PATTERN = /\$\{[-_.@a-zA-Z0-9]+\}/
+      CHUNK_KEY_PLACEHOLDER_PATTERN = /\$\{[-_.@$a-zA-Z0-9]+\}/
       CHUNK_TAG_PLACEHOLDER_PATTERN = /\$\{(tag(?:\[\d+\])?)\}/
 
       CHUNKING_FIELD_WARN_NUM = 4
@@ -161,7 +162,7 @@ module Fluent
       attr_reader :num_errors, :emit_count, :emit_records, :write_count, :rollback_count
 
       # for tests
-      attr_reader :buffer, :retry, :secondary, :chunk_keys, :chunk_key_time, :chunk_key_tag
+      attr_reader :buffer, :retry, :secondary, :chunk_keys, :chunk_key_accessors, :chunk_key_time, :chunk_key_tag
       attr_accessor :output_enqueue_thread_waiting, :dequeued_chunks, :dequeued_chunks_mutex
       # output_enqueue_thread_waiting: for test of output.rb itself
       attr_accessor :retry_for_error_chunk # if true, error flush will be retried even if under_plugin_development is true
@@ -203,7 +204,7 @@ module Fluent
         @output_flush_threads = nil
 
         @simple_chunking = nil
-        @chunk_keys = @chunk_key_time = @chunk_key_tag = nil
+        @chunk_keys = @chunk_key_accessors = @chunk_key_time = @chunk_key_tag = nil
         @flush_mode = nil
         @timekey_zone = nil
 
@@ -276,8 +277,25 @@ module Fluent
           @chunk_keys = @buffer_config.chunk_keys.dup
           @chunk_key_time = !!@chunk_keys.delete('time')
           @chunk_key_tag = !!@chunk_keys.delete('tag')
-          if @chunk_keys.any?{ |key| key !~ CHUNK_KEY_PATTERN }
+          if @chunk_keys.any? { |key|
+              begin
+                k = Fluent::PluginHelper::RecordAccessor::Accessor.parse_parameter(key)
+                if k.is_a?(String)
+                  k !~ CHUNK_KEY_PATTERN
+                else
+                  if key.start_with?('$[')
+                    raise Fluent::ConfigError, "in chunk_keys: bracket notation is not allowed"
+                  else
+                    false
+                  end
+                end
+              rescue => e
+                raise Fluent::ConfigError, "in chunk_keys: #{e.message}"
+              end
+            }
             raise Fluent::ConfigError, "chunk_keys specification includes invalid char"
+          else
+            @chunk_key_accessors = Hash[@chunk_keys.map { |key| [key.to_sym, Fluent::PluginHelper::RecordAccessor::Accessor.new(key)] }]
           end
 
           if @chunk_key_time
@@ -778,7 +796,7 @@ module Fluent
                     else
                       nil
                     end
-          pairs = Hash[@chunk_keys.map{|k| [k.to_sym, record[k]]}]
+          pairs = Hash[@chunk_key_accessors.map { |k, a| [k, a.call(record)] }]
           @buffer.metadata(timekey: timekey, tag: (@chunk_key_tag ? tag : nil), variables: pairs)
         end
       end

data/lib/fluent/plugin_helper/compat_parameters.rb

@@ -65,7 +65,9 @@ module Fluent
         "label_delimiter"  => "label_delimiter", # LabeledTSVParser
         "format_firstline" => "format_firstline", # MultilineParser
         "message_key"      => "message_key", # NoneParser
-        "with_priority"    => "with_priority", # SyslogParser
+        "with_priority"       => "with_priority", # SyslogParser
+        "message_format"      => "message_format", # SyslogParser
+        "rfc5424_time_format" => "rfc5424_time_format", # SyslogParser
         # There has been no parsers which can handle timezone in v0.12
       }
 

data/lib/fluent/supervisor.rb

@@ -173,7 +173,9 @@ module Fluent
 
     def kill_worker
       if config[:worker_pid]
-        config[:worker_pid].each do |pid|
+        pids = config[:worker_pid].clone
+        config[:worker_pid].clear
+        pids.each do |pid|
           if Fluent.windows?
             Process.kill :KILL, pid
           else

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '0.14.20'
+  VERSION = '0.14.21'
 
 end

data/test/config/test_element.rb

@@ -2,6 +2,7 @@ require_relative '../helper'
 require 'fluent/config/element'
 require 'fluent/config/configure_proxy'
 require 'fluent/configurable'
+require 'pp'
 
 class TestConfigElement < ::Test::Unit::TestCase
   def element(name = 'ROOT', arg = '', attrs = {}, elements = [], unused = nil)
@@ -463,4 +464,13 @@ CONF
       assert_false e.for_another_worker?
     end
   end
+
+  sub_test_case '#pretty_print' do
+    test 'prints inspect to pp object' do
+      q = PP.new
+      e = element()
+      e.pretty_print(q)
+      assert_equal e.inspect, q.output
+    end
+  end
 end

data/test/plugin/test_filter_parser.rb

@@ -390,6 +390,29 @@ class ParserFilterTest < Test::Unit::TestCase
     assert_equal 'value"ThreeYes!', first[1]['key3']
   end
 
+  def test_filter_with_nested_record
+    d = create_driver(%[
+      key_name $.data.log
+      <parse>
+        @type csv
+        keys key1,key2,key3
+      </parse>
+    ])
+    time = @default_time.to_i
+    d.run do
+      d.feed(@tag, time, {'data' => {'log' => 'value1,"value2","value""ThreeYes!"'}, 'xxx' => 'x', 'yyy' => 'y'})
+    end
+    filtered = d.filtered
+    assert_equal 1, filtered.length
+
+    first = filtered[0]
+    assert_equal time, first[0]
+    assert_nil first[1]['data']
+    assert_equal 'value1', first[1]['key1']
+    assert_equal 'value2', first[1]['key2']
+    assert_equal 'value"ThreeYes!', first[1]['key3']
+  end
+
   CONFIG_HASH_VALUE_FIELD = %[
     key_name data
     hash_value_field parsed

data/test/plugin/test_filter_record_transformer.rb

@@ -137,6 +137,18 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
       end
     end
 
+    test 'keep_keys that are not present in the original record should not be included in the result record' do
+      config = %[renew_record true\nkeep_keys foo, bar, baz, message]
+      msgs = ['1', '2', nil]
+      filtered = filter(config, msgs)
+      filtered.each_with_index do |(_t, r), i|
+        assert_equal('bar', r['foo'])
+        assert_equal(msgs[i], r['message'])
+        assert_equal(false, r.has_key?('bar'))
+        assert_equal(false, r.has_key?('baz'))
+      end
+    end
+
     test 'enable_ruby' do
       config = %[
         enable_ruby yes

data/test/plugin/test_in_tail.rb

@@ -972,6 +972,20 @@ class TailInputTest < Test::Unit::TestCase
       plugin.receive_lines(['foo', 'bar'], DummyWatcher.new('foo.bar.log'))
     end
 
+    def test_tag_with_only_star
+      config = config_element("", "", {
+                                "tag" => "*",
+                                "path" => "test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log",
+                                "format" => "none",
+                                "read_from_head" => true
+                              })
+      d = create_driver(config, false)
+      d.run {}
+      plugin = d.instance
+      mock(plugin.router).emit_stream('foo.bar.log', anything).once
+      plugin.receive_lines(['foo', 'bar'], DummyWatcher.new('foo.bar.log'))
+    end
+
     def test_tag_prefix
       config = config_element("", "", {
                                 "tag" => "pre.*",

data/test/plugin/test_output.rb

@@ -267,6 +267,18 @@ class OutputTest < Test::Unit::TestCase
       assert_equal "/mypath/%Y/%m/%d/%H-%M/${tag}/${tag[1]}/${tag[2]}/value1/value2/tail", @i.extract_placeholders(tmpl, m)
     end
 
+    test '#extract_placeholders can extract nested variables if variables are configured with dot notation' do
+      @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'key,$.nest.key', {})]))
+      assert !@i.chunk_key_time
+      assert !@i.chunk_key_tag
+      assert_equal ['key','$.nest.key'], @i.chunk_keys
+      tmpl = "/mypath/%Y/%m/%d/%H-%M/${tag}/${tag[1]}/${tag[2]}/${key}/${$.nest.key}/tail"
+      t = event_time('2016-04-11 20:30:00 +0900')
+      v = {:key => "value1", :"$.nest.key" => "value2"}
+      m = create_metadata(timekey: t, tag: 'fluentd.test.output', variables: v)
+      assert_equal "/mypath/%Y/%m/%d/%H-%M/${tag}/${tag[1]}/${tag[2]}/value1/value2/tail", @i.extract_placeholders(tmpl, m)
+    end
+
     test '#extract_placeholders can extract all chunk keys if configured' do
       @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'time,tag,key1,key2', {'timekey' => 60*30, 'timekey_zone' => "+0900"})]))
       assert @i.chunk_key_time
@@ -493,11 +505,21 @@ class OutputTest < Test::Unit::TestCase
       assert_equal ['.hidden', '0001', '@timestamp', 'a_key', 'my-domain'], @i.get_placeholders_keys("http://${my-domain}/${.hidden}/${0001}/${a_key}?timestamp=${@timestamp}")
     end
 
+    data('include space' => 'ke y',
+         'bracket notation' => "$['key']",
+         'invalid notation' => "$.ke y")
+    test 'configure checks invalid chunk keys' do |chunk_keys|
+      i = create_output(:buffered)
+      assert_raise Fluent::ConfigError do
+        i.configure(config_element('ROOT' , '', {}, [config_element('buffer', chunk_keys)]))
+      end
+    end
+
     test '#metadata returns object which contains tag/timekey/variables from records as specified in configuration' do
       tag = 'test.output'
       time = event_time('2016-04-12 15:31:23 -0700')
       timekey = event_time('2016-04-12 15:00:00 -0700')
-      record = {"key1" => "value1", "num1" => 1, "message" => "my message"}
+      record = {"key1" => "value1", "num1" => 1, "message" => "my message", "nest" => {"key" => "nested value"}}
 
       i1 = create_output(:buffered)
       i1.configure(config_element('ROOT','',{},[config_element('buffer', '')]))
@@ -530,6 +552,10 @@ class OutputTest < Test::Unit::TestCase
       i8 = create_output(:buffered)
       i8.configure(config_element('ROOT','',{},[config_element('buffer', 'time,tag,key1', {"timekey" => 3600, "timekey_zone" => "-0700"})]))
       assert_equal create_metadata(timekey: timekey, tag: tag, variables: {key1: "value1"}), i8.metadata(tag, time, record)
+
+      i9 = create_output(:buffered)
+      i9.configure(config_element('ROOT','',{},[config_element('buffer', 'key1,$.nest.key', {})]))
+      assert_equal create_metadata(variables: {:key1 => "value1", :"$.nest.key" => 'nested value'}), i9.metadata(tag, time, record)
     end
 
     test '#emit calls #process via #emit_sync for non-buffered output' do

data/test/plugin_helper/test_compat_parameters.rb

@@ -328,4 +328,26 @@ class CompatParameterTest < Test::Unit::TestCase
       # TODO:
     end
   end
+
+  sub_test_case 'parser plugins' do
+    test 'syslog parser parameters' do
+      hash = {
+        'format' => 'syslog',
+        'message_format' => 'rfc5424',
+        'with_priority' => 'true',
+        'rfc5424_time_format' => '%Y'
+      }
+      conf = config_element('ROOT', '', hash)
+      @i = DummyI0.new
+      @i.configure(conf)
+      @i.start
+      @i.after_start
+
+      parser = @i.parser
+      assert_kind_of(Fluent::Plugin::SyslogParser, parser)
+      assert_equal :rfc5424, parser.message_format
+      assert_equal true, parser.with_priority
+      assert_equal '%Y', parser.rfc5424_time_format
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 0.14.20
+  version: 0.14.21
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-07-31 00:00:00.000000000 Z
+date: 2017-09-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -322,6 +322,7 @@ files:
 - CONTRIBUTING.md
 - COPYING
 - ChangeLog
+- ENTERPRISE_PROVIDERS.md
 - Gemfile
 - MAINTAINERS.md
 - README.md