fluentd 0.12.13 → 0.12.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e8d2b210d487f331113e6ba6be6f9607d3cb0129
-  data.tar.gz: 83237ccc84a3af2e52d8824fdda9bc5ee14485a8
+  metadata.gz: ca3a80d093b1051c5b2bda3a5565419a041a5ca4
+  data.tar.gz: 6ed4dc8a876080c1a0189bafddf2ab1de739601d
 SHA512:
-  metadata.gz: 206acfb5ddab70b8c7b20b1b36c25a62f827c899c851bbbb98484159309184266597cbebd691c53cb23aa5fac7a6e77ebcbe1611ada8bf720583ba93ed26faf6
-  data.tar.gz: b5381b0ec5c92f33a30e021db328767582aa2753d651695073a27c84099c4cb9445c127a0dc9a31350dd441a1ad53d05ef3fe79d488d644faa0a228150dfe706
+  metadata.gz: be8eaf2c1dc61eff8aa6ddd358a2ae4e0b5dd3dd9c5c94e77ea4f6a183e561b65368af361289d73743f7d39cf48556a91adc73f154af6b5634c85b45d53b7e65
+  data.tar.gz: db0f69056992f3ff4d2884aa483715fec20d8a3904816f5583c5ba37d2588f3152cda7a354e504c8c033eed365ac4e92b358b49d07b6327a9f5e403868db6f8b

data/ChangeLog

@@ -1,5 +1,21 @@
 # v0.12
 
+## Release 0.12.14 - 2015/07/17
+
+### New features / Enhancement
+
+* config: Log unused section configuration as warn level
+  https://github.com/fluent/fluentd/pull/629
+* config: Add '@' to log_level. Keep log_level for backward compatibility
+  https://github.com/fluent/fluentd/pull/630
+* parser: Add time_key option for RegexpParser
+  https://github.com/fluent/fluentd/pull/588
+
+### Bug fixes
+
+* out_forward: Add 'dns_round_robin' option
+  https://github.com/fluent/fluentd/pull/632
+
 ## Release 0.12.13 - 2015/07/09
 
 ### New features / Enhancement

data/lib/fluent/config/element.rb

@@ -30,9 +30,10 @@ module Fluent
         @unused = unused || attrs.keys
         @v1_config = false
         @corresponding_proxies = [] # some plugins use flat parameters, e.g. in_http doesn't provide <format> section for parser.
+        @unused_in = false # if this element is not used in plugins, correspoing plugin name and parent element name is set, e.g. [source, plugin class].
       end
 
-      attr_accessor :name, :arg, :elements, :unused, :v1_config, :corresponding_proxies
+      attr_accessor :name, :arg, :elements, :unused, :v1_config, :corresponding_proxies, :unused_in
 
       def add_element(name, arg='')
         e = Element.new(name, arg, {}, [])

data/lib/fluent/config/section.rb

@@ -18,6 +18,7 @@ require 'json'
 
 module Fluent
   require 'fluent/config/error'
+  require 'fluent/config/v1_parser'
 
   module Config
     class Section < BasicObject
@@ -75,7 +76,7 @@ module Fluent
     end
 
     module SectionGenerator
-      def self.generate(proxy, conf, logger, stack = [])
+      def self.generate(proxy, conf, logger, plugin_class, stack = [])
         return nil if conf.nil?
 
         section_stack = ""
@@ -118,6 +119,8 @@ module Fluent
           end
         end
 
+        check_unused_section(proxy, conf, plugin_class)
+
         proxy.sections.each do |name, subproxy|
           varname = subproxy.param_name.to_sym
           elements = (conf.respond_to?(:elements) ? conf.elements : []).select{ |e| e.name == subproxy.name.to_s || e.name == subproxy.alias.to_s }
@@ -131,18 +134,34 @@ module Fluent
             raise ConfigError, "'<#{subproxy.name}>' sections are required" + section_stack
           end
           if subproxy.multi?
-            section_params[varname] = elements.map{ |e| generate(subproxy, e, logger, stack + [subproxy.name]) }
+            section_params[varname] = elements.map{ |e| generate(subproxy, e, logger, plugin_class, stack + [subproxy.name]) }
           else
             if elements.size > 1
               logger.error "config error in:\n#{conf}"
               raise ConfigError, "'<#{subproxy.name}>' section cannot be written twice or more" + section_stack
             end
-            section_params[varname] = generate(subproxy, elements.first, logger, stack + [subproxy.name])
+            section_params[varname] = generate(subproxy, elements.first, logger, plugin_class, stack + [subproxy.name])
           end
         end
 
         Section.new(section_params)
       end
+
+      def self.check_unused_section(proxy, conf, plugin_class)
+        elems = conf.respond_to?(:elements) ? conf.elements : []
+        elems.each { |e|
+          next if plugin_class.nil? && Fluent::Config::V1Parser::ELEM_SYMBOLS.include?(e.name) # skip pre-defined non-plugin elements because it doens't have proxy section
+
+          unless proxy.sections.any? { |name, subproxy| e.name == subproxy.name.to_s || e.name == subproxy.alias.to_s }
+            parent_name = if conf.arg.empty?
+                            conf.name
+                          else
+                            "#{conf.name} #{conf.arg}"
+                          end
+            e.unused_in = [parent_name, plugin_class]
+          end
+        }
+      end
     end
   end
 end

data/lib/fluent/config/v1_parser.rb

@@ -51,7 +51,7 @@ module Fluent
       end
 
       ELEM_SYMBOLS = ['match', 'source', 'filter', 'system']
-      RESERVED_PARAMS = %W(@type @id @label)
+      RESERVED_PARAMS = %W(@type @id @label @log_level)
 
       def parse_element(root_element, elem_name, attrs = {}, elems = [])
         while true

data/lib/fluent/configurable.rb

@@ -50,7 +50,9 @@ module Fluent
       proxy = self.class.merged_configure_proxy
       conf.corresponding_proxies << proxy
 
-      root = Fluent::Config::SectionGenerator.generate(proxy, conf, logger)
+      # In the nested section, can't get plugin class through proxies so get plugin class here
+      plugin_class = Plugin.lookup_name_from_class(proxy.name.to_s)
+      root = Fluent::Config::SectionGenerator.generate(proxy, conf, logger, plugin_class)
       @config_root_section = root
 
       root.instance_eval{ @params.keys }.each do |param_name|

data/lib/fluent/engine.rb

@@ -76,6 +76,16 @@ module Fluent
     def run_configure(conf)
       configure(conf)
       conf.check_not_fetched { |key, e|
+        parent_name, plugin_name = e.unused_in
+        if parent_name
+          message = if plugin_name
+                      "section <#{e.name}> is not used in <#{parent_name}> of #{plugin_name} plugin"
+                    else
+                      "section <#{e.name}> is not used in <#{parent_name}>"
+                    end
+          $log.warn message
+          next
+        end
         unless e.name == 'system'
           unless @without_source && e.name == 'source'
             $log.warn "parameter '#{key}' in #{e.to_s.strip} is not used."

data/lib/fluent/log.rb

@@ -343,7 +343,7 @@ module Fluent
   module PluginLoggerMixin
     def self.included(klass)
       klass.instance_eval {
-        config_param :log_level, :string, :default => nil
+        config_param :log_level, :string, :default => nil, :alias => :@log_level
       }
     end
 

data/lib/fluent/parser.rb

@@ -162,6 +162,7 @@ module Fluent
     class RegexpParser < Parser
       include TypeConverter
 
+      config_param :time_key, :string, :default => 'time'
       config_param :time_format, :string, :default => nil
 
       def initialize(regexp, conf={})
@@ -202,7 +203,7 @@ module Fluent
         m.names.each {|name|
           if value = m[name]
             case name
-            when "time"
+            when @time_key
               time = @mutex.synchronize { @time_parser.parse(value) }
               if @keep_time_key
                 record[name] = if @type_converters.nil?

data/lib/fluent/plugin.rb

@@ -92,6 +92,26 @@ module Fluent
       try_load_plugin(name, type)
     end
 
+    def lookup_name_from_class(klass_or_str)
+      klass = if klass_or_str.class == String
+                eval(klass_or_str) # const_get can't handle A::B
+              else
+                klass_or_str
+              end
+
+      @input.each { |name, plugin|
+        return name if plugin == klass
+      }
+      @output.each { |name, plugin|
+        return name if plugin == klass
+      }
+      @filter.each { |name, plugin|
+        return name if plugin == klass
+      }
+
+      nil
+    end
+
     private
     def register_impl(name, map, type, klass)
       map[type] = klass

data/lib/fluent/plugin/out_forward.rb

@@ -62,6 +62,7 @@ module Fluent
     config_param :ack_response_timeout, :time, :default => 190  # 0 means do not wait for ack responses
     # Linux default tcp_syn_retries is 5 (in many environment)
     # 3 + 6 + 12 + 24 + 48 + 96 -> 189 (sec)
+    config_param :dns_round_robin, :bool, :default => false # heartbeat_type 'udp' is not available for this
 
     attr_reader :nodes
 
@@ -92,6 +93,13 @@ module Fluent
                                   else
                                     false
                                   end
+
+      if @dns_round_robin
+        if @heartbeat_type == :udp
+          raise ConfigError, "forward output heartbeat type must be 'tcp' or 'none' to use dns_round_robin option"
+        end
+      end
+
       conf.elements.each {|e|
         next if e.name != "server"
 
@@ -112,7 +120,7 @@ module Fluent
         failure = FailureDetector.new(@heartbeat_interval, @hard_timeout, Time.now.to_i.to_f)
 
         node_conf = NodeConfig.new(name, host, port, weight, standby, failure,
-          @phi_threshold, recover_sample_size, @expire_dns_cache, @phi_failure_detector)
+          @phi_threshold, recover_sample_size, @expire_dns_cache, @phi_failure_detector, @dns_round_robin)
 
         if @heartbeat_type == :none
           @nodes << NoneHeartbeatNode.new(log, node_conf)
@@ -414,7 +422,7 @@ module Fluent
     end
 
     NodeConfig = Struct.new("NodeConfig", :name, :host, :port, :weight, :standby, :failure,
-      :phi_threshold, :recover_sample_size, :expire_dns_cache, :phi_failure_detector)
+      :phi_threshold, :recover_sample_size, :expire_dns_cache, :phi_failure_detector, :dns_round_robin)
 
     class Node
       def initialize(log, conf)
@@ -471,8 +479,10 @@ module Fluent
       end
 
       def resolve_dns!
-        # sample to support dns round robin
-        Socket.getaddrinfo(@host, @port, nil, Socket::SOCK_STREAM).sample[3]
+        addrinfo_list = Socket.getaddrinfo(@host, @port, nil, Socket::SOCK_STREAM)
+        addrinfo = @conf.dns_round_robin ? addrinfo_list.sample : addrinfo_list.first
+        @sockaddr = Socket.pack_sockaddr_in(addrinfo[1], addrinfo[3]) # used by on_heartbeat
+        addrinfo[3]
       end
       private :resolve_dns!
 

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '0.12.13'
+  VERSION = '0.12.14'
 
 end

data/test/config/test_configurable.rb

@@ -722,6 +722,15 @@ module Fluent::Config
         }
       end
 
+      test 'get plugin name when found unknown section' do
+        @conf = e('ROOT', '', {'normal_param' => 'normal', 'secret_param' => 'secret'}, [e('unknown', '', {'normal_param2' => 'normal', 'secret_param2' => 'secret'} )])
+        @example = ConfigurableSpec::Example5.new
+        @example.configure(@conf)
+        @conf.elements.each { |e|
+          assert_equal(['ROOT', nil], e.unused_in)
+        }
+      end
+
       def assert_secret_param(key, value)
         case key
         when 'normal_param', 'normal_param2'

data/test/plugin/test_out_forward.rb

@@ -64,6 +64,20 @@ class ForwardOutputTest < Test::Unit::TestCase
     assert_equal :none, d.instance.heartbeat_type
   end
 
+  def test_configure_dns_round_robin
+    assert_raise(Fluent::ConfigError) do
+      create_driver(CONFIG + "\nheartbeat_type udp\ndns_round_robin true")
+    end
+
+    d = create_driver(CONFIG + "\nheartbeat_type tcp\ndns_round_robin true")
+    assert_equal true, d.instance.dns_round_robin
+    assert_equal true, d.instance.nodes.first.conf.dns_round_robin
+
+    d = create_driver(CONFIG + "\nheartbeat_type none\ndns_round_robin true")
+    assert_equal true, d.instance.dns_round_robin
+    assert_equal true, d.instance.nodes.first.conf.dns_round_robin
+  end
+
   def test_phi_failure_detector
     d = create_driver(CONFIG + %[phi_failure_detector false \n phi_threshold 0])
     node = d.instance.nodes.first

data/test/test_parser.rb

@@ -105,6 +105,18 @@ module ParserTest
       internal_test_case(TextParser::RegexpParser.new(Regexp.new(%q!^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$!), 'time_format'=>"%d/%b/%Y:%H:%M:%S %z", 'types'=>'user|string,date|time|%d/%b/%Y:%H:%M:%S %z,flag|bool,path|array,code|float,size|integer', 'types_label_delimiter'=>'|'))
     end
 
+    def test_parse_with_time_key
+      parser = TextParser::RegexpParser.new(/(?<logtime>[^\]]*)/)
+      parser.configure(
+        'time_format'=>"%Y-%m-%d %H:%M:%S %z",
+        'time_key'=>'logtime',
+      )
+      text = '2013-02-28 12:00:00 +0900'
+      parser.parse(text) do |time, record|
+        assert_equal Time.parse(text).to_i, time
+      end
+    end
+
     def test_parse_without_time
       time_at_start = Time.now.to_i
       text = "tagomori_satoshi tagomoris 34\n"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 0.12.13
+  version: 0.12.14
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-07-09 00:00:00.000000000 Z
+date: 2015-07-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -463,7 +463,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.2.3
 signing_key: 
 specification_version: 4
 summary: Fluentd event collector