fluentd 0.14.8 → 0.14.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 830d1ce6df9602f508f6f28f57197d5ed8fc3979
-  data.tar.gz: 0f0f897b0a13526974df16bbfa55abde73929c68
+  metadata.gz: 22cf802055c4ce65d338d4ffdc223dfbd737d115
+  data.tar.gz: c7458bd96af9c44f679a501fbea6507c19d5a9e1
 SHA512:
-  metadata.gz: 6f3675e97aac37d204dbe2501ff6ec1629d0aec10d2ed534b6af4ebd5160153e459e9fb9707203f9b09142cf068cbd65272f57d4be214a590fe09109503270a9
-  data.tar.gz: fcdc64eed9af5917723efd395ac292e9200fffb0276c5c6471a8a86c19f513ec9515abd9f3d66650232ac4f60e51bf4c4627a2f6376ef3c1032a4903c804264b
+  metadata.gz: 72db4f8537104b07b0f328e5cdb581601602b66ba8be885cb8288d9e96509e43524986c12d703556bfcb09c9a6a74f9be1b66144b8177c097882768798cac597
+  data.tar.gz: 21aae0faa3e878c709b5acac57d9920092e99542d09ee948f0cc84b8efb1d933914c9f84824624dff7dfe1177eaaa0918c75673b888112281e6887e928711ee6

data/.gitignore

@@ -24,3 +24,5 @@ Gemfile.local
 *.swp
 coverage/*
 .vagrant/
+cov-int/
+cov-fluentd.tar.gz

data/CONTRIBUTING.md

@@ -7,8 +7,13 @@ We'd love your contribution. Here are the guidelines!
 RESOURCES of [Official site](http://www.fluentd.org/) and [Fluentd documentation](http://docs.fluentd.org/) may help you.
 
 If you have further questions about Fluentd and plugins, please direct these to [Mailing List](https://groups.google.com/forum/#!forum/fluentd).
+Don't use Github issue for asking questions. Here are examples:
 
-Don't use Github issue for asking questions.
+- I installed xxx plugin but it doesn't work. Why?
+- Fluentd starts but logs are not sent to xxx. Am I wrong?
+- I want to do xxx. How to realize it with plugins?
+
+We may close such questions to keep clear repository for developers and users.
 Github issue is mainly for submitting a bug report or feature request. See below.
 
 ## Found a bug?

data/ChangeLog

@@ -1,5 +1,43 @@
 # v0.14
 
+## Release v0.14.9 - 2016/11/15
+
+### New features / Enhancement
+
+* filter_parser: Port fluent-plugin-parser into built-in plugin
+  https://github.com/fluent/fluentd/pull/1191
+* parser/formatter plugin helpers with default @type in plugin side
+  https://github.com/fluent/fluentd/pull/1267
+* parser: Reconstruct Parser related classes
+  https://github.com/fluent/fluentd/pull/1286
+* filter_record_transformer: Remove old behaviours
+  https://github.com/fluent/fluentd/pull/1311
+* Migrate some built-in plugins into v0.14 API
+  https://github.com/fluent/fluentd/pull/1257 (out_file)
+  https://github.com/fluent/fluentd/pull/1297 (out_exec, out_exec_filter)
+  https://github.com/fluent/fluentd/pull/1306 (in_forward, out_forward)
+  https://github.com/fluent/fluentd/pull/1308 (in_http)
+* test: Improve test drivers
+  https://github.com/fluent/fluentd/pull/1302
+  https://github.com/fluent/fluentd/pull/1305
+
+### Bug fixes
+
+* log: Avoid name conflict between Fluent::Logger
+  https://github.com/fluent/fluentd/pull/1274
+* fluent-cat: Fix fluent-cat command to send sub-second precision time
+  https://github.com/fluent/fluentd/pull/1277
+* config: Fix a bug not to overwrite default value with nil
+  https://github.com/fluent/fluentd/pull/1296
+* output: Fix timezone for compat timesliced output plugins
+  https://github.com/fluent/fluentd/pull/1307
+* out_forward: fix not to raise error when out_forward is initialized as secondary
+  https://github.com/fluent/fluentd/pull/1313
+* output: Event router for secondary output
+  https://github.com/fluent/fluentd/pull/1283
+* test: fix to return the block value as expected by many rubyists
+  https://github.com/fluent/fluentd/pull/1284
+
 ## Release v0.14.8 - 2016/10/13
 
 ### Bug fixes

data/Rakefile

@@ -24,6 +24,9 @@ end
 
 desc 'Run test_unit based test'
 Rake::TestTask.new(:base_test) do |t|
+  # To run test with dumping all test case names (to find never ending test case)
+  #  $ bundle exec rake test TESTOPTS=-v
+  #
   # To run test for only one file (or file path pattern)
   #  $ bundle exec rake base_test TEST=test/test_specified_path.rb
   #  $ bundle exec rake base_test TEST=test/test_*.rb
@@ -50,4 +53,22 @@ task :coverage do |t|
   Rake::Task["test"].invoke
 end
 
+desc 'Build Coverity tarball & upload it'
+task :coverity do
+  # https://scan.coverity.com/projects/fluentd?tab=overview
+  # See "View Defects" after sign-in.
+  #
+  # Setup steps:
+  # 1. get coverity build tool and set PATH to bin/: https://scan.coverity.com/download
+  # 2. set environment variables:
+  #    * $COVERITY_USER (your email address)
+  #    * $COVERITY_TOKEN (token for Fluentd project: https://scan.coverity.com/projects/fluentd?tab=project_settings)
+  sh "cov-build --dir cov-int --no-command --fs-capture-search ./"
+  sh "tar czf cov-fluentd.tar.gz cov-int"
+  user = ENV['COVERITY_USER']
+  token = ENV['COVERITY_TOKEN']
+  sh "curl --form token=#{token} --form email=#{user} --form file=@cov-fluentd.tar.gz --form version=\"Master\" --form description=\"GIT Master\" https://scan.coverity.com/builds?project=Fluentd"
+  FileUtils.rm_rf(['./cov-int', 'cov-fluentd.tar.gz'])
+end
+
 task default: [:test, :build]

data/example/out_exec_filter.conf

@@ -0,0 +1,42 @@
+<source>
+  @type dummy
+  @label @exec
+  tag exec_input
+  rate 10
+  auto_increment_key num
+  dummy {"data":"mydata"}
+</source>
+
+<label @exec>
+  <match exec_input>
+    @type exec_filter
+    @label @stdout
+    tag result
+    command ruby -e 'STDOUT.sync = true; proc = ->(){line = STDIN.readline.chomp; puts line + "\t" + Process.pid.to_s}; 1000.times{ proc.call }'
+    num_children 3
+    child_respawn -1
+    <inject>
+      time_key time
+      time_type float
+    </inject>
+    <format>
+      @type tsv
+      keys data, num, time
+    </format>
+    <parse>
+      @type tsv
+      keys data, num, time, pid
+    </parse>
+    <extract>
+      time_key time
+      time_type float
+    </extract>
+  </match>
+</label>
+
+<label @stdout>
+  <match result>
+    @type stdout
+  </match>
+</label>
+

data/lib/fluent/agent.rb

@@ -123,7 +123,7 @@ module Fluent
       log.info "adding match#{@context.nil? ? '' : " in #{@context}"}", pattern: pattern, type: type
 
       output = Plugin.new_output(type)
-      output.router = @event_router if output.respond_to?(:router=)
+      output.context_router = @event_router
       output.configure(conf)
       @outputs << output
       if output.respond_to?(:outputs) && output.respond_to?(:multi_output?) && output.multi_output?
@@ -144,7 +144,7 @@ module Fluent
       log.info "adding filter#{@context.nil? ? '' : " in #{@context}"}", pattern: pattern, type: type
 
       filter = Plugin.new_filter(type)
-      filter.router = @event_router
+      filter.context_router = @event_router
       filter.configure(conf)
       @filters << filter
       @event_router.add_rule(pattern, filter)

data/lib/fluent/command/binlog_reader.rb

@@ -164,7 +164,7 @@ module BinlogReaderCommand
     def call
       @formatter = lookup_formatter(@options[:format], @options[:config_params])
 
-      File.open(@path, 'r') do |io|
+      File.open(@path, 'rb') do |io|
         i = 1
         Fluent::MessagePackFactory.unpacker(io).each do |(time, record)|
           print @formatter.format(@path, time, record) # path is used for tag

data/lib/fluent/command/cat.rb

@@ -16,6 +16,8 @@
 
 require 'optparse'
 require 'fluent/env'
+require 'fluent/time'
+require 'fluent/msgpack_factory'
 
 op = OptionParser.new
 
@@ -143,7 +145,7 @@ class Writer
       raise ArgumentError, "Input must be a map (got #{record.class})"
     end
 
-    entry = [Time.now.to_i, record]
+    entry = [Fluent::EventTime.now, record]
     synchronize {
       unless write_impl([entry])
         # write failed
@@ -200,7 +202,8 @@ class Writer
     end
 
     begin
-      socket.write [@tag, array].to_msgpack
+      packer = Fluent::MessagePackFactory.packer
+      socket.write packer.pack([@tag, array])
       socket.flush
     rescue
       $stderr.puts "write failed: #{$!}"

data/lib/fluent/compat/output.rb

@@ -617,17 +617,16 @@ module Fluent
           end
 
           if conf['timezone']
-            @timezone = conf['timezone']
-            Fluent::Timezone.validate!(@timezone)
+            Fluent::Timezone.validate!(conf['timezone'])
           elsif conf['utc']
-            @timezone = "+0000"
-            @localtime = false
+            conf['timezone'] = "+0000"
+            conf['localtime'] = "false"
           elsif conf['localtime']
-            @timezone = Time.now.strftime('%z')
-            @localtime = true
+            conf['timezone'] = Time.now.strftime('%z')
+            conf['localtime'] = "true"
           else
-            @timezone = "+0000" # v0.12 assumes UTC without any configuration
-            @localtime = false
+            conf['timezone'] = "+0000" # v0.12 assumes UTC without any configuration
+            conf['localtime'] = "false"
           end
 
           @_timekey = case conf['time_slice_format']

data/lib/fluent/compat/parser.rb

@@ -18,6 +18,9 @@ require 'fluent/plugin'
 require 'fluent/plugin/parser'
 require 'fluent/mixin'
 
+require 'fluent/config'
+require 'fluent/compat/type_converter'
+
 require 'fluent/plugin/parser_regexp'
 require 'fluent/plugin/parser_json'
 require 'fluent/plugin/parser_tsv'
@@ -56,13 +59,14 @@ module Fluent
         format = conf['format']
 
         @parser = TextParser.lookup(format)
-        if ! @estimate_current_event.nil? && @parser.respond_to?(:'estimate_current_event=')
-          @parser.estimate_current_event = @estimate_current_event
-        end
 
         if @parser.respond_to?(:configure)
           @parser.configure(conf)
         end
+        if !@estimate_current_event.nil? && @parser.respond_to?(:'estimate_current_event=')
+          # external code sets parser.estimate_current_event = false
+          @parser.estimate_current_event = @estimate_current_event
+        end
 
         return true
       end
@@ -116,23 +120,57 @@ module Fluent
         end
       end
 
-      class TimeParser < Fluent::Plugin::Parser::TimeParser
+      module TypeConverterCompatParameters
+        def convert_type_converter_parameters!(conf)
+          if conf["types"]
+            delimiter = conf["types_delimiter"] || ','
+            label_delimiter = conf["types_label_delimiter"] || ':'
+            types = {}
+            conf['types'].split(delimiter).each do |pair|
+              key, value = pair.split(label_delimiter, 2)
+              if value.start_with?("time#{label_delimiter}")
+                value = value.split(label_delimiter, 2).join(':')
+              elsif value.start_with?("array#{label_delimiter}")
+                value = value.split(label_delimiter, 2).join(':')
+              end
+              types[key] = value
+            end
+            conf["types"] = JSON.dump(types)
+          end
+        end
+      end
+
+      class TimeParser < Fluent::TimeParser
         # TODO: warn when deprecated
       end
 
       class RegexpParser < Fluent::Plugin::RegexpParser
+        include TypeConverterCompatParameters
+
         # TODO: warn when deprecated
         def initialize(regexp, conf = {})
           super()
 
+          @stored_regexp = regexp
+          @manually_configured = false
           unless conf.empty?
-            unless conf.is_a?(Config::Element)
-              conf = Config::Element.new('default_regexp_conf', '', conf, [])
-            end
-            configure(conf)
+            conf_init = if conf.is_a?(Fluent::Config::Element)
+                          conf
+                        else
+                          Fluent::Config::Element.new('parse', '', conf, [])
+                        end
+            self.configure(conf_init)
+            @manually_configured = true
           end
+        end
+
+        def configure(conf)
+          return if @manually_configured # not to run twice
+
+          conf['expression'] ||= @stored_regexp.source
+          convert_type_converter_parameters!(conf)
 
-          @regexp = regexp
+          super
         end
 
         def patterns
@@ -140,24 +178,114 @@ module Fluent
         end
       end
 
-      class ValuesParser < Fluent::Plugin::ValuesParser
-        # TODO: warn when deprecated
+      class ValuesParser < Parser
+        include Fluent::Compat::TypeConverter
+
+        config_param :keys, :array, default: []
+        config_param :time_key, :string, default: nil
+        config_param :null_value_pattern, :string, default: nil
+        config_param :null_empty_string, :bool, default: false
+
+        def configure(conf)
+          super
+
+          if @time_key && !@keys.include?(@time_key) && @estimate_current_event
+            raise Fluent::ConfigError, "time_key (#{@time_key.inspect}) is not included in keys (#{@keys.inspect})"
+          end
+
+          if @time_format && !@time_key
+            raise Fluent::ConfigError, "time_format parameter is ignored because time_key parameter is not set. at #{conf.inspect}"
+          end
+
+          @time_parser = time_parser_create
+
+          if @null_value_pattern
+            @null_value_pattern = Regexp.new(@null_value_pattern)
+          end
+
+          @mutex = Mutex.new
+        end
+
+        def values_map(values)
+          record = Hash[keys.zip(values.map { |value| convert_value_to_nil(value) })]
+
+          if @time_key
+            value = @keep_time_key ? record[@time_key] : record.delete(@time_key)
+            time = if value.nil?
+                     if @estimate_current_event
+                       Fluent::EventTime.now
+                     else
+                       nil
+                     end
+                   else
+                     @mutex.synchronize { @time_parser.parse(value) }
+                   end
+          elsif @estimate_current_event
+            time = Fluent::EventTime.now
+          else
+            time = nil
+          end
+
+          convert_field_type!(record) if @type_converters
+
+          return time, record
+        end
+
+        private
+
+        def convert_field_type!(record)
+          @type_converters.each_key { |key|
+            if value = record[key]
+              record[key] = convert_type(key, value)
+            end
+          }
+        end
+
+        def convert_value_to_nil(value)
+          if value and @null_empty_string
+            value = (value == '') ? nil : value
+          end
+          if value and @null_value_pattern
+            value = ::Fluent::StringUtil.match_regexp(@null_value_pattern, value) ? nil : value
+          end
+          value
+        end
       end
 
       class JSONParser < Fluent::Plugin::JSONParser
+        include TypeConverterCompatParameters
         # TODO: warn when deprecated
+        def configure(conf)
+          convert_type_converter_parameters!(conf)
+          super
+        end
       end
 
       class TSVParser < Fluent::Plugin::TSVParser
+        include TypeConverterCompatParameters
         # TODO: warn when deprecated
+        def configure(conf)
+          convert_type_converter_parameters!(conf)
+          super
+        end
       end
 
       class LabeledTSVParser < Fluent::Plugin::LabeledTSVParser
+        include TypeConverterCompatParameters
         # TODO: warn when deprecated
+        def configure(conf)
+          convert_type_converter_parameters!(conf)
+          super
+        end
       end
 
       class CSVParser < Fluent::Plugin::CSVParser
+        include TypeConverterCompatParameters
         # TODO: warn when deprecated
+        def configure(conf)
+          convert_type_converter_parameters!(conf)
+          super
+        end
       end
 
       class NoneParser < Fluent::Plugin::NoneParser

data/lib/fluent/config/configure_proxy.rb

@@ -123,7 +123,7 @@ module Fluent
                  end
 
         # configured_in MUST be kept
-        merged.configured_in_section = self.configured_in_section
+        merged.configured_in_section = self.configured_in_section || other.configured_in_section
 
         merged.argument = other.argument || self.argument
         merged.params = other.params.merge(self.params)
@@ -177,7 +177,7 @@ module Fluent
                    self.class.new(@name, options)
                  end
 
-        merged.configured_in_section = self.configured_in_section
+        merged.configured_in_section = self.configured_in_section || other.configured_in_section
 
         merged.argument = self.argument || other.argument
         merged.params = other.params.merge(self.params)
@@ -339,15 +339,6 @@ module Fluent
         sub_proxy = ConfigureProxy.new(name, type_lookup: @type_lookup, **kwargs)
         sub_proxy.instance_exec(&block)
 
-        if sub_proxy.init?
-          if sub_proxy.argument && !sub_proxy.defaults.has_key?(sub_proxy.argument.first)
-            raise ArgumentError, "#{name}: init is specified, but default value of argument is missing"
-          end
-          if sub_proxy.params.keys.any?{|param_name| !sub_proxy.defaults.has_key?(param_name)}
-            raise ArgumentError, "#{name}: init is specified, but there're parameters without default values"
-          end
-        end
-
         @params.delete(name)
         @sections[name] = sub_proxy