fluentd 0.14.1 → 0.14.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3eae91c70d2df47e3cd6aeb7b1e80d44fd8ee41b
-  data.tar.gz: 926661743f2177e7d116c1c7238d802b57fee63d
+  metadata.gz: 7907d31d5c45acece1bda509c0f893ce21196121
+  data.tar.gz: 37e214f67d98969b7cf77448600983411317b37b
 SHA512:
-  metadata.gz: 3584a4e41dc1188a43c0f5b75ea8505662bae178bdc796dd3a2cc8d28395000c18cb9b13c55cc34c113bf5d704c571634ba0ea1709325cc255e88da2a0150e74
-  data.tar.gz: 2f3836aea63fbaa48d761f2bce898f2925e130228c0c1b40e7115f2ccb0ce424aa7cd6200c0381b279f4358f3eef2e3d90e0f50ee0114441aaecc15250f1e2bd
+  metadata.gz: 37c4a7f40cc99f86f68b8418eff89443c7a9e45303ee2e873964992798df1639a35a67f8ea749e473408206d39770e51a7517442ba3db72432d7278dcc18c207
+  data.tar.gz: 38919ba08777c7b324026843e90b14a29acc14f4f8a8a069cc7acbeaa655a46a3eb03adf8ff115f52f76eb1e1a2bba4ce57b2c182719772d59ca3b13caad2bd7

data/ChangeLog

@@ -1,5 +1,114 @@
 # v0.14
 
+## Release 0.14.1 - 2016/06/30
+
+### New features / Enhancement
+
+* Fix to split large event stream into some/many chunks in buffers
+  https://github.com/fluent/fluentd/pull/1062
+* Add parser and filter support in compat_parameters plugin helper
+  https://github.com/fluent/fluentd/pull/1079
+* Add a RPC call to flush buffers and stop workers
+  https://github.com/fluent/fluentd/pull/1134
+* Update forward protocol to pass the number of events in a payload
+  https://github.com/fluent/fluentd/pull/1137
+* Improve performance of some built-in formatter plugins
+  https://github.com/fluent/fluentd/pull/1082
+  https://github.com/fluent/fluentd/pull/1086
+* Migrate some built-in plugins and plugin util modules into v0.14 API
+  https://github.com/fluent/fluentd/pull/1058
+  https://github.com/fluent/fluentd/pull/1061
+  https://github.com/fluent/fluentd/pull/1076
+  https://github.com/fluent/fluentd/pull/1078
+  https://github.com/fluent/fluentd/pull/1081
+  https://github.com/fluent/fluentd/pull/1083
+  https://github.com/fluent/fluentd/pull/1091
+* Register RegExpParser as a parser plugin explicitly
+  https://github.com/fluent/fluentd/pull/1094
+* Add delimiter option to CSV parser
+  https://github.com/fluent/fluentd/pull/1108
+* Add an option to receive longer udp syslog messages
+  https://github.com/fluent/fluentd/pull/1127
+* Add a option to suspend internal status in dummy plugin
+  https://github.com/fluent/fluentd/pull/900
+* Add a feature to capture filtered records in test driver for Filter plugins
+  https://github.com/fluent/fluentd/pull/1077
+* Add some utility methods to plugin test drivers
+  https://github.com/fluent/fluentd/pull/1114
+
+### Bug fixes
+
+* Fix bug to read non buffer-chunk files as buffer chunks when Fluentd resumed
+  https://github.com/fluent/fluentd/pull/1124
+* Fix bug not to load Filter plugins which are specified in configurations
+  https://github.com/fluent/fluentd/pull/1118
+* Fix bug to ignore `-p` option to specify directories of plugins
+  https://github.com/fluent/fluentd/pull/1133
+* Fix bug to overwrite base class configuration section definitions by subclasses
+  https://github.com/fluent/fluentd/pull/1119
+* Fix to stop Fluentd worker process by Ctrl-C when --no-supervisor specified
+  https://github.com/fluent/fluentd/pull/1089
+* Fix regression about RPC call to reload configuration
+  https://github.com/fluent/fluentd/pull/1093
+* Specify to ensure Oj JSON parser to use strict mode
+  https://github.com/fluent/fluentd/pull/1147
+* Fix unexisting path handling in Windows environment
+  https://github.com/fluent/fluentd/pull/1104
+
+## Release 0.14.1 - 2016/06/30
+
+### New features / Enhancement
+
+* Add plugin helpers for parsers and formatters
+  https://github.com/fluent/fluentd/pull/1023
+* Extract some mixins into compat modules
+  https://github.com/fluent/fluentd/pull/1044
+  https://github.com/fluent/fluentd/pull/1052
+* Add utility methods for tests and test drivers
+  https://github.com/fluent/fluentd/pull/1047
+* Migrate some built-in plugins to v0.14 APIs
+  https://github.com/fluent/fluentd/pull/1049
+  https://github.com/fluent/fluentd/pull/1057
+  https://github.com/fluent/fluentd/pull/1060
+  https://github.com/fluent/fluentd/pull/1064
+* Add support of X-Forwarded-For header in in_http plugin
+  https://github.com/fluent/fluentd/pull/1051
+* Warn not to create too many staged chunks at configure
+  https://github.com/fluent/fluentd/pull/1054
+* Add a plugin helper to inject tag/time/hostname
+  https://github.com/fluent/fluentd/pull/1063
+
+### Bug fixes
+
+* Fix in_monitor_agent for v0.14 plugins
+  https://github.com/fluent/fluentd/pull/1003
+* Fix to call #format_stream of plugins themselves when RecordFilter mixin included
+  https://github.com/fluent/fluentd/pull/1005
+* Fix shutdown sequence to wait force flush
+  https://github.com/fluent/fluentd/pull/1009
+* Fix a deadlock bug in shutdown
+  https://github.com/fluent/fluentd/pull/1010
+* Fix to require DetachProcessMixin in default for compat plugins
+  https://github.com/fluent/fluentd/pull/1014
+* Fix to overwrite configure_proxy name only for root sections for debugging
+  https://github.com/fluent/fluentd/pull/1015
+* Rename file for in_unix plugin
+  https://github.com/fluent/fluentd/pull/1017
+* Fix a bug not to create pid file when daemonized
+  https://github.com/fluent/fluentd/pull/1021
+* Fix wrong DEFAULT_PLUGIN_PATH
+  https://github.com/fluent/fluentd/pull/1028
+* Fix a bug not to use primary plugin type for secondary in default
+  https://github.com/fluent/fluentd/pull/1032
+* Add --run-worker option to distinguish to run as worker without supervisor
+  https://github.com/fluent/fluentd/pull/1033
+* Fix regression of fluent-debug command
+  https://github.com/fluent/fluentd/pull/1046
+* Update windows-pr dependency to 1.2.5
+  https://github.com/fluent/fluentd/pull/1065
+* Fix supervisor to pass RUBYOPT to worker processes
+  https://github.com/fluent/fluentd/pull/1066
+
 ## Release 0.14.0 - 2016/05/25
 
 ### New features / Enhancement
@@ -81,7 +190,7 @@ This list includes changes of 0.14.0.pre.1 and release candidates.
 
 * Fix dependencies between files
   https://github.com/fluent/fluentd/pull/799
-  https://github.com/fluent/fluentd/pull/808  
+  https://github.com/fluent/fluentd/pull/808
   https://github.com/fluent/fluentd/pull/823
   https://github.com/fluent/fluentd/pull/824
   https://github.com/fluent/fluentd/pull/825

data/Rakefile

@@ -13,7 +13,11 @@ Rake::TestTask.new(:base_test) do |t|
   #  $ bundle exec rake base_test TEST=test/test_specified_path.rb
   #  $ bundle exec rake base_test TEST=test/test_*.rb
   t.libs << "test"
-  t.test_files = Dir["test/**/test_*.rb"].sort
+  t.test_files = if ENV["WIN_RAPID"]
+                   ["test/test_event.rb", "test/test_supervisor.rb", "test/plugin_helper/test_event_loop.rb"]
+                 else
+                   Dir["test/**/test_*.rb"].sort
+                 end
   t.verbose = true
   t.warning = true
   t.ruby_opts = ["-Eascii-8bit:ascii-8bit"]

data/appveyor.yml

@@ -11,7 +11,7 @@ install:
   - bundle install
 build: off
 test_script:
-  - bundle exec rake test TESTOPTS=-v
+  - bundle exec rake test
 
 branches:
   only:
@@ -22,14 +22,20 @@ branches:
 
 environment:
   matrix:
+    - ruby_version: "23-x64"
+      devkit: C:\Ruby23-x64\DevKit
+    - ruby_version: "23"
+      devkit: C:\Ruby23\DevKit
     - ruby_version: "22-x64"
       devkit: C:\Ruby23-x64\DevKit
     - ruby_version: "22"
       devkit: C:\Ruby23\DevKit
+      WIN_RAPID: true
     - ruby_version: "21-x64"
       devkit: C:\Ruby23-x64\DevKit
     - ruby_version: "21"
       devkit: C:\Ruby23\DevKit
+      WIN_RAPID: true
 matrix:
   allow_failures:
     - ruby_version: "21"

data/example/in_forward.conf

@@ -1,3 +1,7 @@
+<system>
+  rpc_endpoint 0.0.0.0:24444
+</system>
+
 <source>
   @type forward
 </source>

data/lib/fluent/compat/exec_util.rb

@@ -0,0 +1,129 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'msgpack'
+require 'yajl'
+
+require 'fluent/engine'
+require 'fluent/plugin'
+require 'fluent/parser'
+
+module Fluent
+  module Compat
+    module ExecUtil
+      SUPPORTED_FORMAT = {
+        'tsv' => :tsv,
+        'json' => :json,
+        'msgpack' => :msgpack,
+      }
+
+      class Parser
+        def initialize(on_message)
+          @on_message = on_message
+        end
+      end
+
+      class TextParserWrapperParser < Parser
+        def initialize(conf, on_message)
+          @parser = Plugin.new_parser(conf['format'])
+          @parser.configure(conf)
+          super(on_message)
+        end
+
+        def call(io)
+          io.each_line(&method(:each_line))
+        end
+
+        def each_line(line)
+          line.chomp!
+          @parser.parse(line) { |time, record|
+            @on_message.call(record, time)
+          }
+        end
+      end
+
+      class TSVParser < Parser
+        def initialize(keys, on_message)
+          @keys = keys
+          super(on_message)
+        end
+
+        def call(io)
+          io.each_line(&method(:each_line))
+        end
+
+        def each_line(line)
+          line.chomp!
+          vals = line.split("\t")
+
+          record = Hash[@keys.zip(vals)]
+
+          @on_message.call(record)
+        end
+      end
+
+      class JSONParser < Parser
+        def call(io)
+          y = Yajl::Parser.new
+          y.on_parse_complete = @on_message
+          y.parse(io)
+        end
+      end
+
+      class MessagePackParser < Parser
+        def call(io)
+          @u = Fluent::Engine.msgpack_factory.unpacker(io)
+          begin
+            @u.each(&@on_message)
+          rescue EOFError
+          end
+        end
+      end
+
+      class Formatter
+      end
+
+      class TSVFormatter < Formatter
+        def initialize(in_keys)
+          @in_keys = in_keys
+          super()
+        end
+
+        def call(record, out)
+          last = @in_keys.length-1
+          for i in 0..last
+            key = @in_keys[i]
+            out << record[key].to_s
+            out << "\t" if i != last
+          end
+          out << "\n"
+        end
+      end
+
+      class JSONFormatter < Formatter
+        def call(record, out)
+          out << Yajl.dump(record) << "\n"
+        end
+      end
+
+      class MessagePackFormatter < Formatter
+        def call(record, out)
+          record.to_msgpack(out)
+        end
+      end
+    end
+  end
+end

data/lib/fluent/compat/file_util.rb

@@ -0,0 +1,54 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module Fluent
+  module Compat
+    module FileUtil
+      # Check file is writable if file exists
+      # Check directory is writable if file does not exist
+      #
+      # @param [String] path File path
+      # @return [Boolean] file is writable or not
+      def writable?(path)
+        return false if File.directory?(path)
+        return File.writable?(path) if File.exist?(path)
+
+        dirname = File.dirname(path)
+        return false if !File.directory?(dirname)
+        File.writable?(dirname)
+      end
+      module_function :writable?
+
+      # Check file is writable in conjunction wtih mkdir_p(dirname(path))
+      #
+      # @param [String] path File path
+      # @return [Boolean] file writable or not
+      def writable_p?(path)
+        return false if File.directory?(path)
+        return File.writable?(path) if File.exist?(path)
+
+        dirname = File.dirname(path)
+        until File.exist?(dirname)
+          dirname = File.dirname(dirname)
+        end
+
+        return false if !File.directory?(dirname)
+        File.writable?(dirname)
+      end
+      module_function :writable_p?
+    end
+  end
+end

data/lib/fluent/compat/filter.rb

@@ -17,25 +17,43 @@
 require 'fluent/plugin'
 require 'fluent/plugin/filter'
 require 'fluent/compat/call_super_mixin'
+require 'fluent/compat/formatter_utils'
+require 'fluent/compat/parser_utils'
 
 module Fluent
   module Compat
     class Filter < Fluent::Plugin::Filter
       # TODO: warn when deprecated
 
+      helpers :inject
+
       def initialize
         super
         unless self.class.ancestors.include?(Fluent::Compat::CallSuperMixin)
-          self.class.module_eval do
-            prepend Fluent::Compat::CallSuperMixin
-          end
+          self.class.prepend Fluent::Compat::CallSuperMixin
         end
       end
 
+      def configure(conf)
+        ParserUtils.convert_parser_conf(conf)
+        FormatterUtils.convert_formatter_conf(conf)
+
+        super
+      end
+
       # These definitions are to get instance methods of superclass of 3rd party plugins
       # to make it sure to call super
       def start
         super
+
+        if instance_variable_defined?(:@formatter) && @inject_config
+          unless @formatter.class.ancestors.include?(Fluent::Compat::HandleTagAndTimeMixin)
+            if @formatter.respond_to?(:owner) && !@formatter.owner
+              @formatter.owner = self
+              @formatter.singleton_class.prepend FormatterUtils::InjectMixin
+            end
+          end
+        end
       end
 
       def before_shutdown

data/lib/fluent/compat/formatter.rb

@@ -16,6 +16,8 @@
 
 require 'fluent/plugin'
 require 'fluent/plugin/formatter'
+require 'fluent/compat/handle_tag_and_time_mixin'
+require 'fluent/compat/structured_format_mixin'
 
 require 'fluent/plugin/formatter_out_file'
 require 'fluent/plugin/formatter_stdout'
@@ -66,8 +68,8 @@ module Fluent
         formatter
       end
 
-      HandleTagAndTimeMixin = Fluent::Plugin::Formatter::HandleTagAndTimeMixin
-      StructuredFormatMixin = Fluent::Plugin::Formatter::StructuredFormatMixin
+      HandleTagAndTimeMixin = Fluent::Compat::HandleTagAndTimeMixin
+      StructuredFormatMixin = Fluent::Compat::StructuredFormatMixin
 
       class ProcWrappedFormatter < Fluent::Plugin::ProcWrappedFormatter
         # TODO: warn when deprecated

data/lib/fluent/compat/formatter_utils.rb

@@ -0,0 +1,85 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin_helper/compat_parameters'
+
+module Fluent
+  module Compat
+    module FormatterUtils
+      INJECT_PARAMS = Fluent::PluginHelper::CompatParameters::INJECT_PARAMS
+      FORMATTER_PARAMS = Fluent::PluginHelper::CompatParameters::FORMATTER_PARAMS
+
+      module InjectMixin
+        def format(tag, time, record)
+          r = owner.inject_values_to_record(tag, time, record)
+          super(tag, time, r)
+        end
+      end
+
+      def self.convert_formatter_conf(conf)
+        return if conf.elements(name: 'inject').first || conf.elements(name: 'format').first
+
+        inject_params = {}
+        INJECT_PARAMS.each do |older, newer|
+          next unless newer
+          if conf.has_key?(older)
+            inject_params[newer] = conf[older]
+          end
+        end
+
+        if conf.has_key?('include_time_key') && Fluent::Config.bool_value(conf['include_time_key'])
+          inject_params['time_key'] ||= 'time'
+          inject_params['time_type'] ||= 'string'
+        end
+        if conf.has_key?('time_as_epoch') && Fluent::Config.bool_value(conf['time_as_epoch'])
+          inject_params['time_type'] = 'unixtime'
+        end
+        if conf.has_key?('localtime') || conf.has_key?('utc')
+          if conf.has_key?('localtime') && conf.has_key?('utc')
+            raise Fluent::ConfigError, "both of utc and localtime are specified, use only one of them"
+          elsif conf.has_key?('localtime')
+            inject_params['localtime'] = Fluent::Config.bool_value(conf['localtime'])
+          elsif conf.has_key?('utc')
+            inject_params['localtime'] = !(Fluent::Config.bool_value(conf['utc']))
+            # Specifying "localtime false" means using UTC in TimeFormatter
+            # And specifying "utc" is different from specifying "timezone +0000"(it's not always UTC).
+            # There are difference between "Z" and "+0000" in timezone formatting.
+            # TODO: add kwargs to TimeFormatter to specify "using localtime", "using UTC" or "using specified timezone" in more explicit way
+          end
+        end
+
+        if conf.has_key?('include_tag_key') && Fluent::Config.bool_value(conf['include_tag_key'])
+          inject_params['tag_key'] ||= 'tag'
+        end
+
+        unless inject_params.empty?
+          conf.elements << Fluent::Config::Element.new('inject', '', inject_params, [])
+        end
+
+        formatter_params = {}
+        FORMATTER_PARAMS.each do |older, newer|
+          next unless newer
+          if conf.has_key?(older)
+            formatter_params[newer] = conf[older]
+          end
+        end
+        unless formatter_params.empty?
+          conf.elements << Fluent::Config::Element.new('format', '', formatter_params, [])
+        end
+      end
+    end
+  end
+end