fluentd 0.14.3 → 0.14.4

data/Rakefile

@@ -7,6 +7,21 @@ require 'rake/clean'
 
 task test: [:base_test]
 
+# 1. update ChangeLog and lib/fluent/version.rb
+# 2. bundle && bundle exec rake build:all
+# 3. release 3 packages built on pkg/ directory
+namespace :build do
+  desc 'Build gems for all platforms'
+  task :all do
+    Bundler.with_clean_env do
+      %w[ruby x86-mingw32 x64-mingw32].each do |name|
+        ENV['GEM_BUILD_FAKE_PLATFORM'] = name
+        Rake::Task["build"].execute
+      end
+    end
+  end
+end
+
 desc 'Run test_unit based test'
 Rake::TestTask.new(:base_test) do |t|
   # To run test for only one file (or file path pattern)

data/appveyor.yml

@@ -28,11 +28,11 @@ environment:
       devkit: C:\Ruby23\DevKit
     - ruby_version: "22-x64"
       devkit: C:\Ruby23-x64\DevKit
+    - ruby_version: "21-x64"
+      devkit: C:\Ruby23-x64\DevKit
     - ruby_version: "22"
       devkit: C:\Ruby23\DevKit
       WIN_RAPID: true
-    - ruby_version: "21-x64"
-      devkit: C:\Ruby23-x64\DevKit
     - ruby_version: "21"
       devkit: C:\Ruby23\DevKit
       WIN_RAPID: true

data/example/multi_filters.conf

@@ -0,0 +1,61 @@
+# This example is to measure optimized filter pipeline performance.
+
+<source>
+  @type dummy
+  tag   test
+  size 1000
+</source>
+
+<filter test>
+  @type grep
+  exclude1 hello .
+</filter>
+
+<filter test>
+  @type grep
+  exclude1 hello .
+</filter>
+
+<filter test>
+  @type grep
+  exclude1 hello .
+</filter>
+
+<filter test>
+  @type grep
+  exclude1 hello .
+</filter>
+
+<filter test>
+  @type grep
+  exclude1 hello .
+</filter>
+
+<filter test>
+  @type grep
+  exclude1 hello .
+</filter>
+
+<filter test>
+  @type grep
+  exclude1 hello .
+</filter>
+
+<filter test>
+  @type grep
+  exclude1 hello .
+</filter>
+
+<filter test>
+  @type grep
+  exclude1 hello .
+</filter>
+
+<filter test>
+  @type grep
+  exclude1 hello .
+</filter>
+
+<match test>
+  @type buffered_null
+</match>

data/fluentd.gemspec

@@ -19,30 +19,33 @@ Gem::Specification.new do |gem|
 
   gem.required_ruby_version = '>= 2.1'
 
-  gem.add_runtime_dependency("msgpack", [">= 0.7.0"])
-  gem.add_runtime_dependency("json", [">= 1.4.3"])
+  gem.add_runtime_dependency("msgpack", [">= 0.7.0", "< 2.0.0"])
   gem.add_runtime_dependency("yajl-ruby", ["~> 1.0"])
-  gem.add_runtime_dependency("cool.io", [">= 1.4.3", "< 2.0.0"])
-  gem.add_runtime_dependency("serverengine", [">= 1.6.4", "< 2.0.0"])
+  gem.add_runtime_dependency("cool.io", ["~> 1.4.5"])
+  gem.add_runtime_dependency("serverengine", ["~> 2.0"])
   gem.add_runtime_dependency("http_parser.rb", [">= 0.5.1", "< 0.7.0"])
   gem.add_runtime_dependency("sigdump", ["~> 0.2.2"])
-  gem.add_runtime_dependency("tzinfo", [">= 1.0.0"])
-  gem.add_runtime_dependency("tzinfo-data", [">= 1.0.0"])
-  if /mswin|mingw/ =~ RUBY_PLATFORM
+  gem.add_runtime_dependency("tzinfo", ["~> 1.0"])
+  gem.add_runtime_dependency("tzinfo-data", ["~> 1.0"])
+  gem.add_runtime_dependency("strptime", ["~> 0.1.7"])
+
+  # build gem for a certain platform. see also Rakefile
+  fake_platform = ENV['GEM_BUILD_FAKE_PLATFORM'].to_s
+  gem.platform = fake_platform unless fake_platform.empty?
+  if /mswin|mingw/ =~ fake_platform || (/mswin|mingw/ =~ RUBY_PLATFORM && fake_platform.empty?)
     gem.add_runtime_dependency("win32-service", ["~> 0.8.3"])
     gem.add_runtime_dependency("win32-ipc", ["~> 0.6.1"])
     gem.add_runtime_dependency("win32-event", ["~> 0.6.1"])
     gem.add_runtime_dependency("windows-pr", ["~> 1.2.5"])
   end
-  gem.add_runtime_dependency("strptime", [">= 0.1.7"])
 
-  gem.add_development_dependency("rake", [">= 0.9.2"])
-  gem.add_development_dependency("flexmock", ["~> 2.0.5"])
-  gem.add_development_dependency("parallel_tests", [">= 0.15.3"])
-  gem.add_development_dependency("simplecov", ["~> 0.6.4"])
-  gem.add_development_dependency("rr", [">= 1.0.0"])
-  gem.add_development_dependency("timecop", [">= 0.3.0"])
-  gem.add_development_dependency("test-unit", ["~> 3.1.4"])
-  gem.add_development_dependency("test-unit-rr", ["~> 1.0.3"])
+  gem.add_development_dependency("rake", ["~> 11.0"])
+  gem.add_development_dependency("flexmock", ["~> 2.0"])
+  gem.add_development_dependency("parallel_tests", ["~> 0.15.3"])
+  gem.add_development_dependency("simplecov", ["~> 0.7"])
+  gem.add_development_dependency("rr", ["~> 1.0"])
+  gem.add_development_dependency("timecop", ["~> 0.3"])
+  gem.add_development_dependency("test-unit", ["~> 3.2"])
+  gem.add_development_dependency("test-unit-rr", ["~> 1.0"])
   gem.add_development_dependency("oj", ["~> 2.14"])
 end

data/lib/fluent/agent.rb

@@ -134,6 +134,9 @@ module Fluent
       output.router = @event_router if output.respond_to?(:router=)
       output.configure(conf)
       @outputs << output
+      if output.respond_to?(:outputs) && (output.is_a?(Fluent::Plugin::MultiOutput) || output.is_a?(Fluent::MultiOutput))
+        @outputs.push(*output.outputs)
+      end
       @event_router.add_rule(pattern, output)
 
       output

data/lib/fluent/command/debug.rb

@@ -63,6 +63,7 @@ end
 require 'fluent/log'
 require 'fluent/engine'
 require 'fluent/system_config'
+require 'serverengine'
 
 include Fluent::SystemConfig::Mixin
 
@@ -90,13 +91,12 @@ include Fluent
 
 puts "Connected to #{uri}."
 puts "Usage:"
-puts "    Engine.match('some.tag').output  : get an output plugin instance"
-puts "    Engine.sources[i]                : get input plugin instances"
-puts "    Plugin.load_plugin(type,name)    : load plugin class (use this if you get DRb::DRbUnknown)"
+puts "    Fluent::Engine.match('some.tag').output  : get an output plugin instance"
+puts "    Fluent::Engine.sources[i]                : get input plugin instances"
+puts "    Fluent::Plugin.load_plugin(type,name)    : load plugin class (use this if you get DRb::DRbUnknown)"
 puts ""
 
 Encoding.default_internal = nil if Encoding.respond_to?(:default_internal)
 
 require 'irb'
 IRB.start
-

data/lib/fluent/compat/handle_tag_and_time_mixin.rb

@@ -14,6 +14,8 @@
 #    limitations under the License.
 #
 
+require 'fluent/time' # TimeFormatter
+
 module Fluent
   module Compat
     module HandleTagAndTimeMixin

data/lib/fluent/compat/output.rb

@@ -19,8 +19,10 @@ require 'fluent/plugin/output'
 require 'fluent/plugin/bare_output'
 require 'fluent/compat/call_super_mixin'
 require 'fluent/compat/formatter_utils'
+require 'fluent/compat/handle_tag_and_time_mixin'
 require 'fluent/compat/parser_utils'
 require 'fluent/compat/propagate_default'
+require 'fluent/compat/record_filter_mixin'
 require 'fluent/compat/output_chain'
 require 'fluent/timezone'
 require 'fluent/mixin'
@@ -265,7 +267,7 @@ module Fluent
           conf.elements << Fluent::Config::Element.new('buffer', '', buf_params, [])
         end
 
-        @includes_record_filter = self.class.ancestors.include?(Fluent::RecordFilterMixin) # TODO rename Compat::RecordFilterMixin
+        @includes_record_filter = self.class.ancestors.include?(Fluent::Compat::RecordFilterMixin)
 
         methods_of_plugin = self.class.instance_methods(false)
         @overrides_emit = methods_of_plugin.include?(:emit)
@@ -361,11 +363,12 @@ module Fluent
         end
 
         meta = metadata(nil, nil, nil)
+        size = es.size
         data = es.map{|time,record| format(tag, time, record) }
         write_guard do
           @buffer.write({meta => data}, enqueue: enqueue)
         end
-        @counters_monitor.synchronize{ @emit_records += es_size }
+        @counters_monitor.synchronize{ @emit_records += size }
         [meta]
       end
 

data/lib/fluent/config/configure_proxy.rb

@@ -208,6 +208,12 @@ module Fluent
         end
       end
 
+      def option_value_type!(name, opts, key, klass)
+        if opts.has_key?(key) && !opts[key].is_a?(klass)
+          raise ArgumentError, "#{name}: #{key} must be a #{klass}, but #{opts[key].class}"
+        end
+      end
+
       def parameter_configuration(name, type = nil, **kwargs, &block)
         name = name.to_sym
 
@@ -216,7 +222,7 @@ module Fluent
         opts.merge!(kwargs)
 
         if block && type
-          raise ArgumentError, "#{self.name}: both of block and type cannot be specified"
+          raise ArgumentError, "#{name}: both of block and type cannot be specified"
         end
 
         begin
@@ -224,8 +230,19 @@ module Fluent
           block ||= @type_lookup.call(type)
         rescue ConfigError
           # override error message
-          raise ArgumentError, "#{self.name}: unknown config_argument type `#{type}'"
+          raise ArgumentError, "#{name}: unknown config_argument type `#{type}'"
+        end
+
+        option_value_type!(name, opts, :desc, String)
+        option_value_type!(name, opts, :alias, Symbol)
+        option_value_type!(name, opts, :deprecated, String)
+        option_value_type!(name, opts, :obsoleted, String)
+        if type == :enum
+          if !opts.has_key?(:list) || !opts[:list].all?{|v| v.is_a?(Symbol) }
+            raise ArgumentError, "#{name}: enum parameter requires :list of Symbols"
+          end
         end
+        option_value_type!(name, opts, :value_type, Symbol) # hash, array
 
         if opts.has_key?(:default)
           config_set_default(name, opts[:default])
@@ -235,6 +252,10 @@ module Fluent
           config_set_desc(name, opts[:desc])
         end
 
+        if opts[:deprecated] && opts[:obsoleted]
+          raise ArgumentError, "#{name}: both of deprecated and obsoleted cannot be specified at once"
+        end
+
         [name, block, opts]
       end
 
@@ -296,7 +317,7 @@ module Fluent
 
       def config_section(name, **kwargs, &block)
         unless block_given?
-          raise ArgumentError, "#{self.name}: config_section requires block parameter"
+          raise ArgumentError, "#{name}: config_section requires block parameter"
         end
         name = name.to_sym
 
@@ -305,10 +326,10 @@ module Fluent
 
         if sub_proxy.init?
           if sub_proxy.argument && !sub_proxy.defaults.has_key?(sub_proxy.argument.first)
-            raise ArgumentError, "#{self.name}: init is specified, but default value of argument is missing"
+            raise ArgumentError, "#{name}: init is specified, but default value of argument is missing"
           end
           if sub_proxy.params.keys.any?{|param_name| !sub_proxy.defaults.has_key?(param_name)}
-            raise ArgumentError, "#{self.name}: init is specified, but there're parameters without default values"
+            raise ArgumentError, "#{name}: init is specified, but there're parameters without default values"
           end
         end
 

data/lib/fluent/config/error.rb

@@ -20,4 +20,7 @@ module Fluent
 
   class ConfigParseError < ConfigError
   end
+
+  class ObsoletedParameterError < ConfigError
+  end
 end

data/lib/fluent/config/section.rb

@@ -126,6 +126,7 @@ module Fluent
             logger.error "config error in:\n#{conf}" if logger # logger should exist, but somethimes it's nil (e.g, in tests)
             raise ConfigError, "'<#{proxy.name} ARG>' section requires argument" + section_stack
           end
+          # argument should NOT be deprecated... (argument always has a value: '')
         end
 
         proxy.params.each_pair do |name, defval|
@@ -138,6 +139,20 @@ module Fluent
                     conf[opts[:alias].to_s]
                   end
             section_params[varname] = self.instance_exec(val, opts, name, &block)
+
+            # Source of definitions of deprecated/obsoleted:
+            # https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Deprecated_and_obsolete_features
+            #
+            # Deprecated: These deprecated features can still be used, but should be used with caution
+            #             because they are expected to be removed entirely sometime in the future.
+            # Obsoleted: These obsolete features have been entirely removed from JavaScript and can no longer be used.
+            if opts[:deprecated]
+              logger.warn "'#{name}' parameter is deprecated: #{opts[:deprecated]}"
+            end
+            if opts[:obsoleted]
+              logger.error "config error in:\n#{conf}" if logger
+              raise ObsoletedParameterError, "'#{name}' parameter is already removed: #{opts[:obsoleted]}" + section_stack
+            end
           end
           unless section_params.has_key?(varname)
             logger.error "config error in:\n#{conf}" if logger

data/lib/fluent/event_router.rb

@@ -136,10 +136,12 @@ module Fluent
       def initialize
         @filters = []
         @output = nil
+        @optimizer = FilterOptimizer.new
       end
 
       def add_filter(filter)
         @filters << filter
+        @optimizer.filters = @filters
       end
 
       def set_output(output)
@@ -147,12 +149,83 @@ module Fluent
       end
 
       def emit_events(tag, es)
-        processed = es
-        @filters.each { |filter|
-          processed = filter.filter_stream(tag, processed)
-        }
+        processed = @optimizer.filter_stream(tag, es)
         @output.emit_events(tag, processed)
       end
+
+      class FilterOptimizer
+        def initialize(filters = [])
+          @filters = filters
+          @optimizable = nil
+        end
+
+        def filters=(filters)
+          @filters = filters
+          reset_optimization
+        end
+
+        def filter_stream(tag, es)
+          if optimizable?
+            optimized_filter_stream(tag, es)
+          else
+            @filters.reduce(es) { |acc, filter| filter.filter_stream(tag, acc) }
+          end
+        end
+
+        private
+
+        def optimized_filter_stream(tag, es)
+          new_es = MultiEventStream.new
+          es.each do |time, record|
+            filtered_record = record
+            filtered_time = time
+
+            catch :break_loop do
+              @filters.each do |filter|
+                if filter.has_filter_with_time
+                  begin
+                    filtered_time, filtered_record = filter.filter_with_time(tag, filtered_time, filtered_record)
+                    throw :break_loop unless filtered_record && filtered_time
+                  rescue => e
+                    filter.router.emit_error_event(tag, filtered_time, filtered_record, e)
+                  end
+                else
+                  begin
+                    filtered_record = filter.filter(tag, filtered_time, filtered_record)
+                    throw :break_loop unless filtered_record
+                  rescue => e
+                    filter.router.emit_error_event(tag, filtered_time, filtered_record, e)
+                  end
+                end
+              end
+
+              new_es.add(filtered_time, filtered_record)
+            end
+          end
+          new_es
+        end
+
+        def optimizable?
+          return @optimizable unless @optimizable.nil?
+          @optimizable = if filters_having_filter_stream.empty?
+                           true
+                         else
+                           $log.info "Filtering works with worse performance, because #{filters_having_filter_stream.map(&:class)} uses `#filter_stream` method."
+                           false
+                         end
+        end
+
+        def filters_having_filter_stream
+          @filters_having_filter_stream ||= @filters.select do |filter|
+            filter.class.instance_methods(false).include?(:filter_stream)
+          end
+        end
+
+        def reset_optimization
+          @optimizable = nil
+          @filters_having_filter_stream = nil
+        end
+      end
     end
 
     def find(tag)

data/lib/fluent/plugin/base.rb

@@ -24,11 +24,11 @@ module Fluent
       include Configurable
       include SystemConfig::Mixin
 
-      State = Struct.new(:configure, :start, :stop, :before_shutdown, :shutdown, :after_shutdown, :close, :terminate)
+      State = Struct.new(:configure, :start, :after_start, :stop, :before_shutdown, :shutdown, :after_shutdown, :close, :terminate)
 
       def initialize
         super
-        @_state = State.new(false, false, false, false, false, false, false, false)
+        @_state = State.new(false, false, false, false, false, false, false, false, false)
       end
 
       def has_router?
@@ -37,7 +37,7 @@ module Fluent
 
       def configure(conf)
         super
-        @_state ||= State.new(false, false, false, false, false, false, false, false)
+        @_state ||= State.new(false, false, false, false, false, false, false, false, false)
         @_state.configure = true
         self
       end
@@ -47,6 +47,11 @@ module Fluent
         self
       end
 
+      def after_start
+        @_state.after_start = true
+        self
+      end
+
       def stop
         @_state.stop = true
         self
@@ -85,6 +90,10 @@ module Fluent
         @_state.start
       end
 
+      def after_started?
+        @_state.after_start
+      end
+
       def stopped?
         @_state.stop
       end