fluentd 0.14.1 → 0.14.2

data/lib/fluent/compat/handle_tag_and_time_mixin.rb

@@ -0,0 +1,60 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module Fluent
+  module Compat
+    module HandleTagAndTimeMixin
+      def self.included(klass)
+        klass.instance_eval {
+          config_param :include_time_key, :bool, default: false
+          config_param :time_key, :string, default: 'time'
+          config_param :time_format, :string, default: nil
+          config_param :time_as_epoch, :bool, default: false
+          config_param :include_tag_key, :bool, default: false
+          config_param :tag_key, :string, default: 'tag'
+          config_param :localtime, :bool, default: true
+          config_param :timezone, :string, default: nil
+        }
+      end
+
+      def configure(conf)
+        super
+
+        if conf['utc']
+          @localtime = false
+        end
+        @timef = Fluent::TimeFormatter.new(@time_format, @localtime, @timezone)
+        if @time_as_epoch && !@include_time_key
+          log.warn "time_as_epoch will be ignored because include_time_key is false"
+        end
+      end
+
+      def filter_record(tag, time, record)
+        if @include_tag_key
+          record[@tag_key] = tag
+        end
+        if @include_time_key
+          if @time_as_epoch
+            record[@time_key] = time.to_i
+          else
+            record[@time_key] = @timef.format(time)
+          end
+        end
+      end
+    end
+  end
+end
+

data/lib/fluent/compat/input.rb

@@ -27,9 +27,7 @@ module Fluent
       def initialize
         super
         unless self.class.ancestors.include?(Fluent::Compat::CallSuperMixin)
-          self.class.module_eval do
-            prepend Fluent::Compat::CallSuperMixin
-          end
+          self.class.prepend Fluent::Compat::CallSuperMixin
         end
       end
 

data/lib/fluent/compat/output.rb

@@ -18,6 +18,8 @@ require 'fluent/plugin'
 require 'fluent/plugin/output'
 require 'fluent/plugin/bare_output'
 require 'fluent/compat/call_super_mixin'
+require 'fluent/compat/formatter_utils'
+require 'fluent/compat/parser_utils'
 require 'fluent/compat/propagate_default'
 require 'fluent/compat/output_chain'
 require 'fluent/timezone'
@@ -109,6 +111,10 @@ module Fluent
       def size
         self.bytesize
       end
+
+      def size_of_events
+        @size + @adding_size
+      end
     end
 
     module BufferedChunkMixin
@@ -136,7 +142,7 @@ module Fluent
     class Output < Fluent::Plugin::Output
       # TODO: warn when deprecated
 
-      helpers :event_emitter
+      helpers :event_emitter, :inject
 
       def support_in_v12_style?(feature)
         case feature
@@ -154,8 +160,26 @@ module Fluent
       def initialize
         super
         unless self.class.ancestors.include?(Fluent::Compat::CallSuperMixin)
-          self.class.module_eval do
-            prepend Fluent::Compat::CallSuperMixin
+          self.class.prepend Fluent::Compat::CallSuperMixin
+        end
+      end
+
+      def configure(conf)
+        ParserUtils.convert_parser_conf(conf)
+        FormatterUtils.convert_formatter_conf(conf)
+
+        super
+      end
+
+      def start
+        super
+
+        if instance_variable_defined?(:@formatter) && @inject_config
+          unless @formatter.class.ancestors.include?(Fluent::Compat::HandleTagAndTimeMixin)
+            if @formatter.respond_to?(:owner) && !@formatter.owner
+              @formatter.owner = self
+              @formatter.singleton_class.prepend FormatterUtils::InjectMixin
+            end
           end
         end
       end
@@ -174,7 +198,7 @@ module Fluent
     class BufferedOutput < Fluent::Plugin::Output
       # TODO: warn when deprecated
 
-      helpers :event_emitter
+      helpers :event_emitter, :inject
 
       def support_in_v12_style?(feature)
         case feature
@@ -212,10 +236,10 @@ module Fluent
 
       config_param :flush_at_shutdown, :bool, default: true
 
-      PARAMS_MAP = Fluent::PluginHelper::CompatParameters::PARAMS_MAP
+      BUFFER_PARAMS = Fluent::PluginHelper::CompatParameters::BUFFER_PARAMS
 
       def self.propagate_default_params
-        PARAMS_MAP
+        BUFFER_PARAMS
       end
       include PropagateDefault
 
@@ -227,7 +251,7 @@ module Fluent
             "flush_mode" => "interval",
             "retry_type" => "exponential_backoff",
           }
-          PARAMS_MAP.each do |older, newer|
+          BUFFER_PARAMS.each do |older, newer|
             next unless newer
             if conf.has_key?(older)
               if older == 'buffer_queue_full_action' && conf[older] == 'exception'
@@ -248,6 +272,9 @@ module Fluent
         # RecordFilter mixin uses its own #format_stream method implementation
         @overrides_format_stream = methods_of_plugin.include?(:format_stream) || @includes_record_filter
 
+        ParserUtils.convert_parser_conf(conf)
+        FormatterUtils.convert_formatter_conf(conf)
+
         super
 
         if config_style == :v1
@@ -256,9 +283,7 @@ module Fluent
           end
         end
 
-        (class << self; self; end).module_eval do
-          prepend BufferedChunkMixin
-        end
+        self.extend BufferedChunkMixin
 
         if @overrides_emit
           self.singleton_class.module_eval do
@@ -271,9 +296,7 @@ module Fluent
               output_plugin.last_emit_via_buffer = [key, data]
             end
           end
-          @buffer.singleton_class.module_eval do
-            prepend m
-          end
+          @buffer.extend m
         end
       end
 
@@ -320,7 +343,7 @@ module Fluent
           # on-the-fly key assignment can be done, and it's not configurable if Plugin#emit does it dynamically
           meta = @buffer.metadata(variables: (key && !key.empty? ? {key: key} : nil))
           write_guard do
-            @buffer.write({meta => [data, size]}, bulk: true, enqueue: enqueue)
+            @buffer.write({meta => data}, format: ->(_data){ _data }, size: ->(){ size }, enqueue: enqueue)
           end
           @counters_monitor.synchronize{ @emit_records += size }
           return [meta]
@@ -331,17 +354,16 @@ module Fluent
           size = es.size
           bulk = format_stream(tag, es)
           write_guard do
-            @buffer.write({meta => [bulk, size]}, bulk: true, enqueue: enqueue)
+            @buffer.write({meta => bulk}, format: ->(_data){ _data }, size: ->(){ size }, enqueue: enqueue)
           end
           @counters_monitor.synchronize{ @emit_records += size }
           return [meta]
         end
 
         meta = metadata(nil, nil, nil)
-        es_size = 0
-        es_bulk = es.map{|time,record| es_size += 1; format(tag, time, record) }.join
+        data = es.map{|time,record| format(tag, time, record) }
         write_guard do
-          @buffer.write({meta => [es_bulk, es_size]}, bulk: true, enqueue: enqueue)
+          @buffer.write({meta => data}, enqueue: enqueue)
         end
         @counters_monitor.synchronize{ @emit_records += es_size }
         [meta]
@@ -354,8 +376,19 @@ module Fluent
       def initialize
         super
         unless self.class.ancestors.include?(Fluent::Compat::CallSuperMixin)
-          self.class.module_eval do
-            prepend Fluent::Compat::CallSuperMixin
+          self.class.prepend Fluent::Compat::CallSuperMixin
+        end
+      end
+
+      def start
+        super
+
+        if instance_variable_defined?(:@formatter) && @inject_config
+          unless @formatter.class.ancestors.include?(Fluent::Compat::HandleTagAndTimeMixin)
+            if @formatter.respond_to?(:owner) && !@formatter.owner
+              @formatter.owner = self
+              @formatter.singleton_class.prepend FormatterUtils::InjectMixin
+            end
           end
         end
       end
@@ -364,7 +397,7 @@ module Fluent
     class ObjectBufferedOutput < Fluent::Plugin::Output
       # TODO: warn when deprecated
 
-      helpers :event_emitter
+      helpers :event_emitter, :inject
 
       # This plugin cannot inherit BufferedOutput because #configure sets chunk_key 'tag'
       # to flush chunks per tags, but BufferedOutput#configure doesn't allow setting chunk_key
@@ -408,10 +441,10 @@ module Fluent
 
       config_set_default :time_as_integer, true
 
-      PARAMS_MAP = Fluent::PluginHelper::CompatParameters::PARAMS_MAP
+      BUFFER_PARAMS = Fluent::PluginHelper::CompatParameters::BUFFER_PARAMS
 
       def self.propagate_default_params
-        PARAMS_MAP
+        BUFFER_PARAMS
       end
       include PropagateDefault
 
@@ -423,7 +456,7 @@ module Fluent
             "flush_mode" => "interval",
             "retry_type" => "exponential_backoff",
           }
-          PARAMS_MAP.each do |older, newer|
+          BUFFER_PARAMS.each do |older, newer|
             next unless newer
             if conf.has_key?(older)
               if older == 'buffer_queue_full_action' && conf[older] == 'exception'
@@ -437,6 +470,9 @@ module Fluent
           conf.elements << Fluent::Config::Element.new('buffer', 'tag', buf_params, [])
         end
 
+        ParserUtils.convert_parser_conf(conf)
+        FormatterUtils.convert_formatter_conf(conf)
+
         super
 
         if config_style == :v1
@@ -445,9 +481,7 @@ module Fluent
           end
         end
 
-        (class << self; self; end).module_eval do
-          prepend BufferedChunkMixin
-        end
+        self.extend BufferedChunkMixin
       end
 
       def format_stream(tag, es) # for BufferedOutputTestDriver
@@ -465,8 +499,19 @@ module Fluent
       def initialize
         super
         unless self.class.ancestors.include?(Fluent::Compat::CallSuperMixin)
-          self.class.module_eval do
-            prepend Fluent::Compat::CallSuperMixin
+          self.class.prepend Fluent::Compat::CallSuperMixin
+        end
+      end
+
+      def start
+        super
+
+        if instance_variable_defined?(:@formatter) && @inject_config
+          unless @formatter.class.ancestors.include?(Fluent::Compat::HandleTagAndTimeMixin)
+            if @formatter.respond_to?(:owner) && !@formatter.owner
+              @formatter.owner = self
+              @formatter.singleton_class.prepend FormatterUtils::InjectMixin
+            end
           end
         end
       end
@@ -475,7 +520,7 @@ module Fluent
     class TimeSlicedOutput < Fluent::Plugin::Output
       # TODO: warn when deprecated
 
-      helpers :event_emitter
+      helpers :event_emitter, :inject
 
       def support_in_v12_style?(feature)
         case feature
@@ -526,21 +571,19 @@ module Fluent
         config_set_default :@type, 'file'
       end
 
-      PARAMS_MAP = Fluent::PluginHelper::CompatParameters::PARAMS_MAP.merge(Fluent::PluginHelper::CompatParameters::TIME_SLICED_PARAMS)
+      BUFFER_PARAMS = Fluent::PluginHelper::CompatParameters::BUFFER_PARAMS.merge(Fluent::PluginHelper::CompatParameters::BUFFER_TIME_SLICED_PARAMS)
 
       def initialize
         super
         @localtime = true
 
         unless self.class.ancestors.include?(Fluent::Compat::CallSuperMixin)
-          self.class.module_eval do
-            prepend Fluent::Compat::CallSuperMixin
-          end
+          self.class.prepend Fluent::Compat::CallSuperMixin
         end
       end
 
       def self.propagate_default_params
-        PARAMS_MAP
+        BUFFER_PARAMS
       end
       include PropagateDefault
 
@@ -552,7 +595,7 @@ module Fluent
             "flush_mode" => (conf['flush_interval'] ? "interval" : "lazy"),
             "retry_type" => "exponential_backoff",
           }
-          PARAMS_MAP.each do |older, newer|
+          BUFFER_PARAMS.each do |older, newer|
             next unless newer
             if conf.has_key?(older)
               if older == 'buffer_queue_full_action' && conf[older] == 'exception'
@@ -591,6 +634,9 @@ module Fluent
           conf.elements << Fluent::Config::Element.new('buffer', 'time', buf_params, [])
         end
 
+        ParserUtils.convert_parser_conf(conf)
+        FormatterUtils.convert_formatter_conf(conf)
+
         super
 
         if config_style == :v1
@@ -599,8 +645,19 @@ module Fluent
           end
         end
 
-        (class << self; self; end).module_eval do
-          prepend TimeSliceChunkMixin
+        self.extend TimeSliceChunkMixin
+      end
+
+      def start
+        super
+
+        if instance_variable_defined?(:@formatter) && @inject_config
+          unless @formatter.class.ancestors.include?(Fluent::Compat::HandleTagAndTimeMixin)
+            if @formatter.respond_to?(:owner) && !@formatter.owner
+              @formatter.owner = self
+              @formatter.singleton_class.prepend FormatterUtils::InjectMixin
+            end
+          end
         end
       end
 
@@ -615,4 +672,3 @@ module Fluent
     end
   end
 end
-

data/lib/fluent/compat/parser.rb

@@ -18,6 +18,7 @@ require 'fluent/plugin'
 require 'fluent/plugin/parser'
 require 'fluent/mixin'
 
+require 'fluent/plugin/parser_regexp'
 require 'fluent/plugin/parser_json'
 require 'fluent/plugin/parser_tsv'
 require 'fluent/plugin/parser_ltsv'
@@ -121,6 +122,22 @@ module Fluent
 
       class RegexpParser < Fluent::Plugin::RegexpParser
         # TODO: warn when deprecated
+        def initialize(regexp, conf = {})
+          super()
+
+          unless conf.empty?
+            unless conf.is_a?(Config::Element)
+              conf = Config::Element.new('default_regexp_conf', '', conf, [])
+            end
+            configure(conf)
+          end
+
+          @regexp = regexp
+        end
+
+        def patterns
+          {'format' => @regexp, 'time_format' => @time_format}
+        end
       end
 
       class ValuesParser < Fluent::Plugin::ValuesParser

data/lib/fluent/compat/parser_utils.rb

@@ -0,0 +1,40 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin_helper/compat_parameters'
+
+module Fluent
+  module Compat
+    module ParserUtils
+      PARSER_PARAMS = Fluent::PluginHelper::CompatParameters::PARSER_PARAMS
+
+      def self.convert_parser_conf(conf)
+        return if conf.elements(name: 'parse').first
+
+        parser_params = {}
+        PARSER_PARAMS.each do |older, newer|
+          next unless newer
+          if conf.has_key?(older)
+            parser_params[newer] = conf[older]
+          end
+        end
+        unless parser_params.empty?
+          conf.elements << Fluent::Config::Element.new('parse', '', parser_params, [])
+        end
+      end
+    end
+  end
+end