fluentd 0.14.0 → 0.14.1

data/lib/fluent/plugin/buffer.rb

@@ -413,8 +413,8 @@ module Fluent
           write_step_by_step(metadata, data, data.size / 3, &block)
         end
       rescue ShouldRetry
-        enqueue_list.each do |metadata|
-          enqueue_chunk(metadata)
+        enqueue_list.each do |m|
+          enqueue_chunk(m)
         end
         retry
       end

data/lib/fluent/plugin/in_dummy.rb

@@ -89,7 +89,11 @@ module Fluent::Plugin
     end
 
     def emit(num)
-      num.times { router.emit(@tag, Fluent::Engine.now, generate()) }
+      begin
+        num.times { router.emit(@tag, Fluent::Engine.now, generate()) }
+      rescue => _
+        # ignore all errors not to stop emits by emit errors
+      end
     end
 
     def generate

data/lib/fluent/plugin/in_gc_stat.rb

@@ -14,13 +14,13 @@
 #    limitations under the License.
 #
 
-require 'cool.io'
+require 'fluent/plugin/input'
 
-require 'fluent/input'
+module Fluent::Plugin
+  class GCStatInput < Fluent::Plugin::Input
+    Fluent::Plugin.register_input('gc_stat', self)
 
-module Fluent
-  class GCStatInput < Input
-    Plugin.register_input('gc_stat', self)
+    helpers :timer
 
     def initialize
       super
@@ -29,22 +29,6 @@ module Fluent
     config_param :emit_interval, :time, default: 60
     config_param :tag, :string
 
-    class TimerWatcher < Coolio::TimerWatcher
-      def initialize(interval, repeat, log, &callback)
-        @callback = callback
-        @log = log
-        super(interval, repeat)
-      end
-
-      def on_timer
-        @callback.call
-      rescue
-        # TODO log?
-        @log.error $!.to_s
-        @log.error_backtrace
-      end
-    end
-
     def configure(conf)
       super
     end
@@ -52,29 +36,15 @@ module Fluent
     def start
       super
 
-      @loop = Coolio::Loop.new
-      @timer = TimerWatcher.new(@emit_interval, true, log, &method(:on_timer))
-      @loop.attach(@timer)
-      @thread = Thread.new(&method(:run))
+      timer_execute(:in_gc_stat, @emit_interval, &method(:on_timer))
     end
 
     def shutdown
-      @loop.watchers.each {|w| w.detach }
-      @loop.stop
-      @thread.join
-
       super
     end
 
-    def run
-      @loop.run
-    rescue
-      log.error "unexpected error", error: $!.to_s
-      log.error_backtrace
-    end
-
     def on_timer
-      now = Engine.now
+      now = Fluent::EventTime.now
       record = GC.stat
       router.emit(@tag, now, record)
     end

data/lib/fluent/plugin/in_http.rb

@@ -309,6 +309,8 @@ module Fluent
             end
           when /Origin/i
             @origin  = v
+          when /X-Forwarded-For/i
+            @remote_addr = v.split(",").first
           end
         }
         if expect

data/lib/fluent/plugin/out_buffered_stdout.rb

@@ -0,0 +1,60 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/output'
+
+module Fluent::Plugin
+  class BufferedStdoutOutput < Output
+    Fluent::Plugin.register_output('buffered_stdout', self)
+
+    desc 'Output format.(json,hash)'
+    config_param :output_type, default: 'json'
+    config_section :buffer do
+      config_set_default :chunk_keys, ['tag']
+      config_set_default :flush_at_shutdown, true
+      config_set_default :chunk_limit_size, 10 * 1024
+    end
+
+    attr_accessor :delayed
+
+    def initialize
+      super
+      @delayed = false
+    end
+
+    def prefer_delayed_commit
+      @delayed
+    end
+
+    def configure(conf)
+      super
+      @formatter = Fluent::Plugin.new_formatter(@output_type, parent: self)
+      @formatter.configure(conf)
+    end
+
+    def write(chunk)
+      chunk.write_to($log)
+    end
+
+    def try_write(chunk)
+      chunk.write_to($log)
+    end
+
+    def format(tag, time, record)
+      "#{Time.at(time).localtime} #{tag}: #{@formatter.format(tag, time, record).chomp}\n"
+    end
+  end
+end

data/lib/fluent/plugin/out_copy.rb

@@ -14,72 +14,29 @@
 #    limitations under the License.
 #
 
-require 'fluent/output'
+require 'fluent/plugin/multi_output'
 require 'fluent/config/error'
 require 'fluent/event'
 
-module Fluent
+module Fluent::Plugin
   class CopyOutput < MultiOutput
-    Plugin.register_output('copy', self)
+    Fluent::Plugin.register_output('copy', self)
 
     desc 'If true, pass different record to each `store` plugin.'
     config_param :deep_copy, :bool, default: false
 
-    def initialize
-      super
-      @outputs = []
-    end
-
-    attr_reader :outputs
-
-    def configure(conf)
-      super
-      conf.elements.select {|e|
-        e.name == 'store'
-      }.each {|e|
-        type = e['@type']
-        unless type
-          raise ConfigError, "Missing 'type' parameter on <store> directive"
-        end
-        log.debug "adding store type=#{type.dump}"
-
-        output = Plugin.new_output(type)
-        output.router = router
-        output.configure(e)
-        @outputs << output
-      }
-    end
-
-    def start
-      super
-
-      @outputs.each do |o|
-        o.start unless o.started?
-      end
-    end
-
-    def shutdown
-      @outputs.each do |o|
-        o.shutdown unless o.shutdown?
-      end
-
-      super
-    end
-
-    def emit(tag, es, chain)
+    def process(tag, es)
       unless es.repeatable?
-        m = MultiEventStream.new
+        m = Fluent::MultiEventStream.new
         es.each {|time,record|
           m.add(time, record)
         }
         es = m
       end
-      if @deep_copy
-        chain = CopyOutputChain.new(@outputs, tag, es, chain)
-      else
-        chain = OutputChain.new(@outputs, tag, es, chain)
+
+      outputs.each do |output|
+        output.emit_events(tag, @deep_copy ? es.dup : es)
       end
-      chain.next
     end
   end
 end

data/lib/fluent/plugin/out_null.rb

@@ -14,14 +14,14 @@
 #    limitations under the License.
 #
 
-require 'fluent/output'
+require 'fluent/plugin/output'
 
-module Fluent
+module Fluent::Plugin
   class NullOutput < Output
-    Plugin.register_output('null', self)
+    Fluent::Plugin.register_output('null', self)
 
-    def emit(tag, es, chain)
-      chain.next
+    def process(tag, es)
+      # Do nothing
     end
   end
 end

data/lib/fluent/plugin/out_relabel.rb

@@ -14,15 +14,15 @@
 #    limitations under the License.
 #
 
-require 'fluent/output'
+require 'fluent/plugin/output'
 
-module Fluent
+module Fluent::Plugin
   class RelabelOutput < Output
-    Plugin.register_output('relabel', self)
+    Fluent::Plugin.register_output('relabel', self)
+    helpers :event_emitter
 
-    def emit(tag, es, chain)
+    def process(tag, es)
       router.emit_stream(tag, es)
-      chain.next
     end
   end
 end

data/lib/fluent/plugin/out_roundrobin.rb

@@ -14,68 +14,41 @@
 #    limitations under the License.
 #
 
-require 'fluent/output'
+require 'fluent/plugin/multi_output'
 require 'fluent/config/error'
 
-module Fluent
+module Fluent::Plugin
   class RoundRobinOutput < MultiOutput
-    Plugin.register_output('roundrobin', self)
+    Fluent::Plugin.register_output('roundrobin', self)
+
+    config_section :store do
+      config_param :weight, :integer, default: 1
+    end
 
     def initialize
       super
-
-      @outputs = []
       @weights = []
     end
 
-    attr_reader :outputs, :weights
-    attr_accessor :rand_seed
+    attr_reader :weights
 
     def configure(conf)
       super
 
-      conf.elements.select {|e|
-        e.name == 'store'
-      }.each {|e|
-        type = e['@type']
-        unless type
-          raise ConfigError, "Missing 'type' parameter on <store> directive"
-        end
-
-        weight = e['weight']
-        weight = weight ? weight.to_i : 1
-        log.debug "adding store type=#{type.dump}, weight=#{weight}"
-
-        output = Plugin.new_output(type)
-        output.router = router
-        output.configure(e)
-        @outputs << output
-        @weights << weight
-      }
+      @stores.each do |store|
+        @weights << store.weight
+      end
       @rr = -1  # starts from @output[0]
       @rand_seed = Random.new.seed
     end
 
     def start
       super
-
       rebuild_weight_array
-
-      @outputs.each do |o|
-        o.start unless o.started?
-      end
-    end
-
-    def shutdown
-      @outputs.each do |o|
-        o.shutdown unless o.shutdown?
-      end
-
-      super
     end
 
-    def emit(tag, es, chain)
-      next_output.emit(tag, es, chain)
+    def process(tag, es)
+      next_output.emit_events(tag, es)
     end
 
     private

data/lib/fluent/plugin/output.rb

@@ -37,6 +37,8 @@ module Fluent
       CHUNK_KEY_PATTERN = /^[-_.@a-zA-Z0-9]+$/
       CHUNK_KEY_PLACEHOLDER_PATTERN = /\$\{[-_.@a-zA-Z0-9]+\}/
 
+      CHUNKING_FIELD_WARN_NUM = 4
+
       config_param :time_as_integer, :bool, default: false
 
       # `<buffer>` and `<secondary>` sections are available only when '#format' and '#write' are implemented
@@ -253,6 +255,10 @@ module Fluent
             @output_time_formatter_cache = {}
           end
 
+          if (@chunk_key_tag ? 1 : 0) + @chunk_keys.size >= CHUNKING_FIELD_WARN_NUM
+            log.warn "many chunk keys specified, and it may cause too many chunks on your system."
+          end
+
           # no chunk keys or only tags (chunking can be done without iterating event stream)
           @simple_chunking = !@chunk_key_time && @chunk_keys.empty?
 
@@ -287,6 +293,9 @@ module Fluent
           raise Fluent::ConfigError, "<secondary> section and 'retry_forever' are exclusive" if @buffer_config.retry_forever
 
           secondary_type = @secondary_config[:@type]
+          unless secondary_type
+            secondary_type = conf['@type'] # primary plugin type
+          end
           secondary_conf = conf.elements(name: 'secondary').first
           @secondary = Plugin.new_output(secondary_type)
           @secondary.acts_as_secondary(self)

data/lib/fluent/plugin_helper.rb

@@ -20,6 +20,8 @@ require 'fluent/plugin_helper/event_loop'
 require 'fluent/plugin_helper/timer'
 require 'fluent/plugin_helper/child_process'
 require 'fluent/plugin_helper/storage'
+require 'fluent/plugin_helper/parser'
+require 'fluent/plugin_helper/formatter'
 require 'fluent/plugin_helper/retry_state'
 require 'fluent/plugin_helper/compat_parameters'
 

data/lib/fluent/plugin_helper/formatter.rb

@@ -0,0 +1,138 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin'
+require 'fluent/plugin/formatter'
+require 'fluent/config/element'
+
+module Fluent
+  module PluginHelper
+    module Formatter
+      def formatter_create(usage: '', type: nil, conf: nil)
+        formatter = @_formatters[usage]
+        return formatter if formatter
+
+        if !type
+          raise ArgumentError, "BUG: both type and conf are not specified" unless conf
+          raise Fluent::ConfigError, "@type is required in <format>" unless conf['@type']
+          type = conf['@type']
+        end
+        formatter = Fluent::Plugin.new_formatter(type, parent: self)
+        config = case conf
+                 when Fluent::Config::Element
+                   conf
+                 when Hash
+                   # in code, programmer may use symbols as keys, but Element needs strings
+                   conf = Hash[conf.map{|k,v| [k.to_s, v]}]
+                   Fluent::Config::Element.new('format', usage, conf, [])
+                 when nil
+                   Fluent::Config::Element.new('format', usage, {}, [])
+                 else
+                   raise ArgumentError, "BUG: conf must be a Element, Hash (or unspecified), but '#{conf.class}'"
+                 end
+        formatter.configure(config)
+        if @_formatters_started
+          formatter.start
+        end
+
+        @_formatters[usage] = formatter
+        formatter
+      end
+
+      def self.included(mod)
+        mod.instance_eval do
+          # minimum section definition to instantiate formatter plugin instances
+          config_section :format, required: false, multi: true, param_name: :formatter_configs do
+            config_argument :usage, :string, default: ''
+            config_param    :@type, :string
+          end
+        end
+      end
+
+      attr_reader :_formatters # for tests
+
+      def initialize
+        super
+        @_formatters_started = false
+        @_formatters = {} # usage => formatter
+      end
+
+      def configure(conf)
+        super
+
+        @formatter_configs.each do |section|
+          if @_formatters[section.usage]
+            raise Fluent::ConfigError, "duplicated formatter configured: #{section.usage}"
+          end
+          formatter = Plugin.new_formatter(section[:@type], parent: self)
+          formatter.configure(section.corresponding_config_element)
+          @_formatters[section.usage] = formatter
+        end
+      end
+
+      def start
+        super
+        @_formatters_started = true
+        @_formatters.each_pair do |usage, formatter|
+          formatter.start
+        end
+      end
+
+      def formatter_operate(method_name, &block)
+        @_formatters.each_pair do |usage, formatter|
+          begin
+            formatter.send(method_name)
+            block.call(formatter) if block_given?
+          rescue => e
+            log.error "unexpected error while #{method_name}", usage: usage, formatter: formatter, error: e
+          end
+        end
+      end
+
+      def stop
+        super
+        formatter_operate(:stop)
+      end
+
+      def before_shutdown
+        formatter_operate(:before_shutdown)
+        super
+      end
+
+      def shutdown
+        formatter_operate(:shutdown)
+        super
+      end
+
+      def after_shutdown
+        formatter_operate(:after_shutdown)
+        super
+      end
+
+      def close
+        formatter_operate(:close)
+        super
+      end
+
+      def terminate
+        formatter_operate(:terminate)
+        @_formatters_started = false
+        @_formatters = {}
+        super
+      end
+    end
+  end
+end