fluentd 0.12.20 → 0.12.21

data/lib/fluent/config/v1_parser.rb

@@ -14,14 +14,16 @@
 #    limitations under the License.
 #
 
-module Fluent
-  module Config
+require 'strscan'
+require 'uri'
 
-    require 'strscan'
-    require 'fluent/config/error'
-    require 'fluent/config/literal_parser'
-    require 'fluent/config/element'
+require 'fluent/config/error'
+require 'fluent/config/basic_parser'
+require 'fluent/config/literal_parser'
+require 'fluent/config/element'
 
+module Fluent
+  module Config
     class V1Parser < LiteralParser
       ELEMENT_NAME = /[a-zA-Z0-9_]+/
 

data/lib/fluent/configurable.rb

@@ -14,12 +14,14 @@
 #    limitations under the License.
 #
 
-module Fluent
-  require 'fluent/config/configure_proxy'
-  require 'fluent/config/section'
-  require 'fluent/config/error'
-  require 'fluent/registry'
+require 'fluent/config/configure_proxy'
+require 'fluent/config/section'
+require 'fluent/config/error'
+require 'fluent/registry'
+require 'fluent/plugin'
+require 'fluent/mixin'
 
+module Fluent
   module Configurable
     def self.included(mod)
       mod.extend(ClassMethods)
@@ -51,7 +53,7 @@ module Fluent
       conf.corresponding_proxies << proxy
 
       # In the nested section, can't get plugin class through proxies so get plugin class here
-      plugin_class = Plugin.lookup_name_from_class(proxy.name.to_s)
+      plugin_class = Fluent::Plugin.lookup_name_from_class(proxy.name.to_s)
       root = Fluent::Config::SectionGenerator.generate(proxy, conf, logger, plugin_class)
       @config_root_section = root
 

data/lib/fluent/engine.rb

@@ -14,11 +14,20 @@
 #    limitations under the License.
 #
 
-module Fluent
-  require 'fluent/event_router'
-  require 'fluent/root_agent'
-  require 'fluent/time'
+require 'socket'
+
+require 'msgpack'
+require 'cool.io'
+
+require 'fluent/config'
+require 'fluent/event'
+require 'fluent/event_router'
+require 'fluent/root_agent'
+require 'fluent/time'
+require 'fluent/system_config'
+require 'fluent/plugin'
 
+module Fluent
   class EngineClass
     class DummyMessagePackFactory
       def packer(*args)
@@ -51,8 +60,11 @@ module Fluent
     attr_reader :root_agent
     attr_reader :matches, :sources
     attr_reader :msgpack_factory
+    attr_reader :system_config
+
+    def init(system_config)
+      @system_config = system_config
 
-    def init(opts = {})
       BasicSocket.do_not_reverse_lookup = true
       Plugin.load_plugins
       if defined?(Encoding)
@@ -60,11 +72,11 @@ module Fluent
         Encoding.default_external = 'ASCII-8BIT' if Encoding.respond_to?(:default_external)
       end
 
-      suppress_interval(opts[:suppress_interval]) if opts[:suppress_interval]
-      @suppress_config_dump = opts[:suppress_config_dump] if opts[:suppress_config_dump]
-      @without_source = opts[:without_source] if opts[:without_source]
+      suppress_interval(system_config.emit_error_log_interval) unless system_config.emit_error_log_interval.nil?
+      @suppress_config_dump = system_config.suppress_config_dump unless system_config.suppress_config_dump.nil?
+      @without_source = system_config.without_source unless system_config.without_source.nil?
 
-      @root_agent = RootAgent.new(opts)
+      @root_agent = RootAgent.new(@system_config)
 
       self
     end
@@ -164,7 +176,7 @@ module Fluent
           begin
             @event_router.emit(tag, time, record)
           rescue => e
-            $log.error "failed to emit fluentd's log event", :tag => tag, :event => record, :error_class => e.class, :error => e
+            $log.error "failed to emit fluentd's log event", tag: tag, event: record, error_class: e.class, error: e
           end
         }
       end
@@ -193,7 +205,7 @@ module Fluent
         end
 
       rescue => e
-        $log.error "unexpected error", :error_class=>e.class, :error=>e
+        $log.error "unexpected error", error_class: e.class, error: e
         $log.error_backtrace
       ensure
         $log.info "shutting down fluentd"

data/lib/fluent/event.rb

@@ -14,6 +14,8 @@
 #    limitations under the License.
 #
 
+require 'fluent/engine'
+
 module Fluent
   class EventStream
     include Enumerable

data/lib/fluent/event_router.rb

@@ -13,9 +13,11 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 #
-module Fluent
-  require 'fluent/match'
 
+require 'fluent/match'
+require 'fluent/event'
+
+module Fluent
   #
   # EventRouter is responsible to route events to a collector.
   #

data/lib/fluent/filter.rb

@@ -14,6 +14,12 @@
 #    limitations under the License.
 #
 
+require 'fluent/config'
+require 'fluent/configurable'
+require 'fluent/engine'
+require 'fluent/event'
+require 'fluent/log'
+
 module Fluent
   class Filter
     include Configurable

data/lib/fluent/formatter.rb

@@ -14,9 +14,11 @@
 #    limitations under the License.
 #
 
-module Fluent
-  require 'fluent/registry'
+require 'fluent/configurable'
+require 'fluent/registry'
+require 'fluent/mixin'
 
+module Fluent
   class Formatter
     include Configurable
 
@@ -33,13 +35,13 @@ module Fluent
     module HandleTagAndTimeMixin
       def self.included(klass)
         klass.instance_eval {
-          config_param :include_time_key, :bool, :default => false
-          config_param :time_key, :string, :default => 'time'
-          config_param :time_format, :string, :default => nil
-          config_param :include_tag_key, :bool, :default => false
-          config_param :tag_key, :string, :default => 'tag'
-          config_param :localtime, :bool, :default => true
-          config_param :timezone, :string, :default => nil
+          config_param :include_time_key, :bool, default: false
+          config_param :time_key, :string, default: 'time'
+          config_param :time_format, :string, default: nil
+          config_param :include_tag_key, :bool, default: false
+          config_param :tag_key, :string, default: 'tag'
+          config_param :localtime, :bool, default: true
+          config_param :timezone, :string, default: nil
         }
       end
 
@@ -65,9 +67,9 @@ module Fluent
     class OutFileFormatter < Formatter
       include HandleTagAndTimeMixin
 
-      config_param :output_time, :bool, :default => true
-      config_param :output_tag, :bool, :default => true
-      config_param :delimiter, :default => "\t" do |val|
+      config_param :output_time, :bool, default: true
+      config_param :output_tag, :bool, default: true
+      config_param :delimiter, default: "\t" do |val|
         case val
         when /SPACE/i then ' '
         when /COMMA/i then ','
@@ -85,7 +87,7 @@ module Fluent
     end
 
     class StdoutFormatter < Formatter
-      config_param :output_type, :string, :default => 'json'
+      config_param :output_type, :string, default: 'json'
 
       def configure(conf)
         super
@@ -103,7 +105,7 @@ module Fluent
     module StructuredFormatMixin
       def self.included(klass)
         klass.instance_eval {
-          config_param :time_as_epoch, :bool, :default => false
+          config_param :time_as_epoch, :bool, default: false
         }
       end
 
@@ -131,7 +133,7 @@ module Fluent
       include HandleTagAndTimeMixin
       include StructuredFormatMixin
 
-      config_param :json_parser, :string, :default => 'oj'
+      config_param :json_parser, :string, default: 'oj'
 
       def configure(conf)
         super
@@ -139,7 +141,7 @@ module Fluent
         begin
           raise LoadError unless @json_parser == 'oj'
           require 'oj'
-          Oj.default_options = {:mode => :compat}
+          Oj.default_options = {mode: :compat}
           @dump_proc = Oj.method(:dump)
         rescue LoadError
           @dump_proc = Yajl.method(:dump)
@@ -172,8 +174,8 @@ module Fluent
     class LabeledTSVFormatter < Formatter
       include HandleTagAndTimeMixin
 
-      config_param :delimiter, :string, :default => "\t"
-      config_param :label_delimiter, :string, :default =>  ":"
+      config_param :delimiter, :string, default: "\t"
+      config_param :label_delimiter, :string, default: ":"
 
       def format(tag, time, record)
         filter_record(tag, time, record)
@@ -189,11 +191,11 @@ module Fluent
     class CsvFormatter < Formatter
       include HandleTagAndTimeMixin
 
-      config_param :delimiter, :default => ',' do |val|
+      config_param :delimiter, default: ',' do |val|
         ['\t', 'TAB'].include?(val) ? "\t" : val
       end
-      config_param :force_quotes, :bool, :default => true
-      config_param :fields, :default => [] do |val|
+      config_param :force_quotes, :bool, default: true
+      config_param :fields, default: [] do |val|
         val.split(',').map do |f|
           f.strip!
           f.size > 0 ? f : nil
@@ -211,14 +213,14 @@ module Fluent
             memo << record[key]
             memo
         end
-        CSV.generate_line(row, :col_sep => @delimiter,
-                          :force_quotes => @force_quotes)
+        CSV.generate_line(row, col_sep: @delimiter,
+                          force_quotes: @force_quotes)
       end
     end
 
     class SingleValueFormatter < Formatter
-      config_param :message_key, :string, :default => 'message'
-      config_param :add_newline, :bool, :default => true
+      config_param :message_key, :string, default: 'message'
+      config_param :add_newline, :bool, default: true
 
       def format(tag, time, record)
         text = record[@message_key].to_s.dup

data/lib/fluent/input.rb

@@ -14,6 +14,11 @@
 #    limitations under the License.
 #
 
+require 'fluent/config'
+require 'fluent/configurable'
+require 'fluent/engine'
+require 'fluent/log'
+
 module Fluent
   class Input
     include Configurable

data/lib/fluent/label.rb

@@ -14,9 +14,9 @@
 #    limitations under the License.
 #
 
-module Fluent
-  require 'fluent/agent'
+require 'fluent/agent'
 
+module Fluent
   class Label < Agent
     def initialize(name, opts = {})
       super(opts)

data/lib/fluent/log.rb

@@ -65,6 +65,9 @@ module Fluent
       # TODO: This variable name is unclear so we should change to better name.
       @threads_exclude_events = []
 
+      # Fluent::Engine requires Fluent::Log, so we must take that object lazily
+      @engine = Fluent.const_get('Engine')
+
       if opts.has_key?(:suppress_repeated_stacktrace)
         @suppress_repeated_stacktrace = opts[:suppress_repeated_stacktrace]
       end
@@ -285,7 +288,7 @@ module Fluent
           record[key] = record[key].inspect unless record[key].respond_to?(:to_msgpack)
         }
         record['message'] = message.dup
-        Engine.push_log_event("#{@tag}.#{level}", time.to_i, record)
+        @engine.push_log_event("#{@tag}.#{level}", time.to_i, record)
       end
 
       return time, message
@@ -344,7 +347,7 @@ module Fluent
     def self.included(klass)
       klass.instance_eval {
         desc 'Allows the user to set different levels of logging for each plugin.'
-        config_param :log_level, :string, :default => nil, :alias => :@log_level
+        config_param :log_level, :string, default: nil, alias: :@log_level
       }
     end
 

data/lib/fluent/mixin.rb

@@ -14,10 +14,12 @@
 #    limitations under the License.
 #
 
+require 'fluent/timezone'
+require 'fluent/time'
+require 'fluent/config/error'
+
 module Fluent
   class TimeFormatter
-    require 'fluent/timezone'
-
     def initialize(format, localtime, timezone = nil)
       @tc1 = 0
       @tc1_str = nil

data/lib/fluent/output.rb

@@ -14,6 +14,16 @@
 #    limitations under the License.
 #
 
+require 'thread'
+
+require 'fluent/config'
+require 'fluent/configurable'
+require 'fluent/engine'
+require 'fluent/log'
+require 'fluent/plugin'
+require 'fluent/status'
+require 'fluent/timezone'
+
 module Fluent
   class OutputChain
     def initialize(array, tag, es, chain=NullOutputChain.instance)
@@ -88,12 +98,11 @@ module Fluent
 
     def secondary_init(primary)
       if primary.class != self.class
-        $log.warn "type of secondary output should be same as primary output", :primary=>primary.class.to_s, :secondary=>self.class.to_s
+        $log.warn "type of secondary output should be same as primary output", primary: primary.class.to_s, secondary: self.class.to_s
       end
     end
   end
 
-
   class OutputThread
     def initialize(output)
       @output = output
@@ -152,7 +161,7 @@ module Fluent
         @mutex.unlock
       end
     rescue
-      $log.error "error on output thread", :error=>$!.to_s
+      $log.error "error on output thread", error: $!.to_s
       $log.error_backtrace
       raise
     ensure
@@ -180,22 +189,22 @@ module Fluent
     end
 
     desc 'The buffer type (memory, file)'
-    config_param :buffer_type, :string, :default => 'memory'
+    config_param :buffer_type, :string, default: 'memory'
     desc 'The interval between data flushes.'
-    config_param :flush_interval, :time, :default => 60
-    config_param :try_flush_interval, :float, :default => 1
+    config_param :flush_interval, :time, default: 60
+    config_param :try_flush_interval, :float, default: 1
     desc 'If true, the value of `retry_value` is ignored and there is no limit'
-    config_param :disable_retry_limit, :bool, :default => false
+    config_param :disable_retry_limit, :bool, default: false
     desc 'The limit on the number of retries before buffered data is discarded'
-    config_param :retry_limit, :integer, :default => 17
+    config_param :retry_limit, :integer, default: 17
     desc 'The initial intervals between write retries.'
-    config_param :retry_wait, :time, :default => 1.0
+    config_param :retry_wait, :time, default: 1.0
     desc 'The maximum intervals between write retries.'
-    config_param :max_retry_wait, :time, :default => nil
+    config_param :max_retry_wait, :time, default: nil
     desc 'The number of threads to flush the buffer.'
-    config_param :num_threads, :integer, :default => 1
+    config_param :num_threads, :integer, default: 1
     desc 'The interval between data flushes for queued chunk.'
-    config_param :queued_chunk_flush_interval, :time, :default => 1
+    config_param :queued_chunk_flush_interval, :time, default: 1
 
     def configure(conf)
       super
@@ -334,7 +343,7 @@ module Fluent
           @num_errors = 0
           # Note: don't notify to other threads to prevent
           #       burst to recovered server
-          $log.warn "retry succeeded.", :plugin_id=>plugin_id
+          $log.warn "retry succeeded.", plugin_id: plugin_id
         end
 
         if has_next
@@ -359,20 +368,20 @@ module Fluent
         end
 
         if @disable_retry_limit || error_count < @retry_limit
-          $log.warn "temporarily failed to flush the buffer.", :next_retry=>Time.at(@next_retry_time), :error_class=>e.class.to_s, :error=>e.to_s, :plugin_id=>plugin_id
+          $log.warn "temporarily failed to flush the buffer.", next_retry: Time.at(@next_retry_time), error_class: e.class.to_s, error: e.to_s, plugin_id: plugin_id
           $log.warn_backtrace e.backtrace
 
         elsif @secondary
           if error_count == @retry_limit
-            $log.warn "failed to flush the buffer.", :error_class=>e.class.to_s, :error=>e.to_s, :plugin_id=>plugin_id
+            $log.warn "failed to flush the buffer.", error_class: e.class.to_s, error: e.to_s, plugin_id: plugin_id
             $log.warn "retry count exceededs limit. falling back to secondary output."
             $log.warn_backtrace e.backtrace
             retry  # retry immediately
           elsif error_count <= @retry_limit + @secondary_limit
-            $log.warn "failed to flush the buffer, next retry will be with secondary output.", :next_retry=>Time.at(@next_retry_time), :error_class=>e.class.to_s, :error=>e.to_s, :plugin_id=>plugin_id
+            $log.warn "failed to flush the buffer, next retry will be with secondary output.", next_retry: Time.at(@next_retry_time), error_class: e.class.to_s, error: e.to_s, plugin_id: plugin_id
             $log.warn_backtrace e.backtrace
           else
-            $log.warn "failed to flush the buffer.", :error_class=>e.class, :error=>e.to_s, :plugin_id=>plugin_id
+            $log.warn "failed to flush the buffer.", error_class: e.class, error: e.to_s, plugin_id: plugin_id
             $log.warn "secondary retry count exceededs limit."
             $log.warn_backtrace e.backtrace
             write_abort
@@ -380,7 +389,7 @@ module Fluent
           end
 
         else
-          $log.warn "failed to flush the buffer.", :error_class=>e.class.to_s, :error=>e.to_s, :plugin_id=>plugin_id
+          $log.warn "failed to flush the buffer.", error_class: e.class.to_s, error: e.to_s, plugin_id: plugin_id
           $log.warn "retry count exceededs limit."
           $log.warn_backtrace e.backtrace
           write_abort
@@ -403,7 +412,7 @@ module Fluent
       begin
         @buffer.before_shutdown(self)
       rescue
-        $log.warn "before_shutdown failed", :error=>$!.to_s
+        $log.warn "before_shutdown failed", error: $!.to_s
         $log.warn_backtrace
       end
     end
@@ -425,7 +434,7 @@ module Fluent
       begin
         @buffer.clear!
       rescue
-        $log.error "unexpected error while aborting", :error=>$!.to_s
+        $log.error "unexpected error while aborting", error: $!.to_s
         $log.error_backtrace
       end
     end
@@ -483,11 +492,11 @@ module Fluent
     end
 
     desc 'The time format used as part of the file name.'
-    config_param :time_slice_format, :string, :default => '%Y%m%d'
+    config_param :time_slice_format, :string, default: '%Y%m%d'
     desc 'The amount of time Fluentd will wait for old logs to arrive.'
-    config_param :time_slice_wait, :time, :default => 10*60
+    config_param :time_slice_wait, :time, default: 10*60
     desc 'Parse the time value in the specified timezone'
-    config_param :timezone, :string, :default => nil
+    config_param :timezone, :string, default: nil
     config_set_default :buffer_type, 'file'  # overwrite default buffer_type
     config_set_default :buffer_chunk_limit, 256*1024*1024  # overwrite default buffer_chunk_limit
     config_set_default :flush_interval, nil
@@ -552,14 +561,20 @@ module Fluent
       @emit_count += 1
       formatted_data = {}
       es.each {|time,record|
-        tc = time / @time_slice_cache_interval
-        if @before_tc == tc
-          key = @before_key
-        else
-          @before_tc = tc
-          key = @time_slicer.call(time)
-          @before_key = key
+        begin
+          tc = time / @time_slice_cache_interval
+          if @before_tc == tc
+            key = @before_key
+          else
+            @before_tc = tc
+            key = @time_slicer.call(time)
+            @before_key = key
+          end
+        rescue => e
+          @router.emit_error_event(tag, Engine.now, {'time' => time, 'record' => record}, e)
+          next
         end
+
         formatted_data[key] ||= ''
         formatted_data[key] << format(tag, time, record)
       }
@@ -603,4 +618,3 @@ module Fluent
     #end
   end
 end
-