fluentd 0.12.43 → 0.14.0

data/lib/fluent/daemon.rb

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+# -*- coding: utf-8 -*-
+
+here = File.dirname(__FILE__)
+$LOAD_PATH << File.expand_path(File.join(here, '..'))
+
+require 'serverengine'
+require 'fluent/supervisor'
+
+server_module = Fluent.const_get(ARGV[0])
+worker_module = Fluent.const_get(ARGV[1])
+# it doesn't call ARGV in block because when reloading config, params will be initialized and then it can't use previous config.
+config_path = ARGV[2]
+params = JSON.parse(ARGV[3])
+ServerEngine::Daemon.run_server(server_module, worker_module) { Fluent::Supervisor.load_config(config_path, params) }

data/lib/fluent/engine.rb

@@ -16,12 +16,12 @@
 
 require 'socket'
 
-require 'msgpack'
 require 'cool.io'
 
 require 'fluent/config'
 require 'fluent/event'
 require 'fluent/event_router'
+require 'fluent/msgpack_factory'
 require 'fluent/root_agent'
 require 'fluent/time'
 require 'fluent/system_config'
@@ -29,15 +29,7 @@ require 'fluent/plugin'
 
 module Fluent
   class EngineClass
-    class DummyMessagePackFactory
-      def packer(*args)
-        MessagePack::Packer.new(*args)
-      end
-
-      def unpacker(*args)
-        MessagePack::Unpacker.new(*args)
-      end
-    end
+    include Fluent::MessagePackFactory::Mixin
 
     def initialize
       @root_agent = nil
@@ -51,32 +43,30 @@ module Fluent
 
       @suppress_config_dump = false
 
-      @msgpack_factory = DummyMessagePackFactory.new
+      @system_config = SystemConfig.new
     end
 
+    MAINLOOP_SLEEP_INTERVAL = 0.3
+
     MATCH_CACHE_SIZE = 1024
     LOG_EMIT_INTERVAL = 0.1
 
     attr_reader :root_agent
     attr_reader :matches, :sources
-    attr_reader :msgpack_factory
     attr_reader :system_config
 
     def init(system_config)
       @system_config = system_config
 
       BasicSocket.do_not_reverse_lookup = true
-      Plugin.load_plugins
-      if defined?(Encoding)
-        Encoding.default_internal = 'ASCII-8BIT' if Encoding.respond_to?(:default_internal)
-        Encoding.default_external = 'ASCII-8BIT' if Encoding.respond_to?(:default_external)
-      end
 
       suppress_interval(system_config.emit_error_log_interval) unless system_config.emit_error_log_interval.nil?
       @suppress_config_dump = system_config.suppress_config_dump unless system_config.suppress_config_dump.nil?
       @without_source = system_config.without_source unless system_config.without_source.nil?
 
-      @root_agent = RootAgent.new(@system_config)
+      @root_agent = RootAgent.new(log: log, system_config: @system_config)
+
+      MessagePackFactory.init
 
       self
     end
@@ -134,22 +124,20 @@ module Fluent
       end
     end
 
-    def load_plugin_dir(dir)
-      Plugin.load_plugin_dir(dir)
+    def add_plugin_dir(dir)
+      Plugin.add_plugin_dir(dir)
     end
 
     def emit(tag, time, record)
-      unless record.nil?
-        emit_stream tag, OneEventStream.new(time, record)
-      end
+      raise "BUG: use router.emit instead of Engine.emit"
     end
 
     def emit_array(tag, array)
-      emit_stream tag, ArrayEventStream.new(array)
+      raise "BUG: use router.emit_array instead of Engine.emit_array"
     end
 
     def emit_stream(tag, es)
-      @event_router.emit_stream(tag, es)
+      raise "BUG: use router.emit_stream instead of Engine.emit_stream"
     end
 
     def flush!
@@ -158,7 +146,7 @@ module Fluent
 
     def now
       # TODO thread update
-      Time.now.to_i
+      Fluent::EventTime.now
     end
 
     def log_event_loop
@@ -176,7 +164,7 @@ module Fluent
           begin
             @event_router.emit(tag, time, record)
           rescue => e
-            $log.error "failed to emit fluentd's log event", tag: tag, event: record, error_class: e.class, error: e
+            $log.error "failed to emit fluentd's log event", tag: tag, event: record, error: e
           end
         }
       end
@@ -191,38 +179,24 @@ module Fluent
           @log_emit_thread = Thread.new(&method(:log_event_loop))
         end
 
-        unless @engine_stopped
-          # for empty loop
-          @default_loop = Coolio::Loop.default
-          @default_loop.attach Coolio::TimerWatcher.new(1, true)
-          # TODO attach async watch for thread pool
-          @default_loop.run
-        end
-
-        if @engine_stopped and @default_loop
-          @default_loop.stop
-          @default_loop = nil
-        end
+        sleep MAINLOOP_SLEEP_INTERVAL until @engine_stopped
 
-      rescue => e
-        $log.error "unexpected error", error_class: e.class, error: e
+      rescue Exception => e
+        $log.error "unexpected error", error: e
         $log.error_backtrace
-      ensure
-        $log.info "shutting down fluentd"
-        shutdown
-        if @log_emit_thread
-          @log_event_loop_stop = true
-          @log_emit_thread.join
-        end
+        raise
+      end
+
+      $log.info "shutting down fluentd"
+      shutdown
+      if @log_emit_thread
+        @log_event_loop_stop = true
+        @log_emit_thread.join
       end
     end
 
     def stop
       @engine_stopped = true
-      if @default_loop
-        @default_loop.stop
-        @default_loop = nil
-      end
       nil
     end
 

data/lib/fluent/env.rb

@@ -18,8 +18,10 @@ module Fluent
   DEFAULT_CONFIG_PATH = ENV['FLUENT_CONF'] || '/etc/fluent/fluent.conf'
   DEFAULT_PLUGIN_DIR = ENV['FLUENT_PLUGIN'] || '/etc/fluent/plugin'
   DEFAULT_SOCKET_PATH = ENV['FLUENT_SOCKET'] || '/var/run/fluent/fluent.sock'
-  DEFAULT_OJ_OPTIONS = {bigdecimal_load: :float, mode: :compat, use_to_json: true}
-  DEFAULT_LISTEN_PORT = 24224
-  DEFAULT_FILE_PERMISSION = 0644
-  DEFAULT_DIR_PERMISSION = 0755
+  IS_WINDOWS = /mswin|mingw/ === RUBY_PLATFORM
+  private_constant :IS_WINDOWS
+
+  def self.windows?
+    IS_WINDOWS
+  end
 end

data/lib/fluent/event.rb

@@ -14,11 +14,17 @@
 #    limitations under the License.
 #
 
-require 'fluent/engine'
+require 'fluent/msgpack_factory'
 
 module Fluent
   class EventStream
     include Enumerable
+    include MessagePackFactory::Mixin
+
+    def size
+      raise NotImplementedError, "DO NOT USE THIS CLASS directly."
+    end
+    alias :length :size
 
     def repeatable?
       false
@@ -28,15 +34,23 @@ module Fluent
       raise NotImplementedError, "DO NOT USE THIS CLASS directly."
     end
 
-    def to_msgpack_stream
-      out = Fluent::Engine.msgpack_factory.packer
+    def to_msgpack_stream(time_int: false)
+      return to_msgpack_stream_forced_integer if time_int
+      out = msgpack_packer
       each {|time,record|
         out.write([time,record])
       }
       out.to_s
     end
-  end
 
+    def to_msgpack_stream_forced_integer
+      out = msgpack_packer
+      each {|time,record|
+        out.write([time.to_i,record])
+      }
+      out.to_s
+    end
+  end
 
   class OneEventStream < EventStream
     def initialize(time, record)
@@ -48,6 +62,10 @@ module Fluent
       OneEventStream.new(@time, @record.dup)
     end
 
+    def size
+      1
+    end
+
     def repeatable?
       true
     end
@@ -72,6 +90,10 @@ module Fluent
       ArrayEventStream.new(entries)
     end
 
+    def size
+      @entries.size
+    end
+
     def repeatable?
       true
     end
@@ -93,7 +115,7 @@ module Fluent
   #
   # Use this class as below, in loop of data-enumeration:
   #  1. initialize blank stream:
-  #     streams[tag] ||= MultiEventStream
+  #     streams[tag] ||= MultiEventStream.new
   #  2. add events
   #     stream[tag].add(time, record)
   class MultiEventStream < EventStream
@@ -110,6 +132,10 @@ module Fluent
       es
     end
 
+    def size
+      @time_array.size
+    end
+
     def add(time, record)
       @time_array << time
       @record_array << record
@@ -135,8 +161,13 @@ module Fluent
 
   class MessagePackEventStream < EventStream
     # Keep cached_unpacker argument for existence plugins
-    def initialize(data, cached_unpacker = nil)
+    def initialize(data, cached_unpacker = nil, size = 0)
       @data = data
+      @size = size
+    end
+
+    def size
+      @size
     end
 
     def repeatable?
@@ -144,15 +175,31 @@ module Fluent
     end
 
     def each(&block)
-      # TODO format check
-      unpacker = Fluent::Engine.msgpack_factory.unpacker
-      unpacker.feed_each(@data, &block)
+      msgpack_unpacker.feed_each(@data, &block)
       nil
     end
 
-    def to_msgpack_stream
+    def to_msgpack_stream(time_int: false)
+      # time_int is always ignored because @data is always packed binary in this class
       @data
     end
   end
-end
 
+  module ChunkMessagePackEventStreamer
+    include MessagePackFactory::Mixin
+    # chunk.extend(ChunkEventStreamer)
+    #  => chunk.each{|time, record| ... }
+    def each(&block)
+      open do |io|
+        msgpack_unpacker(io).each(&block)
+      end
+      nil
+    end
+    alias :msgpack_each :each
+
+    def to_msgpack_stream(time_int: false)
+      # time_int is always ignored because data is already packed and written in chunk
+      read
+    end
+  end
+end

data/lib/fluent/event_router.rb

@@ -16,6 +16,7 @@
 
 require 'fluent/match'
 require 'fluent/event'
+require 'fluent/filter'
 
 module Fluent
   #
@@ -45,7 +46,6 @@ module Fluent
       @match_cache = MatchCache.new
       @default_collector = default_collector
       @emit_error_handler = emit_error_handler
-      @chain = NullOutputChain.instance
     end
 
     attr_accessor :default_collector
@@ -87,7 +87,7 @@ module Fluent
     end
 
     def emit_stream(tag, es)
-      match(tag).emit(tag, es, @chain)
+      match(tag).emit_events(tag, es)
     rescue => e
       @emit_error_handler.handle_emits_error(tag, es, e)
     end
@@ -102,7 +102,7 @@ module Fluent
 
     def match(tag)
       collector = @match_cache.get(tag) {
-        c = find(tag) || @default_collector
+        find(tag) || @default_collector
       }
       collector
     end
@@ -146,12 +146,12 @@ module Fluent
         @output = output
       end
 
-      def emit(tag, es, chain)
+      def emit_events(tag, es)
         processed = es
         @filters.each { |filter|
           processed = filter.filter_stream(tag, processed)
         }
-        @output.emit(tag, processed, chain)
+        @output.emit_events(tag, processed)
       end
     end
 

data/lib/fluent/filter.rb

@@ -14,56 +14,8 @@
 #    limitations under the License.
 #
 
-require 'fluent/config'
-require 'fluent/configurable'
-require 'fluent/engine'
-require 'fluent/event'
-require 'fluent/log'
+require 'fluent/compat/filter'
 
 module Fluent
-  class Filter
-    include Configurable
-    include PluginId
-    include PluginLoggerMixin
-
-    attr_accessor :router
-
-    def initialize
-      super
-    end
-
-    def configure(conf)
-      super
-
-      if label_name = conf['@label']
-        label = Engine.root_agent.find_label(label_name)
-        @router = label.event_router
-      elsif @router.nil?
-        @router = Engine.root_agent.event_router
-      end
-    end
-
-    def start
-    end
-
-    def shutdown
-    end
-
-    def filter(tag, time, record)
-      raise NotImplementedError, "Implement this method in child class"
-    end
-
-    def filter_stream(tag, es)
-      new_es = MultiEventStream.new
-      es.each { |time, record|
-        begin
-          filtered_record = filter(tag, time, record)
-          new_es.add(time, filtered_record) if filtered_record
-        rescue => e
-          router.emit_error_event(tag, time, record, e)
-        end
-      }
-      new_es
-    end
-  end
+  Filter = Fluent::Compat::Filter
 end

data/lib/fluent/formatter.rb

@@ -14,299 +14,10 @@
 #    limitations under the License.
 #
 
-require 'fluent/configurable'
-require 'fluent/env'
-require 'fluent/registry'
-require 'fluent/mixin'
+require 'fluent/compat/formatter'
 
 module Fluent
-  class Formatter
-    include Configurable
-
-    def configure(conf)
-      super
-    end
-
-    def format(tag, time, record)
-      raise NotImplementedError, "Implement this method in child class"
-    end
-  end
-
-  module TextFormatter
-    module HandleTagAndTimeMixin
-      def self.included(klass)
-        klass.instance_eval {
-          config_param :include_time_key, :bool, default: false
-          config_param :time_key, :string, default: 'time'
-          config_param :time_format, :string, default: nil
-          config_param :include_tag_key, :bool, default: false
-          config_param :tag_key, :string, default: 'tag'
-          config_param :localtime, :bool, default: true
-          config_param :timezone, :string, default: nil
-        }
-      end
-
-      def configure(conf)
-        super
-
-        if conf['utc']
-          @localtime = false
-        end
-        @timef = TimeFormatter.new(@time_format, @localtime, @timezone)
-      end
-
-      def filter_record(tag, time, record)
-        if @include_tag_key
-          record[@tag_key] = tag
-        end
-        if @include_time_key
-          record[@time_key] = @timef.format(time)
-        end
-      end
-    end
-
-    class OutFileFormatter < Formatter
-      include HandleTagAndTimeMixin
-
-      config_param :output_time, :bool, default: true
-      config_param :output_tag, :bool, default: true
-      config_param :delimiter, default: "\t" do |val|
-        case val
-        when /SPACE/i then ' '
-        when /COMMA/i then ','
-        else "\t"
-        end
-      end
-
-      def format(tag, time, record)
-        filter_record(tag, time, record)
-        header = ''
-        header << "#{@timef.format(time)}#{@delimiter}" if @output_time
-        header << "#{tag}#{@delimiter}" if @output_tag
-        "#{header}#{Yajl.dump(record)}\n"
-      end
-    end
-
-    class StdoutFormatter < Formatter
-      config_param :output_type, :string, default: 'json'
-
-      def configure(conf)
-        super
-
-        @formatter = Plugin.new_formatter(@output_type)
-        @formatter.configure(conf)
-      end
-
-      def format(tag, time, record)
-        header = "#{Time.now.localtime} #{tag}: "
-        "#{header}#{@formatter.format(tag, time, record)}"
-      end
-    end
-
-    module StructuredFormatMixin
-      def self.included(klass)
-        klass.instance_eval {
-          config_param :time_as_epoch, :bool, default: false
-        }
-      end
-
-      def configure(conf)
-        super
-
-        if @time_as_epoch
-          if @include_time_key
-            @include_time_key = false
-          else
-            $log.warn "include_time_key is false so ignore time_as_epoch"
-            @time_as_epoch = false
-          end
-        end
-      end
-
-      def format(tag, time, record)
-        filter_record(tag, time, record)
-        record[@time_key] = time if @time_as_epoch
-        format_record(record)
-      end
-    end
-
-    class JSONFormatter < Formatter
-      include HandleTagAndTimeMixin
-      include StructuredFormatMixin
-
-      config_param :json_parser, :string, default: 'oj'
-      config_param :add_newline, :bool, default: true
-
-      def configure(conf)
-        super
-
-        begin
-          raise LoadError unless @json_parser == 'oj'
-          require 'oj'
-          Oj.default_options = Fluent::DEFAULT_OJ_OPTIONS
-          @dump_proc = Oj.method(:dump)
-        rescue LoadError
-          @dump_proc = Yajl.method(:dump)
-        end
-
-        # format json is used on various highload environment, so re-define method to skip if check
-        unless @add_newline
-          define_singleton_method(:format, method(:format_without_nl))
-        end
-      end
-
-      def format_record(record)
-        "#{@dump_proc.call(record)}\n"
-      end
-
-      def format_without_nl(tag, time, record)
-        @dump_proc.call(record)
-      end
-    end
-
-    class HashFormatter < Formatter
-      include HandleTagAndTimeMixin
-      include StructuredFormatMixin
-
-      config_param :add_newline, :bool, default: true
-
-      def format_record(record)
-        if @add_newline
-          "#{record.to_s}\n"
-        else
-          record.to_s
-        end
-      end
-    end
-
-    class MessagePackFormatter < Formatter
-      include HandleTagAndTimeMixin
-      include StructuredFormatMixin
-
-      def format_record(record)
-        record.to_msgpack
-      end
-    end
-
-    class LabeledTSVFormatter < Formatter
-      include HandleTagAndTimeMixin
-
-      config_param :delimiter, :string, default: "\t"
-      config_param :label_delimiter, :string, default: ":"
-      config_param :add_newline, :bool, default: true
-
-      def format(tag, time, record)
-        filter_record(tag, time, record)
-        formatted = record.inject('') { |result, pair|
-          result << @delimiter if result.length.nonzero?
-          result << "#{pair.first}#{@label_delimiter}#{pair.last}"
-        }
-        formatted << "\n".freeze if @add_newline
-        formatted
-      end
-    end
-
-    class CsvFormatter < Formatter
-      include HandleTagAndTimeMixin
-
-      config_param :delimiter, default: ',' do |val|
-        ['\t', 'TAB'].include?(val) ? "\t" : val
-      end
-      config_param :force_quotes, :bool, default: true
-      config_param :fields, :array, value_type: :string
-      config_param :add_newline, :bool, default: true
-
-      def initialize
-        super
-        require 'csv'
-      end
-
-      def configure(conf)
-        super
-        @fields = fields.select { |f| !f.empty? }
-        raise ConfigError, "empty value is specified in fields parameter" if @fields.empty?
-
-        @generate_opts = {col_sep: @delimiter, force_quotes: @force_quotes}
-      end
-
-      def format(tag, time, record)
-        filter_record(tag, time, record)
-        row = @fields.inject([]) do |memo, key|
-            memo << record[key]
-            memo
-        end
-        line = CSV.generate_line(row, @generate_opts)
-        line.chomp! unless @add_newline
-        line
-      end
-    end
-
-    class SingleValueFormatter < Formatter
-      config_param :message_key, :string, default: 'message'
-      config_param :add_newline, :bool, default: true
-
-      def format(tag, time, record)
-        text = record[@message_key].to_s.dup
-        text << "\n" if @add_newline
-        text
-      end
-    end
-
-    class ProcWrappedFormatter < Formatter
-      def initialize(proc)
-        @proc = proc
-      end
-
-      def configure(conf)
-      end
-
-      def format(tag, time, record)
-        @proc.call(tag, time, record)
-      end
-    end
-
-    TEMPLATE_REGISTRY = Registry.new(:formatter_type, 'fluent/plugin/formatter_')
-    {
-      'out_file' => Proc.new { OutFileFormatter.new },
-      'stdout' => Proc.new { StdoutFormatter.new },
-      'json' => Proc.new { JSONFormatter.new },
-      'hash' => Proc.new { HashFormatter.new },
-      'msgpack' => Proc.new { MessagePackFormatter.new },
-      'ltsv' => Proc.new { LabeledTSVFormatter.new },
-      'csv' => Proc.new { CsvFormatter.new },
-      'single_value' => Proc.new { SingleValueFormatter.new },
-    }.each { |name, factory|
-      TEMPLATE_REGISTRY.register(name, factory)
-    }
-
-    def self.register_template(name, factory_or_proc)
-      factory = if factory_or_proc.is_a?(Class) # XXXFormatter
-                  Proc.new { factory_or_proc.new }
-                elsif factory_or_proc.arity == 3 # Proc.new { |tag, time, record| }
-                  Proc.new { ProcWrappedFormatter.new(factory_or_proc) }
-                else # Proc.new { XXXFormatter.new }
-                  factory_or_proc
-                end
-
-      TEMPLATE_REGISTRY.register(name, factory)
-    end
-
-    def self.lookup(format)
-      TEMPLATE_REGISTRY.lookup(format).call
-    end
-
-    # Keep backward-compatibility
-    def self.create(conf)
-      format = conf['format']
-      if format.nil?
-        raise ConfigError, "'format' parameter is required"
-      end
-
-      formatter = lookup(format)
-      if formatter.respond_to?(:configure)
-        formatter.configure(conf)
-      end
-      formatter
-    end
-  end
+  Formatter = Fluent::Compat::Formatter
+  TextFormatter = Fluent::Compat::TextFormatter
+  # deprecate_constant is ruby 2.3 feature
 end