fluentd 0.12.40 → 1.6.2

data/lib/fluent/engine.rb

@@ -14,14 +14,10 @@
 #    limitations under the License.
 #
 
-require 'socket'
-
-require 'msgpack'
-require 'cool.io'
-
 require 'fluent/config'
 require 'fluent/event'
 require 'fluent/event_router'
+require 'fluent/msgpack_factory'
 require 'fluent/root_agent'
 require 'fluent/time'
 require 'fluent/system_config'
@@ -29,54 +25,51 @@ require 'fluent/plugin'
 
 module Fluent
   class EngineClass
-    class DummyMessagePackFactory
-      def packer(*args)
-        MessagePack::Packer.new(*args)
-      end
-
-      def unpacker(*args)
-        MessagePack::Unpacker.new(*args)
-      end
-    end
+    include Fluent::MessagePackFactory::Mixin
 
     def initialize
       @root_agent = nil
-      @event_router = nil
       @default_loop = nil
       @engine_stopped = false
+      @_worker_id = nil
 
+      @log_event_router = nil
       @log_emit_thread = nil
       @log_event_loop_stop = false
+      @log_event_loop_graceful_stop = false
       @log_event_queue = []
+      @log_event_verbose = false
 
       @suppress_config_dump = false
 
-      @msgpack_factory = DummyMessagePackFactory.new
+      @system_config = SystemConfig.new
+
+      @dry_run_mode = false
     end
 
+    MAINLOOP_SLEEP_INTERVAL = 0.3
+
     MATCH_CACHE_SIZE = 1024
     LOG_EMIT_INTERVAL = 0.1
 
     attr_reader :root_agent
     attr_reader :matches, :sources
-    attr_reader :msgpack_factory
     attr_reader :system_config
 
+    attr_accessor :dry_run_mode
+
     def init(system_config)
       @system_config = system_config
 
-      BasicSocket.do_not_reverse_lookup = true
-      Plugin.load_plugins
-      if defined?(Encoding)
-        Encoding.default_internal = 'ASCII-8BIT' if Encoding.respond_to?(:default_internal)
-        Encoding.default_external = 'ASCII-8BIT' if Encoding.respond_to?(:default_external)
-      end
-
       suppress_interval(system_config.emit_error_log_interval) unless system_config.emit_error_log_interval.nil?
       @suppress_config_dump = system_config.suppress_config_dump unless system_config.suppress_config_dump.nil?
       @without_source = system_config.without_source unless system_config.without_source.nil?
 
-      @root_agent = RootAgent.new(@system_config)
+      @log_event_verbose = system_config.log_event_verbose unless system_config.log_event_verbose.nil?
+
+      @root_agent = RootAgent.new(log: log, system_config: @system_config)
+
+      MessagePackFactory.init
 
       self
     end
@@ -109,12 +102,21 @@ module Fluent
                     else
                       "section <#{e.name}> is not used in <#{parent_name}>"
                     end
-          $log.warn message
+          if e.for_every_workers?
+            $log.warn :worker0, message
+          elsif e.for_this_worker?
+            $log.warn message
+          end
           next
         end
         unless e.name == 'system'
           unless @without_source && e.name == 'source'
-            $log.warn "parameter '#{key}' in #{e.to_s.strip} is not used."
+            message = "parameter '#{key}' in #{e.to_s.strip} is not used."
+            if e.for_every_workers?
+              $log.warn :worker0, message
+            elsif e.for_this_worker?
+              $log.warn message
+            end
           end
         end
       }
@@ -123,33 +125,60 @@ module Fluent
     def configure(conf)
       # plugins / configuration dumps
       Gem::Specification.find_all.select{|x| x.name =~ /^fluent(d|-(plugin|mixin)-.*)$/}.each do |spec|
-        $log.info "gem '#{spec.name}' version '#{spec.version}'"
+        $log.info :worker0, "gem '#{spec.name}' version '#{spec.version}'"
       end
 
       @root_agent.configure(conf)
-      @event_router = @root_agent.event_router
+
+      begin
+        log_event_agent = @root_agent.find_label(Fluent::Log::LOG_EVENT_LABEL)
+        log_event_router = log_event_agent.event_router
+
+        # suppress mismatched tags only for <label @FLUENT_LOG> label.
+        # it's not suppressed in default event router for non-log-event events
+        log_event_router.suppress_missing_match!
+
+        @log_event_router = log_event_router
+
+        unmatched_tags = Fluent::Log.event_tags.select{|t| !@log_event_router.match?(t) }
+        unless unmatched_tags.empty?
+          $log.warn "match for some tags of log events are not defined (to be ignored)", tags: unmatched_tags
+        end
+      rescue ArgumentError # ArgumentError "#{label_name} label not found"
+        # use default event router if <label @FLUENT_LOG> is missing in configuration
+        log_event_router = @root_agent.event_router
+
+        if Fluent::Log.event_tags.any?{|t| log_event_router.match?(t) }
+          @log_event_router = log_event_router
+
+          unmatched_tags = Fluent::Log.event_tags.select{|t| !@log_event_router.match?(t) }
+          unless unmatched_tags.empty?
+            $log.warn "match for some tags of log events are not defined (to be ignored)", tags: unmatched_tags
+          end
+        end
+      end
+
+      $log.enable_event(true) if @log_event_router
 
       unless @suppress_config_dump
-        $log.info "using configuration file: #{conf.to_s.rstrip}"
+        $log.info :supervisor, "using configuration file: #{conf.to_s.rstrip}"
       end
     end
 
-    def load_plugin_dir(dir)
-      Plugin.load_plugin_dir(dir)
+    def add_plugin_dir(dir)
+      Plugin.add_plugin_dir(dir)
     end
 
     def emit(tag, time, record)
-      unless record.nil?
-        emit_stream tag, OneEventStream.new(time, record)
-      end
+      raise "BUG: use router.emit instead of Engine.emit"
     end
 
     def emit_array(tag, array)
-      emit_stream tag, ArrayEventStream.new(array)
+      raise "BUG: use router.emit_array instead of Engine.emit_array"
     end
 
     def emit_stream(tag, es)
-      @event_router.emit_stream(tag, es)
+      raise "BUG: use router.emit_stream instead of Engine.emit_stream"
     end
 
     def flush!
@@ -158,7 +187,7 @@ module Fluent
 
     def now
       # TODO thread update
-      Time.now.to_i
+      Fluent::EventTime.now
     end
 
     def log_event_loop
@@ -166,6 +195,7 @@ module Fluent
 
       while sleep(LOG_EMIT_INTERVAL)
         break if @log_event_loop_stop
+        break if @log_event_loop_graceful_stop && @log_event_queue.empty?
         next if @log_event_queue.empty?
 
         # NOTE: thead-safe of slice! depends on GVL
@@ -174,9 +204,10 @@ module Fluent
 
         events.each {|tag,time,record|
           begin
-            @event_router.emit(tag, time, record)
+            @log_event_router.emit(tag, time, record)
           rescue => e
-            $log.error "failed to emit fluentd's log event", tag: tag, event: record, error_class: e.class, error: e
+            # This $log.error doesn't emit log events, because of `$log.disable_events(Thread.current)` above
+            $log.error "failed to emit fluentd's log event", tag: tag, event: record, error: e
           end
         }
       end
@@ -184,45 +215,44 @@ module Fluent
 
     def run
       begin
+        $log.info "starting fluentd worker", pid: Process.pid, ppid: Process.ppid, worker: worker_id
         start
 
-        if @event_router.match?($log.tag)
-          $log.enable_event
+        if @log_event_router
+          $log.enable_event(true)
           @log_emit_thread = Thread.new(&method(:log_event_loop))
+          @log_emit_thread.abort_on_exception = true
         end
 
-        unless @engine_stopped
-          # for empty loop
-          @default_loop = Coolio::Loop.default
-          @default_loop.attach Coolio::TimerWatcher.new(1, true)
-          # TODO attach async watch for thread pool
-          @default_loop.run
-        end
-
-        if @engine_stopped and @default_loop
-          @default_loop.stop
-          @default_loop = nil
-        end
+        $log.info "fluentd worker is now running", worker: worker_id
+        sleep MAINLOOP_SLEEP_INTERVAL until @engine_stopped
+        $log.info "fluentd worker is now stopping", worker: worker_id
 
-      rescue => e
-        $log.error "unexpected error", error_class: e.class, error: e
+      rescue Exception => e
+        $log.error "unexpected error", error: e
         $log.error_backtrace
-      ensure
-        $log.info "shutting down fluentd"
-        shutdown
+        raise
+      end
+
+      unless @log_event_verbose
+        $log.enable_event(false)
         if @log_emit_thread
-          @log_event_loop_stop = true
+          # to make sure to emit all log events into router, before shutting down
+          @log_event_loop_graceful_stop = true
           @log_emit_thread.join
+          @log_emit_thread = nil
         end
       end
+      $log.info "shutting down fluentd worker", worker: worker_id
+      shutdown
+      if @log_emit_thread
+        @log_event_loop_stop = true
+        @log_emit_thread.join
+      end
     end
 
     def stop
       @engine_stopped = true
-      if @default_loop
-        @default_loop.stop
-        @default_loop = nil
-      end
       nil
     end
 
@@ -231,8 +261,15 @@ module Fluent
       @log_event_queue.push([tag, time, record])
     end
 
-    private
+    def worker_id
+      return @_worker_id if @_worker_id
+      # if ENV doesn't have SERVERENGINE_WORKER_ID, it is a worker under --no-supervisor or in tests
+      # so it's (almost) a single worker, worker_id=0
+      @_worker_id = (ENV['SERVERENGINE_WORKER_ID'] || 0).to_i
+      @_worker_id
+    end
 
+    private
     def start
       @root_agent.start
     end

data/lib/fluent/env.rb

@@ -14,12 +14,16 @@
 #    limitations under the License.
 #
 
+require 'serverengine/utils'
+
 module Fluent
   DEFAULT_CONFIG_PATH = ENV['FLUENT_CONF'] || '/etc/fluent/fluent.conf'
   DEFAULT_PLUGIN_DIR = ENV['FLUENT_PLUGIN'] || '/etc/fluent/plugin'
   DEFAULT_SOCKET_PATH = ENV['FLUENT_SOCKET'] || '/var/run/fluent/fluent.sock'
+  DEFAULT_BACKUP_DIR = ENV['FLUENT_BACKUP_DIR'] || '/tmp/fluent'
   DEFAULT_OJ_OPTIONS = {bigdecimal_load: :float, mode: :compat, use_to_json: true}
-  DEFAULT_LISTEN_PORT = 24224
-  DEFAULT_FILE_PERMISSION = 0644
-  DEFAULT_DIR_PERMISSION = 0755
+
+  def self.windows?
+    ServerEngine.windows?
+  end
 end

data/lib/fluent/error.rb

@@ -0,0 +1,30 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module Fluent
+  class UnrecoverableError < StandardError
+    def initialize(error_message = nil)
+      @message = error_message || "an unrecoverable error occurs in Fluentd process"
+    end
+
+    def to_s
+      @message
+    end
+  end
+
+  class InvalidRootDirectory < UnrecoverableError
+  end
+end

data/lib/fluent/event.rb

@@ -14,29 +14,68 @@
 #    limitations under the License.
 #
 
-require 'fluent/engine'
+require 'fluent/msgpack_factory'
+require 'fluent/plugin/compressable'
 
 module Fluent
   class EventStream
     include Enumerable
+    include MessagePackFactory::Mixin
+    include Fluent::Plugin::Compressable
+
+    # dup does deep copy for event stream
+    def dup
+      raise NotImplementedError, "DO NOT USE THIS CLASS directly."
+    end
+
+    def size
+      raise NotImplementedError, "DO NOT USE THIS CLASS directly."
+    end
+    alias :length :size
+
+    def empty?
+      size == 0
+    end
+
+    # for tests
+    def ==(other)
+      other.is_a?(EventStream) && self.to_msgpack_stream == other.to_msgpack_stream
+    end
 
     def repeatable?
       false
     end
 
+    def slice(index, num)
+      raise NotImplementedError, "DO NOT USE THIS CLASS directly."
+    end
+
     def each(&block)
       raise NotImplementedError, "DO NOT USE THIS CLASS directly."
     end
 
-    def to_msgpack_stream
-      out = Fluent::Engine.msgpack_factory.packer
+    def to_msgpack_stream(time_int: false)
+      return to_msgpack_stream_forced_integer if time_int
+      out = msgpack_packer
       each {|time,record|
         out.write([time,record])
       }
       out.to_s
     end
-  end
 
+    def to_compressed_msgpack_stream(time_int: false)
+      packed = to_msgpack_stream(time_int: time_int)
+      compress(packed)
+    end
+
+    def to_msgpack_stream_forced_integer
+      out = msgpack_packer
+      each {|time,record|
+        out.write([time.to_i,record])
+      }
+      out.to_s
+    end
+  end
 
   class OneEventStream < EventStream
     def initialize(time, record)
@@ -48,10 +87,22 @@ module Fluent
       OneEventStream.new(@time, @record.dup)
     end
 
+    def size
+      1
+    end
+
     def repeatable?
       true
     end
 
+    def slice(index, num)
+      if index > 0 || num == 0
+        ArrayEventStream.new([])
+      else
+        self.dup
+      end
+    end
+
     def each(&block)
       block.call(@time, @record)
       nil
@@ -68,10 +119,14 @@ module Fluent
     end
 
     def dup
-      entries = @entries.map { |entry| entry.dup } # @entries.map(:dup) doesn't work by ArgumentError
+      entries = @entries.map{ |time, record| [time, record.dup] }
       ArrayEventStream.new(entries)
     end
 
+    def size
+      @entries.size
+    end
+
     def repeatable?
       true
     end
@@ -80,6 +135,10 @@ module Fluent
       @entries.empty?
     end
 
+    def slice(index, num)
+      ArrayEventStream.new(@entries.slice(index, num))
+    end
+
     def each(&block)
       @entries.each(&block)
       nil
@@ -93,21 +152,21 @@ module Fluent
   #
   # Use this class as below, in loop of data-enumeration:
   #  1. initialize blank stream:
-  #     streams[tag] ||= MultiEventStream
+  #     streams[tag] ||= MultiEventStream.new
   #  2. add events
   #     stream[tag].add(time, record)
   class MultiEventStream < EventStream
-    def initialize
-      @time_array = []
-      @record_array = []
+    def initialize(time_array = [], record_array = [])
+      @time_array = time_array
+      @record_array = record_array
     end
 
     def dup
-      es = MultiEventStream.new
-      @time_array.zip(@record_array).each { |time, record|
-        es.add(time, record.dup)
-      }
-      es
+      MultiEventStream.new(@time_array.dup, @record_array.map(&:dup))
+    end
+
+    def size
+      @time_array.size
     end
 
     def add(time, record)
@@ -123,6 +182,10 @@ module Fluent
       @time_array.empty?
     end
 
+    def slice(index, num)
+      MultiEventStream.new(@time_array.slice(index, num), @record_array.slice(index, num))
+    end
+
     def each(&block)
       time_array = @time_array
       record_array = @record_array
@@ -134,25 +197,138 @@ module Fluent
   end
 
   class MessagePackEventStream < EventStream
-    # Keep cached_unpacker argument for existence plugins
-    def initialize(data, cached_unpacker = nil)
+    # https://github.com/msgpack/msgpack-ruby/issues/119
+
+    # Keep cached_unpacker argument for existing plugins
+    def initialize(data, cached_unpacker = nil, size = 0, unpacked_times: nil, unpacked_records: nil)
       @data = data
+      @size = size
+      @unpacked_times = unpacked_times
+      @unpacked_records = unpacked_records
+    end
+
+    def empty?
+      @data.empty?
+    end
+
+    def dup
+      if @unpacked_times
+        self.class.new(@data.dup, nil, @size, unpacked_times: @unpacked_times, unpacked_records: @unpacked_records.map(&:dup))
+      else
+        self.class.new(@data.dup, nil, @size)
+      end
+    end
+
+    def size
+      # @size is unbelievable always when @size == 0
+      # If the number of events is really zero, unpacking events takes very short time.
+      ensure_unpacked! if @size == 0
+      @size
     end
 
     def repeatable?
       true
     end
 
+    def ensure_unpacked!
+      return if @unpacked_times && @unpacked_records
+      @unpacked_times = []
+      @unpacked_records = []
+      msgpack_unpacker.feed_each(@data) do |time, record|
+        @unpacked_times << time
+        @unpacked_records << record
+      end
+      # @size should be updated always right after unpack.
+      # The real size of unpacked objects are correct, rather than given size.
+      @size = @unpacked_times.size
+    end
+
+    # This method returns MultiEventStream, because there are no reason
+    # to surve binary serialized by msgpack.
+    def slice(index, num)
+      ensure_unpacked!
+      MultiEventStream.new(@unpacked_times.slice(index, num), @unpacked_records.slice(index, num))
+    end
+
     def each(&block)
-      # TODO format check
-      unpacker = Fluent::Engine.msgpack_factory.unpacker
-      unpacker.feed_each(@data, &block)
+      if @unpacked_times
+        @unpacked_times.each_with_index do |time, i|
+          block.call(time, @unpacked_records[i])
+        end
+      else
+        @unpacked_times = []
+        @unpacked_records = []
+        msgpack_unpacker.feed_each(@data) do |time, record|
+          @unpacked_times << time
+          @unpacked_records << record
+          block.call(time, record)
+        end
+        @size = @unpacked_times.size
+      end
       nil
     end
 
-    def to_msgpack_stream
+    def to_msgpack_stream(time_int: false)
+      # time_int is always ignored because @data is always packed binary in this class
       @data
     end
   end
-end
 
+  class CompressedMessagePackEventStream < MessagePackEventStream
+    def initialize(data, cached_unpacker = nil, size = 0, unpacked_times: nil, unpacked_records: nil)
+      super
+      @decompressed_data = nil
+      @compressed_data = data
+    end
+
+    def empty?
+      ensure_decompressed!
+      super
+    end
+
+    def ensure_unpacked!
+      ensure_decompressed!
+      super
+    end
+
+    def each(&block)
+      ensure_decompressed!
+      super
+    end
+
+    def to_msgpack_stream(time_int: false)
+      ensure_decompressed!
+      super
+    end
+
+    def to_compressed_msgpack_stream(time_int: false)
+      # time_int is always ignored because @data is always packed binary in this class
+      @compressed_data
+    end
+
+    private
+
+    def ensure_decompressed!
+      return if @decompressed_data
+      @data = @decompressed_data = decompress(@data)
+    end
+  end
+
+  module ChunkMessagePackEventStreamer
+    include MessagePackFactory::Mixin
+    # chunk.extend(ChunkEventStreamer)
+    #  => chunk.each{|time, record| ... }
+    def each(&block)
+      open do |io|
+        msgpack_unpacker(io).each(&block)
+      end
+      nil
+    end
+    alias :msgpack_each :each
+
+    def to_msgpack_stream(time_int: false)
+      # time_int is always ignored because data is already packed and written in chunk
+      read
+    end
+  end
+end