fluentd 0.14.6 → 0.14.7

data/lib/fluent/plugin/compressable.rb

@@ -0,0 +1,91 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'zlib'
+
+module Fluent
+  module Plugin
+    module Compressable
+      def compress(data, **kwargs)
+        output_io = kwargs[:output_io]
+        io = output_io || StringIO.new
+        Zlib::GzipWriter.wrap(io) do |gz|
+          gz.write data
+        end
+
+        output_io || io.string
+      end
+
+      # compressed_data is String like `compress(data1) + compress(data2) + ... + compress(dataN)`
+      # https://www.ruby-forum.com/topic/971591#979503
+      def decompress(compressed_data = nil, output_io: nil, input_io: nil)
+        case
+        when input_io && output_io
+          io_decompress(input_io, output_io)
+        when input_io
+          output_io = StringIO.new
+          io = io_decompress(input_io, output_io)
+          io.string
+        when compressed_data.nil? || compressed_data.empty?
+          # check compressed_data(String) is 0 length
+          compressed_data
+        when output_io
+          # exeucte after checking compressed_data is empty or not
+          io = StringIO.new(compressed_data)
+          io_decompress(io, output_io)
+        else
+          string_decompress(compressed_data)
+        end
+      end
+
+      private
+
+      def string_decompress(compressed_data)
+        io = StringIO.new(compressed_data)
+
+        out = ''
+        loop do
+          gz = Zlib::GzipReader.new(io)
+          out += gz.read
+          unused = gz.unused
+          gz.finish
+
+          break if unused.nil?
+          adjust = unused.length
+          io.pos -= adjust
+        end
+
+        out
+      end
+
+      def io_decompress(input, output)
+        loop do
+          gz = Zlib::GzipReader.new(input)
+          v = gz.read
+          output.write(v)
+          unused = gz.unused
+          gz.finish
+
+          break if unused.nil?
+          adjust = unused.length
+          input.pos -= adjust
+        end
+
+        output
+      end
+    end
+  end
+end

data/lib/fluent/plugin/filter_grep.rb

@@ -38,8 +38,8 @@ module Fluent::Plugin
       (1..REGEXP_MAX_NUM).each do |i|
         next unless conf["regexp#{i}"]
         key, regexp = conf["regexp#{i}"].split(/ /, 2)
-        raise ConfigError, "regexp#{i} does not contain 2 parameters" unless regexp
-        raise ConfigError, "regexp#{i} contains a duplicated key, #{key}" if @regexps[key]
+        raise Fluent::ConfigError, "regexp#{i} does not contain 2 parameters" unless regexp
+        raise Fluent::ConfigError, "regexp#{i} contains a duplicated key, #{key}" if @regexps[key]
         @regexps[key] = Regexp.compile(regexp)
       end
 
@@ -47,8 +47,8 @@ module Fluent::Plugin
       (1..REGEXP_MAX_NUM).each do |i|
         next unless conf["exclude#{i}"]
         key, exclude = conf["exclude#{i}"].split(/ /, 2)
-        raise ConfigError, "exclude#{i} does not contain 2 parameters" unless exclude
-        raise ConfigError, "exclude#{i} contains a duplicated key, #{key}" if @excludes[key]
+        raise Fluent::ConfigError, "exclude#{i} does not contain 2 parameters" unless exclude
+        raise Fluent::ConfigError, "exclude#{i} contains a duplicated key, #{key}" if @excludes[key]
         @excludes[key] = Regexp.compile(exclude)
       end
     end

data/lib/fluent/plugin/formatter.rb

@@ -16,13 +16,13 @@
 
 require 'fluent/plugin/base'
 require 'fluent/plugin/owned_by_mixin'
-
-require 'fluent/mixin' # for TimeFormatter
+require 'fluent/time'
 
 module Fluent
   module Plugin
     class Formatter < Base
       include OwnedByMixin
+      include TimeMixin::Formatter
 
       configured_in :format
 

data/lib/fluent/plugin/formatter_json.rb

@@ -15,6 +15,7 @@
 #
 
 require 'fluent/plugin/formatter'
+require 'fluent/env'
 
 module Fluent
   module Plugin
@@ -29,7 +30,7 @@ module Fluent
         begin
           raise LoadError unless @json_parser == 'oj'
           require 'oj'
-          Oj.default_options = {mode: :compat}
+          Oj.default_options = Fluent::DEFAULT_OJ_OPTIONS
           @dump_proc = Oj.method(:dump)
         rescue LoadError
           @dump_proc = Yajl.method(:dump)

data/lib/fluent/plugin/formatter_out_file.rb

@@ -32,39 +32,12 @@ module Fluent
         else "\t"
         end
       end
-      config_param :time_type, :enum, list: [:float, :unixtime, :string], default: :string
-      config_param :time_format, :string, default: nil
-      config_param :localtime, :bool, default: true # if localtime is false and timezone is nil, then utc
-      config_param :timezone, :string, default: nil
+      config_set_default :time_type, :string
+      config_set_default :time_format, nil # time_format nil => iso8601
 
       def configure(conf)
-        # TODO: make a utility method in TimeFormatter to handle these conversion
-        #       copies of this code: plugin_helper/compat_parameters, compat/formatter_utils and here
-        if conf.has_key?('time_as_epoch') && Fluent::Config.bool_value(conf['time_as_epoch'])
-          conf['time_type'] = 'unixtime'
-        end
-        if conf.has_key?('localtime') || conf.has_key?('utc')
-          if conf.has_key?('localtime') && conf.has_key?('utc')
-            raise Fluent::ConfigError, "both of utc and localtime are specified, use only one of them"
-          elsif conf.has_key?('localtime')
-            conf['localtime'] = Fluent::Config.bool_value(conf['localtime'])
-          elsif conf.has_key?('utc')
-            conf['localtime'] = !(Fluent::Config.bool_value(conf['utc']))
-            # Specifying "localtime false" means using UTC in TimeFormatter
-            # And specifying "utc" is different from specifying "timezone +0000"(it's not always UTC).
-            # There are difference between "Z" and "+0000" in timezone formatting.
-            # TODO: add kwargs to TimeFormatter to specify "using localtime", "using UTC" or "using specified timezone" in more explicit way
-          end
-        end
-
         super
-
-        @timef = case @time_type
-                 when :float then ->(time){ time.to_r.to_f }
-                 when :unixtime then ->(time){ time.to_i }
-                 else
-                   Fluent::TimeFormatter.new(@time_format, @localtime, @timezone)
-                 end
+        @timef = time_formatter_create
       end
 
       def format(tag, time, record)

data/lib/fluent/plugin/in_forward.rb

@@ -153,7 +153,7 @@ module Fluent
       # In test cases it occasionally appeared that when detaching a watcher, another watcher is also detached.
       # In the case in the iteration of watchers, a watcher that has been already detached is intended to be detached
       # and therfore RuntimeError occurs saying that it is not attached to a loop.
-      # It occures only when testing for sending responses to ForwardOutput.
+      # It occurs only when testing for sending responses to ForwardOutput.
       # Sending responses needs to write the socket that is previously used only to read
       # and a handler has 2 watchers that is used to read and to write.
       # This problem occurs possibly because those watchers are thought to be related to each other
@@ -338,7 +338,8 @@ module Fluent
         # PackedForward
         option = msg[2]
         size = (option && option['size']) || 0
-        es = MessagePackEventStream.new(entries, nil, size.to_i)
+        es_class = (option && option['compressed'] == 'gzip') ? CompressedMessagePackEventStream : MessagePackEventStream
+        es = es_class.new(entries, nil, size.to_i)
         es = check_and_skip_invalid_event(tag, es, peeraddr) if @skip_invalid_event
         es = add_source_host(es, peeraddr[2]) if @source_hostname_key
         router.emit_stream(tag, es)

data/lib/fluent/plugin/in_monitor_agent.rb

@@ -262,34 +262,20 @@ module Fluent::Plugin
       array.concat Fluent::Engine.root_agent.inputs
 
       # get all output plugins
-      Fluent::Engine.root_agent.outputs.each { |o|
-        MonitorAgentInput.collect_children(o, array)
-      }
+      array.concat Fluent::Engine.root_agent.outputs
+
       # get all filter plugins
-      Fluent::Engine.root_agent.filters.each { |f|
-        MonitorAgentInput.collect_children(f, array)
-      }
+      array.concat Fluent::Engine.root_agent.filters
+
       Fluent::Engine.root_agent.labels.each { |name, l|
         # TODO: Add label name to outputs / filters for identifing plugins
-        l.outputs.each { |o| MonitorAgentInput.collect_children(o, array) }
-        l.filters.each { |f| MonitorAgentInput.collect_children(f, array) }
+        array.concat l.outputs
+        array.concat l.filters
       }
 
       array
     end
 
-    # get nexted plugins (such as <store> of the copy plugin)
-    # from the plugin `pe` recursively
-    def self.collect_children(pe, array=[])
-      array << pe
-      if pe.is_a?(Fluent::Plugin::MultiOutput) || pe.is_a?(Fluent::MultiOutput) && pe.respond_to?(:outputs)
-        pe.outputs.each {|nop|
-          collect_children(nop, array)
-        }
-      end
-      array
-    end
-
     # try to match the tag and get the info from the matched output plugin
     # TODO: Support output in label
     def plugin_info_by_tag(tag, opts={})
@@ -376,7 +362,7 @@ module Fluent::Plugin
       case pe
       when Fluent::Plugin::Input
         'input'.freeze
-      when Fluent::Plugin::Output, Fluent::Plugin::BareOutput
+      when Fluent::Plugin::Output, Fluent::Plugin::MultiOutput, Fluent::Plugin::BareOutput
         'output'.freeze
       when Fluent::Plugin::Filter
         'filter'.freeze

data/lib/fluent/plugin/in_syslog.rb

@@ -86,7 +86,7 @@ module Fluent::Plugin
       when 'udp'
         :udp
       else
-        raise ConfigError, "syslog input protocol type should be 'tcp' or 'udp'"
+        raise Fluent::ConfigError, "syslog input protocol type should be 'tcp' or 'udp'"
       end
     end
     desc 'If true, add source host to event record.'

data/lib/fluent/plugin/in_tail.rb

@@ -208,7 +208,7 @@ module Fluent::Plugin
       line_buffer_timer_flusher = (@multiline_mode && @multiline_flush_interval) ? TailWatcher::LineBufferTimerFlusher.new(log, @multiline_flush_interval, &method(:flush_buffer)) : nil
       tw = TailWatcher.new(path, @rotate_wait, pe, log, @read_from_head, @enable_watch_timer, @read_lines_limit, method(:update_watcher), line_buffer_timer_flusher, &method(:receive_lines))
       tw.attach do |watcher|
-        timer_execute(:in_tail_timer_trigger, 1, &watcher.method(:on_notify)) if watcher.enable_watch_timer
+        watcher.timer_trigger = timer_execute(:in_tail_timer_trigger, 1, &watcher.method(:on_notify)) if watcher.enable_watch_timer
         event_loop_attach(watcher.stat_trigger)
       end
       tw
@@ -248,6 +248,12 @@ module Fluent::Plugin
 
     # refresh_watchers calls @tails.keys so we don't use stop_watcher -> start_watcher sequence for safety.
     def update_watcher(path, pe)
+      if @pf
+        unless pe.read_inode == @pf[path].read_inode
+          log.trace "Skip update_watcher because watcher has been already updated by other inotify event"
+          return
+        end
+      end
       rotated_tw = @tails[path]
       @tails[path] = setup_watcher(path, pe)
       close_watcher_after_rotate_wait(rotated_tw) if rotated_tw
@@ -398,6 +404,7 @@ module Fluent::Plugin
         @update_watcher = update_watcher
 
         @stat_trigger = StatWatcher.new(path, log, &method(:on_notify))
+        @timer_trigger = nil
 
         @rotate_handler = RotateHandler.new(path, log, &method(:on_rotate))
         @io_handler = nil
@@ -421,11 +428,12 @@ module Fluent::Plugin
       end
 
       def attach
-        yield self
         on_notify
+        yield self
       end
 
       def detach
+        @timer_trigger.detach if @enable_watch_timer && @timer_trigger.attached?
         @stat_trigger.detach if @stat_trigger.attached?
       end
 

data/lib/fluent/plugin/multi_output.rb

@@ -32,7 +32,7 @@ module Fluent
         config_param :@type, :string, default: nil
       end
 
-      attr_reader :outputs
+      attr_reader :outputs, :outputs_statically_created
 
       def process(tag, es)
         raise NotImplementedError, "BUG: output plugins MUST implement this method"
@@ -41,8 +41,7 @@ module Fluent
       def initialize
         super
         @outputs = []
-
-        @compat = false
+        @outputs_statically_created = false
 
         @counters_monitor = Monitor.new
         # TODO: well organized counters
@@ -78,11 +77,72 @@ module Fluent
         end
       end
 
+      def static_outputs
+        @outputs_statically_created = true
+        @outputs
+      end
+
       # Child plugin's lifecycles are controlled by agent automatically.
       # It calls `outputs` to traverse plugins, and invoke start/stop/*shutdown/close/terminate on these directly.
       # * `start` of this plugin will be called after child plugins
       # * `stop`, `*shutdown`, `close` and `terminate` of this plugin will be called before child plugins
 
+      # But when MultiOutput plugins are created dynamically (by forest plugin or others), agent cannot find
+      # sub-plugins. So child plugins' lifecycles MUST be controlled by MultiOutput plugin itself.
+      # TODO: this hack will be removed at v2.
+      def call_lifecycle_method(method_name, checker_name)
+        return if @outputs_statically_created
+        @outputs.each do |o|
+          begin
+            log.debug "calling #{method_name} on output plugin dynamically created", type: Fluent::Plugin.lookup_type_from_class(o.class), plugin_id: o.plugin_id
+            o.send(method_name) unless o.send(checker_name)
+          rescue Exception => e
+            log.warn "unexpected error while calling #{method_name} on output plugin dynamically created", plugin: o.class, plugin_id: o.plugin_id, error: e
+            log.warn_backtrace
+          end
+        end
+      end
+
+      def start
+        super
+        call_lifecycle_method(:start, :started?)
+      end
+
+      def after_start
+        super
+        call_lifecycle_method(:after_start, :after_started?)
+      end
+
+      def stop
+        super
+        call_lifecycle_method(:stop, :stopped?)
+      end
+
+      def before_shutdown
+        super
+        call_lifecycle_method(:before_shutdown, :before_shutdown?)
+      end
+
+      def shutdown
+        super
+        call_lifecycle_method(:shutdown, :shutdown?)
+      end
+
+      def after_shutdown
+        super
+        call_lifecycle_method(:after_shutdown, :after_shutdown?)
+      end
+
+      def close
+        super
+        call_lifecycle_method(:close, :closed?)
+      end
+
+      def terminate
+        super
+        call_lifecycle_method(:terminate, :terminated?)
+      end
+
       def emit_sync(tag, es)
         @counters_monitor.synchronize{ @emit_count += 1 }
         begin

data/lib/fluent/plugin/out_exec.rb

@@ -42,7 +42,7 @@ module Fluent::Plugin
     desc "The format used to map the incoming events to the program input. (#{Fluent::ExecUtil::SUPPORTED_FORMAT.keys.join(',')})"
     config_param :format, default: :tsv, skip_accessor: true do |val|
       f = Fluent::ExecUtil::SUPPORTED_FORMAT[val]
-      raise ConfigError, "Unsupported format '#{val}'" unless f
+      raise Fluent::ConfigError, "Unsupported format '#{val}'" unless f
       f
     end
     config_param :localtime, :bool, default: false

data/lib/fluent/plugin/out_file.rb

@@ -59,7 +59,11 @@ module Fluent
 
       def generate_chunk(metadata)
         chunk = super
-        latest_chunk = metadata_list.sort_by(&:timekey).last
+        # "symlink" feature is to link from symlink_path to the latest file chunk. Records with latest
+        # timekey will be appended into that file chunk. On the other side, resumed file chunks might NOT
+        # have timekey, especially in the cases that resumed file chunks are generated by Fluentd v0.12.
+        # These chunks will be enqueued immediately, and will be flushed soon.
+        latest_chunk = metadata_list.select{|m| m.timekey }.sort_by(&:timekey).last
         if chunk.metadata == latest_chunk
           FileUtils.ln_sf(chunk.path, @_symlink_path)
         end

data/lib/fluent/plugin/out_forward.rb

@@ -80,6 +80,9 @@ module Fluent
     desc 'Enable client-side DNS round robin.'
     config_param :dns_round_robin, :bool, default: false # heartbeat_type 'udp' is not available for this
 
+    desc 'Compress buffered data.'
+    config_param :compress, :enum, list: [:text, :gzip], default: :text
+
     config_section :security, required: false, multi: false do
       desc 'The hostname'
       config_param :self_hostname, :string
@@ -137,6 +140,12 @@ module Fluent
         end
       end
 
+      if @compress == :gzip && @buffer.compress == :text
+        @buffer.compress = :gzip
+      elsif @compress == :text && @buffer.compress == :gzip
+        log.info "buffer is compressed.  If you also want to save the bandwidth of a network, Add `compress` configuration in <match>"
+      end
+
       if @nodes.empty?
         raise ConfigError, "forward output plugin requires at least one <server> is required"
       end
@@ -331,6 +340,7 @@ module Fluent
       def initialize(sender, server, failure:)
         @sender = sender
         @log = sender.log
+        @compress = sender.compress
 
         @name = server.name
         @host = server.host
@@ -432,7 +442,7 @@ module Fluent
             raise ForwardOutputConnectionClosedError, "failed to establish connection with node #{@name}"
           end
 
-          option = { 'size' => chunk.size_of_events }
+          option = { 'size' => chunk.size_of_events, 'compressed' => @compress }
           option['chunk'] = Base64.encode64(chunk.unique_id) if @sender.require_ack_response
 
           # out_forward always uses Raw32 type for content.
@@ -440,13 +450,15 @@ module Fluent
 
           sock.write @sender.forward_header        # beginArray(3)
           sock.write tag.to_msgpack                # 1. writeRaw(tag)
-          sock.write [0xdb, chunk.size].pack('CN') # 2. beginRaw(size) raw32
-          chunk.write_to(sock)                     #    writeRawBody(packed_es)
+          chunk.open(compressed: @compress) do |chunk_io|
+            sock.write [0xdb, chunk_io.size].pack('CN') # 2. beginRaw(size) raw32
+            IO.copy_stream(chunk_io, sock)              # writeRawBody(packed_es)
+          end
           sock.write option.to_msgpack             # 3. writeOption(option)
 
           if @sender.require_ack_response
             # Waiting for a response here results in a decrease of throughput because a chunk queue is locked.
-            # To avoid a decrease of troughput, it is necessary to prepare a list of chunks that wait for responses
+            # To avoid a decrease of throughput, it is necessary to prepare a list of chunks that wait for responses
             # and process them asynchronously.
             if IO.select([sock], nil, nil, @sender.ack_response_timeout)
               raw_data = begin
@@ -510,7 +522,7 @@ module Fluent
             sock.close
           end
         when :udp
-          @usock.send "\0", 0, Socket.pack_sockaddr_in(n.port, n.resolved_host)
+          @usock.send "\0", 0, Socket.pack_sockaddr_in(@port, resolved_host)
         when :none # :none doesn't use this class
           raise "BUG: heartbeat_type none must not use Node"
         else