fluentd 0.14.5-x64-mingw32 → 0.14.7-x64-mingw32

data/lib/fluent/plugin/out_forward.rb

@@ -80,6 +80,9 @@ module Fluent
     desc 'Enable client-side DNS round robin.'
     config_param :dns_round_robin, :bool, default: false # heartbeat_type 'udp' is not available for this
 
+    desc 'Compress buffered data.'
+    config_param :compress, :enum, list: [:text, :gzip], default: :text
+
     config_section :security, required: false, multi: false do
       desc 'The hostname'
       config_param :self_hostname, :string
@@ -137,6 +140,12 @@ module Fluent
         end
       end
 
+      if @compress == :gzip && @buffer.compress == :text
+        @buffer.compress = :gzip
+      elsif @compress == :text && @buffer.compress == :gzip
+        log.info "buffer is compressed.  If you also want to save the bandwidth of a network, Add `compress` configuration in <match>"
+      end
+
       if @nodes.empty?
         raise ConfigError, "forward output plugin requires at least one <server> is required"
       end
@@ -331,6 +340,7 @@ module Fluent
       def initialize(sender, server, failure:)
         @sender = sender
         @log = sender.log
+        @compress = sender.compress
 
         @name = server.name
         @host = server.host
@@ -432,7 +442,7 @@ module Fluent
             raise ForwardOutputConnectionClosedError, "failed to establish connection with node #{@name}"
           end
 
-          option = { 'size' => chunk.size_of_events }
+          option = { 'size' => chunk.size_of_events, 'compressed' => @compress }
           option['chunk'] = Base64.encode64(chunk.unique_id) if @sender.require_ack_response
 
           # out_forward always uses Raw32 type for content.
@@ -440,13 +450,15 @@ module Fluent
 
           sock.write @sender.forward_header        # beginArray(3)
           sock.write tag.to_msgpack                # 1. writeRaw(tag)
-          sock.write [0xdb, chunk.size].pack('CN') # 2. beginRaw(size) raw32
-          chunk.write_to(sock)                     #    writeRawBody(packed_es)
+          chunk.open(compressed: @compress) do |chunk_io|
+            sock.write [0xdb, chunk_io.size].pack('CN') # 2. beginRaw(size) raw32
+            IO.copy_stream(chunk_io, sock)              # writeRawBody(packed_es)
+          end
           sock.write option.to_msgpack             # 3. writeOption(option)
 
           if @sender.require_ack_response
             # Waiting for a response here results in a decrease of throughput because a chunk queue is locked.
-            # To avoid a decrease of troughput, it is necessary to prepare a list of chunks that wait for responses
+            # To avoid a decrease of throughput, it is necessary to prepare a list of chunks that wait for responses
             # and process them asynchronously.
             if IO.select([sock], nil, nil, @sender.ack_response_timeout)
               raw_data = begin
@@ -510,7 +522,7 @@ module Fluent
             sock.close
           end
         when :udp
-          @usock.send "\0", 0, Socket.pack_sockaddr_in(n.port, n.resolved_host)
+          @usock.send "\0", 0, Socket.pack_sockaddr_in(@port, resolved_host)
         when :none # :none doesn't use this class
           raise "BUG: heartbeat_type none must not use Node"
         else

data/lib/fluent/plugin/out_stdout.rb

@@ -23,6 +23,7 @@ module Fluent::Plugin
     helpers :inject, :formatter, :compat_parameters
 
     DEFAULT_FORMAT_TYPE = 'json'
+    TIME_FORMAT = '%Y-%m-%d %H:%M:%S.%9N %z'
 
     config_section :buffer do
       config_set_default :chunk_keys, ['tag']
@@ -69,7 +70,7 @@ module Fluent::Plugin
 
     def format(tag, time, record)
       record = inject_values_to_record(tag, time, record)
-      "#{Time.at(time).localtime} #{tag}: #{@formatter.format(tag, time, record).chomp}\n"
+      "#{Time.at(time).localtime.strftime(TIME_FORMAT)} #{tag}: #{@formatter.format(tag, time, record).chomp}\n"
     end
 
     def write(chunk)

data/lib/fluent/plugin/output.rb

@@ -36,6 +36,7 @@ module Fluent
 
       CHUNK_KEY_PATTERN = /^[-_.@a-zA-Z0-9]+$/
       CHUNK_KEY_PLACEHOLDER_PATTERN = /\$\{[-_.@a-zA-Z0-9]+\}/
+      CHUNK_TAG_PLACEHOLDER_PATTERN = /\$\{(tag(?:\[\d+\])?)\}/
 
       CHUNKING_FIELD_WARN_NUM = 4
 
@@ -74,12 +75,12 @@ module Fluent
         config_param :retry_max_times, :integer, default: nil, desc: 'The maximum number of times to retry to flush while failing.'
 
         config_param :retry_secondary_threshold, :float, default: 0.8, desc: 'ratio of retry_timeout to switch to use secondary while failing.'
-        # expornential backoff sequence will be initialized at the time of this threshold
+        # exponential backoff sequence will be initialized at the time of this threshold
 
         desc 'How to wait next retry to flush buffer.'
         config_param :retry_type, :enum, list: [:exponential_backoff, :periodic], default: :exponential_backoff
         ### Periodic -> fixed :retry_wait
-        ### Exponencial backoff: k is number of retry times
+        ### Exponential backoff: k is number of retry times
         # c: constant factor, @retry_wait
         # b: base factor, @retry_exponential_backoff_base
         # k: times
@@ -118,6 +119,12 @@ module Fluent
         raise NotImplementedError, "BUG: output plugins MUST implement this method"
       end
 
+      def formatted_to_msgpack_binary
+        # To indicate custom format method (#format) returns msgpack binary or not.
+        # If #format returns msgpack binary, override this method to return true.
+        false
+      end
+
       def prefer_buffered_processing
         # override this method to return false only when all of these are true:
         #  * plugin has both implementation for buffered and non-buffered methods
@@ -175,11 +182,13 @@ module Fluent
           @buffering = true
         end
         @custom_format = implement?(:custom_format)
+        @enable_msgpack_streamer = false # decided later
 
         @buffer = nil
         @secondary = nil
         @retry = nil
         @dequeued_chunks = nil
+        @output_enqueue_thread = nil
         @output_flush_threads = nil
 
         @simple_chunking = nil
@@ -326,6 +335,9 @@ module Fluent
           @buffering = prefer_buffered_processing
           if !@buffering && @buffer
             @buffer.terminate # it's not started, so terminate will be enough
+            # At here, this plugin works as non-buffered plugin.
+            # Un-assign @buffer not to show buffering metrics (e.g., in_monitor_agent)
+            @buffer = nil
           end
         end
 
@@ -336,6 +348,7 @@ module Fluent
           end
 
           @custom_format = implement?(:custom_format)
+          @enable_msgpack_streamer = @custom_format ? formatted_to_msgpack_binary : true
           @delayed_commit = if implement?(:buffered) && implement?(:delayed_commit)
                               prefer_delayed_commit
                             else
@@ -355,6 +368,9 @@ module Fluent
 
           @buffer.start
 
+          @output_enqueue_thread = nil
+          @output_enqueue_thread_running = true
+
           @output_flush_threads = []
           @output_flush_threads_mutex = Mutex.new
           @output_flush_threads_running = true
@@ -385,7 +401,7 @@ module Fluent
 
           unless @in_tests
             if @flush_mode == :interval || @chunk_key_time
-              thread_create(:enqueue_thread, &method(:enqueue_thread_run))
+              @output_enqueue_thread = thread_create(:enqueue_thread, &method(:enqueue_thread_run))
             end
           end
         end
@@ -412,6 +428,12 @@ module Fluent
             force_flush
           end
           @buffer.before_shutdown
+          # Need to ensure to stop enqueueing ... after #shutdown, we cannot write any data
+          @output_enqueue_thread_running = false
+          if @output_enqueue_thread && @output_enqueue_thread.alive?
+            @output_enqueue_thread.wakeup
+            @output_enqueue_thread.join
+          end
         end
 
         super
@@ -483,12 +505,146 @@ module Fluent
         end
       end
 
+      def placeholder_validate!(name, str)
+        placeholder_validators(name, str).each do |v|
+          v.validate!
+        end
+      end
+
+      def placeholder_validators(name, str, time_key = (@chunk_key_time && @buffer_config.timekey), tag_key = @chunk_key_tag, chunk_keys = @chunk_keys)
+        validators = []
+
+        sec, title, example = get_placeholders_time(str)
+        if sec || time_key
+          validators << PlaceholderValidator.new(name, str, :time, {sec: sec, title: title, example: example, timekey: time_key})
+        end
+
+        parts = get_placeholders_tag(str)
+        if tag_key || !parts.empty?
+          validators << PlaceholderValidator.new(name, str, :tag, {parts: parts, tagkey: tag_key})
+        end
+
+        keys = get_placeholders_keys(str)
+        if chunk_keys && !chunk_keys.empty? || !keys.empty?
+          validators << PlaceholderValidator.new(name, str, :keys, {keys: keys, chunkkeys: chunk_keys})
+        end
+
+        validators
+      end
+
+      class PlaceholderValidator
+        attr_reader :name, :string, :type, :argument
+
+        def initialize(name, str, type, arg)
+          @name = name
+          @string = str
+          @type = type
+          raise ArgumentError, "invalid type:#{type}" if @type != :time && @type != :tag && @type != :keys
+          @argument = arg
+        end
+
+        def time?
+          @type == :time
+        end
+
+        def tag?
+          @type == :tag
+        end
+
+        def keys?
+          @type == :keys
+        end
+
+        def validate!
+          case @type
+          when :time then validate_time!
+          when :tag  then validate_tag!
+          when :keys then validate_keys!
+          end
+        end
+
+        def validate_time!
+          sec = @argument[:sec]
+          title = @argument[:title]
+          example = @argument[:example]
+          timekey = @argument[:timekey]
+          if !sec && timekey
+            raise Fluent::ConfigError, "Parameter '#{name}' doesn't have timestamp placeholders for timekey #{timekey.to_i}"
+          end
+          if sec && !timekey
+            raise Fluent::ConfigError, "Parameter '#{name}' has timestamp placeholders, but chunk key 'time' is not configured"
+          end
+          if sec && timekey && timekey < sec
+            raise Fluent::ConfigError, "Parameter '#{@name}' doesn't have timestamp placeholder for #{title}('#{example}') for timekey #{timekey.to_i}"
+          end
+        end
+
+        def validate_tag!
+          parts = @argument[:parts]
+          tagkey = @argument[:tagkey]
+          if tagkey && parts.empty?
+            raise Fluent::ConfigError, "Parameter '#{@name}' doesn't have tag placeholder"
+          end
+          if !tagkey && !parts.empty?
+            raise Fluent::ConfigError, "Parameter '#{@name}' has tag placeholders, but chunk key 'tag' is not configured"
+          end
+        end
+
+        def validate_keys!
+          keys = @argument[:keys]
+          chunk_keys = @argument[:chunkkeys]
+          if (chunk_keys - keys).size > 0
+            not_specified = (chunk_keys - keys).sort
+            raise Fluent::ConfigError, "Parameter '#{@name}' doesn't have enough placeholders for keys #{not_specified.join(',')}"
+          end
+          if (keys - chunk_keys).size > 0
+            not_satisfied = (keys - chunk_keys).sort
+            raise Fluent::ConfigError, "Parameter '#{@name}' has placeholders, but chunk keys doesn't have keys #{not_satisfied.join(',')}"
+          end
+        end
+      end
+
+      TIME_KEY_PLACEHOLDER_THRESHOLDS = [
+        [1, :second, '%S'],
+        [60, :minute, '%M'],
+        [3600, :hour, '%H'],
+        [86400, :day, '%d'],
+      ]
+      TIMESTAMP_CHECK_BASE_TIME = Time.parse("2016-01-01 00:00:00 UTC")
+      # it's not validated to use timekey larger than 1 day
+      def get_placeholders_time(str)
+        base_str = TIMESTAMP_CHECK_BASE_TIME.strftime(str)
+        TIME_KEY_PLACEHOLDER_THRESHOLDS.each do |triple|
+          sec = triple.first
+          return triple if (TIMESTAMP_CHECK_BASE_TIME + sec).strftime(str) != base_str
+        end
+        nil
+      end
+
+      # -1 means whole tag
+      def get_placeholders_tag(str)
+        # [["tag"],["tag[0]"]]
+        parts = []
+        str.scan(CHUNK_TAG_PLACEHOLDER_PATTERN).map(&:first).each do |ph|
+          if ph == "tag"
+            parts << -1
+          elsif ph =~ /^tag\[(\d+)\]$/
+            parts << $1.to_i
+          end
+        end
+        parts.sort
+      end
+
+      def get_placeholders_keys(str)
+        str.scan(CHUNK_KEY_PLACEHOLDER_PATTERN).map{|ph| ph[2..-2]}.reject{|s| s == "tag"}.sort
+      end
+
       # TODO: optimize this code
       def extract_placeholders(str, metadata)
         if metadata.empty?
           str
         else
-          rvalue = str
+          rvalue = str.dup
           # strftime formatting
           if @chunk_key_time # this section MUST be earlier than rest to use raw 'str'
             @output_time_formatter_cache[str] ||= Fluent::Timezone.formatter(@timekey_zone, str)
@@ -496,14 +652,18 @@ module Fluent
           end
           # ${tag}, ${tag[0]}, ${tag[1]}, ...
           if @chunk_key_tag
-            if str =~ /\$\{tag\[\d+\]\}/
-              hash = {'${tag}' => metadata.tag}
+            if str.include?('${tag}')
+              rvalue = rvalue.gsub('${tag}', metadata.tag)
+            end
+            if str =~ CHUNK_TAG_PLACEHOLDER_PATTERN
+              hash = {}
               metadata.tag.split('.').each_with_index do |part, i|
                 hash["${tag[#{i}]}"] = part
               end
-              rvalue = rvalue.gsub(/\$\{tag(\[\d+\])?\}/, hash)
-            elsif str.include?('${tag}')
-              rvalue = rvalue.gsub('${tag}', metadata.tag)
+              rvalue = rvalue.gsub(CHUNK_TAG_PLACEHOLDER_PATTERN, hash)
+            end
+            if rvalue =~ CHUNK_TAG_PLACEHOLDER_PATTERN
+              log.warn "tag placeholder '#{$1}' not replaced. tag:#{metadata.tag}, template:#{str}"
             end
           end
           # ${a_chunk_key}, ...
@@ -514,6 +674,9 @@ module Fluent
             end
             rvalue = rvalue.gsub(CHUNK_KEY_PLACEHOLDER_PATTERN, hash)
           end
+          if rvalue =~ CHUNK_KEY_PLACEHOLDER_PATTERN
+            log.warn "chunk key placeholder '#{$1}' not replaced. templace:#{str}"
+          end
           rvalue
         end
       end
@@ -595,6 +758,13 @@ module Fluent
         end
       end
 
+      def metadata_for_test(tag, time, record)
+        raise "BUG: #metadata_for_test is available only when no actual metadata exists" unless @buffer.metadata_list.empty?
+        m = metadata(tag, time, record)
+        @buffer.metadata_list_clear!
+        m
+      end
+
       def execute_chunking(tag, es, enqueue: false)
         if @simple_chunking
           handle_stream_simple(tag, es, enqueue: enqueue)
@@ -641,7 +811,17 @@ module Fluent
       end
 
       FORMAT_MSGPACK_STREAM = ->(e){ e.to_msgpack_stream }
+      FORMAT_COMPRESSED_MSGPACK_STREAM = ->(e){ e.to_compressed_msgpack_stream }
       FORMAT_MSGPACK_STREAM_TIME_INT = ->(e){ e.to_msgpack_stream(time_int: true) }
+      FORMAT_COMPRESSED_MSGPACK_STREAM_TIME_INT = ->(e){ e.to_compressed_msgpack_stream(time_int: true) }
+
+      def generate_format_proc
+        if @buffer && @buffer.compress == :gzip
+          @time_as_integer ? FORMAT_COMPRESSED_MSGPACK_STREAM_TIME_INT : FORMAT_COMPRESSED_MSGPACK_STREAM
+        else
+          @time_as_integer ? FORMAT_MSGPACK_STREAM_TIME_INT : FORMAT_MSGPACK_STREAM
+        end
+      end
 
       # metadata_and_data is a Hash of:
       #  (standard format) metadata => event stream
@@ -651,7 +831,7 @@ module Fluent
       #   `@buffer.write` will do this splitting.
       # For custom format, formatting will be done here. Custom formatting always requires
       #   iteration of event stream, and it should be done just once even if total event stream size
-      #   is biggar than chunk_limit_size because of performance.
+      #   is bigger than chunk_limit_size because of performance.
       def handle_stream_with_custom_format(tag, es, enqueue: false)
         meta_and_data = {}
         records = 0
@@ -669,7 +849,7 @@ module Fluent
       end
 
       def handle_stream_with_standard_format(tag, es, enqueue: false)
-        format_proc = @time_as_integer ? FORMAT_MSGPACK_STREAM_TIME_INT : FORMAT_MSGPACK_STREAM
+        format_proc = generate_format_proc
         meta_and_data = {}
         records = 0
         es.each do |time, record|
@@ -697,7 +877,7 @@ module Fluent
             records += 1
           end
         else
-          format_proc = @time_as_integer ? FORMAT_MSGPACK_STREAM_TIME_INT : FORMAT_MSGPACK_STREAM
+          format_proc = generate_format_proc
           data = es
         end
         write_guard do
@@ -793,7 +973,7 @@ module Fluent
           using_secondary = true
         end
 
-        unless @custom_format
+        if @enable_msgpack_streamer
           chunk.extend ChunkMessagePackEventStreamer
         end
 
@@ -821,6 +1001,10 @@ module Fluent
           log.debug "taking back chunk for errors.", plugin_id: plugin_id, chunk: dump_unique_id_hex(chunk.unique_id)
           @buffer.takeback_chunk(chunk.unique_id)
 
+          if @under_plugin_development
+            raise
+          end
+
           @retry_mutex.synchronize do
             if @retry
               @counters_monitor.synchronize{ @num_errors += 1 }
@@ -941,7 +1125,7 @@ module Fluent
         log.debug "enqueue_thread actually running"
 
         begin
-          while @output_flush_threads_running
+          while @output_enqueue_thread_running
             now_int = Time.now.to_i
             if @output_flush_interrupted
               sleep interval
@@ -965,16 +1149,18 @@ module Fluent
                 @buffer.enqueue_all{ |metadata, chunk| metadata.timekey < current_timekey && metadata.timekey + timekey_unit + timekey_wait <= now_int }
               end
             rescue => e
-              log.error "unexpected error while checking flushed chunks. ignored.", plugin_id: plugin_id, error_class: e.class, error: e
+              raise if @under_plugin_development
+              log.error "unexpected error while checking flushed chunks. ignored.", plugin_id: plugin_id, error: e
               log.error_backtrace
+            ensure
+              @output_enqueue_thread_waiting = false
+              @output_enqueue_thread_mutex.unlock
             end
-            @output_enqueue_thread_waiting = false
-            @output_enqueue_thread_mutex.unlock
             sleep interval
           end
         rescue => e
           # normal errors are rescued by inner begin-rescue clause.
-          log.error "error on enqueue thread", plugin_id: plugin_id, error_class: e.class, error: e
+          log.error "error on enqueue thread", plugin_id: plugin_id, error: e
           log.error_backtrace
           raise
         end
@@ -1003,7 +1189,7 @@ module Fluent
               # next_flush_interval uses flush_thread_interval or flush_thread_burst_interval (or retrying)
               interval = next_flush_time.to_f - Time.now.to_f
               # TODO: if secondary && delayed-commit, next_flush_time will be much longer than expected (because @retry still exists)
-              #   @retry should be cleard if delayed commit is enabled? Or any other solution?
+              #   @retry should be cleared if delayed commit is enabled? Or any other solution?
               state.next_time = Process.clock_gettime(clock_id) + interval
             end