fluentd 0.14.6 → 0.14.7

data/lib/fluent/plugin/out_stdout.rb

@@ -23,6 +23,7 @@ module Fluent::Plugin
     helpers :inject, :formatter, :compat_parameters
 
     DEFAULT_FORMAT_TYPE = 'json'
+    TIME_FORMAT = '%Y-%m-%d %H:%M:%S.%9N %z'
 
     config_section :buffer do
       config_set_default :chunk_keys, ['tag']
@@ -69,7 +70,7 @@ module Fluent::Plugin
 
     def format(tag, time, record)
       record = inject_values_to_record(tag, time, record)
-      "#{Time.at(time).localtime} #{tag}: #{@formatter.format(tag, time, record).chomp}\n"
+      "#{Time.at(time).localtime.strftime(TIME_FORMAT)} #{tag}: #{@formatter.format(tag, time, record).chomp}\n"
     end
 
     def write(chunk)

data/lib/fluent/plugin/output.rb

@@ -36,6 +36,7 @@ module Fluent
 
       CHUNK_KEY_PATTERN = /^[-_.@a-zA-Z0-9]+$/
       CHUNK_KEY_PLACEHOLDER_PATTERN = /\$\{[-_.@a-zA-Z0-9]+\}/
+      CHUNK_TAG_PLACEHOLDER_PATTERN = /\$\{(tag(?:\[\d+\])?)\}/
 
       CHUNKING_FIELD_WARN_NUM = 4
 
@@ -74,12 +75,12 @@ module Fluent
         config_param :retry_max_times, :integer, default: nil, desc: 'The maximum number of times to retry to flush while failing.'
 
         config_param :retry_secondary_threshold, :float, default: 0.8, desc: 'ratio of retry_timeout to switch to use secondary while failing.'
-        # expornential backoff sequence will be initialized at the time of this threshold
+        # exponential backoff sequence will be initialized at the time of this threshold
 
         desc 'How to wait next retry to flush buffer.'
         config_param :retry_type, :enum, list: [:exponential_backoff, :periodic], default: :exponential_backoff
         ### Periodic -> fixed :retry_wait
-        ### Exponencial backoff: k is number of retry times
+        ### Exponential backoff: k is number of retry times
         # c: constant factor, @retry_wait
         # b: base factor, @retry_exponential_backoff_base
         # k: times
@@ -118,6 +119,12 @@ module Fluent
         raise NotImplementedError, "BUG: output plugins MUST implement this method"
       end
 
+      def formatted_to_msgpack_binary
+        # To indicate custom format method (#format) returns msgpack binary or not.
+        # If #format returns msgpack binary, override this method to return true.
+        false
+      end
+
       def prefer_buffered_processing
         # override this method to return false only when all of these are true:
         #  * plugin has both implementation for buffered and non-buffered methods
@@ -175,11 +182,13 @@ module Fluent
           @buffering = true
         end
         @custom_format = implement?(:custom_format)
+        @enable_msgpack_streamer = false # decided later
 
         @buffer = nil
         @secondary = nil
         @retry = nil
         @dequeued_chunks = nil
+        @output_enqueue_thread = nil
         @output_flush_threads = nil
 
         @simple_chunking = nil
@@ -326,6 +335,9 @@ module Fluent
           @buffering = prefer_buffered_processing
           if !@buffering && @buffer
             @buffer.terminate # it's not started, so terminate will be enough
+            # At here, this plugin works as non-buffered plugin.
+            # Un-assign @buffer not to show buffering metrics (e.g., in_monitor_agent)
+            @buffer = nil
           end
         end
 
@@ -336,6 +348,7 @@ module Fluent
           end
 
           @custom_format = implement?(:custom_format)
+          @enable_msgpack_streamer = @custom_format ? formatted_to_msgpack_binary : true
           @delayed_commit = if implement?(:buffered) && implement?(:delayed_commit)
                               prefer_delayed_commit
                             else
@@ -355,6 +368,9 @@ module Fluent
 
           @buffer.start
 
+          @output_enqueue_thread = nil
+          @output_enqueue_thread_running = true
+
           @output_flush_threads = []
           @output_flush_threads_mutex = Mutex.new
           @output_flush_threads_running = true
@@ -385,7 +401,7 @@ module Fluent
 
           unless @in_tests
             if @flush_mode == :interval || @chunk_key_time
-              thread_create(:enqueue_thread, &method(:enqueue_thread_run))
+              @output_enqueue_thread = thread_create(:enqueue_thread, &method(:enqueue_thread_run))
             end
           end
         end
@@ -412,6 +428,12 @@ module Fluent
             force_flush
           end
           @buffer.before_shutdown
+          # Need to ensure to stop enqueueing ... after #shutdown, we cannot write any data
+          @output_enqueue_thread_running = false
+          if @output_enqueue_thread && @output_enqueue_thread.alive?
+            @output_enqueue_thread.wakeup
+            @output_enqueue_thread.join
+          end
         end
 
         super
@@ -483,12 +505,146 @@ module Fluent
         end
       end
 
+      def placeholder_validate!(name, str)
+        placeholder_validators(name, str).each do |v|
+          v.validate!
+        end
+      end
+
+      def placeholder_validators(name, str, time_key = (@chunk_key_time && @buffer_config.timekey), tag_key = @chunk_key_tag, chunk_keys = @chunk_keys)
+        validators = []
+
+        sec, title, example = get_placeholders_time(str)
+        if sec || time_key
+          validators << PlaceholderValidator.new(name, str, :time, {sec: sec, title: title, example: example, timekey: time_key})
+        end
+
+        parts = get_placeholders_tag(str)
+        if tag_key || !parts.empty?
+          validators << PlaceholderValidator.new(name, str, :tag, {parts: parts, tagkey: tag_key})
+        end
+
+        keys = get_placeholders_keys(str)
+        if chunk_keys && !chunk_keys.empty? || !keys.empty?
+          validators << PlaceholderValidator.new(name, str, :keys, {keys: keys, chunkkeys: chunk_keys})
+        end
+
+        validators
+      end
+
+      class PlaceholderValidator
+        attr_reader :name, :string, :type, :argument
+
+        def initialize(name, str, type, arg)
+          @name = name
+          @string = str
+          @type = type
+          raise ArgumentError, "invalid type:#{type}" if @type != :time && @type != :tag && @type != :keys
+          @argument = arg
+        end
+
+        def time?
+          @type == :time
+        end
+
+        def tag?
+          @type == :tag
+        end
+
+        def keys?
+          @type == :keys
+        end
+
+        def validate!
+          case @type
+          when :time then validate_time!
+          when :tag  then validate_tag!
+          when :keys then validate_keys!
+          end
+        end
+
+        def validate_time!
+          sec = @argument[:sec]
+          title = @argument[:title]
+          example = @argument[:example]
+          timekey = @argument[:timekey]
+          if !sec && timekey
+            raise Fluent::ConfigError, "Parameter '#{name}' doesn't have timestamp placeholders for timekey #{timekey.to_i}"
+          end
+          if sec && !timekey
+            raise Fluent::ConfigError, "Parameter '#{name}' has timestamp placeholders, but chunk key 'time' is not configured"
+          end
+          if sec && timekey && timekey < sec
+            raise Fluent::ConfigError, "Parameter '#{@name}' doesn't have timestamp placeholder for #{title}('#{example}') for timekey #{timekey.to_i}"
+          end
+        end
+
+        def validate_tag!
+          parts = @argument[:parts]
+          tagkey = @argument[:tagkey]
+          if tagkey && parts.empty?
+            raise Fluent::ConfigError, "Parameter '#{@name}' doesn't have tag placeholder"
+          end
+          if !tagkey && !parts.empty?
+            raise Fluent::ConfigError, "Parameter '#{@name}' has tag placeholders, but chunk key 'tag' is not configured"
+          end
+        end
+
+        def validate_keys!
+          keys = @argument[:keys]
+          chunk_keys = @argument[:chunkkeys]
+          if (chunk_keys - keys).size > 0
+            not_specified = (chunk_keys - keys).sort
+            raise Fluent::ConfigError, "Parameter '#{@name}' doesn't have enough placeholders for keys #{not_specified.join(',')}"
+          end
+          if (keys - chunk_keys).size > 0
+            not_satisfied = (keys - chunk_keys).sort
+            raise Fluent::ConfigError, "Parameter '#{@name}' has placeholders, but chunk keys doesn't have keys #{not_satisfied.join(',')}"
+          end
+        end
+      end
+
+      TIME_KEY_PLACEHOLDER_THRESHOLDS = [
+        [1, :second, '%S'],
+        [60, :minute, '%M'],
+        [3600, :hour, '%H'],
+        [86400, :day, '%d'],
+      ]
+      TIMESTAMP_CHECK_BASE_TIME = Time.parse("2016-01-01 00:00:00 UTC")
+      # it's not validated to use timekey larger than 1 day
+      def get_placeholders_time(str)
+        base_str = TIMESTAMP_CHECK_BASE_TIME.strftime(str)
+        TIME_KEY_PLACEHOLDER_THRESHOLDS.each do |triple|
+          sec = triple.first
+          return triple if (TIMESTAMP_CHECK_BASE_TIME + sec).strftime(str) != base_str
+        end
+        nil
+      end
+
+      # -1 means whole tag
+      def get_placeholders_tag(str)
+        # [["tag"],["tag[0]"]]
+        parts = []
+        str.scan(CHUNK_TAG_PLACEHOLDER_PATTERN).map(&:first).each do |ph|
+          if ph == "tag"
+            parts << -1
+          elsif ph =~ /^tag\[(\d+)\]$/
+            parts << $1.to_i
+          end
+        end
+        parts.sort
+      end
+
+      def get_placeholders_keys(str)
+        str.scan(CHUNK_KEY_PLACEHOLDER_PATTERN).map{|ph| ph[2..-2]}.reject{|s| s == "tag"}.sort
+      end
+
       # TODO: optimize this code
       def extract_placeholders(str, metadata)
         if metadata.empty?
           str
         else
-          rvalue = str
+          rvalue = str.dup
           # strftime formatting
           if @chunk_key_time # this section MUST be earlier than rest to use raw 'str'
             @output_time_formatter_cache[str] ||= Fluent::Timezone.formatter(@timekey_zone, str)
@@ -496,14 +652,18 @@ module Fluent
           end
           # ${tag}, ${tag[0]}, ${tag[1]}, ...
           if @chunk_key_tag
-            if str =~ /\$\{tag\[\d+\]\}/
-              hash = {'${tag}' => metadata.tag}
+            if str.include?('${tag}')
+              rvalue = rvalue.gsub('${tag}', metadata.tag)
+            end
+            if str =~ CHUNK_TAG_PLACEHOLDER_PATTERN
+              hash = {}
               metadata.tag.split('.').each_with_index do |part, i|
                 hash["${tag[#{i}]}"] = part
               end
-              rvalue = rvalue.gsub(/\$\{tag(\[\d+\])?\}/, hash)
-            elsif str.include?('${tag}')
-              rvalue = rvalue.gsub('${tag}', metadata.tag)
+              rvalue = rvalue.gsub(CHUNK_TAG_PLACEHOLDER_PATTERN, hash)
+            end
+            if rvalue =~ CHUNK_TAG_PLACEHOLDER_PATTERN
+              log.warn "tag placeholder '#{$1}' not replaced. tag:#{metadata.tag}, template:#{str}"
             end
           end
           # ${a_chunk_key}, ...
@@ -514,6 +674,9 @@ module Fluent
             end
             rvalue = rvalue.gsub(CHUNK_KEY_PLACEHOLDER_PATTERN, hash)
           end
+          if rvalue =~ CHUNK_KEY_PLACEHOLDER_PATTERN
+            log.warn "chunk key placeholder '#{$1}' not replaced. templace:#{str}"
+          end
           rvalue
         end
       end
@@ -595,6 +758,13 @@ module Fluent
         end
       end
 
+      def metadata_for_test(tag, time, record)
+        raise "BUG: #metadata_for_test is available only when no actual metadata exists" unless @buffer.metadata_list.empty?
+        m = metadata(tag, time, record)
+        @buffer.metadata_list_clear!
+        m
+      end
+
       def execute_chunking(tag, es, enqueue: false)
         if @simple_chunking
           handle_stream_simple(tag, es, enqueue: enqueue)
@@ -641,7 +811,17 @@ module Fluent
       end
 
       FORMAT_MSGPACK_STREAM = ->(e){ e.to_msgpack_stream }
+      FORMAT_COMPRESSED_MSGPACK_STREAM = ->(e){ e.to_compressed_msgpack_stream }
       FORMAT_MSGPACK_STREAM_TIME_INT = ->(e){ e.to_msgpack_stream(time_int: true) }
+      FORMAT_COMPRESSED_MSGPACK_STREAM_TIME_INT = ->(e){ e.to_compressed_msgpack_stream(time_int: true) }
+
+      def generate_format_proc
+        if @buffer && @buffer.compress == :gzip
+          @time_as_integer ? FORMAT_COMPRESSED_MSGPACK_STREAM_TIME_INT : FORMAT_COMPRESSED_MSGPACK_STREAM
+        else
+          @time_as_integer ? FORMAT_MSGPACK_STREAM_TIME_INT : FORMAT_MSGPACK_STREAM
+        end
+      end
 
       # metadata_and_data is a Hash of:
       #  (standard format) metadata => event stream
@@ -651,7 +831,7 @@ module Fluent
       #   `@buffer.write` will do this splitting.
       # For custom format, formatting will be done here. Custom formatting always requires
       #   iteration of event stream, and it should be done just once even if total event stream size
-      #   is biggar than chunk_limit_size because of performance.
+      #   is bigger than chunk_limit_size because of performance.
       def handle_stream_with_custom_format(tag, es, enqueue: false)
         meta_and_data = {}
         records = 0
@@ -669,7 +849,7 @@ module Fluent
       end
 
       def handle_stream_with_standard_format(tag, es, enqueue: false)
-        format_proc = @time_as_integer ? FORMAT_MSGPACK_STREAM_TIME_INT : FORMAT_MSGPACK_STREAM
+        format_proc = generate_format_proc
         meta_and_data = {}
         records = 0
         es.each do |time, record|
@@ -697,7 +877,7 @@ module Fluent
             records += 1
           end
         else
-          format_proc = @time_as_integer ? FORMAT_MSGPACK_STREAM_TIME_INT : FORMAT_MSGPACK_STREAM
+          format_proc = generate_format_proc
           data = es
         end
         write_guard do
@@ -793,7 +973,7 @@ module Fluent
           using_secondary = true
         end
 
-        unless @custom_format
+        if @enable_msgpack_streamer
           chunk.extend ChunkMessagePackEventStreamer
         end
 
@@ -821,6 +1001,10 @@ module Fluent
           log.debug "taking back chunk for errors.", plugin_id: plugin_id, chunk: dump_unique_id_hex(chunk.unique_id)
           @buffer.takeback_chunk(chunk.unique_id)
 
+          if @under_plugin_development
+            raise
+          end
+
           @retry_mutex.synchronize do
             if @retry
               @counters_monitor.synchronize{ @num_errors += 1 }
@@ -941,7 +1125,7 @@ module Fluent
         log.debug "enqueue_thread actually running"
 
         begin
-          while @output_flush_threads_running
+          while @output_enqueue_thread_running
             now_int = Time.now.to_i
             if @output_flush_interrupted
               sleep interval
@@ -965,16 +1149,18 @@ module Fluent
                 @buffer.enqueue_all{ |metadata, chunk| metadata.timekey < current_timekey && metadata.timekey + timekey_unit + timekey_wait <= now_int }
               end
             rescue => e
-              log.error "unexpected error while checking flushed chunks. ignored.", plugin_id: plugin_id, error_class: e.class, error: e
+              raise if @under_plugin_development
+              log.error "unexpected error while checking flushed chunks. ignored.", plugin_id: plugin_id, error: e
               log.error_backtrace
+            ensure
+              @output_enqueue_thread_waiting = false
+              @output_enqueue_thread_mutex.unlock
             end
-            @output_enqueue_thread_waiting = false
-            @output_enqueue_thread_mutex.unlock
             sleep interval
           end
         rescue => e
           # normal errors are rescued by inner begin-rescue clause.
-          log.error "error on enqueue thread", plugin_id: plugin_id, error_class: e.class, error: e
+          log.error "error on enqueue thread", plugin_id: plugin_id, error: e
           log.error_backtrace
           raise
         end
@@ -1003,7 +1189,7 @@ module Fluent
               # next_flush_interval uses flush_thread_interval or flush_thread_burst_interval (or retrying)
               interval = next_flush_time.to_f - Time.now.to_f
               # TODO: if secondary && delayed-commit, next_flush_time will be much longer than expected (because @retry still exists)
-              #   @retry should be cleard if delayed commit is enabled? Or any other solution?
+              #   @retry should be cleared if delayed commit is enabled? Or any other solution?
               state.next_time = Process.clock_gettime(clock_id) + interval
             end
 

data/lib/fluent/plugin/parser.rb

@@ -21,12 +21,11 @@ require 'fluent/mixin' # for TypeConverter
 require 'fluent/time'
 require 'fluent/plugin/string_util'
 
-require 'strptime'
-
 module Fluent
   module Plugin
     class Parser < Base
       include OwnedByMixin
+      include TimeMixin::Parser
 
       class ParserError < StandardError; end
 
@@ -52,49 +51,7 @@ module Fluent
         parse(*a, &b)
       end
 
-      class TimeParser
-        def initialize(time_format)
-          @cache1_key = nil
-          @cache1_time = nil
-          @cache2_key = nil
-          @cache2_time = nil
-          @parser =
-            if time_format
-              begin
-                strptime = Strptime.new(time_format)
-                Proc.new { |value| Fluent::EventTime.from_time(strptime.exec(value)) }
-              rescue
-                Proc.new { |value| Fluent::EventTime.from_time(Time.strptime(value, time_format)) }
-              end
-            else
-              Proc.new { |value| Fluent::EventTime.parse(value) }
-            end
-        end
-
-        # TODO: new cache mechanism using format string
-        def parse(value)
-          unless value.is_a?(String)
-            raise ParserError, "value must be string: #{value}"
-          end
-
-          if @cache1_key == value
-            return @cache1_time
-          elsif @cache2_key == value
-            return @cache2_time
-          else
-            begin
-              time = @parser.call(value)
-            rescue => e
-              raise ParserError, "invalid time format: value = #{value}, error_class = #{e.class.name}, error = #{e.message}"
-            end
-            @cache1_key = @cache2_key
-            @cache1_time = @cache2_time
-            @cache2_key = value
-            @cache2_time = time
-            return time
-          end
-        end
-      end
+      TimeParser = Fluent::TimeParser
     end
 
     class ValuesParser < Parser
@@ -102,7 +59,6 @@ module Fluent
 
       config_param :keys, :array, default: []
       config_param :time_key, :string, default: nil
-      config_param :time_format, :string, default: nil
       config_param :null_value_pattern, :string, default: nil
       config_param :null_empty_string, :bool, default: false
 
@@ -110,14 +66,14 @@ module Fluent
         super
 
         if @time_key && !@keys.include?(@time_key) && @estimate_current_event
-          raise ConfigError, "time_key (#{@time_key.inspect}) is not included in keys (#{@keys.inspect})"
+          raise Fluent::ConfigError, "time_key (#{@time_key.inspect}) is not included in keys (#{@keys.inspect})"
         end
 
         if @time_format && !@time_key
-          raise ConfigError, "time_format parameter is ignored because time_key parameter is not set. at #{conf.inspect}"
+          raise Fluent::ConfigError, "time_format parameter is ignored because time_key parameter is not set. at #{conf.inspect}"
         end
 
-        @time_parser = TimeParser.new(@time_format)
+        @time_parser = time_parser_create
 
         if @null_value_pattern
           @null_value_pattern = Regexp.new(@null_value_pattern)