fluentd 0.14.6 → 0.14.7

data/lib/fluent/config/element.rb

@@ -102,7 +102,7 @@ module Fluent
       end
 
       def has_key?(key)
-        @unused_in = false # some sections, e.g. <store> in copy, is not defined by config_section so clear unused flag for better warning message in chgeck_not_fetched.
+        @unused_in = false # some sections, e.g. <store> in copy, is not defined by config_section so clear unused flag for better warning message in check_not_fetched.
         @unused.delete(key)
         super
       end

data/lib/fluent/config/section.rb

@@ -165,7 +165,7 @@ module Fluent
         proxy.sections.each do |name, subproxy|
           varname = subproxy.variable_name
           elements = (conf.respond_to?(:elements) ? conf.elements : []).select{ |e| e.name == subproxy.name.to_s || e.name == subproxy.alias.to_s }
-          if elements.empty? && subproxy.init? && !subproxy.multi?
+          if elements.empty? && subproxy.init?
             elements << Fluent::Config::Element.new(subproxy.name.to_s, '', {}, [])
           end
 

data/lib/fluent/config/v1_parser.rb

@@ -106,7 +106,7 @@ module Fluent
           else
             k = scan_string(SPACING)
             spacing_without_comment
-            if prev_match.include?("\n") # support 'tag_mapped' like "without value" configuration
+            if prev_match.include?("\n") || eof? # support 'tag_mapped' like "without value" configuration
               attrs[k] = ""
             else
               if k == '@include'

data/lib/fluent/env.rb

@@ -18,6 +18,7 @@ module Fluent
   DEFAULT_CONFIG_PATH = ENV['FLUENT_CONF'] || '/etc/fluent/fluent.conf'
   DEFAULT_PLUGIN_DIR = ENV['FLUENT_PLUGIN'] || '/etc/fluent/plugin'
   DEFAULT_SOCKET_PATH = ENV['FLUENT_SOCKET'] || '/var/run/fluent/fluent.sock'
+  DEFAULT_OJ_OPTIONS = {bigdecimal_load: :float, mode: :compat}
   IS_WINDOWS = /mswin|mingw/ === RUBY_PLATFORM
   private_constant :IS_WINDOWS
 

data/lib/fluent/event.rb

@@ -15,11 +15,13 @@
 #
 
 require 'fluent/msgpack_factory'
+require 'fluent/plugin/compressable'
 
 module Fluent
   class EventStream
     include Enumerable
     include MessagePackFactory::Mixin
+    include Fluent::Plugin::Compressable
 
     # dup does deep copy for event stream
     def dup
@@ -61,6 +63,11 @@ module Fluent
       out.to_s
     end
 
+    def to_compressed_msgpack_stream(time_int: false)
+      packed = to_msgpack_stream(time_int: time_int)
+      compress(packed)
+    end
+
     def to_msgpack_stream_forced_integer
       out = msgpack_packer
       each {|time,record|
@@ -206,9 +213,9 @@ module Fluent
 
     def dup
       if @unpacked_times
-        MessagePackEventStream.new(@data.dup, nil, @size, unpacked_times: @unpacked_times, unpacked_records: @unpacked_records.map(&:dup))
+        self.class.new(@data.dup, nil, @size, unpacked_times: @unpacked_times, unpacked_records: @unpacked_records.map(&:dup))
       else
-        MessagePackEventStream.new(@data.dup, nil, @size)
+        self.class.new(@data.dup, nil, @size)
       end
     end
 
@@ -267,6 +274,46 @@ module Fluent
     end
   end
 
+  class CompressedMessagePackEventStream < MessagePackEventStream
+    def initialize(data, cached_unpacker = nil, size = 0, unpacked_times: nil, unpacked_records: nil)
+      super
+      @decompressed_data = nil
+      @compressed_data = data
+    end
+
+    def empty?
+      ensure_decompressed!
+      super
+    end
+
+    def ensure_unpacked!
+      ensure_decompressed!
+      super
+    end
+
+    def each(&block)
+      ensure_decompressed!
+      super
+    end
+
+    def to_msgpack_stream(time_int: false)
+      ensure_decompressed!
+      super
+    end
+
+    def to_compressed_msgpack_stream(time_int: false)
+      # time_int is always ignored because @data is always packed binary in this class
+      @compressed_data
+    end
+
+    private
+
+    def ensure_decompressed!
+      return if @decompressed_data
+      @data = @decompressed_data = decompress(@data)
+    end
+  end
+
   module ChunkMessagePackEventStreamer
     include MessagePackFactory::Mixin
     # chunk.extend(ChunkEventStreamer)

data/lib/fluent/event_router.rb

@@ -207,10 +207,14 @@ module Fluent
 
         def optimizable?
           return @optimizable unless @optimizable.nil?
-          @optimizable = if filters_having_filter_stream.empty?
+          fs_filters = filters_having_filter_stream
+          @optimizable = if fs_filters.empty?
                            true
                          else
-                           $log.info "Filtering works with worse performance, because #{filters_having_filter_stream.map(&:class)} uses `#filter_stream` method."
+                           # skip log message when filter is only 1, because its performace is same as non optimized chain.
+                           if @filters.size > 1 && fs_filters.size >= 1
+                             $log.info "disable filter chain optimization because #{fs_filters.map(&:class)} uses `#filter_stream` method."
+                           end
                            false
                          end
         end

data/lib/fluent/label.rb

@@ -27,6 +27,14 @@ module Fluent
 
     attr_accessor :root_agent
 
+    def configure(conf)
+      super
+
+      if conf.elements('match').size == 0
+        raise ConfigError, "Missing <match> sections in <label #{@context}> section"
+      end
+    end
+
     def emit_error_event(tag, time, record, e)
       @root_agent.emit_error_event(tag, time, record, e)
     end

data/lib/fluent/log.rb

@@ -15,6 +15,7 @@
 #
 
 require 'forwardable'
+require 'logger'
 
 module Fluent
   class Log
@@ -122,7 +123,7 @@ module Fluent
     end
 
     def reopen!
-      # do noting in @logger.reopen! because it's already reopened in Supervisor.load_config
+      # do nothing in @logger.reopen! because it's already reopened in Supervisor.load_config
       @logger.reopen! if @logger
       nil
     end
@@ -442,4 +443,32 @@ module Fluent
       @log.reset
     end
   end
+
+  # This class delegetes some methods which are used in `Fluent::Logger` to a instance variable(`dev`) in `Logger::LogDevice` class
+  # https://github.com/ruby/ruby/blob/7b2d47132ff8ee950b0f978ab772dee868d9f1b0/lib/logger.rb#L661
+  class LogDeviceIO < Logger::LogDevice
+    def flush
+      if @dev.respond_to?(:flush)
+        @dev.flush
+      else
+        super
+      end
+    end
+
+    def tty?
+      if @dev.respond_to?(:tty?)
+        @dev.tty?
+      else
+        super
+      end
+    end
+
+    def sync=(v)
+      if @dev.respond_to?(:sync=)
+        @dev.sync = v
+      else
+        super
+      end
+    end
+  end
 end

data/lib/fluent/plugin.rb

@@ -30,7 +30,7 @@ module Fluent
     OUTPUT_REGISTRY    = Registry.new(:output,    'fluent/plugin/out_',        dir_search_prefix: 'out_')
     FILTER_REGISTRY    = Registry.new(:filter,    'fluent/plugin/filter_',     dir_search_prefix: 'filter_')
 
-    # feature plugin: second class plugins (instanciated by Plugins or Helpers)
+    # feature plugin: second class plugins (instantiated by Plugins or Helpers)
     BUFFER_REGISTRY    = Registry.new(:buffer,    'fluent/plugin/buf_',        dir_search_prefix: 'buf_')
     PARSER_REGISTRY    = Registry.new(:parser,    'fluent/plugin/parser_',     dir_search_prefix: 'parser_')
     FORMATTER_REGISTRY = Registry.new(:formatter, 'fluent/plugin/formatter_',  dir_search_prefix: 'formatter_')

data/lib/fluent/plugin/base.rb

@@ -26,9 +26,12 @@ module Fluent
 
       State = Struct.new(:configure, :start, :after_start, :stop, :before_shutdown, :shutdown, :after_shutdown, :close, :terminate)
 
+      attr_accessor :under_plugin_development
+
       def initialize
         super
         @_state = State.new(false, false, false, false, false, false, false, false, false)
+        @under_plugin_development = false
       end
 
       def has_router?

data/lib/fluent/plugin/buf_file.rb

@@ -144,9 +144,9 @@ module Fluent
       def generate_chunk(metadata)
         # FileChunk generates real path with unique_id
         if @file_permission
-          Fluent::Plugin::Buffer::FileChunk.new(metadata, @path, :create, perm: @file_permission)
+          Fluent::Plugin::Buffer::FileChunk.new(metadata, @path, :create, perm: @file_permission, compress: @compress)
         else
-          Fluent::Plugin::Buffer::FileChunk.new(metadata, @path, :create)
+          Fluent::Plugin::Buffer::FileChunk.new(metadata, @path, :create, compress: @compress)
         end
       end
     end

data/lib/fluent/plugin/buf_memory.rb

@@ -27,7 +27,7 @@ module Fluent
       end
 
       def generate_chunk(metadata)
-        Fluent::Plugin::Buffer::MemoryChunk.new(metadata)
+        Fluent::Plugin::Buffer::MemoryChunk.new(metadata, compress: @compress)
       end
     end
   end

data/lib/fluent/plugin/buffer.rb

@@ -55,6 +55,9 @@ module Fluent
       # if chunk size (or records) is 95% or more after #write, then that chunk will be enqueued
       config_param :chunk_full_threshold, :float, default: DEFAULT_CHUNK_FULL_THRESHOLD
 
+      desc 'Compress buffered data.'
+      config_param :compress, :enum, list: [:text, :gzip], default: :text
+
       Metadata = Struct.new(:timekey, :tag, :variables) do
         def empty?
           timekey.nil? && tag.nil? && variables.nil?
@@ -157,6 +160,13 @@ module Fluent
         end
       end
 
+      # it's too dangerous, and use it so carefully to remove metadata for tests
+      def metadata_list_clear!
+        synchronize do
+          @metadata_list.clear
+        end
+      end
+
       def new_metadata(timekey: nil, tag: nil, variables: nil)
         Metadata.new(timekey, tag, variables)
       end
@@ -458,7 +468,7 @@ module Fluent
               serialized = format.call(data)
               chunk.concat(serialized, size ? size.call : data.size)
             else
-              chunk.append(data)
+              chunk.append(data, compress: @compress)
             end
             adding_bytesize = chunk.bytesize - original_bytesize
 
@@ -558,7 +568,7 @@ module Fluent
                 if format
                   chunk.concat(format.call(split), split.size)
                 else
-                  chunk.append(split)
+                  chunk.append(split, compress: @compress)
                 end
 
                 if chunk_size_over?(chunk) # split size is larger than difference between size_full? and size_over?

data/lib/fluent/plugin/buffer/chunk.rb

@@ -15,18 +15,20 @@
 #
 
 require 'fluent/plugin/buffer'
+require 'fluent/plugin/compressable'
 require 'fluent/unique_id'
 require 'fluent/event'
 
 require 'monitor'
+require 'tempfile'
+require 'zlib'
 
 module Fluent
   module Plugin
-    class Buffer # fluent/plugin/buffer is alread loaded
+    class Buffer # fluent/plugin/buffer is already loaded
       class Chunk
         include MonitorMixin
         include UniqueId::Mixin
-        include ChunkMessagePackEventStreamer
 
         # Chunks has 2 part:
         # * metadata: contains metadata which should be restored after resume (if possible)
@@ -46,7 +48,7 @@ module Fluent
 
         # TODO: CompressedPackedMessage of forward protocol?
 
-        def initialize(metadata)
+        def initialize(metadata, compress: :text)
           super()
           @unique_id = generate_unique_id
           @metadata = metadata
@@ -57,12 +59,15 @@ module Fluent
           @size = 0
           @created_at = Time.now
           @modified_at = Time.now
+
+          extend Decompressable if compress == :gzip
         end
 
         attr_reader :unique_id, :metadata, :created_at, :modified_at, :state
 
         # data is array of formatted record string
-        def append(data)
+        def append(data, **kwargs)
+          raise ArgumentError, '`compress: gzip` can be used for Compressable module' if kwargs[:compress] == :gzip
           adding = ''.b
           data.each do |d|
             adding << d.b
@@ -141,19 +146,75 @@ module Fluent
           self
         end
 
-        def read
+        def read(**kwargs)
+          raise ArgumentError, '`compressed: gzip` can be used for Compressable module' if kwargs[:compressed] == :gzip
           raise NotImplementedError, "Implement this method in child class"
         end
 
-        def open(&block)
+        def open(**kwargs, &block)
+          raise ArgumentError, '`compressed: gzip` can be used for Compressable module' if kwargs[:compressed] == :gzip
           raise NotImplementedError, "Implement this method in child class"
         end
 
-        def write_to(io)
+        def write_to(io, **kwargs)
+          raise ArgumentError, '`compressed: gzip` can be used for Compressable module' if kwargs[:compressed] == :gzip
           open do |i|
             IO.copy_stream(i, io)
           end
         end
+
+        module Decompressable
+          include Fluent::Plugin::Compressable
+
+          def append(data, **kwargs)
+            if kwargs[:compress] == :gzip
+              io = StringIO.new
+              Zlib::GzipWriter.wrap(io) do |gz|
+                data.each do |d|
+                  gz.write d
+                end
+              end
+              concat(io.string, data.size)
+            else
+              super
+            end
+          end
+
+          def open(**kwargs, &block)
+            if kwargs[:compressed] == :gzip
+              super
+            else
+              super(kwargs) do |chunk_io|
+                output_io = if chunk_io.is_a?(StringIO)
+                              StringIO.new
+                            else
+                              Tempfile.new('decompressed-data')
+                            end
+                decompress(input_io: chunk_io, output_io: output_io)
+                output_io.seek(0, IO::SEEK_SET)
+                yield output_io
+              end
+            end
+          end
+
+          def read(**kwargs)
+            if kwargs[:compressed] == :gzip
+              super
+            else
+              decompress(super)
+            end
+          end
+
+          def write_to(io, **kwargs)
+            open(compressed: :gzip) do |chunk_io|
+              if kwargs[:compressed] == :gzip
+                IO.copy_stream(chunk_io, io)
+              else
+                decompress(input_io: chunk_io, output_io: io)
+              end
+            end
+          end
+        end
       end
     end
   end

data/lib/fluent/plugin/buffer/file_chunk.rb

@@ -40,8 +40,8 @@ module Fluent
 
         attr_reader :path, :permission
 
-        def initialize(metadata, path, mode, perm: system_config.file_permission || FILE_PERMISSION)
-          super(metadata)
+        def initialize(metadata, path, mode, perm: system_config.file_permission || FILE_PERMISSION, compress: :text)
+          super(metadata, compress: compress)
           @permission = perm
           @bytesize = @size = @adding_bytes = @adding_size = 0
           @meta = nil
@@ -133,12 +133,12 @@ module Fluent
           File.unlink(@path, @meta_path)
         end
 
-        def read
+        def read(**kwargs)
           @chunk.seek(0, IO::SEEK_SET)
           @chunk.read
         end
 
-        def open(&block)
+        def open(**kwargs, &block)
           @chunk.seek(0, IO::SEEK_SET)
           val = yield @chunk
           @chunk.seek(0, IO::SEEK_END) if self.staged?

data/lib/fluent/plugin/buffer/memory_chunk.rb

@@ -20,7 +20,7 @@ module Fluent
   module Plugin
     class Buffer
       class MemoryChunk < Chunk
-        def initialize(metadata)
+        def initialize(metadata, compress: :text)
           super
           @chunk = ''.force_encoding(Encoding::ASCII_8BIT)
           @chunk_bytes = 0
@@ -72,15 +72,15 @@ module Fluent
           true
         end
 
-        def read
+        def read(**kwargs)
           @chunk
         end
 
-        def open(&block)
+        def open(**kwargs, &block)
           StringIO.open(@chunk, &block)
         end
 
-        def write_to(io)
+        def write_to(io, **kwargs)
           # re-implementation to optimize not to create StringIO
           io.write @chunk
         end