fluentd 1.7.4-x64-mingw32 → 1.8.0-x64-mingw32

data/lib/fluent/event.rb

@@ -20,7 +20,6 @@ require 'fluent/plugin/compressable'
 module Fluent
   class EventStream
     include Enumerable
-    include MessagePackFactory::Mixin
     include Fluent::Plugin::Compressable
 
     # dup does deep copy for event stream
@@ -56,7 +55,7 @@ module Fluent
 
     def to_msgpack_stream(time_int: false, packer: nil)
       return to_msgpack_stream_forced_integer(packer: packer) if time_int
-      out = packer || msgpack_packer
+      out = packer || Fluent::MessagePackFactory.msgpack_packer
       each {|time,record|
         out.write([time,record])
       }
@@ -69,7 +68,7 @@ module Fluent
     end
 
     def to_msgpack_stream_forced_integer(packer: nil)
-      out = packer || msgpack_packer
+      out = packer || Fluent::MessagePackFactory.msgpack_packer
       each {|time,record|
         out.write([time.to_i,record])
       }
@@ -238,7 +237,7 @@ module Fluent
       return if @unpacked_times && @unpacked_records
       @unpacked_times = []
       @unpacked_records = []
-      (unpacker || msgpack_unpacker).feed_each(@data) do |time, record|
+      (unpacker || Fluent::MessagePackFactory.msgpack_unpacker).feed_each(@data) do |time, record|
         @unpacked_times << time
         @unpacked_records << record
       end
@@ -262,7 +261,7 @@ module Fluent
       else
         @unpacked_times = []
         @unpacked_records = []
-        (unpacker || msgpack_unpacker).feed_each(@data) do |time, record|
+        (unpacker || Fluent::MessagePackFactory.msgpack_unpacker).feed_each(@data) do |time, record|
           @unpacked_times << time
           @unpacked_records << record
           block.call(time, record)
@@ -319,12 +318,11 @@ module Fluent
   end
 
   module ChunkMessagePackEventStreamer
-    include MessagePackFactory::Mixin
     # chunk.extend(ChunkEventStreamer)
     #  => chunk.each{|time, record| ... }
     def each(unpacker: nil, &block)
       open do |io|
-        (unpacker || msgpack_unpacker(io)).each(&block)
+        (unpacker || Fluent::MessagePackFactory.msgpack_unpacker(io)).each(&block)
       end
       nil
     end

data/lib/fluent/fluent_log_event_router.rb

@@ -0,0 +1,141 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/log'
+
+module Fluent
+  # DO NOT write any logic here
+  class NullFluentLogEventRouter
+    def start; end
+
+    def stop; end
+
+    def graceful_stop; end
+
+    def emit_event(_event); end
+
+    def emittable?
+      self.class != NullFluentLogEventRouter
+    end
+  end
+
+  # This class is for handling fluentd's inner log
+  # e.g. <label @FLUNT_LOG> section and <match fluent.**> section
+  class FluentLogEventRouter < NullFluentLogEventRouter
+    # @param root_agent [Fluent::RootAgent]
+    def self.build(root_agent)
+      log_event_router = nil
+
+      begin
+        log_event_agent = root_agent.find_label(Fluent::Log::LOG_EVENT_LABEL)
+        log_event_router = log_event_agent.event_router
+
+        # suppress mismatched tags only for <label @FLUENT_LOG> label.
+        # it's not suppressed in default event router for non-log-event events
+        log_event_router.suppress_missing_match!
+
+        log_event_router = log_event_router
+
+        unmatched_tags = Fluent::Log.event_tags.select { |t| !log_event_router.match?(t) }
+        unless unmatched_tags.empty?
+          $log.warn "match for some tags of log events are not defined in @FLUENT_LOG label (to be ignored)", tags: unmatched_tags
+        end
+
+      rescue ArgumentError # ArgumentError "#{label_name} label not found"
+        # use default event router if <label @FLUENT_LOG> is missing in configuration
+        root_log_event_router = root_agent.event_router
+        event_tags = Fluent::Log.event_tags
+        if event_tags.any? { |t| root_log_event_router.match?(t) }
+          log_event_router = root_log_event_router
+
+          unmatched_tags = event_tags.select { |t| !log_event_router.match?(t) }
+          if unmatched_tags.empty?
+            $log.warn "define <match fluent.**> to capture fluentd logs in top level is deprecated. Use <label @FLUENT_LOG> instead"
+          else
+            matched_sections = (event_tags - unmatched_tags).map { |tag| "<match #{tag}>" }.join(', ')
+            $log.warn "define #{matched_sections} to capture fluentd logs in top level is deprecated. Use <label @FLUENT_LOG> instead"
+            $log.warn "match for some tags of log events are not defined in top level (to be ignored)", tags: unmatched_tags
+          end
+        end
+      end
+
+      if log_event_router
+        FluentLogEventRouter.new(log_event_router)
+      else
+        $log.debug('No fluent logger for internal event')
+        NullFluentLogEventRouter.new
+      end
+    end
+
+    STOP = :stop
+    GRACEFUL_STOP = :graceful_stop
+
+    # @param event_router [Fluent::EventRouter]
+    def initialize(event_router)
+      @event_router = event_router
+      @thread = nil
+      @graceful_stop = false
+      @event_queue = Queue.new
+    end
+
+    def start
+      @thread = Thread.new do
+        $log.disable_events(Thread.current)
+
+        loop do
+          event = @event_queue.pop
+
+          case event
+          when GRACEFUL_STOP
+            @graceful_stop = true
+          when STOP
+            break
+          else
+            begin
+              tag, time, record = event
+              @event_router.emit(tag, time, record)
+            rescue => e
+              # This $log.error doesn't emit log events, because of `$log.disable_events(Thread.current)` above
+              $log.error "failed to emit fluentd's log event", tag: tag, event: record, error: e
+            end
+          end
+
+          if @graceful_stop && @event_queue.empty?
+            break
+          end
+        end
+      end
+
+      @thread.abort_on_exception = true
+    end
+
+    def stop
+      @event_queue.push(STOP)
+      # there is no problem calling Thread#join multiple times.
+      @thread && @thread.join
+    end
+
+    def graceful_stop
+      # to make sure to emit all log events into router, before shutting down
+      @event_queue.push(GRACEFUL_STOP)
+      @thread && @thread.join
+    end
+
+    def emit_event(event)
+      @event_queue.push(event)
+    end
+  end
+end

data/lib/fluent/msgpack_factory.rb

@@ -23,15 +23,24 @@ module Fluent
 
     module Mixin
       def msgpack_factory
+        if $log
+          $log.warn('Deprecated method: this method is going to be deleted. Use Fluent::MessagePackFactory.engine_factory')
+        end
         MessagePackFactory.engine_factory
       end
 
       def msgpack_packer(*args)
-        msgpack_factory.packer(*args)
+        if $log
+          $log.warn('Deprecated method: this method is going to be deleted. Use Fluent::MessagePackFactory.msgpack_packer')
+        end
+        MessagePackFactory.msgpack_packer(*args)
       end
 
       def msgpack_unpacker(*args)
-        msgpack_factory.unpacker(*args)
+        if $log
+          $log.warn('Deprecated method: this method is going to be deleted. Use Fluent::MessagePackFactory.msgpack_unpacker')
+        end
+        MessagePackFactory.msgpack_unpacker(*args)
       end
     end
 
@@ -39,6 +48,14 @@ module Fluent
       @@engine_factory || factory
     end
 
+    def self.msgpack_packer(*args)
+      engine_factory.packer(*args)
+    end
+
+    def self.msgpack_unpacker(*args)
+      engine_factory.unpacker(*args)
+    end
+
     def self.factory
       factory = MessagePack::Factory.new
       factory.register_type(Fluent::EventTime::TYPE, Fluent::EventTime)

data/lib/fluent/plugin.rb

@@ -35,8 +35,9 @@ module Fluent
     PARSER_REGISTRY    = Registry.new(:parser,    'fluent/plugin/parser_',     dir_search_prefix: 'parser_')
     FORMATTER_REGISTRY = Registry.new(:formatter, 'fluent/plugin/formatter_',  dir_search_prefix: 'formatter_')
     STORAGE_REGISTRY   = Registry.new(:storage,   'fluent/plugin/storage_',    dir_search_prefix: 'storage_')
+    SD_REGISTRY        = Registry.new(:sd,        'fluent/plugin/sd_',         dir_search_prefix: 'sd_')
 
-    REGISTRIES = [INPUT_REGISTRY, OUTPUT_REGISTRY, FILTER_REGISTRY, BUFFER_REGISTRY, PARSER_REGISTRY, FORMATTER_REGISTRY, STORAGE_REGISTRY]
+    REGISTRIES = [INPUT_REGISTRY, OUTPUT_REGISTRY, FILTER_REGISTRY, BUFFER_REGISTRY, PARSER_REGISTRY, FORMATTER_REGISTRY, STORAGE_REGISTRY, SD_REGISTRY]
 
     def self.register_input(type, klass)
       register_impl('input', INPUT_REGISTRY, type, klass)
@@ -54,6 +55,10 @@ module Fluent
       register_impl('buffer', BUFFER_REGISTRY, type, klass)
     end
 
+    def self.register_sd(type, klass)
+      register_impl('sd', SD_REGISTRY, type, klass)
+    end
+
     def self.register_parser(type, klass_or_proc)
       if klass_or_proc.is_a?(Regexp)
         # This usage is not recommended for new API
@@ -112,6 +117,10 @@ module Fluent
       new_impl('buffer', BUFFER_REGISTRY, type, parent)
     end
 
+    def self.new_sd(type, parent: nil)
+      new_impl('sd', SD_REGISTRY, type, parent)
+    end
+
     def self.new_parser(type, parent: nil)
       if type[0] == '/' && type[-1] == '/'
         # This usage is not recommended for new API... create RegexpParser directly

data/lib/fluent/plugin/base.rb

@@ -52,7 +52,7 @@ module Fluent
       end
 
       def configure(conf)
-        if conf.respond_to?(:for_this_worker?) && conf.for_this_worker?
+        if Fluent::Engine.supervisor_mode || (conf.respond_to?(:for_this_worker?) && conf.for_this_worker?)
           workers = if conf.target_worker_ids && !conf.target_worker_ids.empty?
                       conf.target_worker_ids.size
                     else
@@ -60,7 +60,7 @@ module Fluent
                     end
           system_config_override(workers: workers)
         end
-        super
+        super(conf, system_config.strict_config_value)
         @_state ||= State.new(false, false, false, false, false, false, false, false, false)
         @_state.configure = true
         self

data/lib/fluent/plugin/buf_file.rb

@@ -50,6 +50,7 @@ module Fluent
         @symlink_path = nil
         @multi_workers_available = false
         @additional_resume_path = nil
+        @buffer_path = nil
       end
 
       def configure(conf)
@@ -73,7 +74,8 @@ module Fluent
           raise ConfigError, "Other '#{type_using_this_path}' plugin already use same buffer path: type = #{type_of_owner}, buffer path = #{@path}"
         end
 
-        @@buffer_paths[@path] = type_of_owner
+        @buffer_path = @path
+        @@buffer_paths[@buffer_path] = type_of_owner
 
         specified_directory_exists = File.exist?(@path) && File.directory?(@path)
         unexisting_path_for_directory = !File.exist?(@path) && !@path.include?('.*')
@@ -122,6 +124,12 @@ module Fluent
         super
       end
 
+      def stop
+        @@buffer_paths.delete(@buffer_path)
+
+        super
+      end
+
       def persistent?
         true
       end
@@ -167,12 +175,8 @@ module Fluent
 
       def generate_chunk(metadata)
         # FileChunk generates real path with unique_id
-        if @file_permission
-          chunk = Fluent::Plugin::Buffer::FileChunk.new(metadata, @path, :create, perm: @file_permission, compress: @compress)
-        else
-          chunk = Fluent::Plugin::Buffer::FileChunk.new(metadata, @path, :create, compress: @compress)
-        end
-
+        perm = @file_permission || system_config.file_permission
+        chunk = Fluent::Plugin::Buffer::FileChunk.new(metadata, @path, :create, perm: perm, compress: @compress)
         log.debug "Created new chunk", chunk_id: dump_unique_id_hex(chunk.unique_id), metadata: metadata
 
         return chunk

data/lib/fluent/plugin/buf_file_single.rb

@@ -144,6 +144,12 @@ module Fluent
         super
       end
 
+      def stop
+        @@buffer_paths.delete(@path)
+
+        super
+      end
+
       def persistent?
         true
       end
@@ -189,11 +195,8 @@ module Fluent
 
       def generate_chunk(metadata)
         # FileChunk generates real path with unique_id
-        if @file_permission
-          chunk = Fluent::Plugin::Buffer::FileSingleChunk.new(metadata, @path, :create, @key_in_path, perm: @file_permission, compress: @compress)
-        else
-          chunk = Fluent::Plugin::Buffer::FileSingleChunk.new(metadata, @path, :create, @key_in_path, compress: @compress)
-        end
+        perm = @file_permission || system_config.file_permission
+        chunk = Fluent::Plugin::Buffer::FileSingleChunk.new(metadata, @path, :create, @key_in_path, perm: perm, compress: @compress)
 
         log.debug "Created new chunk", chunk_id: dump_unique_id_hex(chunk.unique_id), metadata: metadata
 

data/lib/fluent/plugin/buffer/chunk.rb

@@ -202,7 +202,7 @@ module Fluent
             if kwargs[:compressed] == :gzip
               super
             else
-              super(kwargs) do |chunk_io|
+              super(**kwargs) do |chunk_io|
                 output_io = if chunk_io.is_a?(StringIO)
                               StringIO.new
                             else

data/lib/fluent/plugin/buffer/file_chunk.rb

@@ -37,15 +37,13 @@ module Fluent
         # path_prefix: path prefix string, ended with '.'
         # path_suffix: path suffix string, like '.log' (or any other user specified)
 
-        include SystemConfig::Mixin
-        include MessagePackFactory::Mixin
-
         FILE_PERMISSION = 0644
 
         attr_reader :path, :permission
 
-        def initialize(metadata, path, mode, perm: system_config.file_permission || FILE_PERMISSION, compress: :text)
+        def initialize(metadata, path, mode, perm: nil, compress: :text)
           super(metadata, compress: compress)
+          perm ||= FILE_PERMISSION
           @permission = perm.is_a?(String) ? perm.to_i(8) : perm
           @bytesize = @size = @adding_bytes = @adding_size = 0
           @meta = nil
@@ -221,7 +219,7 @@ module Fluent
 
           unless data
             # old type of restore
-            data = msgpack_unpacker(symbolize_keys: true).feed(bindata).read rescue {}
+            data = Fluent::MessagePackFactory.msgpack_unpacker(symbolize_keys: true).feed(bindata).read rescue {}
           end
 
           now = Fluent::Clock.real_now
@@ -403,7 +401,7 @@ module Fluent
           if chunk.slice(0, 2) == BUFFER_HEADER
             size = chunk.slice(2, 4).unpack('N').first
             if size
-              return msgpack_unpacker(symbolize_keys: true).feed(chunk.slice(6, size)).read rescue nil
+              return Fluent::MessagePackFactory.msgpack_unpacker(symbolize_keys: true).feed(chunk.slice(6, size)).read rescue nil
             end
           end
 

data/lib/fluent/plugin/buffer/file_single_chunk.rb

@@ -29,9 +29,6 @@ module Fluent
         ## buffer chunk path          : /path/to/directory/fsb.key.b513b61c9791029c2513b61c9791029c2.buf
         ## state: b/q - 'b'(on stage), 'q'(enqueued)
 
-        include SystemConfig::Mixin
-        include MessagePackFactory::Mixin
-
         PATH_EXT = 'buf'
         PATH_SUFFIX = ".#{PATH_EXT}"
         PATH_REGEXP = /\.(b|q)([0-9a-f]+)\.#{PATH_EXT}*\Z/n  # //n switch means explicit 'ASCII-8BIT' pattern
@@ -39,9 +36,10 @@ module Fluent
 
         attr_reader :path, :permission
 
-        def initialize(metadata, path, mode, key, perm: system_config.file_permission || FILE_PERMISSION, compress: :text)
+        def initialize(metadata, path, mode, key, perm: FILE_PERMISSION, compress: :text)
           super(metadata, compress: compress)
           @key = key
+          perm ||= FILE_PERMISSION
           @permission = perm.is_a?(String) ? perm.to_i(8) : perm
           @bytesize = @size = @adding_bytes = @adding_size = 0
 
@@ -216,7 +214,7 @@ module Fluent
           count = 0
           File.open(@path, 'rb') { |f|
             if chunk_format == :msgpack
-              msgpack_unpacker(f).each { |d| count += 1 }
+              Fluent::MessagePackFactory.msgpack_unpacker(f).each { |d| count += 1 }
             else
               f.each_line { |l| count += 1 }
             end

data/lib/fluent/plugin/formatter_csv.rb

@@ -14,6 +14,7 @@
 #    limitations under the License.
 #
 
+require 'fluent/plugin_helper'
 require 'fluent/plugin/formatter'
 require 'csv'
 
@@ -22,6 +23,9 @@ module Fluent
     class CsvFormatter < Formatter
       Plugin.register_formatter('csv', self)
 
+      include PluginHelper::Mixin
+      helpers :record_accessor
+
       config_param :delimiter, default: ',' do |val|
         ['\t', 'TAB'].include?(val) ? "\t" : val
       end
@@ -37,6 +41,14 @@ module Fluent
         @fields = fields.select{|f| !f.empty? }
         raise ConfigError, "empty value is specified in fields parameter" if @fields.empty?
 
+        if @fields.any? { |f| record_accessor_nested?(f) }
+          @accessors = @fields.map { |f| record_accessor_create(f) }
+          mformat = method(:format_with_nested_fields)
+          singleton_class.module_eval do
+            define_method(:format, mformat)
+          end
+        end
+
         @generate_opts = {col_sep: @delimiter, force_quotes: @force_quotes, headers: @fields,
                           row_sep: @add_newline ? :auto : "".force_encoding(Encoding::ASCII_8BIT)}
         # Cache CSV object per thread to avoid internal state sharing
@@ -44,13 +56,23 @@ module Fluent
       end
 
       def format(tag, time, record)
-        csv = (@cache[Thread.current] ||= CSV.new("".force_encoding(Encoding::ASCII_8BIT), @generate_opts))
+        csv = (@cache[Thread.current] ||= CSV.new("".force_encoding(Encoding::ASCII_8BIT), **@generate_opts))
         line = (csv << record).string.dup
         # Need manual cleanup because CSV writer doesn't provide such method.
         csv.rewind
         csv.truncate(0)
         line
       end
+
+      def format_with_nested_fields(tag, time, record)
+        csv = (@cache[Thread.current] ||= CSV.new("".force_encoding(Encoding::ASCII_8BIT), **@generate_opts))
+        values = @accessors.map { |a| a.call(record) }
+        line = (csv << values).string.dup
+        # Need manual cleanup because CSV writer doesn't provide such method.
+        csv.rewind
+        csv.truncate(0)
+        line
+      end
     end
   end
 end