fluentd 1.7.4 → 1.8.0.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b09f1cbc0b3f4be0bc2dedfc142a1f6f0ed195df071ec809c019987de83c0993
-  data.tar.gz: bfdc00fe223cbbef68da3472d1d99518c3a52562dc10e2f720912f93d40cd8d3
+  metadata.gz: 4c1ecaaf9a3cd9d2ff59843b0c6fa388aa9afe180684b0cc0e51045f2509d2a4
+  data.tar.gz: 7c5a881eb1ed5b87b9aa3e74deb792a3d1fc57e85a26b900a10c15a2966d6762
 SHA512:
-  metadata.gz: 1ac20a296903da6869d0cd28baafa4ec2e4ac001b96820f03417f7ec2913a5bf52641db630f9814f02b10ddafa993c5157e461dffae09688fba9cdcd704922e4
-  data.tar.gz: 4bc9cce3c2083b0de6d256cd6ffaa07c57e9390de886491d5f0cbb35b181b9af2b3c3a48ab619b6cecc4f9355a7fb0b71ae87f403db0a49ba13198149770f92b
+  metadata.gz: a21f1004723f63ae0ae306e36446ee3324f0d1dce404780bd42e71b7a935a357806482f94c8381ec8e3220aa9d27756b80aa8e05acf7d4a3e59b11dfcedf9387
+  data.tar.gz: d1789835f28d427233aa489032fd389ad890dea347a3fa4bae7ec4e21bb7341395fbabddcd9008ef3bcd6ea7343a6a1a68e20cdda93f3959554df0d09474fc19

data/CHANGELOG.md

@@ -1,3 +1,29 @@
+# Unreleased
+
+### New feature
+
+* Add service discovery plugin and `out_forward` use it
+  https://github.com/fluent/fluentd/pull/2541
+
+### Enhancement
+
+* formatter_csv: Support nested fields
+  https://github.com/fluent/fluentd/pull/2643
+* record_accessor helper: Make code simple and bit faster
+  https://github.com/fluent/fluentd/pull/2660
+* clean up internal classes / code
+  https://github.com/fluent/fluentd/pull/2647
+  https://github.com/fluent/fluentd/pull/2648
+  https://github.com/fluent/fluentd/pull/2653
+  https://github.com/fluent/fluentd/pull/2654
+  https://github.com/fluent/fluentd/pull/2657
+
+### Bug fixes
+
+* suppress warning
+  https://github.com/fluent/fluentd/pull/2652
+
+
 # v1.7
 
 ## Release v1.7.4 - 2019/10/24

data/example/out_forward_sd.conf

@@ -0,0 +1,17 @@
+<source>
+  @type dummy
+  tag test
+</source>
+
+<match test>
+  @type forward
+
+  <service_discovery>
+    @type file
+    path "#{Dir.pwd}/example/sd.yaml"
+  </service_discovery>
+
+  <buffer>
+    flush_interval 1
+  </buffer>
+</match>

data/example/sd.yaml

@@ -0,0 +1,8 @@
+- 'host': 127.0.0.1
+  'port': 24224
+  'weight': 1
+  'name': server1
+- 'host': 127.0.0.1
+  'port': 24225
+  'weight': 1
+  'name': server2

data/lib/fluent/command/cat.rb

@@ -315,7 +315,7 @@ when 'msgpack'
   require 'fluent/engine'
 
   begin
-    u = Fluent::Engine.msgpack_factory.unpacker($stdin)
+    u = Fluent::MessagePackFactory.msgpack_unpacker($stdin)
     u.each {|record|
       w.write(record)
     }
@@ -340,4 +340,3 @@ else
   $stderr.puts "Unknown format '#{format}'"
   exit 1
 end
-

data/lib/fluent/compat/call_super_mixin.rb

@@ -55,6 +55,15 @@ module Fluent
         end
       end
 
+      def stop
+        klass = self.class
+        @@_super_start.delete(klass)
+        @@_super_before_shutdown.delete(klass)
+        @@_super_shutdown.delete(klass)
+
+        super
+      end
+
       def shutdown
         super
         unless self.shutdown?

data/lib/fluent/compat/exec_util.rb

@@ -85,7 +85,7 @@ module Fluent
 
       class MessagePackParser < Parser
         def call(io)
-          @u = Fluent::Engine.msgpack_factory.unpacker(io)
+          @u = Fluent::MessagePackFactory.msgpack_unpacker(io)
           begin
             @u.each(&@on_message)
           rescue EOFError

data/lib/fluent/counter/base_socket.rb

@@ -20,14 +20,12 @@ require 'fluent/msgpack_factory'
 module Fluent
   module Counter
     class BaseSocket < Coolio::TCPSocket
-      include Fluent::MessagePackFactory::Mixin
-
       def packed_write(data)
         write pack(data)
       end
 
       def on_read(data)
-        msgpack_unpacker.feed_each(data) do |d|
+        Fluent::MessagePackFactory.msgpack_unpacker.feed_each(data) do |d|
           on_message d
         end
       end
@@ -39,7 +37,7 @@ module Fluent
       private
 
       def pack(data)
-        msgpack_packer.pack(data)
+        Fluent::MessagePackFactory.msgpack_packer.pack(data)
       end
     end
   end

data/lib/fluent/engine.rb

@@ -25,6 +25,7 @@ require 'fluent/plugin'
 
 module Fluent
   class EngineClass
+    # For compat. remove it in fluentd v2
     include Fluent::MessagePackFactory::Mixin
 
     def initialize
@@ -61,7 +62,6 @@ module Fluent
     def init(system_config)
       @system_config = system_config
 
-      suppress_interval(system_config.emit_error_log_interval) unless system_config.emit_error_log_interval.nil?
       @suppress_config_dump = system_config.suppress_config_dump unless system_config.suppress_config_dump.nil?
       @without_source = system_config.without_source unless system_config.without_source.nil?
 
@@ -69,8 +69,6 @@ module Fluent
 
       @root_agent = RootAgent.new(log: log, system_config: @system_config)
 
-      MessagePackFactory.init
-
       self
     end
 
@@ -78,11 +76,6 @@ module Fluent
       $log
     end
 
-    def suppress_interval(interval_time)
-      @suppress_emit_error_log_interval = interval_time
-      @next_emit_error_log_time = Time.now.to_i
-    end
-
     def parse_config(io, fname, basepath = Dir.pwd, v1_config = false)
       if fname =~ /\.rb$/
         require 'fluent/config/dsl'

data/lib/fluent/event.rb

@@ -20,7 +20,6 @@ require 'fluent/plugin/compressable'
 module Fluent
   class EventStream
     include Enumerable
-    include MessagePackFactory::Mixin
     include Fluent::Plugin::Compressable
 
     # dup does deep copy for event stream
@@ -56,7 +55,7 @@ module Fluent
 
     def to_msgpack_stream(time_int: false, packer: nil)
       return to_msgpack_stream_forced_integer(packer: packer) if time_int
-      out = packer || msgpack_packer
+      out = packer || Fluent::MessagePackFactory.msgpack_packer
       each {|time,record|
         out.write([time,record])
       }
@@ -69,7 +68,7 @@ module Fluent
     end
 
     def to_msgpack_stream_forced_integer(packer: nil)
-      out = packer || msgpack_packer
+      out = packer || Fluent::MessagePackFactory.msgpack_packer
       each {|time,record|
         out.write([time.to_i,record])
       }
@@ -238,7 +237,7 @@ module Fluent
       return if @unpacked_times && @unpacked_records
       @unpacked_times = []
       @unpacked_records = []
-      (unpacker || msgpack_unpacker).feed_each(@data) do |time, record|
+      (unpacker || Fluent::MessagePackFactory.msgpack_unpacker).feed_each(@data) do |time, record|
         @unpacked_times << time
         @unpacked_records << record
       end
@@ -262,7 +261,7 @@ module Fluent
       else
         @unpacked_times = []
         @unpacked_records = []
-        (unpacker || msgpack_unpacker).feed_each(@data) do |time, record|
+        (unpacker || Fluent::MessagePackFactory.msgpack_unpacker).feed_each(@data) do |time, record|
           @unpacked_times << time
           @unpacked_records << record
           block.call(time, record)
@@ -319,12 +318,11 @@ module Fluent
   end
 
   module ChunkMessagePackEventStreamer
-    include MessagePackFactory::Mixin
     # chunk.extend(ChunkEventStreamer)
     #  => chunk.each{|time, record| ... }
     def each(unpacker: nil, &block)
       open do |io|
-        (unpacker || msgpack_unpacker(io)).each(&block)
+        (unpacker || Fluent::MessagePackFactory.msgpack_unpacker(io)).each(&block)
       end
       nil
     end

data/lib/fluent/msgpack_factory.rb

@@ -23,15 +23,24 @@ module Fluent
 
     module Mixin
       def msgpack_factory
+        if $log
+          $log.warn('Deprecated method: this method is going to be deleted. Use Fluent::MessagePackFactory.engine_factory')
+        end
         MessagePackFactory.engine_factory
       end
 
       def msgpack_packer(*args)
-        msgpack_factory.packer(*args)
+        if $log
+          $log.warn('Deprecated method: this method is going to be deleted. Use Fluent::MessagePackFactory.msgpack_packer')
+        end
+        MessagePackFactory.msgpack_packer(*args)
       end
 
       def msgpack_unpacker(*args)
-        msgpack_factory.unpacker(*args)
+        if $log
+          $log.warn('Deprecated method: this method is going to be deleted. Use Fluent::MessagePackFactory.msgpack_unpacker')
+        end
+        MessagePackFactory.msgpack_unpacker(*args)
       end
     end
 
@@ -39,6 +48,14 @@ module Fluent
       @@engine_factory || factory
     end
 
+    def self.msgpack_packer(*args)
+      engine_factory.packer(*args)
+    end
+
+    def self.msgpack_unpacker(*args)
+      engine_factory.unpacker(*args)
+    end
+
     def self.factory
       factory = MessagePack::Factory.new
       factory.register_type(Fluent::EventTime::TYPE, Fluent::EventTime)

data/lib/fluent/plugin.rb

@@ -35,8 +35,9 @@ module Fluent
     PARSER_REGISTRY    = Registry.new(:parser,    'fluent/plugin/parser_',     dir_search_prefix: 'parser_')
     FORMATTER_REGISTRY = Registry.new(:formatter, 'fluent/plugin/formatter_',  dir_search_prefix: 'formatter_')
     STORAGE_REGISTRY   = Registry.new(:storage,   'fluent/plugin/storage_',    dir_search_prefix: 'storage_')
+    SD_REGISTRY        = Registry.new(:sd,        'fluent/plugin/sd_',         dir_search_prefix: 'sd_')
 
-    REGISTRIES = [INPUT_REGISTRY, OUTPUT_REGISTRY, FILTER_REGISTRY, BUFFER_REGISTRY, PARSER_REGISTRY, FORMATTER_REGISTRY, STORAGE_REGISTRY]
+    REGISTRIES = [INPUT_REGISTRY, OUTPUT_REGISTRY, FILTER_REGISTRY, BUFFER_REGISTRY, PARSER_REGISTRY, FORMATTER_REGISTRY, STORAGE_REGISTRY, SD_REGISTRY]
 
     def self.register_input(type, klass)
       register_impl('input', INPUT_REGISTRY, type, klass)
@@ -54,6 +55,10 @@ module Fluent
       register_impl('buffer', BUFFER_REGISTRY, type, klass)
     end
 
+    def self.register_sd(type, klass)
+      register_impl('sd', SD_REGISTRY, type, klass)
+    end
+
     def self.register_parser(type, klass_or_proc)
       if klass_or_proc.is_a?(Regexp)
         # This usage is not recommended for new API
@@ -112,6 +117,10 @@ module Fluent
       new_impl('buffer', BUFFER_REGISTRY, type, parent)
     end
 
+    def self.new_sd(type, parent: nil)
+      new_impl('sd', SD_REGISTRY, type, parent)
+    end
+
     def self.new_parser(type, parent: nil)
       if type[0] == '/' && type[-1] == '/'
         # This usage is not recommended for new API... create RegexpParser directly

data/lib/fluent/plugin/buf_file.rb

@@ -50,6 +50,7 @@ module Fluent
         @symlink_path = nil
         @multi_workers_available = false
         @additional_resume_path = nil
+        @buffer_path = nil
       end
 
       def configure(conf)
@@ -73,7 +74,8 @@ module Fluent
           raise ConfigError, "Other '#{type_using_this_path}' plugin already use same buffer path: type = #{type_of_owner}, buffer path = #{@path}"
         end
 
-        @@buffer_paths[@path] = type_of_owner
+        @buffer_path = @path
+        @@buffer_paths[@buffer_path] = type_of_owner
 
         specified_directory_exists = File.exist?(@path) && File.directory?(@path)
         unexisting_path_for_directory = !File.exist?(@path) && !@path.include?('.*')
@@ -122,6 +124,12 @@ module Fluent
         super
       end
 
+      def stop
+        @@buffer_paths.delete(@buffer_path)
+
+        super
+      end
+
       def persistent?
         true
       end

data/lib/fluent/plugin/buf_file_single.rb

@@ -144,6 +144,12 @@ module Fluent
         super
       end
 
+      def stop
+        @@buffer_paths.delete(@path)
+
+        super
+      end
+
       def persistent?
         true
       end

data/lib/fluent/plugin/buffer/file_chunk.rb

@@ -38,7 +38,6 @@ module Fluent
         # path_suffix: path suffix string, like '.log' (or any other user specified)
 
         include SystemConfig::Mixin
-        include MessagePackFactory::Mixin
 
         FILE_PERMISSION = 0644
 
@@ -221,7 +220,7 @@ module Fluent
 
           unless data
             # old type of restore
-            data = msgpack_unpacker(symbolize_keys: true).feed(bindata).read rescue {}
+            data = Fluent::MessagePackFactory.msgpack_unpacker(symbolize_keys: true).feed(bindata).read rescue {}
           end
 
           now = Fluent::Clock.real_now
@@ -403,7 +402,7 @@ module Fluent
           if chunk.slice(0, 2) == BUFFER_HEADER
             size = chunk.slice(2, 4).unpack('N').first
             if size
-              return msgpack_unpacker(symbolize_keys: true).feed(chunk.slice(6, size)).read rescue nil
+              return Fluent::MessagePackFactory.msgpack_unpacker(symbolize_keys: true).feed(chunk.slice(6, size)).read rescue nil
             end
           end
 

data/lib/fluent/plugin/buffer/file_single_chunk.rb

@@ -30,7 +30,6 @@ module Fluent
         ## state: b/q - 'b'(on stage), 'q'(enqueued)
 
         include SystemConfig::Mixin
-        include MessagePackFactory::Mixin
 
         PATH_EXT = 'buf'
         PATH_SUFFIX = ".#{PATH_EXT}"
@@ -216,7 +215,7 @@ module Fluent
           count = 0
           File.open(@path, 'rb') { |f|
             if chunk_format == :msgpack
-              msgpack_unpacker(f).each { |d| count += 1 }
+              Fluent::MessagePackFactory.msgpack_unpacker(f).each { |d| count += 1 }
             else
               f.each_line { |l| count += 1 }
             end

data/lib/fluent/plugin/formatter_csv.rb

@@ -14,6 +14,7 @@
 #    limitations under the License.
 #
 
+require 'fluent/plugin_helper'
 require 'fluent/plugin/formatter'
 require 'csv'
 
@@ -22,6 +23,9 @@ module Fluent
     class CsvFormatter < Formatter
       Plugin.register_formatter('csv', self)
 
+      include PluginHelper::Mixin
+      helpers :record_accessor
+
       config_param :delimiter, default: ',' do |val|
         ['\t', 'TAB'].include?(val) ? "\t" : val
       end
@@ -37,6 +41,14 @@ module Fluent
         @fields = fields.select{|f| !f.empty? }
         raise ConfigError, "empty value is specified in fields parameter" if @fields.empty?
 
+        if @fields.any? { |f| record_accessor_nested?(f) }
+          @accessors = @fields.map { |f| record_accessor_create(f) }
+          mformat = method(:format_with_nested_fields)
+          singleton_class.module_eval do
+            define_method(:format, mformat)
+          end
+        end
+
         @generate_opts = {col_sep: @delimiter, force_quotes: @force_quotes, headers: @fields,
                           row_sep: @add_newline ? :auto : "".force_encoding(Encoding::ASCII_8BIT)}
         # Cache CSV object per thread to avoid internal state sharing
@@ -51,6 +63,16 @@ module Fluent
         csv.truncate(0)
         line
       end
+
+      def format_with_nested_fields(tag, time, record)
+        csv = (@cache[Thread.current] ||= CSV.new("".force_encoding(Encoding::ASCII_8BIT), @generate_opts))
+        values = @accessors.map { |a| a.call(record) }
+        line = (csv << values).string.dup
+        # Need manual cleanup because CSV writer doesn't provide such method.
+        csv.rewind
+        csv.truncate(0)
+        line
+      end
     end
   end
 end

data/lib/fluent/plugin/in_forward.rb

@@ -256,7 +256,7 @@ module Fluent::Plugin
             serializer = :to_json.to_proc
             feeder = ->(d){ parser << d }
           else # msgpack
-            parser = Fluent::Engine.msgpack_factory.unpacker
+            parser = Fluent::MessagePackFactory.msgpack_unpacker
             serializer = :to_msgpack.to_proc
             feeder = ->(d){
               parser.feed_each(d){|obj|