fluentd 1.8.1-x86-mingw32 → 1.9.0-x86-mingw32

data/lib/fluent/ext_monitor_require.rb

@@ -0,0 +1,28 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+# To avoid duplicated requirements, extract this logic as file.
+
+if Gem::Version.create(RUBY_VERSION) >= Gem::Version.create('2.7.0')
+  require 'monitor'
+else
+  begin
+    # monitor_ext is bundled since ruby 2.7.0
+    require 'ext_monitor'
+  rescue LoadError => _
+    require 'monitor'
+  end
+end

data/lib/fluent/load.rb

@@ -2,7 +2,6 @@ require 'thread'
 require 'socket'
 require 'fcntl'
 require 'time'
-require 'monitor'
 require 'stringio'
 require 'fileutils'
 require 'json'
@@ -33,3 +32,4 @@ require 'fluent/input'
 require 'fluent/output'
 require 'fluent/filter'
 require 'fluent/match'
+require 'fluent/ext_monitor_require'

data/lib/fluent/msgpack_factory.rb

@@ -44,8 +44,8 @@ module Fluent
       end
     end
 
-    def self.engine_factory
-      @@engine_factory || factory
+    def self.engine_factory(enable_time_support: false)
+      @@engine_factory || factory(enable_time_support: enable_time_support)
     end
 
     def self.msgpack_packer(*args)
@@ -56,9 +56,15 @@ module Fluent
       engine_factory.unpacker(*args)
     end
 
-    def self.factory
+    def self.factory(enable_time_support: false)
       factory = MessagePack::Factory.new
       factory.register_type(Fluent::EventTime::TYPE, Fluent::EventTime)
+      if enable_time_support
+        factory.register_type(
+          MessagePack::Timestamp::TYPE, Time,
+          packer: MessagePack::Time::Packer,
+          unpacker: MessagePack::Time::Unpacker)
+      end
       factory
     end
 
@@ -70,9 +76,15 @@ module Fluent
       factory.unpacker(*args)
     end
 
-    def self.init
+    def self.init(enable_time_support: false)
       factory = MessagePack::Factory.new
       factory.register_type(Fluent::EventTime::TYPE, Fluent::EventTime)
+      if enable_time_support
+        factory.register_type(
+          MessagePack::Timestamp::TYPE, Time,
+          packer: MessagePack::Time::Packer,
+          unpacker: MessagePack::Time::Unpacker)
+      end
       @@engine_factory = factory
     end
 

data/lib/fluent/plugin/base.rb

@@ -187,6 +187,11 @@ module Fluent
         #   https://github.com/ruby/ruby/blob/trunk/gc.c#L788
         "#<%s:%014x>" % [self.class.name, '0x%014x' % (__id__ << 1)]
       end
+
+      def reloadable_plugin?
+        # Engine can't capture all class variables. so it's forbbiden to use class variables in each plugins if enabling reload.
+        self.class.class_variables.empty?
+      end
     end
   end
 end

data/lib/fluent/plugin/buf_file.rb

@@ -19,6 +19,7 @@ require 'fileutils'
 require 'fluent/plugin/buffer'
 require 'fluent/plugin/buffer/file_chunk'
 require 'fluent/system_config'
+require 'fluent/variable_store'
 
 module Fluent
   module Plugin
@@ -43,19 +44,20 @@ module Fluent
       config_param :file_permission, :string, default: nil # '0644'
       config_param :dir_permission,  :string, default: nil # '0755'
 
-      @@buffer_paths = {}
-
       def initialize
         super
         @symlink_path = nil
         @multi_workers_available = false
         @additional_resume_path = nil
         @buffer_path = nil
+        @variable_store = nil
       end
 
       def configure(conf)
         super
 
+        @variable_store = Fluent::VariableStore.fetch_or_build(:buf_file)
+
         multi_workers_configured = owner.system_config.workers > 1 ? true : false
 
         using_plugin_root_dir = false
@@ -69,13 +71,13 @@ module Fluent
         end
 
         type_of_owner = Plugin.lookup_type_from_class(@_owner.class)
-        if @@buffer_paths.has_key?(@path) && !called_in_test?
-          type_using_this_path = @@buffer_paths[@path]
+        if @variable_store.has_key?(@path) && !called_in_test?
+          type_using_this_path = @variable_store[@path]
           raise ConfigError, "Other '#{type_using_this_path}' plugin already use same buffer path: type = #{type_of_owner}, buffer path = #{@path}"
         end
 
         @buffer_path = @path
-        @@buffer_paths[@buffer_path] = type_of_owner
+        @variable_store[@buffer_path] = type_of_owner
 
         specified_directory_exists = File.exist?(@path) && File.directory?(@path)
         unexisting_path_for_directory = !File.exist?(@path) && !@path.include?('.*')
@@ -125,7 +127,9 @@ module Fluent
       end
 
       def stop
-        @@buffer_paths.delete(@buffer_path)
+        if @variable_store
+          @variable_store.delete(@buffer_path)
+        end
 
         super
       end

data/lib/fluent/plugin/buf_file_single.rb

@@ -19,6 +19,7 @@ require 'fileutils'
 require 'fluent/plugin/buffer'
 require 'fluent/plugin/buffer/file_single_chunk'
 require 'fluent/system_config'
+require 'fluent/variable_store'
 
 module Fluent
   module Plugin
@@ -48,18 +49,19 @@ module Fluent
       desc 'The permission of chunk directory. If no specified, <system> setting or 0755 is used'
       config_param :dir_permission, :string, default: nil
 
-      @@buffer_paths = {}
-
       def initialize
         super
 
         @multi_workers_available = false
         @additional_resume_path = nil
+        @variable_store = nil
       end
 
       def configure(conf)
         super
 
+        @variable_store = Fluent::VariableStore.fetch_or_build(:buf_file_single)
+
         if @chunk_format == :auto
           @chunk_format = owner.formatted_to_msgpack_binary? ? :msgpack : :text
         end
@@ -117,12 +119,12 @@ module Fluent
         end
 
         type_of_owner = Plugin.lookup_type_from_class(@_owner.class)
-        if @@buffer_paths.has_key?(@path) && !called_in_test?
-          type_using_this_path = @@buffer_paths[@path]
+        if @variable_store.has_key?(@path) && !called_in_test?
+          type_using_this_path = @variable_store[@path]
           raise Fluent::ConfigError, "Other '#{type_using_this_path}' plugin already uses same buffer path: type = #{type_of_owner}, buffer path = #{@path}"
         end
 
-        @@buffer_paths[@path] = type_of_owner
+        @variable_store[@path] = type_of_owner
         @dir_permission = if @dir_permission
                             @dir_permission.to_i(8)
                           else
@@ -145,7 +147,9 @@ module Fluent
       end
 
       def stop
-        @@buffer_paths.delete(@path)
+        if @variable_store
+          @variable_store.delete(@path)
+        end
 
         super
       end

data/lib/fluent/plugin/buffer.rb

@@ -17,8 +17,7 @@
 require 'fluent/plugin/base'
 require 'fluent/plugin/owned_by_mixin'
 require 'fluent/unique_id'
-
-require 'monitor'
+require 'fluent/ext_monitor_require'
 
 module Fluent
   module Plugin
@@ -139,9 +138,12 @@ module Fluent
         # Actually this overhead is very small but this class is generated *per chunk* (and used in hash object).
         # This means that this class is one of the most called object in Fluentd.
         # See https://github.com/fluent/fluentd/pull/2560
+        # But, this optimization has a side effect on Windows due to differing object_id.
+        # This difference causes flood of buffer files.
+        # So, this optimization should be enabled on non-Windows platform.
         def hash
           timekey.object_id
-        end
+        end unless Fluent.windows?
       end
 
       # for tests
@@ -266,10 +268,10 @@ module Fluent
 
         log.on_trace { log.trace "writing events into buffer", instance: self.object_id, metadata_size: metadata_and_data.size }
 
-        staged_bytesize = 0
         operated_chunks = []
         unstaged_chunks = {} # metadata => [chunk, chunk, ...]
         chunks_to_enqueue = []
+        staged_bytesizes_by_chunk = {}
 
         begin
           # sort metadata to get lock of chunks in same order with other threads
@@ -279,7 +281,13 @@ module Fluent
               chunk.mon_enter # add lock to prevent to be committed/rollbacked from other threads
               operated_chunks << chunk
               if chunk.staged?
-                staged_bytesize += adding_bytesize
+                #
+                # https://github.com/fluent/fluentd/issues/2712
+                # write_once is supposed to write to a chunk only once
+                # but this block **may** run multiple times from write_step_by_step and previous write may be rollbacked
+                # So we should be counting the stage_size only for the last successful write
+                #
+                staged_bytesizes_by_chunk[chunk] = adding_bytesize
               elsif chunk.unstaged?
                 unstaged_chunks[metadata] ||= []
                 unstaged_chunks[metadata] << chunk
@@ -326,27 +334,37 @@ module Fluent
 
           # All locks about chunks are released.
 
-          synchronize do
-            # At here, staged chunks may be enqueued by other threads.
-            @stage_size += staged_bytesize
-
-            chunks_to_enqueue.each do |c|
-              if c.staged? && (enqueue || chunk_size_full?(c))
-                m = c.metadata
-                enqueue_chunk(m)
-                if unstaged_chunks[m]
-                  u = unstaged_chunks[m].pop
+          #
+          # Now update the stage, stage_size with proper locking
+          # FIX FOR stage_size miscomputation - https://github.com/fluent/fluentd/issues/2712
+          #
+          staged_bytesizes_by_chunk.each do |chunk, bytesize|
+            chunk.synchronize do
+              synchronize { @stage_size += bytesize }
+              log.on_trace { log.trace { "chunk #{chunk.path} size_added: #{bytesize} new_size: #{chunk.bytesize}" } }
+            end
+          end
+
+          chunks_to_enqueue.each do |c|
+            if c.staged? && (enqueue || chunk_size_full?(c))
+              m = c.metadata
+              enqueue_chunk(m)
+              if unstaged_chunks[m]
+                u = unstaged_chunks[m].pop
+                u.synchronize do
                   if u.unstaged? && !chunk_size_full?(u)
-                    @stage[m] = u.staged!
-                    @stage_size += u.bytesize
+                    synchronize {
+                      @stage[m] = u.staged!
+                      @stage_size += u.bytesize
+                    }
                   end
                 end
-              elsif c.unstaged?
-                enqueue_unstaged_chunk(c)
-              else
-                # previously staged chunk is already enqueued, closed or purged.
-                # no problem.
               end
+            elsif c.unstaged?
+              enqueue_unstaged_chunk(c)
+            else
+              # previously staged chunk is already enqueued, closed or purged.
+              # no problem.
             end
           end
 

data/lib/fluent/plugin/buffer/chunk.rb

@@ -18,8 +18,8 @@ require 'fluent/plugin/buffer'
 require 'fluent/plugin/compressable'
 require 'fluent/unique_id'
 require 'fluent/event'
+require 'fluent/ext_monitor_require'
 
-require 'monitor'
 require 'tempfile'
 require 'zlib'
 

data/lib/fluent/plugin/in_http.rb

@@ -331,29 +331,29 @@ module Fluent::Plugin
         headers.each_pair {|k,v|
           @env["HTTP_#{k.gsub('-','_').upcase}"] = v
           case k
-          when /Expect/i
+          when /\AExpect\z/i
             expect = v
-          when /Content-Length/i
+          when /\AContent-Length\Z/i
             size = v.to_i
-          when /Content-Type/i
+          when /\AContent-Type\Z/i
             @content_type = v
-          when /Content-Encoding/i
+          when /\AContent-Encoding\Z/i
             @content_encoding = v
-          when /Connection/i
+          when /\AConnection\Z/i
             if v =~ /close/i
               @keep_alive = false
             elsif v =~ /Keep-alive/i
               @keep_alive = true
             end
-          when /Origin/i
+          when /\AOrigin\Z/i
             @origin  = v
-          when /X-Forwarded-For/i
+          when /\AX-Forwarded-For\Z/i
             # For multiple X-Forwarded-For headers. Use first header value.
             v = v.first if v.is_a?(Array)
             @remote_addr = v.split(",").first
-          when /Access-Control-Request-Method/i
+          when /\AAccess-Control-Request-Method\Z/i
             @access_control_request_method = v
-          when /Access-Control-Request-Headers/i
+          when /\AAccess-Control-Request-Headers\Z/i
             @access_control_request_headers = v
           end
         }

data/lib/fluent/plugin/in_tail.rb

@@ -21,6 +21,7 @@ require 'fluent/config/error'
 require 'fluent/event'
 require 'fluent/plugin/buffer'
 require 'fluent/plugin/parser_multiline'
+require 'fluent/variable_store'
 
 if Fluent.windows?
   require_relative 'file_wrapper'
@@ -105,9 +106,8 @@ module Fluent::Plugin
 
     attr_reader :paths
 
-    @@pos_file_paths = {}
-
     def configure(conf)
+      @variable_store = Fluent::VariableStore.fetch_or_build(:in_tail)
       compat_parameters_convert(conf, :parser)
       parser_config = conf.elements('parse').first
       unless parser_config
@@ -139,11 +139,11 @@ module Fluent::Plugin
 
       # TODO: Use plugin_root_dir and storage plugin to store positions if available
       if @pos_file
-        if @@pos_file_paths.has_key?(@pos_file) && !called_in_test?
-          plugin_id_using_this_path = @@pos_file_paths[@pos_file]
+        if @variable_store.key?(@pos_file) && !called_in_test?
+          plugin_id_using_this_path = @variable_store[@pos_file]
           raise Fluent::ConfigError, "Other 'in_tail' plugin already use same pos_file path: plugin_id = #{plugin_id_using_this_path}, pos_file path = #{@pos_file}"
         end
-        @@pos_file_paths[@pos_file] = self.plugin_id
+        @variable_store[@pos_file] = self.plugin_id
       else
         $log.warn "'pos_file PATH' parameter is not set to a 'tail' source."
         $log.warn "this parameter is highly recommended to save the position to resume tailing."
@@ -212,7 +212,9 @@ module Fluent::Plugin
     end
 
     def stop
-      @@pos_file_paths.delete(@pos_file)
+      if @variable_store
+        @variable_store.delete(@pos_file)
+      end
 
       super
     end

data/lib/fluent/plugin/out_http.rb

@@ -212,9 +212,9 @@ module Fluent::Plugin
             end
 
       if res.is_a?(Net::HTTPSuccess)
-        log.debug { "#{res.code} #{res.message}#{res.body}" }
+        log.debug { "#{res.code} #{res.message.rstrip}#{res.body.lstrip}" }
       else
-        msg = "#{res.code} #{res.message}#{res.body}"
+        msg = "#{res.code} #{res.message.rstrip} #{res.body.lstrip}"
 
         if @retryable_response_codes.include?(res.code.to_i)
           raise RetryableResponse, msg

data/lib/fluent/plugin/output.rb

@@ -25,9 +25,9 @@ require 'fluent/plugin_helper'
 require 'fluent/timezone'
 require 'fluent/unique_id'
 require 'fluent/clock'
+require 'fluent/ext_monitor_require'
 
 require 'time'
-require 'monitor'
 
 module Fluent
   module Plugin
@@ -387,7 +387,7 @@ module Fluent
           @secondary.acts_as_secondary(self)
           @secondary.configure(secondary_conf)
           if (self.class != @secondary.class) && (@custom_format || @secondary.implement?(:custom_format))
-            log.warn "secondary type should be same with primary one", primary: self.class.to_s, secondary: @secondary.class.to_s
+            log.warn "Use different plugin for secondary. Check the plugin works with primary like secondary_file", primary: self.class.to_s, secondary: @secondary.class.to_s
           end
         else
           @secondary = nil

data/lib/fluent/plugin/parser.rb

@@ -105,6 +105,12 @@ module Fluent
         :text_per_line
       end
 
+      def initialize
+        super
+
+        @timeout_checker = nil
+      end
+
       def configure(conf)
         super