fluentd 0.14.12 → 0.14.13

data/lib/fluent/compat/detach_process_mixin.rb

@@ -0,0 +1,25 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module Fluent
+  module Compat
+    module DetachProcessMixin
+    end
+
+    module DetachMultiProcessMixin
+    end
+  end
+end

data/lib/fluent/compat/filter.rb

@@ -25,7 +25,7 @@ module Fluent
     class Filter < Fluent::Plugin::Filter
       # TODO: warn when deprecated
 
-      helpers :inject
+      helpers_internal :inject
 
       def initialize
         super

data/lib/fluent/compat/input.rb

@@ -16,6 +16,7 @@
 
 require 'fluent/plugin'
 require 'fluent/plugin/input'
+require 'fluent/process'
 require 'fluent/compat/call_super_mixin'
 
 module Fluent

data/lib/fluent/compat/output.rb

@@ -15,6 +15,7 @@
 #
 
 require 'fluent/plugin'
+require 'fluent/plugin/buffer'
 require 'fluent/plugin/output'
 require 'fluent/plugin/bare_output'
 require 'fluent/compat/call_super_mixin'
@@ -26,6 +27,7 @@ require 'fluent/compat/record_filter_mixin'
 require 'fluent/compat/output_chain'
 require 'fluent/timezone'
 require 'fluent/mixin'
+require 'fluent/process'
 require 'fluent/event'
 
 require 'fluent/plugin_helper/compat_parameters'
@@ -35,6 +37,7 @@ require 'time'
 module Fluent
   module Compat
     NULL_OUTPUT_CHAIN = NullOutputChain.instance
+    BufferQueueLimitError = ::Fluent::Plugin::Buffer::BufferOverflowError
 
     module CompatOutputUtils
       def self.buffer_section(conf)
@@ -146,7 +149,7 @@ module Fluent
     class Output < Fluent::Plugin::Output
       # TODO: warn when deprecated
 
-      helpers :event_emitter, :inject
+      helpers_internal :event_emitter, :inject
 
       def support_in_v12_style?(feature)
         case feature
@@ -202,7 +205,7 @@ module Fluent
     class MultiOutput < Fluent::Plugin::BareOutput
       # TODO: warn when deprecated
 
-      helpers :event_emitter
+      helpers_internal :event_emitter
 
       def process(tag, es)
         emit(tag, es, NULL_OUTPUT_CHAIN)
@@ -212,7 +215,7 @@ module Fluent
     class BufferedOutput < Fluent::Plugin::Output
       # TODO: warn when deprecated
 
-      helpers :event_emitter, :inject
+      helpers_internal :event_emitter, :inject
 
       def support_in_v12_style?(feature)
         case feature
@@ -422,7 +425,7 @@ module Fluent
     class ObjectBufferedOutput < Fluent::Plugin::Output
       # TODO: warn when deprecated
 
-      helpers :event_emitter, :inject
+      helpers_internal :event_emitter, :inject
 
       # This plugin cannot inherit BufferedOutput because #configure sets chunk_key 'tag'
       # to flush chunks per tags, but BufferedOutput#configure doesn't allow setting chunk_key
@@ -559,7 +562,7 @@ module Fluent
     class TimeSlicedOutput < Fluent::Plugin::Output
       # TODO: warn when deprecated
 
-      helpers :event_emitter, :inject
+      helpers_internal :event_emitter, :inject
 
       def support_in_v12_style?(feature)
         case feature

data/lib/fluent/config/configure_proxy.rb

@@ -368,16 +368,40 @@ module Fluent
         name
       end
 
-      def dump(level = 0)
-        dumped_config = ""
-        indent = " " * level
+      def dump_config_definition
+        dumped_config = {}
         @params.each do |name, config|
-          dumped_config << "#{indent}#{name}: #{config[1][:type]}: <#{@defaults[name].inspect}>"
-          dumped_config << " # #{@descriptions[name]}" if @descriptions[name]
-          dumped_config << "\n"
+          dumped_config[name] = config[1]
+          dumped_config[name][:required] = !@defaults.key?(name)
+          dumped_config[name][:default] = @defaults[name] if @defaults.key?(name)
+          dumped_config[name][:desc] = @descriptions[name] if @descriptions.key?(name)
+        end
+        # Overwrite by config_set_default
+        @defaults.each do |name, value|
+          if @params.key?(name)
+            dumped_config[name][:default] = value
+          else
+            dumped_config[name] = { default: value }
+          end
+        end
+        # Overwrite by config_set_desc
+        @descriptions.each do |name, value|
+          if @params.key?(name)
+            dumped_config[name][:desc] = value
+          else
+            dumped_config[name] = { desc: value }
+          end
         end
         @sections.each do |section_name, sub_proxy|
-          dumped_config << "#{indent}#{section_name}\n#{sub_proxy.dump(level + 1)}"
+          if dumped_config.key?(section_name)
+            dumped_config[section_name].update(sub_proxy.dump_config_definition)
+          else
+            dumped_config[section_name] = sub_proxy.dump_config_definition
+            dumped_config[section_name][:required] = sub_proxy.required?
+            dumped_config[section_name][:multi] = sub_proxy.multi?
+            dumped_config[section_name][:alias] = sub_proxy.alias
+            dumped_config[section_name][:section] = true
+          end
         end
         dumped_config
       end
@@ -392,4 +416,3 @@ module Fluent
     end
   end
 end
-

data/lib/fluent/configurable.rb

@@ -192,8 +192,8 @@ module Fluent
         configurables.map{ |a| a.configure_proxy(a.name || a.object_id.to_s) }.reduce(:merge)
       end
 
-      def dump(level = 0)
-        configure_proxy_map[self.to_s].dump(level)
+      def dump_config_definition
+        configure_proxy_map[self.to_s].dump_config_definition
       end
     end
   end

data/lib/fluent/output.rb

@@ -23,4 +23,7 @@ module Fluent
   ObjectBufferedOutput = Fluent::Compat::ObjectBufferedOutput
   TimeSlicedOutput     = Fluent::Compat::TimeSlicedOutput
   MultiOutput          = Fluent::Compat::MultiOutput
+
+  # Some input plugins refer BufferQueueLimitError for throttling
+  BufferQueueLimitError = Fluent::Compat::BufferQueueLimitError
 end

data/lib/fluent/plugin/buffer/file_chunk.rb

@@ -106,10 +106,37 @@ module Fluent
 
           write_metadata(update: false) # re-write metadata w/ finalized records
 
-          file_rename(@chunk, @path, new_chunk_path, ->(new_io){ @chunk = new_io })
-          @path = new_chunk_path
+          begin
+            file_rename(@chunk, @path, new_chunk_path, ->(new_io) { @chunk = new_io })
+          rescue => e
+            begin
+              file_rename(@chunk, new_chunk_path, @path, ->(new_io) { @chunk = new_io }) if File.exist?(new_chunk_path)
+            rescue => re
+              # In this point, restore buffer state is hard because previous `file_rename` failed by resource problem.
+              # Retry is one possible approach but it may cause livelock under limited resources or high load environment.
+              # So we ignore such errors for now and log better message instead.
+              # "Too many open files" should be fixed by proper buffer configuration and system setting.
+              raise "can't enqueue buffer file and failed to restore. This may causes inconsistent state: path = #{@path}, error = '#{e}', retry error = '#{re}'"
+            else
+              raise "can't enqueue buffer file: path = #{@path}, error = '#{e}'"
+            end
+          end
+
+          begin
+            file_rename(@meta, @meta_path, new_meta_path, ->(new_io) { @meta = new_io })
+          rescue => e
+            begin
+              file_rename(@chunk, new_chunk_path, @path, ->(new_io) { @chunk = new_io }) if File.exist?(new_chunk_path)
+              file_rename(@meta, new_meta_path, @meta_path, ->(new_io) { @meta = new_io }) if File.exist?(new_meta_path)
+            rescue => re
+              # See above
+              raise "can't enqueue buffer metadata and failed to restore. This may causes inconsistent state: path = #{@meta_path}, error = '#{e}', retry error = '#{re}'"
+            else
+              raise "can't enqueue buffer metadata: path = #{@meta_path}, error = '#{e}'"
+            end
+          end
 
-          file_rename(@meta, @meta_path, new_meta_path, ->(new_io){ @meta = new_io })
+          @path = new_chunk_path
           @meta_path = new_meta_path
 
           super
@@ -242,14 +269,28 @@ module Fluent
         def create_new_chunk(path, perm)
           @path = self.class.generate_stage_chunk_path(path, @unique_id)
           @meta_path = @path + '.meta'
-          @chunk = File.open(@path, 'wb+', perm)
-          @chunk.set_encoding(Encoding::ASCII_8BIT)
-          @chunk.sync = true
-          @chunk.binmode
-          @meta = File.open(@meta_path, 'wb', perm)
-          @meta.set_encoding(Encoding::ASCII_8BIT)
-          @meta.sync = true
-          @meta.binmode
+          begin
+            @chunk = File.open(@path, 'wb+', perm)
+            @chunk.set_encoding(Encoding::ASCII_8BIT)
+            @chunk.sync = true
+            @chunk.binmode
+          rescue => e
+            # Here assumes "Too many open files" like recoverable error so raising BufferOverflowError.
+            # If other cases are possible, we will change erorr handling with proper classes.
+            raise BufferOverflowError, "can't create buffer file for #{path}. Stop creating buffer files: error = #{e}"
+          end
+          begin
+            @meta = File.open(@meta_path, 'wb', perm)
+            @meta.set_encoding(Encoding::ASCII_8BIT)
+            @meta.sync = true
+            @meta.binmode
+          rescue => e
+            # This case is easier than enqueued!. Just removing pre-create buffer file
+            @chunk.close rescue nil
+            File.unlink(@path) rescue nil
+            # Same as @chunk case. See above
+            raise BufferOverflowError, "can't create buffer metadata for #{path}. Stop creating buffer files: error = #{e}"
+          end
 
           @state = :unstaged
           @bytesize = 0

data/lib/fluent/plugin/filter.rb

@@ -28,7 +28,7 @@ module Fluent
       include PluginLoggerMixin
       include PluginHelper::Mixin
 
-      helpers :event_emitter
+      helpers_internal :event_emitter
 
       attr_reader :has_filter_with_time
 

data/lib/fluent/plugin/filter_record_transformer.rb

@@ -308,10 +308,14 @@ module Fluent::Plugin
         raise "failed to expand `#{str}` : error = #{e}"
       end
 
-      class CleanroomExpander < BasicObject
+      class CleanroomExpander
         def expand(__str_to_eval__, tag, time, record, tag_parts, tag_prefix, tag_suffix, hostname)
           instance_eval(__str_to_eval__)
         end
+
+        (Object.instance_methods).each do |m|
+          undef_method m unless m.to_s =~ /^__|respond_to_missing\?|object_id|public_methods|instance_eval|method_missing|define_singleton_method|respond_to\?|new_ostruct_member/
+        end
       end
     end
   end

data/lib/fluent/plugin/in_tail.rb

@@ -77,6 +77,8 @@ module Fluent::Plugin
     config_param :path_key, :string, default: nil
     desc 'Open and close the file on every update instead of leaving it open until it gets rotated.'
     config_param :open_on_every_update, :bool, default: false
+    desc 'Limit the watching files that the modification time is within the specified time range (when use \'*\' in path).'
+    config_param :limit_recently_modified, :time, default: nil
 
     attr_reader :paths
 
@@ -163,7 +165,8 @@ module Fluent::Plugin
     end
 
     def shutdown
-      stop_watchers(@tails.keys, true)
+      # during shutdown phase, don't close io. It should be done in close after all threads are stopped. See close.
+      stop_watchers(@tails.keys, immediate: true, remove_watcher: false)
       @pf_file.close if @pf_file
 
       super
@@ -184,8 +187,12 @@ module Fluent::Plugin
         path = date.strftime(path)
         if path.include?('*')
           paths += Dir.glob(path).select { |p|
-            if File.readable?(p)
-              true
+            if File.readable?(p) && !File.directory?(p)
+              if @limit_recently_modified && File.mtime(p) < (date - @limit_recently_modified)
+                false
+              else
+                true
+              end
             else
               log.warn "#{p} unreadable. It is excluded and would be examined next time."
               false
@@ -211,7 +218,7 @@ module Fluent::Plugin
       unwatched = existence_paths - target_paths
       added = target_paths - existence_paths
 
-      stop_watchers(unwatched, false, true) unless unwatched.empty?
+      stop_watchers(unwatched, immediate: false, unwatched: true) unless unwatched.empty?
       start_watchers(added) unless added.empty?
     end
 
@@ -243,9 +250,9 @@ module Fluent::Plugin
       }
     end
 
-    def stop_watchers(paths, immediate = false, unwatched = false)
+    def stop_watchers(paths, immediate: false, unwatched: false, remove_watcher: true)
       paths.each { |path|
-        tw = @tails[path]
+        tw = remove_watcher ? @tails.delete(path) : @tails[path]
         if tw
           tw.unwatched = unwatched
           if immediate

data/lib/fluent/plugin/input.rb

@@ -27,7 +27,7 @@ module Fluent
       include PluginLoggerMixin
       include PluginHelper::Mixin
 
-      helpers :event_emitter
+      helpers_internal :event_emitter
 
       def multi_workers_ready?
         false

data/lib/fluent/plugin/output.rb

@@ -33,7 +33,7 @@ module Fluent
       include PluginHelper::Mixin
       include UniqueId::Mixin
 
-      helpers :thread, :retry_state
+      helpers_internal :thread, :retry_state
 
       CHUNK_KEY_PATTERN = /^[-_.@a-zA-Z0-9]+$/
       CHUNK_KEY_PLACEHOLDER_PATTERN = /\$\{[-_.@a-zA-Z0-9]+\}/
@@ -229,6 +229,7 @@ module Fluent
         end
 
         has_buffer_section = (conf.elements(name: 'buffer').size > 0)
+        has_flush_interval = conf.has_key?('flush_interval')
 
         super
 
@@ -290,7 +291,12 @@ module Fluent
 
           @flush_mode = @buffer_config.flush_mode
           if @flush_mode == :default
-            @flush_mode = (@chunk_key_time ? :lazy : :interval)
+            if has_flush_interval
+              log.info "'flush_interval' is configured at out side of <buffer>. 'flush_mode' is set to 'interval' to keep existing behaviour"
+              @flush_mode = :interval
+            else
+              @flush_mode = (@chunk_key_time ? :lazy : :interval)
+            end
           end
 
           buffer_type = @buffer_config[:@type]
@@ -310,6 +316,14 @@ module Fluent
             log.warn "'flush_at_shutdown' is false, and buffer plugin '#{buf_type}' is not persistent buffer."
             log.warn "your configuration will lose buffered data at shutdown. please confirm your configuration again."
           end
+
+          if (@flush_mode != :interval) && buffer_conf.has_key?('flush_interval')
+            if buffer_conf.has_key?('flush_mode')
+              raise Fluent::ConfigError, "'flush_interval' can't be specified when 'flush_mode' is not 'interval' explicitly: '#{@flush_mode}'"
+            else
+              log.warn "'flush_interval' is ignored because default 'flush_mode' is not 'interval': '#{@flush_mode}'"
+            end
+          end
         end
 
         if @secondary_config
@@ -850,8 +864,11 @@ module Fluent
         es.each do |time, record|
           meta = metadata(tag, time, record)
           meta_and_data[meta] ||= []
-          meta_and_data[meta] << format(tag, time, record)
-          records += 1
+          res = format(tag, time, record)
+          if res
+            meta_and_data[meta] << res
+            records += 1
+          end
         end
         write_guard do
           @buffer.write(meta_and_data, enqueue: enqueue)
@@ -885,8 +902,11 @@ module Fluent
           records = 0
           data = []
           es.each do |time, record|
-            data << format(tag, time, record)
-            records += 1
+            res = format(tag, time, record)
+            if res
+              data << res
+              records += 1
+            end
           end
         else
           format_proc = generate_format_proc

data/lib/fluent/plugin/parser_apache.rb

@@ -14,7 +14,7 @@
 #    limitations under the License.
 #
 
-require 'fluent/plugin/parser'
+require 'fluent/plugin/parser_regexp'
 
 module Fluent
   module Plugin

data/lib/fluent/plugin/parser_apache_error.rb

@@ -14,7 +14,7 @@
 #    limitations under the License.
 #
 
-require 'fluent/plugin/parser'
+require 'fluent/plugin/parser_regexp'
 
 module Fluent
   module Plugin

data/lib/fluent/plugin/parser_multiline.rb

@@ -15,6 +15,7 @@
 #
 
 require 'fluent/plugin/parser'
+require 'fluent/plugin/parser_regexp'
 
 module Fluent
   module Plugin