fluentd 0.12.40 → 1.6.2

data/lib/fluent/plugin/bare_output.rb

@@ -0,0 +1,63 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/base'
+
+require 'fluent/log'
+require 'fluent/plugin_id'
+require 'fluent/plugin_helper'
+
+module Fluent
+  module Plugin
+    class BareOutput < Base
+      # DO NOT USE THIS plugin for normal output plugin. Use Output instead.
+      # This output plugin base class is only for meta-output plugins
+      # which cannot be implemented on MultiOutput.
+      # E.g,: forest, config-expander
+
+      include PluginId
+      include PluginLoggerMixin
+      include PluginHelper::Mixin
+
+      attr_reader :num_errors, :emit_count, :emit_records
+
+      def process(tag, es)
+        raise NotImplementedError, "BUG: output plugins MUST implement this method"
+      end
+
+      def initialize
+        super
+        @counters_monitor = Monitor.new
+        # TODO: well organized counters
+        @num_errors = 0
+        @emit_count = 0
+        @emit_records = 0
+      end
+
+      def emit_sync(tag, es)
+        @counters_monitor.synchronize{ @emit_count += 1 }
+        begin
+          process(tag, es)
+          @counters_monitor.synchronize{ @emit_records += es.size }
+        rescue
+          @counters_monitor.synchronize{ @num_errors += 1 }
+          raise
+        end
+      end
+      alias :emit_events :emit_sync
+    end
+  end
+end

data/lib/fluent/plugin/base.rb

@@ -0,0 +1,192 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin'
+require 'fluent/configurable'
+require 'fluent/system_config'
+
+module Fluent
+  module Plugin
+    class Base
+      include Configurable
+      include SystemConfig::Mixin
+
+      State = Struct.new(:configure, :start, :after_start, :stop, :before_shutdown, :shutdown, :after_shutdown, :close, :terminate)
+
+      attr_accessor :under_plugin_development
+
+      def initialize
+        @log = nil
+        super
+        @_state = State.new(false, false, false, false, false, false, false, false, false)
+        @_context_router = nil
+        @_fluentd_worker_id = nil
+        @under_plugin_development = false
+      end
+
+      def has_router?
+        false
+      end
+
+      def plugin_root_dir
+        nil # override this in plugin_id.rb
+      end
+
+      def fluentd_worker_id
+        return @_fluentd_worker_id if @_fluentd_worker_id
+        @_fluentd_worker_id = (ENV['SERVERENGINE_WORKER_ID'] || 0).to_i
+        @_fluentd_worker_id
+      end
+
+      def configure(conf)
+        if conf.respond_to?(:for_this_worker?) && conf.for_this_worker?
+          workers = if conf.target_worker_ids && !conf.target_worker_ids.empty?
+                      conf.target_worker_ids.size
+                    else
+                      1
+                    end
+          system_config_override(workers: workers)
+        end
+        super
+        @_state ||= State.new(false, false, false, false, false, false, false, false, false)
+        @_state.configure = true
+        self
+      end
+
+      def multi_workers_ready?
+        true
+      end
+
+      def string_safe_encoding(str)
+        unless str.valid_encoding?
+          log.info "invalid byte sequence is replaced in `#{str}`" if self.respond_to?(:log)
+          str = str.scrub('?')
+        end
+        yield str
+      end
+
+      def context_router=(router)
+        @_context_router = router
+      end
+
+      def context_router
+        @_context_router
+      end
+
+      def start
+        # By initialization order, plugin logger is created before set log_event_enabled.
+        # It causes '@id' specified plugin, it uses plugin logger instead of global logger, ignores `<label @FLUENT_LOG>` setting.
+        # This is adhoc approach but impact is minimal.
+        if @log.is_a?(Fluent::PluginLogger) && $log.respond_to?(:log_event_enabled) # log_event_enabled check for tests
+          @log.log_event_enabled = $log.log_event_enabled
+        end
+        @_state.start = true
+        self
+      end
+
+      def after_start
+        @_state.after_start = true
+        self
+      end
+
+      def stop
+        @_state.stop = true
+        self
+      end
+
+      def before_shutdown
+        @_state.before_shutdown = true
+        self
+      end
+
+      def shutdown
+        @_state.shutdown = true
+        self
+      end
+
+      def after_shutdown
+        @_state.after_shutdown = true
+        self
+      end
+
+      def close
+        @_state.close = true
+        self
+      end
+
+      def terminate
+        @_state.terminate = true
+        self
+      end
+
+      def configured?
+        @_state.configure
+      end
+
+      def started?
+        @_state.start
+      end
+
+      def after_started?
+        @_state.after_start
+      end
+
+      def stopped?
+        @_state.stop
+      end
+
+      def before_shutdown?
+        @_state.before_shutdown
+      end
+
+      def shutdown?
+        @_state.shutdown
+      end
+
+      def after_shutdown?
+        @_state.after_shutdown
+      end
+
+      def closed?
+        @_state.close
+      end
+
+      def terminated?
+        @_state.terminate
+      end
+
+      def called_in_test?
+        caller_locations.each do |location|
+          # Thread::Backtrace::Location#path returns base filename or absolute path.
+          # #absolute_path returns absolute_path always.
+          # https://bugs.ruby-lang.org/issues/12159
+          if location.absolute_path =~ /\/test_[^\/]+\.rb$/ # location.path =~ /test_.+\.rb$/
+            return true
+          end
+        end
+        false
+      end
+
+      def inspect
+        # Plugin instances are sometimes too big to dump because it may have too many thins (buffer,storage, ...)
+        # Original commit comment says that:
+        #   To emulate normal inspect behavior `ruby -e'o=Object.new;p o;p (o.__id__<<1).to_s(16)'`.
+        #   https://github.com/ruby/ruby/blob/trunk/gc.c#L788
+        "#<%s:%014x>" % [self.class.name, '0x%014x' % (__id__ << 1)]
+      end
+    end
+  end
+end

data/lib/fluent/plugin/buf_file.rb

@@ -15,218 +15,172 @@
 #
 
 require 'fileutils'
-require 'uri'
 
-require 'fluent/env'
-require 'fluent/plugin'
-require 'fluent/buffer'
+require 'fluent/plugin/buffer'
+require 'fluent/plugin/buffer/file_chunk'
+require 'fluent/system_config'
 
 module Fluent
-  class FileBufferChunk < BufferChunk
-    def initialize(key, path, unique_id, mode="a+", symlink_path = nil)
-      super(key)
-      @path = path
-      @unique_id = unique_id
-      @file = File.open(@path, mode, DEFAULT_FILE_PERMISSION)
-      @file.sync = true
-      @size = @file.stat.size
-      FileUtils.ln_sf(@path, symlink_path) if symlink_path
-    end
-
-    attr_reader :unique_id, :path
+  module Plugin
+    class FileBuffer < Fluent::Plugin::Buffer
+      Plugin.register_buffer('file', self)
 
-    def <<(data)
-      @file.write(data)
-      @size += data.bytesize
-    end
+      include SystemConfig::Mixin
 
-    def size
-      @size
-    end
+      DEFAULT_CHUNK_LIMIT_SIZE = 256 * 1024 * 1024        # 256MB
+      DEFAULT_TOTAL_LIMIT_SIZE =  64 * 1024 * 1024 * 1024 #  64GB, same with v0.12 (TimeSlicedOutput + buf_file)
 
-    def empty?
-      @size == 0
-    end
+      DIR_PERMISSION = 0755
 
-    def close
-      stat = @file.stat
-      @file.close
-      if stat.size == 0
-        File.unlink(@path)
-      end
-    end
+      desc 'The path where buffer chunks are stored.'
+      config_param :path, :string, default: nil
 
-    def purge
-      @file.close
-      File.unlink(@path) rescue nil  # TODO rescue?
-    end
+      config_set_default :chunk_limit_size, DEFAULT_CHUNK_LIMIT_SIZE
+      config_set_default :total_limit_size, DEFAULT_TOTAL_LIMIT_SIZE
 
-    def read
-      @file.pos = 0
-      @file.read
-    end
+      config_param :file_permission, :string, default: nil # '0644'
+      config_param :dir_permission,  :string, default: nil # '0755'
 
-    def open(&block)
-      @file.pos = 0
-      yield @file
-    end
+      @@buffer_paths = {}
 
-    def mv(path)
-      File.rename(@path, path)
-      @path = path
-    end
-  end
+      def initialize
+        super
+        @symlink_path = nil
+        @multi_workers_available = false
+        @additional_resume_path = nil
+      end
 
-  class FileBuffer < BasicBuffer
-    Plugin.register_buffer('file', self)
+      def configure(conf)
+        super
 
-    @@buffer_paths = {}
+        multi_workers_configured = owner.system_config.workers > 1 ? true : false
 
-    def initialize
-      require 'uri'
-      super
+        using_plugin_root_dir = false
+        unless @path
+          if root_dir = owner.plugin_root_dir
+            @path = File.join(root_dir, 'buffer')
+            using_plugin_root_dir = true # plugin_root_dir path contains worker id
+          else
+            raise Fluent::ConfigError, "buffer path is not configured. specify 'path' in <buffer>"
+          end
+        end
 
-      @uri_parser = URI::Parser.new
-    end
+        type_of_owner = Plugin.lookup_type_from_class(@_owner.class)
+        if @@buffer_paths.has_key?(@path) && !called_in_test?
+          type_using_this_path = @@buffer_paths[@path]
+          raise ConfigError, "Other '#{type_using_this_path}' plugin already use same buffer path: type = #{type_of_owner}, buffer path = #{@path}"
+        end
 
-    desc 'The path where buffer chunks are stored.'
-    config_param :buffer_path, :string
-    desc 'If true, queued chunks are flushed at shutdown process.'
-    config_param :flush_at_shutdown, :bool, default: false
+        @@buffer_paths[@path] = type_of_owner
 
-    # 'symlink_path' is currently only for out_file.
-    # That is the reason why this is not config_param, but attr_accessor.
-    # See: https://github.com/fluent/fluentd/pull/181
-    attr_accessor :symlink_path
+        specified_directory_exists = File.exist?(@path) && File.directory?(@path)
+        unexisting_path_for_directory = !File.exist?(@path) && !@path.include?('.*')
 
-    def configure(conf)
-      super
+        if specified_directory_exists || unexisting_path_for_directory # directory
+          if using_plugin_root_dir || !multi_workers_configured
+            @path = File.join(@path, 'buffer.*.log')
+          else
+            @path = File.join(@path, "worker#{fluentd_worker_id}", 'buffer.*.log')
+            if fluentd_worker_id == 0
+              # worker 0 always checks unflushed buffer chunks to be resumed (might be created while non-multi-worker configuration)
+              @additional_resume_path = File.join(File.expand_path("../../", @path), 'buffer.*.log')
+            end
+          end
+          @multi_workers_available = true
+        else # specified path is file path
+          if File.basename(@path).include?('.*.')
+            # valid file path
+          elsif File.basename(@path).end_with?('.*')
+            @path = @path + '.log'
+          else
+            # existing file will be ignored
+            @path = @path + '.*.log'
+          end
+          @multi_workers_available = false
+        end
 
-      if @@buffer_paths.has_key?(@buffer_path)
-        raise ConfigError, "Other '#{@@buffer_paths[@buffer_path]}' plugin already use same buffer_path: type = #{conf['@type'] || conf['type']}, buffer_path = #{@buffer_path}"
-      else
-        @@buffer_paths[@buffer_path] = conf['@type'] || conf['type']
+        if @dir_permission
+          @dir_permission = @dir_permission.to_i(8) if @dir_permission.is_a?(String)
+        else
+          @dir_permission = system_config.dir_permission || DIR_PERMISSION
+        end
       end
 
-      if pos = @buffer_path.index('*')
-        @buffer_path_prefix = @buffer_path[0, pos]
-        @buffer_path_suffix = @buffer_path[(pos + 1)..-1]
-      else
-        @buffer_path_prefix = @buffer_path + "."
-        @buffer_path_suffix = ".log"
+      # This method is called only when multi worker is configured
+      def multi_workers_ready?
+        unless @multi_workers_available
+          log.error "file buffer with multi workers should be configured to use directory 'path', or system root_dir and plugin id"
+        end
+        @multi_workers_available
       end
 
-    end
-
-    def start
-      FileUtils.mkdir_p File.dirname(@buffer_path_prefix + "path"), mode: DEFAULT_DIR_PERMISSION
-      super
-    end
-
-    # Dots are separator for many cases:
-    #   we should have to escape dots in keys...
-    PATH_MATCH = /^([-_.%0-9a-zA-Z]*)\.(b|q)([0-9a-fA-F]{1,32})$/
+      def start
+        FileUtils.mkdir_p File.dirname(@path), mode: @dir_permission
 
-    def new_chunk(key)
-      encoded_key = encode_key(key)
-      path, tsuffix = make_path(encoded_key, "b")
-      unique_id = tsuffix_to_unique_id(tsuffix)
-      FileBufferChunk.new(key, path, unique_id, "a+", @symlink_path)
-    end
+        super
+      end
 
-    def resume
-      maps = []
-      queues = []
-
-      Dir.glob("#{@buffer_path_prefix}*#{@buffer_path_suffix}") {|path|
-        identifier_part = chunk_identifier_in_path(path)
-        if m = PATH_MATCH.match(identifier_part)
-          key = decode_key(m[1])
-          bq = m[2]
-          tsuffix = m[3]
-          timestamp = m[3].to_i(16)
-          unique_id = tsuffix_to_unique_id(tsuffix)
-
-          if bq == 'b'
-            chunk = FileBufferChunk.new(key, path, unique_id, "a+")
-            maps << [timestamp, chunk]
-          elsif bq == 'q'
-            chunk = FileBufferChunk.new(key, path, unique_id, "r")
-            queues << [timestamp, chunk]
-          end
-        end
-      }
-
-      map = {}
-      maps.sort_by {|(timestamp,chunk)|
-        timestamp
-      }.each {|(timestamp,chunk)|
-        map[chunk.key] = chunk
-      }
-
-      queue = queues.sort_by {|(timestamp,chunk)|
-        timestamp
-      }.map {|(timestamp,chunk)|
-        chunk
-      }
-
-      return queue, map
-    end
+      def persistent?
+        true
+      end
 
-    def chunk_identifier_in_path(path)
-      pos_after_prefix = @buffer_path_prefix.length
-      pos_before_suffix = @buffer_path_suffix.length + 1 # from tail of path
+      def resume
+        stage = {}
+        queue = []
 
-      path.slice(pos_after_prefix..-pos_before_suffix)
-    end
+        patterns = [@path]
+        patterns.unshift @additional_resume_path if @additional_resume_path
+        Dir.glob(patterns) do |path|
+          next unless File.file?(path)
 
-    def enqueue(chunk)
-      path = chunk.path
-      identifier_part = chunk_identifier_in_path(path)
+          log.debug { "restoring buffer file: path = #{path}" }
 
-      m = PATH_MATCH.match(identifier_part)
-      encoded_key = m ? m[1] : ""
-      tsuffix = m[3]
-      npath = "#{@buffer_path_prefix}#{encoded_key}.q#{tsuffix}#{@buffer_path_suffix}"
+          m = new_metadata() # this metadata will be overwritten by resuming .meta file content
+                             # so it should not added into @metadata_list for now
+          mode = Fluent::Plugin::Buffer::FileChunk.assume_chunk_state(path)
+          if mode == :unknown
+            log.debug "unknown state chunk found", path: path
+            next
+          end
 
-      chunk.mv(npath)
-    end
+          begin
+            chunk = Fluent::Plugin::Buffer::FileChunk.new(m, path, mode) # file chunk resumes contents of metadata
+          rescue Fluent::Plugin::Buffer::FileChunk::FileChunkError => e
+            handle_broken_files(path, mode, e)
+            next
+          end
 
-    def before_shutdown(out)
-      if @flush_at_shutdown
-        synchronize do
-          @map.each_key {|key|
-            push(key)
-          }
-          while pop(out)
+          case chunk.state
+          when :staged
+            stage[chunk.metadata] = chunk
+          when :queued
+            queue << chunk
           end
         end
-      end
-    end
 
-    private
+        queue.sort_by!{ |chunk| chunk.modified_at }
 
-    # Dots are separator for many cases:
-    #   we should have to escape dots in keys...
-    def encode_key(key)
-      @uri_parser.escape(key, /[^-_.a-zA-Z0-9]/n) # //n switch means explicit 'ASCII-8BIT' pattern
-    end
+        return stage, queue
+      end
 
-    def decode_key(encoded_key)
-      @uri_parser.unescape(encoded_key)
-    end
+      def generate_chunk(metadata)
+        # FileChunk generates real path with unique_id
+        if @file_permission
+          chunk = Fluent::Plugin::Buffer::FileChunk.new(metadata, @path, :create, perm: @file_permission, compress: @compress)
+        else
+          chunk = Fluent::Plugin::Buffer::FileChunk.new(metadata, @path, :create, compress: @compress)
+        end
 
-    def make_path(encoded_key, bq)
-      now = Time.now.utc
-      timestamp = ((now.to_i * 1000 * 1000 + now.usec) << 12 | rand(0xfff))
-      tsuffix = timestamp.to_s(16)
-      path = "#{@buffer_path_prefix}#{encoded_key}.#{bq}#{tsuffix}#{@buffer_path_suffix}"
-      return path, tsuffix
-    end
+        log.debug "Created new chunk", chunk_id: dump_unique_id_hex(chunk.unique_id), metadata: metadata
 
-    def tsuffix_to_unique_id(tsuffix)
-      # why *2 ? frsyuki said that I forgot why completely.
-      tsuffix.scan(/../).map {|x| x.to_i(16) }.pack('C*') * 2
+        return chunk
+      end
+
+      def handle_broken_files(path, mode, e)
+        log.error "found broken chunk file during resume. Deleted corresponding files:", :path => path, :mode => mode, :err_msg => e.message
+        # After support 'backup_dir' feature, these files are moved to backup_dir instead of unlink.
+        File.unlink(path, path + '.meta') rescue nil
+      end
     end
   end
 end