fluentd 0.14.1 → 0.14.2

data/lib/fluent/plugin/buffer/chunk.rb

@@ -51,8 +51,8 @@ module Fluent
           @unique_id = generate_unique_id
           @metadata = metadata
 
-          # state: staged/queued/closed
-          @state = :staged
+          # state: unstaged/staged/queued/closed
+          @state = :unstaged
 
           @size = 0
           @created_at = Time.now
@@ -63,7 +63,11 @@ module Fluent
 
         # data is array of formatted record string
         def append(data)
-          raise NotImplementedError, "Implement this method in child class"
+          adding = ''.b
+          data.each do |d|
+            adding << d.b
+          end
+          concat(adding, data.size)
         end
 
         # for event streams which is packed or zipped (and we want not to unpack/uncompress)
@@ -92,6 +96,14 @@ module Fluent
           size == 0
         end
 
+        def writable?
+          @state == :staged || @state == :unstaged
+        end
+
+        def unstaged?
+          @state == :unstaged
+        end
+
         def staged?
           @state == :staged
         end
@@ -104,16 +116,29 @@ module Fluent
           @state == :closed
         end
 
+        def staged!
+          @state = :staged
+          self
+        end
+
+        def unstaged!
+          @state = :unstaged
+          self
+        end
+
         def enqueued!
           @state = :queued
+          self
         end
 
         def close
           @state = :closed
+          self
         end
 
         def purge
           @state = :closed
+          self
         end
 
         def read

data/lib/fluent/plugin/buffer/file_chunk.rb

@@ -55,26 +55,8 @@ module Fluent
           end
         end
 
-        def append(data)
-          raise "BUG: appending to non-staged chunk, now '#{self.state}'" unless self.staged?
-
-          bytes = 0
-          adding = ''.force_encoding(Encoding::ASCII_8BIT)
-          data.each do |d|
-            x = d.force_encoding(Encoding::ASCII_8BIT)
-            bytes += x.bytesize
-            adding << x
-          end
-          @chunk.write adding
-
-          @adding_bytes += bytes
-          @adding_size += data.size
-
-          true
-        end
-
         def concat(bulk, bulk_size)
-          raise "BUG: appending to non-staged chunk, now '#{self.state}'" unless self.staged?
+          raise "BUG: concatenating to unwritable chunk, now '#{self.state}'" unless self.writable?
 
           bulk.force_encoding(Encoding::ASCII_8BIT)
           @chunk.write bulk
@@ -167,7 +149,9 @@ module Fluent
           if /\.(b|q)([0-9a-f]+)\.[^\/]*\Z/n =~ path # //n switch means explicit 'ASCII-8BIT' pattern
             $1 == 'b' ? :staged : :queued
           else
-            :queued
+            # files which matches to glob of buffer file pattern
+            # it includes files which are created by out_file
+            :unknown
           end
         end
 
@@ -267,7 +251,7 @@ module Fluent
           @meta.sync = true
           @meta.binmode
 
-          @state = :staged
+          @state = :unstaged
           @bytesize = 0
           @commit_position = @chunk.pos # must be 0
           @adding_bytes = 0

data/lib/fluent/plugin/buffer/memory_chunk.rb

@@ -28,18 +28,8 @@ module Fluent
           @adding_size = 0
         end
 
-        def append(data)
-          raise "BUG: appending to non-staged chunk, now '#{self.state}'" unless self.staged?
-
-          adding = data.join.force_encoding(Encoding::ASCII_8BIT)
-          @chunk << adding
-          @adding_bytes += adding.bytesize
-          @adding_size += data.size
-          true
-        end
-
         def concat(bulk, bulk_size)
-          raise "BUG: appending to non-staged chunk, now '#{self.state}'" unless self.staged?
+          raise "BUG: concatenating to unwritable chunk, now '#{self.state}'" unless self.writable?
 
           bulk.force_encoding(Encoding::ASCII_8BIT)
           @chunk << bulk

data/lib/fluent/plugin/exec_util.rb

@@ -14,119 +14,9 @@
 #    limitations under the License.
 #
 
-require 'msgpack'
-require 'yajl'
-
-require 'fluent/engine'
-require 'fluent/plugin'
-require 'fluent/parser'
+require 'fluent/compat/exec_util'
 
 module Fluent
-  module Plugin
-    module ExecUtil
-      SUPPORTED_FORMAT = {
-        'tsv' => :tsv,
-        'json' => :json,
-        'msgpack' => :msgpack,
-      }
-
-      class Parser
-        def initialize(on_message)
-          @on_message = on_message
-        end
-      end
-
-      class TextParserWrapperParser < Parser
-        def initialize(conf, on_message)
-          @parser = Plugin.new_parser(conf['format'])
-          @parser.configure(conf)
-          super(on_message)
-        end
-
-        def call(io)
-          io.each_line(&method(:each_line))
-        end
-
-        def each_line(line)
-          line.chomp!
-          @parser.parse(line) { |time, record|
-            @on_message.call(record, time)
-          }
-        end
-      end
-
-      class TSVParser < Parser
-        def initialize(keys, on_message)
-          @keys = keys
-          super(on_message)
-        end
-
-        def call(io)
-          io.each_line(&method(:each_line))
-        end
-
-        def each_line(line)
-          line.chomp!
-          vals = line.split("\t")
-
-          record = Hash[@keys.zip(vals)]
-
-          @on_message.call(record)
-        end
-      end
-
-      class JSONParser < Parser
-        def call(io)
-          y = Yajl::Parser.new
-          y.on_parse_complete = @on_message
-          y.parse(io)
-        end
-      end
-
-      class MessagePackParser < Parser
-        def call(io)
-          @u = Fluent::Engine.msgpack_factory.unpacker(io)
-          begin
-            @u.each(&@on_message)
-          rescue EOFError
-          end
-        end
-      end
-
-      class Formatter
-      end
-
-      class TSVFormatter < Formatter
-        def initialize(in_keys)
-          @in_keys = in_keys
-          super()
-        end
-
-        def call(record, out)
-          last = @in_keys.length-1
-          for i in 0..last
-            key = @in_keys[i]
-            out << record[key].to_s
-            out << "\t" if i != last
-          end
-          out << "\n"
-        end
-      end
-
-      class JSONFormatter < Formatter
-        def call(record, out)
-          out << Yajl.dump(record) << "\n"
-        end
-      end
-
-      class MessagePackFormatter < Formatter
-        def call(record, out)
-          record.to_msgpack(out)
-        end
-      end
-    end
-  end
-
   # obsolete
-  ExecUtil = Fluent::Plugin::ExecUtil
+  ExecUtil = Fluent::Compat::ExecUtil
 end

data/lib/fluent/plugin/file_util.rb

@@ -14,44 +14,9 @@
 #    limitations under the License.
 #
 
-module Fluent
-  module Plugin
-    module FileUtil
-      # Check file is writable if file exists
-      # Check directory is writable if file does not exist
-      #
-      # @param [String] path File path
-      # @return [Boolean] file is writable or not
-      def writable?(path)
-        return false if File.directory?(path)
-        return File.writable?(path) if File.exist?(path)
-
-        dirname = File.dirname(path)
-        return false if !File.directory?(dirname)
-        File.writable?(dirname)
-      end
-      module_function :writable?
-
-      # Check file is writable in conjunction wtih mkdir_p(dirname(path))
-      #
-      # @param [String] path File path
-      # @return [Boolean] file writable or not
-      def writable_p?(path)
-        return false if File.directory?(path)
-        return File.writable?(path) if File.exist?(path)
-
-        dirname = File.dirname(path)
-        until File.exist?(dirname)
-          dirname = File.dirname(dirname)
-        end
-
-        return false if !File.directory?(dirname)
-        File.writable?(dirname)
-      end
-      module_function :writable_p?
-    end
-  end
+require 'fluent/compat/file_util'
 
+module Fluent
   # obsolete
-  FileUtil = Fluent::Plugin::FileUtil
+  FileUtil = Fluent::Compat::FileUtil
 end

data/lib/fluent/plugin/file_wrapper.rb

@@ -78,7 +78,7 @@ module Fluent
                      0, creationdisposition, FILE_ATTRIBUTE_NORMAL, 0)
       if @file_handle == INVALID_HANDLE_VALUE
         err = GetLastError.call
-        if err == ERROR_FILE_NOT_FOUND || err == ERROR_ACCESS_DENIED
+        if err == ERROR_FILE_NOT_FOUND || err == ERROR_PATH_NOT_FOUND || err == ERROR_ACCESS_DENIED
           raise SystemCallError.new(2)
         end
         raise SystemCallError.new(err)

data/lib/fluent/plugin/filter_grep.rb

@@ -14,18 +14,14 @@
 #    limitations under the License.
 #
 
-require 'fluent/filter'
+require 'fluent/plugin/filter'
 require 'fluent/config/error'
+require 'fluent/plugin/string_util'
 
-module Fluent
+module Fluent::Plugin
   class GrepFilter < Filter
     Fluent::Plugin.register_filter('grep', self)
 
-    def initialize
-      super
-      require 'fluent/plugin/string_util'
-    end
-
     REGEXP_MAX_NUM = 20
 
     (1..REGEXP_MAX_NUM).each {|i| config_param :"regexp#{i}",  :string, default: nil }

data/lib/fluent/plugin/filter_record_transformer.rb

@@ -18,13 +18,13 @@ require 'socket'
 require 'json'
 require 'ostruct'
 
-require 'fluent/filter'
+require 'fluent/plugin/filter'
 require 'fluent/config/error'
 require 'fluent/event'
 require 'fluent/time'
 
-module Fluent
-  class RecordTransformerFilter < Filter
+module Fluent::Plugin
+  class RecordTransformerFilter < Fluent::Plugin::Filter
     Fluent::Plugin.register_filter('record_transformer', self)
 
     desc 'A comma-delimited list of keys to delete.'
@@ -81,7 +81,7 @@ module Fluent
     end
 
     def filter_stream(tag, es)
-      new_es = MultiEventStream.new
+      new_es = Fluent::MultiEventStream.new
       tag_parts = tag.split('.')
       tag_prefix = tag_prefix(tag_parts)
       tag_suffix = tag_suffix(tag_parts)
@@ -101,7 +101,7 @@ module Fluent
         })
         new_record = reform(record, placeholder_values)
         if @renew_time_key && new_record.has_key?(@renew_time_key)
-          time = EventTime.from_time(Time.at(new_record[@renew_time_key].to_f))
+          time = Fluent::EventTime.from_time(Time.at(new_record[@renew_time_key].to_f))
         end
         new_es.add(time, new_record)
       end

data/lib/fluent/plugin/filter_stdout.rb

@@ -14,31 +14,38 @@
 #    limitations under the License.
 #
 
-require 'fluent/filter'
-require 'fluent/plugin'
+require 'fluent/plugin/filter'
 
-module Fluent
+module Fluent::Plugin
   class StdoutFilter < Filter
-    Plugin.register_filter('stdout', self)
+    Fluent::Plugin.register_filter('stdout', self)
+
+    helpers :formatter, :compat_parameters, :inject
+
+    DEFAULT_FORMAT_TYPE = 'stdout'
+
+    config_section :format do
+      config_set_default :@type, DEFAULT_FORMAT_TYPE
+    end
 
     # for tests
     attr_reader :formatter
 
-    desc 'The format of the output.'
-    config_param :format, :string, default: 'stdout'
-    # config_param :output_type, :string, :default => 'json' (StdoutFormatter defines this)
-
     def configure(conf)
+      compat_parameters_convert(conf, :inject, :formatter)
       super
+    end
 
-      @formatter = Plugin.new_formatter(@format)
-      @formatter.configure(conf)
+    def start
+      @formatter = formatter_create(conf: @config.elements('format').first, default_type: DEFAULT_FORMAT_TYPE)
+      super
     end
 
     def filter_stream(tag, es)
       es.each { |time, record|
         begin
-          log.write @formatter.format(tag, time, record)
+          r = inject_values_to_record(tag, time, record)
+          log.write @formatter.format(tag, time, r)
         rescue => e
           router.emit_error_event(tag, time, record, e)
         end

data/lib/fluent/plugin/formatter.rb

@@ -29,54 +29,6 @@ module Fluent
       def format(tag, time, record)
         raise NotImplementedError, "Implement this method in child class"
       end
-
-      # Mixins for formatter plugins
-      module HandleTagAndTimeMixin
-        def self.included(klass)
-          klass.instance_eval {
-            config_param :include_time_key, :bool, default: false
-            config_param :time_key, :string, default: 'time'
-            config_param :time_format, :string, default: nil
-            config_param :time_as_epoch, :bool, default: false
-            config_param :include_tag_key, :bool, default: false
-            config_param :tag_key, :string, default: 'tag'
-            config_param :localtime, :bool, default: true
-            config_param :timezone, :string, default: nil
-          }
-        end
-
-        def configure(conf)
-          super
-
-          if conf['utc']
-            @localtime = false
-          end
-          @timef = Fluent::TimeFormatter.new(@time_format, @localtime, @timezone)
-          if @time_as_epoch && !@include_time_key
-            log.warn "time_as_epoch will be ignored because include_time_key is false"
-          end
-        end
-
-        def filter_record(tag, time, record)
-          if @include_tag_key
-            record[@tag_key] = tag
-          end
-          if @include_time_key
-            if @time_as_epoch
-              record[@time_key] = time.to_i
-            else
-              record[@time_key] = @timef.format(time)
-            end
-          end
-        end
-      end
-
-      module StructuredFormatMixin
-        def format(tag, time, record)
-          filter_record(tag, time, record)
-          format_record(record)
-        end
-      end
     end
 
     class ProcWrappedFormatter < Formatter