fluentd 0.12.40 → 0.14.0

data/lib/fluent/plugin/buffer/memory_chunk.rb

@@ -0,0 +1,100 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/buffer/chunk'
+
+module Fluent
+  module Plugin
+    class Buffer
+      class MemoryChunk < Chunk
+        def initialize(metadata)
+          super
+          @chunk = ''.force_encoding(Encoding::ASCII_8BIT)
+          @chunk_bytes = 0
+          @adding_bytes = 0
+          @adding_size = 0
+        end
+
+        def append(data)
+          raise "BUG: appending to non-staged chunk, now '#{self.state}'" unless self.staged?
+
+          adding = data.join.force_encoding(Encoding::ASCII_8BIT)
+          @chunk << adding
+          @adding_bytes += adding.bytesize
+          @adding_size += data.size
+          true
+        end
+
+        def concat(bulk, bulk_size)
+          raise "BUG: appending to non-staged chunk, now '#{self.state}'" unless self.staged?
+
+          bulk.force_encoding(Encoding::ASCII_8BIT)
+          @chunk << bulk
+          @adding_bytes += bulk.bytesize
+          @adding_size += bulk_size
+          true
+        end
+
+        def commit
+          @size += @adding_size
+          @chunk_bytes += @adding_bytes
+
+          @adding_bytes = @adding_size = 0
+          @modified_at = Time.now
+          true
+        end
+
+        def rollback
+          @chunk.slice!(@chunk_bytes, @adding_bytes)
+          @adding_bytes = @adding_size = 0
+          true
+        end
+
+        def bytesize
+          @chunk_bytes + @adding_bytes
+        end
+
+        def size
+          @size + @adding_size
+        end
+
+        def empty?
+          @chunk.empty?
+        end
+
+        def purge
+          super
+          @chunk = ''.force_encoding("ASCII-8BIT")
+          @chunk_bytes = @size = @adding_bytes = @adding_size = 0
+          true
+        end
+
+        def read
+          @chunk
+        end
+
+        def open(&block)
+          StringIO.open(@chunk, &block)
+        end
+
+        def write_to(io)
+          # re-implementation to optimize not to create StringIO
+          io.write @chunk
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/exec_util.rb

@@ -22,106 +22,111 @@ require 'fluent/plugin'
 require 'fluent/parser'
 
 module Fluent
-  module ExecUtil
-    SUPPORTED_FORMAT = {
-      'tsv' => :tsv,
-      'json' => :json,
-      'msgpack' => :msgpack,
-    }
-
-    class Parser
-      def initialize(on_message)
-        @on_message = on_message
+  module Plugin
+    module ExecUtil
+      SUPPORTED_FORMAT = {
+        'tsv' => :tsv,
+        'json' => :json,
+        'msgpack' => :msgpack,
+      }
+
+      class Parser
+        def initialize(on_message)
+          @on_message = on_message
+        end
       end
-    end
 
-    class TextParserWrapperParser < Parser
-      def initialize(conf, on_message)
-        @parser = Plugin.new_parser(conf['format'])
-        @parser.configure(conf)
-        super(on_message)
-      end
+      class TextParserWrapperParser < Parser
+        def initialize(conf, on_message)
+          @parser = Plugin.new_parser(conf['format'])
+          @parser.configure(conf)
+          super(on_message)
+        end
 
-      def call(io)
-        io.each_line(&method(:each_line))
-      end
+        def call(io)
+          io.each_line(&method(:each_line))
+        end
 
-      def each_line(line)
-        line.chomp!
-        @parser.parse(line) { |time, record|
-          @on_message.call(record, time)
-        }
+        def each_line(line)
+          line.chomp!
+          @parser.parse(line) { |time, record|
+            @on_message.call(record, time)
+          }
+        end
       end
-    end
 
-    class TSVParser < Parser
-      def initialize(keys, on_message)
-        @keys = keys
-        super(on_message)
-      end
+      class TSVParser < Parser
+        def initialize(keys, on_message)
+          @keys = keys
+          super(on_message)
+        end
 
-      def call(io)
-        io.each_line(&method(:each_line))
-      end
+        def call(io)
+          io.each_line(&method(:each_line))
+        end
 
-      def each_line(line)
-        line.chomp!
-        vals = line.split("\t")
+        def each_line(line)
+          line.chomp!
+          vals = line.split("\t")
 
-        record = Hash[@keys.zip(vals)]
+          record = Hash[@keys.zip(vals)]
 
-        @on_message.call(record)
+          @on_message.call(record)
+        end
       end
-    end
 
-    class JSONParser < Parser
-      def call(io)
-        y = Yajl::Parser.new
-        y.on_parse_complete = @on_message
-        y.parse(io)
+      class JSONParser < Parser
+        def call(io)
+          y = Yajl::Parser.new
+          y.on_parse_complete = @on_message
+          y.parse(io)
+        end
       end
-    end
 
-    class MessagePackParser < Parser
-      def call(io)
-        @u = Fluent::Engine.msgpack_factory.unpacker(io)
-        begin
-          @u.each(&@on_message)
-        rescue EOFError
+      class MessagePackParser < Parser
+        def call(io)
+          @u = Fluent::Engine.msgpack_factory.unpacker(io)
+          begin
+            @u.each(&@on_message)
+          rescue EOFError
+          end
         end
       end
-    end
 
-    class Formatter
-    end
-
-    class TSVFormatter < Formatter
-      def initialize(in_keys)
-        @in_keys = in_keys
-        super()
+      class Formatter
       end
 
-      def call(record, out)
-        last = @in_keys.length-1
-        for i in 0..last
-          key = @in_keys[i]
-          out << record[key].to_s
-          out << "\t" if i != last
+      class TSVFormatter < Formatter
+        def initialize(in_keys)
+          @in_keys = in_keys
+          super()
+        end
+
+        def call(record, out)
+          last = @in_keys.length-1
+          for i in 0..last
+            key = @in_keys[i]
+            out << record[key].to_s
+            out << "\t" if i != last
+          end
+          out << "\n"
         end
-        out << "\n"
       end
-    end
 
-    class JSONFormatter < Formatter
-      def call(record, out)
-        out << Yajl.dump(record) << "\n"
+      class JSONFormatter < Formatter
+        def call(record, out)
+          out << Yajl.dump(record) << "\n"
+        end
       end
-    end
 
-    class MessagePackFormatter < Formatter
-      def call(record, out)
-        record.to_msgpack(out)
+      class MessagePackFormatter < Formatter
+        def call(record, out)
+          record.to_msgpack(out)
+        end
       end
     end
   end
+
+  # obsolete
+  ExecUtil = Fluent::Plugin::ExecUtil
 end

data/lib/fluent/plugin/file_util.rb

@@ -15,38 +15,43 @@
 #
 
 module Fluent
-  module FileUtil
-    # Check file is writable if file exists
-    # Check directory is writable if file does not exist
-    #
-    # @param [String] path File path
-    # @return [Boolean] file is writable or not
-    def writable?(path)
-      return false if File.directory?(path)
-      return File.writable?(path) if File.exist?(path)
+  module Plugin
+    module FileUtil
+      # Check file is writable if file exists
+      # Check directory is writable if file does not exist
+      #
+      # @param [String] path File path
+      # @return [Boolean] file is writable or not
+      def writable?(path)
+        return false if File.directory?(path)
+        return File.writable?(path) if File.exist?(path)
 
-      dirname = File.dirname(path)
-      return false if !File.directory?(dirname)
-      File.writable?(dirname)
-    end
-    module_function :writable?
+        dirname = File.dirname(path)
+        return false if !File.directory?(dirname)
+        File.writable?(dirname)
+      end
+      module_function :writable?
 
-    # Check file is writable in conjunction wtih mkdir_p(dirname(path))
-    #
-    # @param [String] path File path
-    # @return [Boolean] file writable or not
-    def writable_p?(path)
-      return false if File.directory?(path)
-      return File.writable?(path) if File.exist?(path)
+      # Check file is writable in conjunction wtih mkdir_p(dirname(path))
+      #
+      # @param [String] path File path
+      # @return [Boolean] file writable or not
+      def writable_p?(path)
+        return false if File.directory?(path)
+        return File.writable?(path) if File.exist?(path)
 
-      dirname = File.dirname(path)
-      until File.exist?(dirname)
-        dirname = File.dirname(dirname)
-      end
+        dirname = File.dirname(path)
+        until File.exist?(dirname)
+          dirname = File.dirname(dirname)
+        end
 
-      return false if !File.directory?(dirname)
-      File.writable?(dirname)
+        return false if !File.directory?(dirname)
+        File.writable?(dirname)
+      end
+      module_function :writable_p?
     end
-    module_function :writable_p?
   end
+
+  # obsolete
+  FileUtil = Fluent::Plugin::FileUtil
 end

data/lib/fluent/plugin/file_wrapper.rb

@@ -0,0 +1,120 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module Fluent
+  module FileWrapper
+    def self.open(*args)
+      io = WindowsFile.new(*args).io
+      if block_given?
+        v = yield io
+        io.close
+        v
+      else
+        io
+      end
+    end
+
+    def self.stat(path)
+      f = WindowsFile.new(path)
+      s = f.stat
+      f.close
+      s
+    end
+  end
+
+  module WindowsFileExtension
+    attr_reader :path
+
+    def stat
+      s = super
+      s.instance_variable_set :@ino, @ino
+      def s.ino; @ino; end
+      s
+    end
+  end
+
+  # To open and get stat with setting FILE_SHARE_DELETE
+  class WindowsFile
+    require 'windows/file'
+    require 'windows/error'
+    require 'windows/handle'
+    require 'windows/nio'
+
+    include Windows::Error
+    include Windows::File
+    include Windows::Handle
+    include Windows::NIO
+
+    def initialize(path, mode='r', sharemode=FILE_SHARE_READ|FILE_SHARE_WRITE|FILE_SHARE_DELETE)
+      @path = path
+      @file_handle = INVALID_HANDLE_VALUE
+      @mode = mode
+
+
+      access, creationdisposition, seektoend = case mode.delete('b')
+      when "r" ; [FILE_GENERIC_READ                     , OPEN_EXISTING, false]
+      when "r+"; [FILE_GENERIC_READ | FILE_GENERIC_WRITE, OPEN_ALWAYS  , false]
+      when "w" ; [FILE_GENERIC_WRITE                    , CREATE_ALWAYS, false]
+      when "w+"; [FILE_GENERIC_READ | FILE_GENERIC_WRITE, CREATE_ALWAYS, false]
+      when "a" ; [FILE_GENERIC_WRITE                    , OPEN_ALWAYS  , true]
+      when "a+"; [FILE_GENERIC_READ | FILE_GENERIC_WRITE, OPEN_ALWAYS  , true]
+      else raise "unknown mode '#{mode}'"
+      end
+
+      @file_handle = CreateFile.call(@path, access, sharemode,
+                     0, creationdisposition, FILE_ATTRIBUTE_NORMAL, 0)
+      if @file_handle == INVALID_HANDLE_VALUE
+        err = GetLastError.call
+        if err == ERROR_FILE_NOT_FOUND || err == ERROR_ACCESS_DENIED
+          raise SystemCallError.new(2)
+        end
+        raise SystemCallError.new(err)
+      end
+    end
+
+    def close
+      CloseHandle.call(@file_handle)
+      @file_handle = INVALID_HANDLE_VALUE
+    end
+
+    def io
+      fd = _open_osfhandle(@file_handle, 0)
+      raise Errno::ENOENT if fd == -1
+      io = File.for_fd(fd, @mode)
+      io.instance_variable_set :@ino, self.ino
+      io.instance_variable_set :@path, @path
+      io.extend WindowsFileExtension
+      io
+    end
+
+    def ino
+      by_handle_file_information = '\0'*(4+8+8+8+4+4+4+4+4+4)   #72bytes
+
+      unless GetFileInformationByHandle.call(@file_handle, by_handle_file_information)
+        return 0
+      end
+
+      by_handle_file_information.unpack("I11Q1")[11] # fileindex
+    end
+
+    def stat
+      s = File.stat(@path)
+      s.instance_variable_set :@ino, self.ino
+      def s.ino; @ino; end
+      s
+    end
+  end
+end if Fluent.windows?