fluentd 0.10.35 → 0.10.36

data/lib/fluent/plugin/buf_file.rb

@@ -16,192 +16,187 @@
 #    limitations under the License.
 #
 module Fluent
+  class FileBufferChunk < BufferChunk
+    def initialize(key, path, unique_id, mode="a+")
+      super(key)
+      @path = path
+      @unique_id = unique_id
+      @file = File.open(@path, mode, DEFAULT_FILE_PERMISSION)
+      @file.sync = true
+      @size = @file.stat.size
+    end
 
+    attr_reader :unique_id
 
-class FileBufferChunk < BufferChunk
-  def initialize(key, path, unique_id, mode="a+")
-    super(key)
-    @path = path
-    @unique_id = unique_id
-    @file = File.open(@path, mode, DEFAULT_FILE_PERMISSION)
-    @file.sync = true
-    @size = @file.stat.size
-  end
-
-  attr_reader :unique_id
-
-  def <<(data)
-    @file.write(data)
-    @size += data.bytesize
-  end
+    def <<(data)
+      @file.write(data)
+      @size += data.bytesize
+    end
 
-  def size
-    @size
-  end
+    def size
+      @size
+    end
 
-  def empty?
-    @size == 0
-  end
+    def empty?
+      @size == 0
+    end
 
-  def close
-    stat = @file.stat
-    @file.close
-    if stat.size == 0
-      File.unlink(@path)
+    def close
+      stat = @file.stat
+      @file.close
+      if stat.size == 0
+        File.unlink(@path)
+      end
     end
-  end
 
-  def purge
-    @file.close
-    File.unlink(@path) rescue nil  # TODO rescue?
-  end
+    def purge
+      @file.close
+      File.unlink(@path) rescue nil  # TODO rescue?
+    end
 
-  def read
-    @file.pos = 0
-    @file.read
-  end
+    def read
+      @file.pos = 0
+      @file.read
+    end
 
-  def open(&block)
-    @file.pos = 0
-    yield @file
-  end
+    def open(&block)
+      @file.pos = 0
+      yield @file
+    end
 
-  attr_reader :path
+    attr_reader :path
 
-  def mv(path)
-    File.rename(@path, path)
-    @path = path
+    def mv(path)
+      File.rename(@path, path)
+      @path = path
+    end
   end
-end
 
+  class FileBuffer < BasicBuffer
+    Plugin.register_buffer('file', self)
 
-class FileBuffer < BasicBuffer
-  Plugin.register_buffer('file', self)
+    def initialize
+      require 'uri'
+      super
+    end
 
-  def initialize
-    require 'uri'
-    super
-  end
+    config_param :buffer_path, :string
 
-  config_param :buffer_path, :string
+    def configure(conf)
+      super
 
-  def configure(conf)
-    super
+      if pos = @buffer_path.index('*')
+        @buffer_path_prefix = @buffer_path[0,pos]
+        @buffer_path_suffix = @buffer_path[pos+1..-1]
+      else
+        @buffer_path_prefix = @buffer_path+"."
+        @buffer_path_suffix = ".log"
+      end
 
-    if pos = @buffer_path.index('*')
-      @buffer_path_prefix = @buffer_path[0,pos]
-      @buffer_path_suffix = @buffer_path[pos+1..-1]
-    else
-      @buffer_path_prefix = @buffer_path+"."
-      @buffer_path_suffix = ".log"
+      if flush_at_shutdown = conf['flush_at_shutdown']
+        @flush_at_shutdown = true
+      else
+        @flush_at_shutdown = false
+      end
     end
 
-    if flush_at_shutdown = conf['flush_at_shutdown']
-      @flush_at_shutdown = true
-    else
-      @flush_at_shutdown = false
+    def start
+      FileUtils.mkdir_p File.dirname(@buffer_path_prefix+"path")
+      super
     end
-  end
-
-  def start
-    FileUtils.mkdir_p File.dirname(@buffer_path_prefix+"path")
-    super
-  end
 
-  PATH_MATCH = /^(.*)[\._](b|q)([0-9a-fA-F]{1,32})$/
+    PATH_MATCH = /^(.*)[\._](b|q)([0-9a-fA-F]{1,32})$/
 
-  def new_chunk(key)
-    encoded_key = encode_key(key)
-    path, tsuffix = make_path(encoded_key, "b")
-    unique_id = tsuffix_to_unique_id(tsuffix)
-    FileBufferChunk.new(key, path, unique_id)
-  end
+    def new_chunk(key)
+      encoded_key = encode_key(key)
+      path, tsuffix = make_path(encoded_key, "b")
+      unique_id = tsuffix_to_unique_id(tsuffix)
+      FileBufferChunk.new(key, path, unique_id)
+    end
 
-  def resume
-    maps = []
-    queues = []
-
-    Dir.glob("#{@buffer_path_prefix}*#{@buffer_path_suffix}") {|path|
-      match = path[@buffer_path_prefix.length..-(@buffer_path_suffix.length+1)]
-      if m = PATH_MATCH.match(match)
-        key = decode_key(m[1])
-        bq = m[2]
-        tsuffix = m[3]
-        timestamp = m[3].to_i(16)
-        unique_id = tsuffix_to_unique_id(tsuffix)
-
-        if bq == 'b'
-          chunk = FileBufferChunk.new(key, path, unique_id, "a+")
-          maps << [timestamp, chunk]
-        elsif bq == 'q'
-          chunk = FileBufferChunk.new(key, path, unique_id, "r")
-          queues << [timestamp, chunk]
+    def resume
+      maps = []
+      queues = []
+
+      Dir.glob("#{@buffer_path_prefix}*#{@buffer_path_suffix}") {|path|
+        match = path[@buffer_path_prefix.length..-(@buffer_path_suffix.length+1)]
+        if m = PATH_MATCH.match(match)
+          key = decode_key(m[1])
+          bq = m[2]
+          tsuffix = m[3]
+          timestamp = m[3].to_i(16)
+          unique_id = tsuffix_to_unique_id(tsuffix)
+
+          if bq == 'b'
+            chunk = FileBufferChunk.new(key, path, unique_id, "a+")
+            maps << [timestamp, chunk]
+          elsif bq == 'q'
+            chunk = FileBufferChunk.new(key, path, unique_id, "r")
+            queues << [timestamp, chunk]
+          end
         end
-      end
-    }
-
-    map = {}
-    maps.sort_by {|(timestamp,chunk)|
-      timestamp
-    }.each {|(timestamp,chunk)|
-      map[chunk.key] = chunk
-    }
-
-    queue = queues.sort_by {|(timestamp,chunk)|
-      timestamp
-    }.map {|(timestamp,chunk)|
-      chunk
-    }
-
-    return queue, map
-  end
+      }
+
+      map = {}
+      maps.sort_by {|(timestamp,chunk)|
+        timestamp
+      }.each {|(timestamp,chunk)|
+        map[chunk.key] = chunk
+      }
+
+      queue = queues.sort_by {|(timestamp,chunk)|
+        timestamp
+      }.map {|(timestamp,chunk)|
+        chunk
+      }
+
+      return queue, map
+    end
 
-  def enqueue(chunk)
-    path = chunk.path
-    mp = path[@buffer_path_prefix.length..-(@buffer_path_suffix.length+1)]
+    def enqueue(chunk)
+      path = chunk.path
+      mp = path[@buffer_path_prefix.length..-(@buffer_path_suffix.length+1)]
 
-    m = PATH_MATCH.match(mp)
-    encoded_key = m ? m[1] : ""
-    tsuffix = m[3]
-    npath = "#{@buffer_path_prefix}#{encoded_key}.q#{tsuffix}#{@buffer_path_suffix}"
+      m = PATH_MATCH.match(mp)
+      encoded_key = m ? m[1] : ""
+      tsuffix = m[3]
+      npath = "#{@buffer_path_prefix}#{encoded_key}.q#{tsuffix}#{@buffer_path_suffix}"
 
-    chunk.mv(npath)
-  end
+      chunk.mv(npath)
+    end
 
-  def before_shutdown(out)
-    if @flush_at_shutdown
-      synchronize do
-        @map.each_key {|key|
-          push(key)
-        }
-        while pop(out)
+    def before_shutdown(out)
+      if @flush_at_shutdown
+        synchronize do
+          @map.each_key {|key|
+            push(key)
+          }
+          while pop(out)
+          end
         end
       end
     end
-  end
 
-  protected
-  def encode_key(key)
-    URI.escape(key, /[^-_.a-zA-Z0-9]/n)
-  end
+    protected
 
-  def decode_key(encoded_key)
-    URI.unescape(encoded_key)
-  end
-
-  def make_path(encoded_key, bq)
-    now = Time.now.utc
-    timestamp = ((now.to_i*1000*1000+now.usec) << 12 | rand(0xfff))
-    tsuffix = timestamp.to_s(16)
-    path = "#{@buffer_path_prefix}#{encoded_key}.#{bq}#{tsuffix}#{@buffer_path_suffix}"
-    return path, tsuffix
-  end
+    def encode_key(key)
+      URI.escape(key, /[^-_.a-zA-Z0-9]/n)
+    end
 
-  def tsuffix_to_unique_id(tsuffix)
-    tsuffix.scan(/../).map {|x| x.to_i(16) }.pack('C*') * 2
-  end
-end
+    def decode_key(encoded_key)
+      URI.unescape(encoded_key)
+    end
 
+    def make_path(encoded_key, bq)
+      now = Time.now.utc
+      timestamp = ((now.to_i*1000*1000+now.usec) << 12 | rand(0xfff))
+      tsuffix = timestamp.to_s(16)
+      path = "#{@buffer_path_prefix}#{encoded_key}.#{bq}#{tsuffix}#{@buffer_path_suffix}"
+      return path, tsuffix
+    end
 
+    def tsuffix_to_unique_id(tsuffix)
+      tsuffix.scan(/../).map {|x| x.to_i(16) }.pack('C*') * 2
+    end
+  end
 end
-

data/lib/fluent/plugin/buf_memory.rb

@@ -16,93 +16,88 @@
 #    limitations under the License.
 #
 module Fluent
+  class MemoryBufferChunk < BufferChunk
+    def initialize(key, data='')
+      @data = data
+      @data.force_encoding('ASCII-8BIT')
+      now = Time.now.utc
+      u1 = ((now.to_i*1000*1000+now.usec) << 12 | rand(0xfff))
+      @unique_id = [u1 >> 32, u1 & u1 & 0xffffffff, rand(0xffffffff), rand(0xffffffff)].pack('NNNN')
+      super(key)
+    end
 
+    attr_reader :unique_id
 
-class MemoryBufferChunk < BufferChunk
-  def initialize(key, data='')
-    @data = data
-    @data.force_encoding('ASCII-8BIT')
-    now = Time.now.utc
-    u1 = ((now.to_i*1000*1000+now.usec) << 12 | rand(0xfff))
-    @unique_id = [u1 >> 32, u1 & u1 & 0xffffffff, rand(0xffffffff), rand(0xffffffff)].pack('NNNN')
-    super(key)
-  end
-
-  attr_reader :unique_id
-
-  def <<(data)
-    data.force_encoding('ASCII-8BIT')
-    @data << data
-  end
+    def <<(data)
+      data.force_encoding('ASCII-8BIT')
+      @data << data
+    end
 
-  def size
-    @data.bytesize
-  end
+    def size
+      @data.bytesize
+    end
 
-  def close
-  end
+    def close
+    end
 
-  def purge
-  end
+    def purge
+    end
 
-  def read
-    @data
-  end
+    def read
+      @data
+    end
 
-  def open(&block)
-    StringIO.open(@data, &block)
-  end
+    def open(&block)
+      StringIO.open(@data, &block)
+    end
 
-  # optimize
-  def write_to(io)
-    io.write @data
-  end
+    # optimize
+    def write_to(io)
+      io.write @data
+    end
 
-  # optimize
-  def msgpack_each(&block)
-    u = MessagePack::Unpacker.new
-    u.feed_each(@data, &block)
+    # optimize
+    def msgpack_each(&block)
+      u = MessagePack::Unpacker.new
+      u.feed_each(@data, &block)
+    end
   end
-end
 
 
-class MemoryBuffer < BasicBuffer
-  Plugin.register_buffer('memory', self)
+  class MemoryBuffer < BasicBuffer
+    Plugin.register_buffer('memory', self)
 
-  def initialize
-    super
-  end
+    def initialize
+      super
+    end
 
-  # Overwrite default BasicBuffer#buffer_queue_limit
-  # to limit total memory usage upto 512MB.
-  config_set_default :buffer_queue_limit, 64
+    # Overwrite default BasicBuffer#buffer_queue_limit
+    # to limit total memory usage upto 512MB.
+    config_set_default :buffer_queue_limit, 64
 
-  def configure(conf)
-    super
-  end
+    def configure(conf)
+      super
+    end
 
-  def before_shutdown(out)
-    synchronize do
-      @map.each_key {|key|
-        push(key)
-      }
-      while pop(out)
+    def before_shutdown(out)
+      synchronize do
+        @map.each_key {|key|
+          push(key)
+        }
+        while pop(out)
+        end
       end
     end
-  end
 
-  def new_chunk(key)
-    MemoryBufferChunk.new(key)
-  end
+    def new_chunk(key)
+      MemoryBufferChunk.new(key)
+    end
 
-  def resume
-    return [], {}
-  end
+    def resume
+      return [], {}
+    end
 
-  def enqueue(chunk)
+    def enqueue(chunk)
+    end
   end
 end
-
-
-end
-