fluentd 0.10.0

data/lib/fluent/buffer.rb

@@ -0,0 +1,274 @@
+#
+# Fluent
+#
+# Copyright (C) 2011 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+module Fluent
+
+
+class BufferError < StandardError
+end
+
+
+class Buffer
+  include Configurable
+
+  def initialize
+    super
+  end
+
+  def configure(conf)
+    super
+  end
+
+  def start
+  end
+
+  def shutdown
+  end
+
+  def before_shutdown(out)
+  end
+
+  #def emit(key, data, chain)
+  #end
+
+  #def keys
+  #end
+
+  #def push(key)
+  #end
+
+  #def pop(out)
+  #end
+
+  #def clear!
+  #end
+end
+
+
+class BufferChunk
+  include MonitorMixin
+
+  def initialize(key)
+    super()
+    @key = key
+  end
+
+  attr_reader :key
+
+  #def <<(data)
+  #end
+
+  #def size
+  #end
+
+  def empty?
+    size == 0
+  end
+
+  #def close
+  #end
+
+  #def purge
+  #end
+
+  #def read
+  #end
+
+  #def open
+  #end
+
+  def write_to(io)
+    open {|i|
+      FileUtils.copy_stream(i, io)
+    }
+  end
+
+  def msgpack_each(&block)
+    open {|io|
+      u = MessagePack::Unpacker.new(io)
+      begin
+        u.each(&block)
+      rescue EOFError
+      end
+    }
+  end
+end
+
+
+class BasicBuffer < Buffer
+  include MonitorMixin
+
+  def initialize
+    super()
+    @chunk_limit = 16*1024*1024  # TODO default
+    @queue_limit = 64 # TODO default
+    @parallel = false
+  end
+
+  attr_accessor :queue_limit, :chunk_limit
+
+  def configure(conf)
+    super
+
+    if chunk_limit = conf['buffer_chunk_limit']
+      @chunk_limit = Config.size_value(chunk_limit)
+    end
+
+    if queue_limit = conf['buffer_queue_limit']
+      @queue_limit = queue_limit.to_i
+    end
+  end
+
+  def start
+    @queue, @map = resume
+    @queue.extend(MonitorMixin)
+  end
+
+  def shutdown
+    synchronize do
+      @queue.synchronize do
+        until @queue.empty?
+          @queue.shift.close
+        end
+      end
+      @map.each_pair {|key,chunk|
+        chunk.close
+      }
+    end
+  end
+
+  def emit(key, data, chain)
+    key = key.to_s
+
+    synchronize do
+      top = (@map[key] ||= new_chunk(key))  # TODO generate unique chunk id
+
+      if top.size + data.bytesize <= @chunk_limit
+        chain.next
+        top << data
+        return false
+
+      elsif data.bytesize > @chunk_limit
+        # TODO
+        raise BufferError, "received data too large"
+
+      elsif @queue.size >= @queue_limit
+        # TODO
+        raise BufferError, "queue size exceeds limit"
+      end
+
+      nc = new_chunk(key)  # TODO generate unique chunk id
+      ok = false
+
+      begin
+        nc << data
+        chain.next
+
+        @queue.synchronize {
+          enqueue(top)
+          @queue << top
+          @map[key] = nc
+        }
+
+        ok = true
+        return @queue.size == 1
+      ensure
+        nc.purge unless ok
+      end
+
+    end  # synchronize
+  end
+
+  def keys
+    @map.keys
+  end
+
+  def queue_size
+    @queue.size
+  end
+
+  #def new_chunk(key)
+  #end
+
+  #def resume
+  #end
+
+  #def enqueue(chunk)
+  #end
+
+  def push(key)
+    synchronize do
+      top = @map[key]
+      if !top || top.empty?
+        return false
+      end
+
+      @queue.synchronize do
+        enqueue(top)
+        @queue << top
+        @map.delete(key)
+      end
+
+      return true
+    end  # synchronize
+  end
+
+  def pop(out)
+    chunk = nil
+    @queue.synchronize do
+      if @parallel
+        chunk = @queue.find {|c| c.try_mon_enter }
+        return false unless chunk
+      else
+        chunk = @queue.first
+        return false unless chunk
+        return false unless chunk.try_mon_enter
+      end
+    end
+
+    begin
+      if !chunk.empty?
+        write_chunk(chunk, out)
+      end
+
+      @queue.delete_if {|c|
+        c.object_id == chunk.object_id
+      }
+
+      chunk.purge
+
+      return !@queue.empty?
+    ensure
+      chunk.mon_exit
+    end
+  end
+
+  def write_chunk(chunk, out)
+    out.write(chunk)
+  end
+
+  def clear!
+    @queue.delete_if {|chunk|
+      chunk.purge
+      true
+    }
+  end
+end
+
+
+end
+

data/lib/fluent/command/cat.rb

@@ -0,0 +1,299 @@
+#
+# Fluent cat
+#
+# Copyright (C) 2011 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'optparse'
+require 'fluent/env'
+
+op = OptionParser.new
+
+op.banner += " <tag>"
+
+port = Fluent::DEFAULT_LISTEN_PORT
+host = '127.0.0.1'
+unix = false
+socket_path = Fluent::DEFAULT_SOCKET_PATH
+
+config_path = Fluent::DEFAULT_CONFIG_PATH
+format = 'json'
+
+op.on('-p', '--port PORT', "fluent tcp port (default: #{port})", Integer) {|i|
+  port = i
+}
+
+op.on('-h', '--host HOST', "fluent host (default: #{host})") {|s|
+  host = s
+}
+
+op.on('-u', '--unix', "use unix socket instead of tcp", TrueClass) {|b|
+  unix = b
+}
+
+op.on('-s', '--socket PATH', "unix socket path (default: #{socket_path})") {|s|
+  socket_path = s
+}
+
+op.on('-f', '--format FORMAT', "input format (default: #{format})") {|s|
+  format = s
+}
+
+op.on('--json', "same as: -f json", TrueClass) {|b|
+  format = 'json'
+}
+
+op.on('--msgpack', "same as: -f msgpack", TrueClass) {|b|
+  format = 'msgpack'
+}
+
+(class<<self;self;end).module_eval do
+  define_method(:usage) do |msg|
+    puts op.to_s
+    puts "error: #{msg}" if msg
+    exit 1
+  end
+end
+
+begin
+  op.parse!(ARGV)
+
+  if ARGV.length != 1
+    usage nil
+  end
+
+  tag = ARGV.shift
+
+rescue
+  usage $!.to_s
+end
+
+
+require 'thread'
+require 'monitor'
+require 'socket'
+require 'json'
+require 'msgpack'
+
+
+class Writer
+  include MonitorMixin
+
+  class TimerThread
+    def initialize(writer)
+      @writer = writer
+    end
+
+    def start
+      @finish = false
+      @thread = Thread.new(&method(:run))
+    end
+
+    def shutdown
+      @finish = true
+      @thread.join
+    end
+
+    def run
+      until @finish
+        sleep 1
+        @writer.on_timer
+      end
+    end
+  end
+
+  def initialize(tag, connector)
+    @tag = tag
+    @connector = connector
+    @socket = false
+
+    @socket_time = Time.now.to_i
+    @socket_ttl = 10  # TODO
+    @error_history = []
+
+    @pending = []
+    @pending_limit = 1024  # TODO
+    @retry_wait = 1
+    @retry_limit = 5  # TODO
+
+    super()
+  end
+
+  def write(record)
+    if record.class != Hash
+      raise ArgumentError, "Input must be a map (got #{record.class})"
+    end
+
+    entry = [Time.now.to_i, record]
+    synchronize {
+      unless write_impl([entry])
+        # write failed
+        @pending.push(entry)
+
+        while @pending.size > @pending_limit
+          # exceeds pending limit; trash oldest record
+          time, record = @pending.shift
+          abort_message(time, record)
+        end
+      end
+    }
+  end
+
+  def on_timer
+    now = Time.now.to_i
+
+    synchronize {
+      unless @pending.empty?
+        # flush pending records
+        if write_impl(@pending)
+          # write succeeded
+          @pending.clear
+        end
+      end
+
+      if @socket && @socket_time + @socket_ttl < now
+        # socket is not used @socket_ttl seconds
+        close
+      end
+    }
+  end
+
+  def close
+    @socket.close
+    @socket = nil
+  end
+
+  def start
+    @timer = TimerThread.new(self)
+    @timer.start
+    self
+  end
+
+  def shutdown
+    @timer.shutdown
+  end
+
+  private
+  def write_impl(array)
+    socket = get_socket
+    unless socket
+      return false
+    end
+
+    begin
+      socket.write [@tag, array].to_msgpack
+      socket.flush
+    rescue
+      $stderr.puts "write failed: #{$!}"
+      close
+      return false
+    end
+
+    return true
+  end
+
+  def get_socket
+    unless @socket
+      unless try_connect
+        return nil
+      end
+    end
+
+    @socket_time = Time.now.to_i
+    return @socket
+  end
+
+  def try_connect
+    now = Time.now.to_i
+
+    unless @error_history.empty?
+      # wait before re-connecting
+      wait = @retry_wait * (2 ** (@error_history.size-1))
+      if now <= @socket_time + wait
+        return false
+      end
+    end
+
+    begin
+      @socket = @connector.call
+      @error_history.clear
+      return true
+
+    rescue
+      $stderr.puts "connect failed: #{$!}"
+      @error_history << $!
+      @socket_time = now
+
+      if @retry_limit < @error_history.size
+        # abort all pending records
+        @pending.each {|(time, record)|
+          abort_message(time, record)
+        }
+        @pending.clear
+        @error_history.clear
+      end
+
+      return false
+    end
+  end
+
+  def abort_message(time, record)
+    $stdout.puts "!#{time}:#{record.to_json}"
+  end
+end
+
+
+if unix
+  connector = Proc.new {
+    UNIXSocket.open(socket_path)
+  }
+else
+  connector = Proc.new {
+    TCPSocket.new(host, port)
+  }
+end
+
+w = Writer.new(tag, connector)
+w.start
+
+case format
+when 'json'
+  begin
+    while line = $stdin.gets
+      record = JSON.parse(line)
+      w.write(record)
+    end
+  rescue
+    $stderr.puts $!
+    exit 1
+  end
+
+when 'msgpack'
+  begin
+    u = MessagePack::Unpacker.new($stdin)
+    u.each {|record|
+      w.write(record)
+    }
+  rescue EOFError
+  rescue
+    $stderr.puts $!
+    exit 1
+  end
+
+else
+  $stderr.puts "Unknown format '#{format}'"
+  exit 1
+end
+