fluentd 0.10.0

data/lib/fluent/plugin/out_null.rb

@@ -0,0 +1,44 @@
+#
+# Fluent
+#
+# Copyright (C) 2011 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+module Fluent
+
+
+class NullOutput < Output
+  Plugin.register_output('null', self)
+
+  def initialize
+    super
+  end
+
+  def configure(conf)
+    super
+  end
+
+  def start
+  end
+
+  def shutdown
+  end
+
+  def emit(tag, es, chain)
+    chain.next
+  end
+end
+
+
+end

data/lib/fluent/plugin/out_roundrobin.rb

@@ -0,0 +1,72 @@
+#
+# Fluent
+#
+# Copyright (C) 2011 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+module Fluent
+
+
+class RoundRobinOutput < MultiOutput
+  Plugin.register_output('roundrobin', self)
+
+  def initialize
+    @outputs = []
+  end
+
+  attr_reader :outputs
+
+  def configure(conf)
+    conf.elements.select {|e|
+      e.name == 'store'
+    }.each {|e|
+      type = e['type']
+      unless type
+        raise ConfigError, "Missing 'type' parameter on <store> directive"
+      end
+      $log.debug "adding store type=#{type.dump}"
+
+      output = Plugin.new_output(type)
+      output.configure(e)
+      @outputs << output
+    }
+    @rr = -1  # starts from @output[0]
+  end
+
+  def start
+    @outputs.each {|o|
+      o.start
+    }
+  end
+
+  def shutdown
+    @outputs.each {|o|
+      o.shutdown
+    }
+  end
+
+  def emit(tag, es, chain)
+    next_output.emit(tag, es, chain)
+  end
+
+  protected
+  def next_output
+    @rr = 0 if (@rr += 1) >= @outputs.size
+    @outputs[@rr]
+  end
+end
+
+
+end
+

data/lib/fluent/plugin/out_stdout.rb

@@ -0,0 +1,34 @@
+#
+# Fluent
+#
+# Copyright (C) 2011 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+module Fluent
+
+
+class StdoutOutput < Output
+  Plugin.register_output('stdout', self)
+
+  def emit(tag, es, chain)
+    es.each {|time,record|
+      puts "#{Time.at(time).localtime} #{tag}: #{record.to_json}"
+    }
+    chain.next
+  end
+end
+
+
+end
+

data/lib/fluent/plugin/out_stream.rb

@@ -0,0 +1,128 @@
+#
+# Fluent
+#
+# Copyright (C) 2011 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+module Fluent
+
+
+class StreamOutput < BufferedOutput
+  def initialize
+    super
+    require 'socket'
+    require 'fileutils'
+    @send_timeout = 60
+  end
+
+  config_param :send_timeout, :time, :default => 60
+
+  def configure(conf)
+    super
+  end
+
+  def format_stream(tag, es)
+    # use PackedForward
+    [tag, es.to_msgpack_stream].to_msgpack
+  end
+
+  def write(chunk)
+    sock = connect
+    begin
+      opt = [1, @send_timeout.to_i].pack('I!I!')  # { int l_onoff; int l_linger; }
+      sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_LINGER, opt)
+
+      opt = [@send_timeout.to_i, 0].pack('L!L!')  # struct timeval
+      sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_SNDTIMEO, opt)
+
+      chunk.write_to(sock)
+    ensure
+      sock.close
+    end
+  end
+
+  def flush_secondary(secondary)
+    unless secondary.is_a?(StreamOutput)
+      secondary = ReformatWriter.new(secondary)
+    end
+    @buffer.pop(secondary)
+  end
+
+  class ReformatWriter
+    def initialize(secondary)
+      @secondary = secondary
+    end
+
+    def write(chunk)
+      chain = NullOutputChain.instance
+      chunk.open {|io|
+        # TODO use MessagePackIoEventStream
+        u = MessagePack::Unpacker.new(io)
+        begin
+          u.each {|(tag,entries)|
+            es = MultiEventStream.new
+            entries.each {|o|
+              es.add(o[0], o[1])
+            }
+            @secondary.emit(tag, es, chain)
+          }
+        rescue EOFError
+        end
+      }
+    end
+  end
+end
+
+
+class TcpOutput < StreamOutput
+  Plugin.register_output('tcp', self)
+
+  def initialize
+    super
+  end
+
+  config_param :port, :integer, :default => DEFAULT_LISTEN_PORT
+  config_param :host, :string
+
+  def configure(conf)
+    super
+  end
+
+  def connect
+    TCPSocket.new(@host, @port)
+  end
+end
+
+
+class UnixOutput < StreamOutput
+  Plugin.register_output('unix', self)
+
+  def initialize
+    super
+  end
+
+  config_param :path, :string
+
+  def configure(conf)
+    super
+  end
+
+  def connect
+    UNIXSocket.new(@path)
+  end
+end
+
+
+end
+

data/lib/fluent/plugin/out_test.rb

@@ -0,0 +1,68 @@
+#
+# Fluent
+#
+# Copyright (C) 2011 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+module Fluent
+
+
+class TestOutput < Output
+  Plugin.register_output('test', self)
+
+  def initialize
+    @emits = []
+    @name = nil
+  end
+
+  attr_reader :emits, :name
+
+  def events
+    all = []
+    @emits.each {|tag,events|
+      all.concat events
+    }
+    all
+  end
+
+  def records
+    all = []
+    @emits.each {|tag,events|
+      events.each {|time,record|
+        all << record
+      }
+    }
+    all
+  end
+
+  def configure(conf)
+    if name = conf['name']
+      @name = name
+    end
+  end
+
+  def start
+  end
+
+  def shutdown
+  end
+
+  def emit(tag, es, chain)
+    chain.next
+    @emits << [tag, es.to_a]
+  end
+end
+
+
+end

data/lib/fluent/test.rb

@@ -0,0 +1,8 @@
+require 'test/unit'
+require 'fluent/load'
+require 'fluent/test/base'
+require 'fluent/test/input_test'
+require 'fluent/test/output_test'
+
+$log ||= Fluent::Log.new
+

data/lib/fluent/test/base.rb

@@ -0,0 +1,63 @@
+#
+# Fluent
+#
+# Copyright (C) 2011 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+module Fluent
+module Test
+
+
+class TestDriver
+  include ::Test::Unit::Assertions
+
+  def initialize(klass, &block)
+    if klass.is_a?(Class)
+      if block
+        klass = Class.new(klass, &block)
+      end
+      @instance = klass.new
+    else
+      @instance = klass
+    end
+    @config = Config.new
+  end
+
+  attr_reader :instance, :config
+
+  def configure(str)
+    if str.is_a?(Config)
+      @config = @str
+    else
+      @config = Config.parse(str, "(test)")
+    end
+    @instance.configure(@config)
+    self
+  end
+
+  def run(&block)
+    @instance.start
+    begin
+      # wait until thread starts
+      10.times { sleep 0.01 }
+      return yield
+    ensure
+      @instance.shutdown
+    end
+  end
+end
+
+
+end
+end

data/lib/fluent/test/input_test.rb

@@ -0,0 +1,89 @@
+#
+# Fluent
+#
+# Copyright (C) 2011 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+module Fluent
+module Test
+
+
+class InputTestDriver < TestDriver
+  def initialize(klass, &block)
+    super(klass, &block)
+    @emits = []
+    @expects = nil
+  end
+
+  def expect_emit(tag, time, record)
+    (@expects ||= []) << [tag, time, record]
+    self
+  end
+
+  def expected_emits
+    @expects ||= []
+  end
+
+  attr_reader :emits
+
+  def events
+    all = []
+    @emits.each {|tag,events|
+      all.concat events
+    }
+    all
+  end
+
+  def records
+    all = []
+    @emits.each {|tag,events|
+      events.each {|time,record|
+        all << record
+      }
+    }
+    all
+  end
+
+  def run(&block)
+    m = method(:emit_stream)
+    super {
+      Engine.define_singleton_method(:emit_stream) {|tag,es|
+        m.call(tag, es)
+      }
+
+      block.call if block
+
+      if @expects
+        i = 0
+        @emits.each {|tag,events|
+          events.each {|time,record|
+            assert_equal(@expects[i], [tag, time, record])
+            i += 1
+          }
+        }
+        assert_equal @expects.length, i
+      end
+    }
+    self
+  end
+
+  private
+  def emit_stream(tag, es)
+    @emits << [tag, es.to_a]
+  end
+end
+
+
+end
+end