jmoses_fluent-logger 0.4.8

data/lib/fluent/logger/logger_base.rb

@@ -0,0 +1,39 @@
+#
+# Fluent
+#
+# Copyright (C) 2011 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+module Fluent
+module Logger
+
+class LoggerBase
+  def self.open(*args, &block)
+    Fluent::Logger.open(self, *args, &block)
+  end
+
+  def post(tag, map)
+    raise ArgumentError.new("Second argument should kind of Hash (tag: #{map})") unless map.kind_of? Hash
+    post_with_time(tag, map, Time.now)
+  end
+
+  #def post_with_time(tag, map)
+  #end
+
+  def close
+  end
+end
+
+end
+end

data/lib/fluent/logger/null_logger.rb

@@ -0,0 +1,28 @@
+#
+# Fluent
+#
+# Copyright (C) 2011 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+module Fluent
+module Logger
+
+class NullLogger < LoggerBase
+  def post_with_time(tag, map, time)
+    false
+  end
+end
+
+end
+end

data/lib/fluent/logger/test_logger.rb

@@ -0,0 +1,51 @@
+#
+# Fluent
+#
+# Copyright (C) 2011 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+module Fluent
+module Logger
+
+class TestLogger < LoggerBase
+  def initialize(queue=[])
+    @queue = queue
+    @max = 1024
+  end
+
+  attr_accessor :max
+  attr_reader :queue
+
+  def post_with_time(tag, map, time)
+    while @queue.size > @max-1
+      @queue.shift
+    end
+    (class<<map;self;end).module_eval do
+      define_method(:tag) { tag }
+      define_method(:time) { time }
+    end
+    @queue << map
+    true
+  end
+
+  def tag_queue(tag)
+    @queue.find_all {|map| map.tag == tag }
+  end
+
+  def close
+  end
+end
+
+end
+end

data/lib/fluent/logger/text_logger.rb

@@ -0,0 +1,39 @@
+#
+# Fluent
+#
+# Copyright (C) 2011 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+module Fluent
+module Logger
+
+class TextLogger < LoggerBase
+  def initialize
+    require 'yajl'
+    @time_format = "%b %e %H:%M:%S"
+  end
+
+  def post_with_time(tag, map, time)
+    a = [time.strftime(@time_format), " ", tag, ":"]
+    map.each_pair {|k,v|
+      a << " #{k}="
+      a << Yajl::Encoder.encode(v)
+    }
+    post_text a.join
+    true
+  end
+end
+
+end
+end

data/lib/fluent/logger/version.rb

@@ -0,0 +1,7 @@
+module Fluent
+module Logger
+
+VERSION = '0.4.8'
+
+end
+end

data/lib/fluent/logger.rb

@@ -0,0 +1,69 @@
+#
+# Fluent
+#
+# Copyright (C) 2011 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+module Fluent
+
+module Logger
+  autoload :ConsoleLogger , 'fluent/logger/console_logger'
+  autoload :FluentLogger  , 'fluent/logger/fluent_logger'
+  autoload :LoggerBase    , 'fluent/logger/logger_base'
+  autoload :TestLogger    , 'fluent/logger/test_logger'
+  autoload :TextLogger    , 'fluent/logger/text_logger'
+  autoload :NullLogger    , 'fluent/logger/null_logger'
+  autoload :VERSION       , 'fluent/logger/version'
+
+  @@default_logger = nil
+
+  def self.new(*args)
+    if args.first.is_a?(Class) && args.first.ancestors.include?(LoggerBase)
+      type = args.shift
+    else
+      type = FluentLogger
+    end
+    type.new(*args)
+  end
+
+  def self.open(*args)
+    close
+    @@default_logger = new(*args)
+  end
+
+  def self.close
+    if @@default_logger
+      @@default_logger.close
+      @@default_logger = nil
+    end
+  end
+
+  def self.post(tag, map)
+    @@default_logger.post(tag, map)
+  end
+
+  def self.post_with_time(tag, map, time)
+    @@default_logger.post_with_time(tag, map, time)
+  end
+
+  def self.default
+    @@default_logger ||= ConsoleLogger.new(STDOUT)
+  end
+
+  def self.default=(logger)
+    @@default_logger = logger
+  end
+end
+
+end

data/lib/fluent-logger.rb

@@ -0,0 +1 @@
+require File.join(File.dirname(__FILE__), 'fluent', 'logger')

data/spec/console_logger_spec.rb

@@ -0,0 +1,69 @@
+
+require 'spec_helper'
+require 'stringio'
+require 'tempfile'
+require 'pathname'
+
+describe Fluent::Logger::ConsoleLogger do
+  before(:each) {
+    Timecop.freeze Time.local(2008, 9, 1, 10, 5, 0)
+  }
+  after(:each) {
+    Timecop.return
+  }
+
+  context "IO output" do
+    let(:io) { StringIO.new }
+    let(:logger) { Fluent::Logger::ConsoleLogger.new(io) }
+
+    subject {
+      io
+    }
+
+    context "post and read" do
+      before do
+        logger.post('example', {:foo => :bar})
+        io.rewind
+      end
+      its(:read)  { should eq %Q!Sep  1 10:05:00 example: foo="bar"\n! }
+    end
+  end
+
+  context "Filename output" do
+    let(:path) {
+      @tmp = Tempfile.new('fluent-logger') # ref instance var because Tempfile.close(true) check GC
+      filename = @tmp.path
+      @tmp.close(true)
+      Pathname.new(filename)
+    }
+    let(:logger) { Fluent::Logger::ConsoleLogger.new(path.to_s) }
+
+    subject { path }
+    after { path.unlink }
+
+    context "post and read" do
+      before do
+        logger.post('example', {:foo => :bar})
+        logger.close
+      end
+      its(:read)  { should eq %Q!Sep  1 10:05:00 example: foo="bar"\n! }
+    end
+
+    context "reopen" do
+      before do
+        logger.post('example', {:foo => :baz})
+        logger.close
+        logger.reopen!
+      end
+      its(:read)  { should eq %Q!Sep  1 10:05:00 example: foo="baz"\n! }
+    end
+  end
+
+  context "Invalid output" do
+    it {
+      expect {
+        Fluent::Logger::ConsoleLogger.new(nil)
+      }.to raise_error
+    }
+  end
+end

data/spec/fluent_logger_spec.rb

@@ -0,0 +1,297 @@
+
+require 'spec_helper'
+if RUBY_VERSION < "1.9.2"
+
+describe Fluent::Logger::FluentLogger do
+  pending "fluentd don't work RUBY < 1.9.2"
+end
+
+else
+
+require 'fluent/load'
+require 'tempfile'
+require 'logger'
+require 'socket'
+require 'stringio'
+require 'fluent/logger/fluent_logger/cui'
+
+$log = Fluent::Log.new(StringIO.new) # XXX should remove $log from fluentd 
+
+describe Fluent::Logger::FluentLogger do
+  WAIT = ENV['WAIT'] ? ENV['WAIT'].to_f : 0.1
+
+  let(:fluentd_port) {
+    port = 60001
+    loop do
+      begin
+        TCPServer.open('localhost', port).close
+        break
+      rescue Errno::EADDRINUSE
+        port += 1
+      end
+    end
+    port
+  }
+
+  let(:logger) {
+    @logger_io = StringIO.new
+    logger = ::Logger.new(@logger_io)
+    Fluent::Logger::FluentLogger.new('logger-test', {
+      :host   => 'localhost', 
+      :port   => fluentd_port,
+      :logger => logger,
+    })
+  }
+
+  let(:logger_io) {
+    @logger_io
+  }
+
+  let(:output) {
+    sleep 0.0001 # next tick
+    Fluent::Engine.match('logger-test').output
+  }
+
+  let(:queue) {
+    queue = []
+    output.emits.each {|tag, time, record|
+      queue << [tag, record]
+    }
+    queue
+  }
+
+  let(:queue_with_time) {
+    queue = []
+    output.emits.each {|tag, time, record|
+      queue << [tag, time, record]
+    }
+    queue
+  }
+
+  after(:each) do
+    output.emits.clear rescue nil
+  end
+
+  def wait_transfer
+    sleep WAIT
+  end
+
+  context "running fluentd" do
+    before(:each) do
+      tmp = Tempfile.new('fluent-logger-config')
+      tmp.close(false)
+
+      File.open(tmp.path, 'w') {|f|
+        f.puts <<EOF
+<source>
+  type tcp
+  port #{fluentd_port}
+</source>
+<match logger-test.**>
+  type test
+</match>
+EOF
+      }
+      Fluent::Test.setup
+      Fluent::Engine.read_config(tmp.path)
+      @coolio_default_loop = nil
+      @thread = Thread.new {
+        @coolio_default_loop = Coolio::Loop.default
+        Fluent::Engine.run
+      }
+      wait_transfer
+    end
+
+    after(:each) do
+      @coolio_default_loop.stop
+      Fluent::Engine.send :shutdown
+      @thread.join
+    end
+
+    context('Post by CUI') do
+      it('post') {
+        args = %W(-h localhost -p #{fluentd_port} -t logger-test.tag -v a=b -v foo=bar)
+        Fluent::Logger::FluentLogger::CUI.post(args)
+        wait_transfer
+        queue.last.should == ['logger-test.tag', {'a' => 'b', 'foo' => 'bar'}]
+      }
+    end
+
+    context('post') do
+      it ('success') { 
+        logger.post('tag', {'a' => 'b'}).should be_true
+        wait_transfer
+        queue.last.should == ['logger-test.tag', {'a' => 'b'}]
+      }
+
+      it ('close after post') {
+        logger.should be_connect
+        logger.close
+        logger.should_not be_connect
+
+        logger.post('tag', {'b' => 'c'})
+        logger.should be_connect
+        wait_transfer
+        queue.last.should == ['logger-test.tag', {'b' => 'c'}]
+      }
+
+      it ('large data') {
+        data = {'a' => ('b' * 1000000)}
+        logger.post('tag', data)
+        wait_transfer
+        queue.last.should == ['logger-test.tag', data]
+      }
+
+      it ('msgpack unsupport data') {
+        data = {
+          'time'   => Time.utc(2008, 9, 1, 10, 5, 0),
+          'object' => Object.new,
+          'proc'   => proc { 1 },
+        }
+        logger.post('tag', data)
+        wait_transfer
+        logger_data = queue.last.last
+        logger_data['time'].should == '2008-09-01 10:05:00 UTC'
+        logger_data['proc'].should be
+        logger_data['object'].should be
+      }
+
+      it ('msgpack and JSON unsupport data') {
+        data = {
+          'time'   => Time.utc(2008, 9, 1, 10, 5, 0),
+          'object' => Object.new,
+          'proc'   => proc { 1 },
+          'NaN'    => (0.0/0.0) # JSON don't convert
+        }
+        logger.post('tag', data)
+        wait_transfer
+        queue.last.should be_nil
+        logger_io.rewind
+        logger_io.read =~ /FluentLogger: Can't convert to msgpack:/
+      }
+
+      it ('batch post') {
+        messages = [
+          ['logger-test.tag1', 'message 1', Time.utc(2008, 9, 1, 10, 5, 0)],
+          ['logger-test.tag2', 'message 2', Time.utc(2008, 9, 1, 10, 6, 0)],
+        ]
+
+        logger.batch_post_with_time(messages)
+        wait_transfer
+
+        queue_with_time.should have(2).items
+        queue_with_time.first[0].should eq('logger-test.tag1')
+        queue_with_time.first[1].should eq(1220263500)
+        queue_with_time.first[2].should eq('message 1')
+        queue_with_time.last[0].should eq('logger-test.tag2')
+        queue_with_time.last[1].should eq(1220263560)
+        queue_with_time.last[2].should eq('message 2')
+      }
+
+      it ('batches faster than single') {
+        require 'benchmark'
+        require 'digest/md5'
+
+        messages = 1000.times.map do |id|
+          ['logger-test.tag', Digest::MD5.hexdigest(id.to_s), Time.now]
+        end
+
+        single = Benchmark.realtime do
+          output.emits.clear
+          messages.each {|m| logger.post_with_time *m }
+          wait_transfer
+          queue.should have(1000).items
+        end
+
+        batch = Benchmark.realtime do
+          output.emits.clear
+          logger.batch_post_with_time messages
+          wait_transfer
+          queue.should have(1000).items
+        end
+
+        batch.should be < single
+      }
+
+      it ('should raise an error when second argument is non hash object') {
+        data = 'FooBar'
+        expect {
+          logger.post('tag', data)
+        }.to raise_error(ArgumentError)
+
+        data = nil
+        expect {
+          logger.post('tag', data)
+        }.to raise_error(ArgumentError)
+      }
+    end
+
+    context "initializer" do
+      it "backward compatible" do
+        port = fluentd_port
+        fluent_logger = Fluent::Logger::FluentLogger.new('logger-test', 'localhost', port)
+        fluent_logger.method_missing(:instance_eval) { # fluent_logger is delegetor
+          @host.should == 'localhost'
+          @port.should == port
+        }
+      end
+
+      it "hash argument" do
+        port = fluentd_port
+        fluent_logger = Fluent::Logger::FluentLogger.new('logger-test', {
+          :host => 'localhost',
+          :port => port
+        })
+        fluent_logger.method_missing(:instance_eval) { # fluent_logger is delegetor
+          @host.should == 'localhost'
+          @port.should == port
+        }
+      end
+    end
+  end
+  
+  context "not running fluentd" do
+    context('fluent logger interface') do
+      it ('post & close') {
+        logger.post('tag', {'a' => 'b'}).should be_false
+        wait_transfer  # even if wait
+        queue.last.should be_nil
+        logger.close
+        logger_io.rewind
+        log = logger_io.read
+        log.should =~ /Failed to connect/
+        log.should =~ /Can't send logs to/
+      }
+
+      it ('post limit over') do
+        logger.limit = 100
+        logger.post('tag', {'a' => 'b'})
+        wait_transfer  # even if wait
+        queue.last.should be_nil
+
+        logger_io.rewind
+        logger_io.read.should_not =~ /Can't send logs to/
+
+        logger.post('tag', {'a' => ('c' * 1000)})
+        logger_io.rewind
+        logger_io.read.should =~ /Can't send logs to/
+      end
+
+      it ('log connect error once') do
+        Fluent::Logger::FluentLogger.any_instance.stub(:suppress_sec).and_return(-1)
+        logger.log_reconnect_error_threshold = 1
+        Fluent::Logger::FluentLogger.any_instance.should_receive(:log_reconnect_error).once.and_call_original
+
+        logger.post('tag', {'a' => 'b'})
+        wait_transfer  # even if wait
+        logger.post('tag', {'a' => 'b'})
+        wait_transfer  # even if wait
+        logger_io.rewind
+        logger_io.read.should =~ /Failed to connect/
+      end
+    end
+  end
+
+end
+
+end

data/spec/logger_base_spec.rb

@@ -0,0 +1,11 @@
+
+require 'spec_helper'
+
+describe Fluent::Logger::LoggerBase do
+  context "subclass" do
+    subject { Class.new(Fluent::Logger::LoggerBase) }
+    its(:open) {
+      should be_kind_of(Fluent::Logger::LoggerBase)
+    }
+  end
+end

data/spec/logger_spec.rb

@@ -0,0 +1,42 @@
+
+require 'spec_helper'
+require 'stringio'
+
+describe Fluent::Logger do
+  context "default logger" do
+    let(:test_logger) {
+      Fluent::Logger::TestLogger.new
+    }
+    before(:each) do
+      Fluent::Logger.default = test_logger
+    end
+
+    it('post') {
+      test_logger.should_receive(:post).with('tag1', {:foo => :bar})
+      Fluent::Logger.post('tag1', {:foo => :bar})
+    }
+
+    it('close') {
+      test_logger.should_receive(:close)
+      Fluent::Logger.close
+    }
+
+    it('open') {
+      test_logger.should_receive(:close)
+      klass = Class.new(Fluent::Logger::LoggerBase)
+      fluent_logger_logger_io = StringIO.new
+      Fluent::Logger.open('tag-prefix', {
+        :logger => ::Logger.new(fluent_logger_logger_io)
+      })
+      # Fluent::Logger::FluentLogger is delegator
+      Fluent::Logger.default.method_missing(:kind_of?, Fluent::Logger::FluentLogger).should be_true
+    }
+
+    it('open with BaseLogger class') {
+      test_logger.should_receive(:close)
+      klass = Class.new(Fluent::Logger::LoggerBase)
+      Fluent::Logger.open(klass)
+      Fluent::Logger.default.class.should == klass
+    }
+  end
+end

data/spec/null_logger_spec.rb

@@ -0,0 +1,15 @@
+
+require 'spec_helper'
+
+describe Fluent::Logger::NullLogger do
+  context "logger method" do
+    let(:logger) { Fluent::Logger::NullLogger.new }
+
+    context "post" do
+      it('false') {
+        logger.post('tag1', {:foo => :bar}).should == false
+        logger.post('tag2', {:foo => :baz}).should == false
+      }
+    end
+  end
+end