fluent-logger 0.3.1 → 0.4.0

data/.gitignore

@@ -0,0 +1,3 @@
+Gemfile.lock
+pkg/*
+coverage/*

data/.gitmodules

@@ -0,0 +1,3 @@
+[submodule "vendor/fluentd"]
+	path = vendor/fluentd
+	url = git://github.com/fluent/fluentd.git

data/.rspec

@@ -0,0 +1 @@
+--color

data/.travis.yml

@@ -0,0 +1,10 @@
+rvm:
+ - 1.8.7
+ - 1.9.2
+ - ree
+ - jruby
+
+before_script:
+  - git submodule update -i
+
+script: bundle exec rake spec

data/AUTHORS

@@ -0,0 +1 @@
+FURUHASHI Sadayuki <frsyuki _at_ gmail.com>

data/COPYING

@@ -0,0 +1,14 @@
+Copyright (C) 2011 FURUHASHI Sadayuki
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+

data/ChangeLog

@@ -1,4 +1,18 @@
 
+Release 0.4.0 - 2011/11/05
+
+  * Wait before reconnecting to fluentd to prevent burst
+  * Flush logs when process stops using finalizer
+  * Added rspec and coverage
+  * Supports objects that don't support to_msgpack by
+    JSON.load(JSON.dump(obj)).to_msgpack
+  * FluentLogger uses IO#sync=true + IO#write instead of IO#syswrite to
+    avoid unexpected blocking
+  * Logger#post(tag, map) -> Logger#post(tag, map, time=Time.now)
+  * Removed Event classes
+  * Added NullLogger
+
+
 Release 0.3.1 - 2011/08/28
 
   * FluentLogger#initialize doesn't raise error when connection is failed.

data/Gemfile

@@ -0,0 +1,9 @@
+
+source :rubygems
+
+gemspec
+
+gem "simplecov", :require => false
+gem 'yajl-ruby' # FIXME ruby 1.8.7 don't work add_dependency('yajl-ruby')
+gem "fluentd", :path => 'vendor/fluentd' if RUBY_VERSION >= "1.9.2"
+

data/README.rdoc

@@ -8,109 +8,47 @@ A structured event loger
 
     require 'fluent-logger'
     
-    EventLogger = Fluent::Logger::ConsoleLogger.new(STDOUT)
+    log = Fluent::Logger::FluentLogger.new(nil, :host=>'localhost', :port=>24224)
+    log.post("myapp.access", {"agent"=>"foo"})
     
-    module Event
-      AccessEvent = EventLogger.create_event('access', :agent, :action=>'access')
-    end
-    
-    #=> access: action="access" agent="foo"
-    Event::AccessEvent.agent('foo').post!
-
+    # output: myapp.access {"agent":"foo"}
 
 === Singleton
 
     require 'fluent-logger'
     
-    Fluent::Logger::ConsoleLogger.open(STDOUT)
+    Fluent::Logger::FluentLogger.open(nil, :host=>'localhost', :port=>24224)
+    Fluent::Logger.post("myapp.access", {"agent"=>"foo"})
     
-    module Event
-      AccessEvent = Fluent::Logger.create_event('access', :agent, :action=>'access')
-    end
-    
-    #=> access: action="access" agent="foo"
-    Event::AccessEvent.agent('foo').post!
-
+    # output: myapp.access {"agent":"foo"}
 
-=== Combine events
+=== Tag prefix
 
     require 'fluent-logger'
     
-    EventLogger = Fluent::Logger::ConsoleLogger.new(STDOUT)
-    
-    module Event
-      User = EventLogger.create_event('user', :name, :age)
-      LoginEvent = EventLogger.create_event('login', :action=>'login')
-      BuyEvent = EventLogger.create_event('buy', :item, :action=>'login')
-    end
-    
-    e_user = Event::User.name('me').age(24)
-    
-    #=> login: action="login" name="me" age=24
-    Event::LoginEvent.with(e_user).post!
+    log = Fluent::Logger::FluentLogger.new('myapp', :host=>'localhost', :port=>24224)
+    log.post("access", {"agent"=>"foo"})
     
-    #=> buy: action="login" name="me" age=24 item="item01"
-    Event::BuyEvent.with(e_user).item("item01").post!
+    # output: myapp.access {"agent":"foo"}
 
-=== Update created events
-
-    require 'fluent-logger'
-    
-    EventLogger = Fluent::Logger::ConsoleLogger.new(STDOUT)
-    
-    module Event
-      User = EventLogger.create_event('user', :name, :age)
-      AgeChangeEvent = EventLogger.create_event('age_change', :changed_age, :action=>'age_change')
-      BuyEvent = EventLogger.create_event('buy', :item, :action=>'buy')
-    end
-    
-    e_user = User.name('me').age(24)
-    
-    #=> age_change: action="age_change" name="me" age=24 changed_age=25
-    Event::AgeChangeEvent.with(e_user).changed_age(25).post!
-    e_user.age!(25)
-    
-    #=> buy: action="buy" name="me" age=25 item="item01"
-    Event::BuyEvent.with(e_user).item("item01").post!
-
-=== Update created events by combining
-
-    require 'fluent-logger'
-    
-    EventLogger = Fluent::Logger::ConsoleLogger.new(STDOUT)
-    
-    module Event
-      User = EventLogger.create_event('user', :name, :age)
-      Browser = EventLogger.create_event('browser', :host, :agent)
-      LoginEvent = EventLogger.create_event('login', :action=>'login')
-    end
-    
-    e_user = Event::User.name('me').age(24)
-    e_browser = Event::Browser.host('remoteip').agent('firefox')
-    e_user.with!(e_browser)
-    
-    #=> login: action="login" name="me" age=24 host="remoteip" agent="firefox"
-    Event::LoginEvent.with(e_user).post!
+== Loggers
 
+=== Fluent
 
-== Loggers
+    Fluent::Logger::FluentLogger.open('tag_prefix', :host=>'localhost', :port=24224)
 
 === Console
 
     Fluent::Logger::ConsoleLogger.open(io)
 
-=== Syslog
-
-    Fluent::Logger::SyslogLogger.open(ident=$0, level=:info)
-
-=== Fluent
+=== Null
 
-    Fluent::Logger::FluentLogger.open('tag_prefix', 'host', port=24224)
+    Fluent::Logger::NullLogger.open
 
 
 Web site::  http://fluent.github.com/
 Documents:: http://fluent.github.com/doc/
-Source repository:: http://github.com/fluent
+Source repository:: https://github.com/fluent/fluent-logger-ruby
 Author:: Sadayuki Furuhashi
 Copyright:: (c) 2011 FURUHASHI Sadayuki
 License:: Apache License, Version 2.0

data/Rakefile

@@ -0,0 +1,18 @@
+
+require 'bundler'
+Bundler::GemHelper.install_tasks
+
+require 'rspec/core'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec) do |spec|
+  spec.pattern = FileList['spec/**/*_spec.rb']
+end
+
+task :coverage do |t|
+  ENV['SIMPLE_COV'] = '1'
+  Rake::Task["spec"].invoke
+end
+
+task :default => :build
+

data/VERSION

@@ -0,0 +1 @@
+0.4.0

data/fluent-logger.gemspec

@@ -0,0 +1,44 @@
+# encoding: utf-8
+$:.push File.expand_path('../lib', __FILE__)
+
+Gem::Specification.new do |gem|
+  version_file = "lib/fluent/logger/version.rb"
+  version = File.read("VERSION").strip
+  File.open(version_file, "w") {|f|
+    f.write <<EOF
+module Fluent
+module Logger
+
+VERSION = '#{version}'
+
+end
+end
+EOF
+  }
+
+  unless File.exist?("vendor/fluentd/Gemfile")
+    puts "git submodule update -i"
+    system("git submodule update -i")
+  end
+
+  gem.name        = %q{fluent-logger}
+  gem.version     = version
+  # gem.platform  = Gem::Platform::RUBY
+  gem.authors     = ["Sadayuki Furuhashi"]
+  gem.email       = %q{frsyuki@gmail.com}
+  gem.homepage    = %q{https://github.com/fluent/fluent-logger-ruby}
+  gem.description = %q{fluent logger for ruby}
+  gem.summary     = gem.description
+
+  gem.files         = `git ls-files`.split("\n")
+  gem.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.require_paths = ['lib']
+
+  gem.add_dependency 'yajl-ruby', '~> 1.0.0'
+  gem.add_dependency 'msgpack', '~> 0.4.4'
+  gem.add_development_dependency 'rake', '>= 0.9.2'
+  gem.add_development_dependency 'rspec', '>= 2.7.0'
+  gem.add_development_dependency 'simplecov', '>= 0.5.4'
+  gem.add_development_dependency 'timecop', '>= 0.3.0'
+end

data/lib/fluent/logger.rb

@@ -17,30 +17,13 @@
 #
 module Fluent
 
-
 module Logger
-  require 'fluent/logger/event'
-  require 'fluent/logger/base'
-
-  class DefaultLogger < LoggerBase
-    INSTANCE = self.new
-
-    def self.instance
-      INSTANCE
-    end
-
-    def self.new
-      INSTANCE
-    end
-
-    def post(tag, map)
-      Fluent::Logger.default.post(tag, map)
-    end
-
-    def close
-      Fluent::Logger.default.close
-    end
-  end
+  autoload :ConsoleLogger , 'fluent/logger/console_logger'
+  autoload :FluentLogger  , 'fluent/logger/fluent_logger'
+  autoload :LoggerBase    , 'fluent/logger/logger_base'
+  autoload :TestLogger    , 'fluent/logger/test_logger'
+  autoload :TextLogger    , 'fluent/logger/text_logger'
+  autoload :NullLogger    , 'fluent/logger/null_logger'
 
   @@default_logger = nil
 
@@ -65,12 +48,8 @@ module Logger
     end
   end
 
-  def self.create_event(*args)
-    DefaultLogger.instance.create_event(*args)
-  end
-
-  def self.post(tag, map)
-    DefaultLogger.instance.post(tag, map)
+  def self.post(tag, map, time=nil)
+    @@default_logger.post(tag, map, time)
   end
 
   def self.default
@@ -80,13 +59,6 @@ module Logger
   def self.default=(logger)
     @@default_logger = logger
   end
-
-  autoload :FluentLogger, 'fluent/logger/fluent'
-  autoload :ConsoleLogger, 'fluent/logger/console'
-  autoload :SyslogLogger, 'fluent/logger/syslog'
-  autoload :TestLogger, 'fluent/logger/test'
 end
 
-
 end
-

data/lib/fluent/logger/{console.rb → console_logger.rb}

@@ -15,10 +15,11 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 #
+require 'fluent/logger/text_logger'
+
 module Fluent
 module Logger
 
-
 class ConsoleLogger < TextLogger
   def initialize(out)
     super()

data/lib/fluent/logger/fluent_logger.rb

@@ -0,0 +1,214 @@
+#
+# Fluent
+#
+# Copyright (C) 2011 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+require 'msgpack'
+require 'socket'
+require 'monitor'
+require 'logger'
+require 'yajl'
+
+module Fluent
+module Logger
+
+
+class FluentLogger < LoggerBase
+  module Finalizable
+    require 'delegate'
+    def new(*args, &block)
+      obj = allocate
+      obj.instance_eval { initialize(*args, &block) }
+      dc = DelegateClass(obj.class).new(obj)
+      ObjectSpace.define_finalizer(dc, finalizer(obj))
+      dc
+    end
+
+    def finalizer(obj)
+      fin = obj.method(:finalize)
+      proc {|id|
+        fin.call
+      }
+    end
+  end
+  extend Finalizable
+
+  BUFFER_LIMIT = 8*1024*1024
+  RECONNECT_WAIT = 0.5
+  RECONNECT_WAIT_INCR_RATE = 1.5
+  RECONNECT_WAIT_MAX = 60
+  RECONNECT_WAIT_MAX_COUNT =
+      (1..100).inject(RECONNECT_WAIT_MAX / RECONNECT_WAIT) {|r,i|
+        break i + 1 if r < RECONNECT_WAIT_INCR_RATE
+        r / RECONNECT_WAIT_INCR_RATE
+      }
+
+  def initialize(tag_prefix, *args)
+    super()
+
+    options = {
+      :host => 'localhost',
+      :port => 24224
+    }
+
+    case args.first
+    when String, Symbol
+      # backward compatible
+      options[:host] = args[0]
+      options[:port] = args[1] if args[1]
+    when Hash
+      options.update args.first
+    end
+
+    @tag_prefix = tag_prefix
+    @host = options[:host]
+    @port = options[:port]
+
+    @mon = Monitor.new
+    @pending = nil
+    @connect_error_history = []
+
+    @limit = options[:buffer_limit] || BUFFER_LIMIT
+    @logger = options[:logger] || ::Logger.new(STDERR)
+
+    begin
+      connect!
+    rescue
+      @logger.error "Failed to connect fluentd: #{$!}"
+      @logger.error "Connection will be retried."
+    end
+  end
+
+  attr_accessor :limit, :logger
+
+  def post(tag, map, time=nil)
+    time ||= Time.now
+    tag = "#{@tag_prefix}.#{tag}" if @tag_prefix
+    write [tag, time.to_i, map]
+  end
+
+  def close
+    @mon.synchronize {
+      if @pending
+        begin
+          send_data(@pending)
+        rescue
+          @logger.error("FluentLogger: Can't send logs to #{@host}:#{@port}: #{$!}")
+        end
+      end
+      @con.close if connect?
+      @con = nil
+      @pending = nil
+    }
+    self
+  end
+
+  def connect?
+    !!@con
+  end
+
+  def finalize
+    close
+  end
+
+  private
+  def to_msgpack(msg)
+    begin
+      msg.to_msgpack
+    rescue NoMethodError
+      Yajl::Parser.parse( Yajl::Encoder.encode(msg) ).to_msgpack
+    end
+  end
+
+  def write(msg)
+    begin
+      data = to_msgpack(msg)
+    rescue
+      @logger.error("FluentLogger: Can't convert to msgpack: #{msg.inspect}: #{$!}")
+      return false
+    end
+
+    @mon.synchronize {
+      if @pending
+        @pending << data
+      else
+        @pending = data
+      end
+
+      # suppress reconnection burst
+      if !@connect_error_history.empty? && @pending.bytesize <= @limit
+        if (sz = @connect_error_history.size) < RECONNECT_WAIT_MAX_COUNT
+          suppress_sec = RECONNECT_WAIT * (RECONNECT_WAIT_INCR_RATE ** (sz - 1))
+        else
+          suppress_sec = RECONNECT_WAIT_MAX
+        end
+        if Time.now.to_i - @connect_error_history.last < suppress_sec
+          return false
+        end
+      end
+
+      begin
+        send_data(@pending)
+        @pending = nil
+        true
+      rescue
+        if @pending.bytesize > @limit
+          @logger.error("FluentLogger: Can't send logs to #{@host}:#{@port}: #{$!}")
+          @pending = nil
+        end
+        @con.close if connect?
+        @con = nil
+        false
+      end
+    }
+  end
+
+  def send_data(data)
+    unless connect?
+      connect!
+    end
+    @con.write data
+    #while true
+    #  puts "sending #{data.length} bytes"
+    #  if data.length > 32*1024
+    #    n = @con.syswrite(data[0..32*1024])
+    #  else
+    #    n = @con.syswrite(data)
+    #  end
+    #  puts "sent #{n}"
+    #  if n >= data.bytesize
+    #    break
+    #  end
+    #  data = data[n..-1]
+    #end
+    true
+  end
+
+  def connect!
+    @con = TCPSocket.new(@host, @port)
+    @con.sync = true
+    @connect_error_history.clear
+  rescue
+    @connect_error_history << Time.now.to_i
+    if @connect_error_history.size > RECONNECT_WAIT_MAX_COUNT
+      @connect_error_history.shift
+    end
+    raise
+  end
+end
+
+
+end
+end