jmoses_fluent-logger 0.4.8

data/.gitignore

@@ -0,0 +1,4 @@
+Gemfile.lock
+pkg/*
+coverage/*
+coverage.vim

data/.gitmodules

@@ -0,0 +1,3 @@
+[submodule "vendor/fluentd"]
+	path = vendor/fluentd
+	url = git://github.com/fluent/fluentd.git

data/.rspec

@@ -0,0 +1 @@
+--color

data/.travis.yml

@@ -0,0 +1,15 @@
+rvm:
+ - 1.8.7
+ - 1.9.2
+ - 1.9.3
+ - 2.0.0
+ - ree
+
+before_install: git submodule update -i
+
+script: bundle exec rake spec
+
+matrix:
+  allow_failures:
+    - rvm: 1.8.7
+    - rvm: ree

data/AUTHORS

@@ -0,0 +1 @@
+FURUHASHI Sadayuki <frsyuki _at_ gmail.com>

data/COPYING

@@ -0,0 +1,14 @@
+Copyright (C) 2011 FURUHASHI Sadayuki
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+

data/ChangeLog

@@ -0,0 +1,81 @@
+Release jmoses_0.4.8 - 2014/01/28
+
+Note that you probably do *not* want this version, but the official version
+instead.
+
+* Support for batch posting events in a single connection attempt
+(semi-tested)
+
+Release 0.4.7 - 2013/11/21
+
+  * Suppress log error message with :log_reconnect_error_threshold option
+  * Add FluentLogger#last_error method to get last exception in user code
+
+Release 0.4.6 - 2013/06/17
+
+  * Raise an ArgumentError when passes invalid argument to post method
+  * Relax msgpack gem version
+
+Release 0.4.5 - 2013/02/26
+
+  * Use https scheme for rubygems source
+  * Fix broken spec
+
+Release 0.4.4 - 2012/12/26
+
+  * Change msgpack dependency version == 0.4.7 for avoding 0.4.8 yanked issue
+
+Release 0.4.3 - 2012/04/24
+
+  * Update yajl dependency version >= 1.0 (thanks shun0102)
+
+Release 0.4.2 - 2012/03/02
+
+  * Added TestLogger#tag_queue(tag_name)
+  * Added bin/fluent-post cli command
+  * Impl default LoggerBase#close
+  * Don't change logger.level if :debug=> option is sepcified
+
+Release 0.4.1 - 2011/11/07
+
+  * added Logger#post_with_time(tag, map, time)
+  * Logger#post(tag, map, time=Time.now) -> Logger#post(tag, map)
+  * FluentLogger supports :debug=>true option to write all events to STDERR
+
+
+Release 0.4.0 - 2011/11/05
+
+  * Wait before reconnecting to fluentd to prevent burst
+  * Flush logs when process stops using finalizer
+  * Added rspec and coverage
+  * Supports objects that don't support to_msgpack by
+    JSON.load(JSON.dump(obj)).to_msgpack
+  * FluentLogger uses IO#sync=true + IO#write instead of IO#syswrite to
+    avoid unexpected blocking
+  * Logger#post(tag, map) -> Logger#post(tag, map, time=Time.now)
+  * Removed Event classes
+  * Added NullLogger
+
+
+Release 0.3.1 - 2011/08/28
+
+  * FluentLogger#initialize doesn't raise error when connection is failed.
+    Instead, it tries to reconnect.
+
+
+Release 0.3.0 - 2011/08/21
+
+  * Added 'tag' for event logs
+
+
+Release 0.2.0 - 2011/08/05
+
+  * Redesigned Event class
+  * Added TestLogger (Fluent.open(:test))
+  * Added test programs
+
+
+Release 0.1.0 - 2011/08/04
+
+  * First release
+

data/Gemfile

@@ -0,0 +1,11 @@
+
+source 'https://rubygems.org/'
+
+gemspec
+
+gem "simplecov", :require => false
+gem "simplecov-vim"
+
+gem 'yajl-ruby' # FIXME ruby 1.8.7 don't work add_dependency('yajl-ruby')
+gem "fluentd", :path => 'vendor/fluentd' if RUBY_VERSION >= "1.9.2"
+

data/README.md

@@ -0,0 +1,63 @@
+# Fluent logger
+A structured event loger
+
+## Examples
+
+### Simple
+
+```ruby
+require 'fluent-logger'
+
+log = Fluent::Logger::FluentLogger.new(nil, :host=>'localhost', :port=>24224)
+unless log.post("myapp.access", {"agent"=>"foo"})
+  p log.last_error # You can get last error object via last_error method
+end
+
+# output: myapp.access {"agent":"foo"}
+```
+
+### Singleton
+```ruby
+require 'fluent-logger'
+
+Fluent::Logger::FluentLogger.open(nil, :host=>'localhost', :port=>24224)
+Fluent::Logger.post("myapp.access", {"agent"=>"foo"})
+
+# output: myapp.access {"agent":"foo"}
+```
+
+### Tag prefix
+```ruby
+require 'fluent-logger'
+
+log = Fluent::Logger::FluentLogger.new('myapp', :host=>'localhost', :port=>24224)
+log.post("access", {"agent"=>"foo"})
+
+# output: myapp.access {"agent":"foo"}
+```
+
+## Loggers
+
+### Fluent
+```ruby
+Fluent::Logger::FluentLogger.open('tag_prefix', :host=>'localhost', :port=24224)
+```
+
+### Console
+```ruby
+Fluent::Logger::ConsoleLogger.open(io)
+```
+
+### Null
+```ruby
+Fluent::Logger::NullLogger.open
+```
+
+|name|description|
+|---|---|
+|Web site|http://fluent.github.com/|
+|Documents|http://fluent.github.com/doc/|
+|Source repository|https://github.com/fluent/fluent-logger-ruby|
+|Author|Sadayuki Furuhashi|
+|Copyright|(c) 2011 FURUHASHI Sadayuki|
+|License|Apache License, Version 2.0|

data/Rakefile

@@ -0,0 +1,18 @@
+
+require 'bundler'
+Bundler::GemHelper.install_tasks
+
+require 'rspec/core'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec) do |spec|
+  spec.pattern = FileList['spec/**/*_spec.rb']
+end
+
+task :coverage do |t|
+  ENV['SIMPLE_COV'] = '1'
+  Rake::Task["spec"].invoke
+end
+
+task :default => :build
+

data/VERSION

@@ -0,0 +1 @@
+0.4.8

data/bin/fluent-post

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+
+require 'fluent/logger/fluent_logger/cui'
+
+res = Fluent::Logger::FluentLogger::CUI.post(ARGV)
+if res[:success]
+  warn "post successed. #=> #{res[:data].inspect}"
+else
+  warn "post failed. #=> #{res[:data].inspect}"
+  exit 1
+end

data/fluent-logger.gemspec

@@ -0,0 +1,44 @@
+# encoding: utf-8
+$:.push File.expand_path('../lib', __FILE__)
+
+Gem::Specification.new do |gem|
+  version_file = "lib/fluent/logger/version.rb"
+  version = File.read("VERSION").strip
+  File.open(version_file, "w") {|f|
+    f.write <<EOF
+module Fluent
+module Logger
+
+VERSION = '#{version}'
+
+end
+end
+EOF
+  }
+
+  unless File.exist?("vendor/fluentd/Gemfile")
+    puts "git submodule update -i"
+    system("git submodule update -i")
+  end
+
+  gem.name        = %q{jmoses_fluent-logger}
+  gem.version     = version
+  # gem.platform  = Gem::Platform::RUBY
+  gem.authors     = ["Jon Moses", "Sadayuki Furuhashi"]
+  gem.email       = %q{jon@burningbush.us frsyuki@gmail.com}
+  gem.homepage    = %q{https://github.com/jmoses/fluent-logger-ruby}
+  gem.description = %q{fluent logger for ruby}
+  gem.summary     = gem.description + " (fork by jmoses)"
+
+  gem.files         = `git ls-files`.split("\n")
+  gem.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.require_paths = ['lib']
+
+  gem.add_dependency 'yajl-ruby', '~> 1.0'
+  gem.add_dependency "msgpack", [">= 0.4.4", "!= 0.5.0", "!= 0.5.1", "!= 0.5.2", "!= 0.5.3", "< 0.6.0"]
+  gem.add_development_dependency 'rake', '>= 0.9.2'
+  gem.add_development_dependency 'rspec', '>= 2.7.0'
+  gem.add_development_dependency 'simplecov', '>= 0.5.4'
+  gem.add_development_dependency 'timecop', '>= 0.3.0'
+end

data/lib/fluent/logger/console_logger.rb

@@ -0,0 +1,57 @@
+#
+# Fluent
+#
+# Copyright (C) 2011 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+require 'fluent/logger/text_logger'
+
+module Fluent
+module Logger
+
+class ConsoleLogger < TextLogger
+  def initialize(out)
+    super()
+    require 'time'
+
+    if out.is_a?(String)
+      @io = File.open(out, "a")
+      @on_reopen = Proc.new { @io.reopen(out, "a") }
+    elsif out.respond_to?(:write)
+      @io = out
+      @on_reopen = Proc.new { }
+    else
+      raise "Invalid output: #{out.inspect}"
+    end
+  end
+
+  attr_accessor :time_format
+
+  def reopen!
+    @on_reopen.call
+  end
+
+  def post_text(text)
+    @io.puts text
+  end
+
+  def close
+    @io.close
+    self
+  end
+end
+
+
+end
+end

data/lib/fluent/logger/fluent_logger/cui.rb

@@ -0,0 +1,46 @@
+
+require 'fluent/logger'
+require 'optparse'
+
+module Fluent
+module Logger
+class FluentLogger
+
+module CUI
+  def post(args)
+    options = {
+      :port => '24224',
+      :host => 'localhost'
+    }
+
+    o = OptionParser.new
+    o.version = Fluent::Logger::VERSION
+    o.on('-t [tag (default nil)]') {|v| options[:tag] = v }
+    o.on('-p [port (default 24224)]') {|v| options[:port] = v }
+    o.on('-h [host (default localhost)]') {|v| options[:host] = v }
+    o.on('-v [key=value]') {|v| 
+      key, value = v.split('=')
+      (options[:data] ||= {})[key] = value
+    }
+    o.banner = 'Usage: fluent-post -t tag.foo.bar -v key1=value1 -v key2=value2'
+    args = args.to_a
+    args << '--help' if args.empty?
+    o.parse(args)
+    
+    f = Fluent::Logger::FluentLogger.new(nil, {
+      :host => options[:host],
+      :port => options[:port]
+    })
+
+    {
+      :success => f.post(options[:tag], options[:data]),
+      :data    => options[:data]
+    }
+  end
+
+  extend self
+end
+
+end
+end
+end

data/lib/fluent/logger/fluent_logger.rb

@@ -0,0 +1,298 @@
+#
+# Fluent
+#
+# Copyright (C) 2011 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+require 'msgpack'
+require 'socket'
+require 'monitor'
+require 'logger'
+require 'yajl'
+
+module Fluent
+module Logger
+
+
+class FluentLogger < LoggerBase
+  module Finalizable
+    require 'delegate'
+    def new(*args, &block)
+      obj = allocate
+      obj.instance_eval { initialize(*args, &block) }
+      dc = DelegateClass(obj.class).new(obj)
+      ObjectSpace.define_finalizer(dc, finalizer(obj))
+      dc
+    end
+
+    def finalizer(obj)
+      fin = obj.method(:finalize)
+      proc {|id|
+        fin.call
+      }
+    end
+  end
+  extend Finalizable
+
+  BUFFER_LIMIT = 8*1024*1024
+  RECONNECT_WAIT = 0.5
+  RECONNECT_WAIT_INCR_RATE = 1.5
+  RECONNECT_WAIT_MAX = 60
+  RECONNECT_WAIT_MAX_COUNT =
+      (1..100).inject(RECONNECT_WAIT_MAX / RECONNECT_WAIT) {|r,i|
+        break i + 1 if r < RECONNECT_WAIT_INCR_RATE
+        r / RECONNECT_WAIT_INCR_RATE
+      }
+
+  def initialize(tag_prefix, *args)
+    super()
+
+    options = {
+      :host => 'localhost',
+      :port => 24224
+    }
+
+    case args.first
+    when String, Symbol
+      # backward compatible
+      options[:host] = args[0]
+      options[:port] = args[1] if args[1]
+    when Hash
+      options.update args.first
+    end
+
+    @tag_prefix = tag_prefix
+    @host = options[:host]
+    @port = options[:port]
+
+    @mon = Monitor.new
+    @pending = nil
+    @connect_error_history = []
+
+    @limit = options[:buffer_limit] || BUFFER_LIMIT
+    @log_reconnect_error_threshold = options[:log_reconnect_error_threshold] ||  RECONNECT_WAIT_MAX_COUNT
+
+    if logger = options[:logger]
+      @logger = logger
+    else
+      @logger = ::Logger.new(STDERR)
+      if options[:debug]
+        @logger.level = ::Logger::DEBUG
+      else
+        @logger.level = ::Logger::INFO
+      end
+    end
+
+    @last_error = {}
+
+    begin
+      connect!
+    rescue => e
+      set_last_error(e)
+      @logger.error "Failed to connect fluentd: #{$!}"
+      @logger.error "Connection will be retried."
+    end
+  end
+
+  attr_accessor :limit, :logger, :log_reconnect_error_threshold
+  attr_reader :last_error
+
+  def last_error
+    @last_error[Thread.current.object_id]
+  end
+
+  def post_with_time(tag, map, time)
+    @logger.debug { "event: #{tag} #{map.to_json}" rescue nil }
+    tag = "#{@tag_prefix}.#{tag}" if @tag_prefix
+    write [tag, time.to_i, map]
+  end
+
+  def batch_post_with_time(messages)
+    # convert each message
+    # batch send
+    # if batch > byte max, send in multiple requests
+    #
+    # how to handle errors? immediate fail batch, or skip bad messageS?
+
+    # what if pending is not null here? if we check needs to be syncronized
+
+    # Reorder since #post_with_time takes a different order than #write
+    # NOTE Should we mangle the time? It should be .to_i here
+    payloads = messages.map do |msg| 
+      proper = msg.values_at(0, 2, 1)
+      proper[1] = proper[1].to_i unless proper[1].is_a?(Fixnum) # Convert time
+      prepare_msg proper
+    end
+
+    # Check for 'false' payloads, those are errors, and last message will be set.
+
+    payload = ""
+
+    payloads.each do |data|
+      if payload.bytesize + data.bytesize > @limit
+        raw_write payload
+        payload = ""
+      end
+
+      payload << data
+    end
+
+    raw_write payload if payload.bytesize > 0
+  end
+
+  def close
+    @mon.synchronize {
+      if @pending
+        begin
+          send_data(@pending)
+        rescue => e
+          set_last_error(e)
+          @logger.error("FluentLogger: Can't send logs to #{@host}:#{@port}: #{$!}")
+        end
+      end
+      @con.close if connect?
+      @con = nil
+      @pending = nil
+    }
+    self
+  end
+
+  def connect?
+    !!@con
+  end
+
+  def finalize
+    close
+  end
+
+  private
+  def to_msgpack(msg)
+    begin
+      msg.to_msgpack
+    rescue NoMethodError
+      Yajl::Parser.parse( Yajl::Encoder.encode(msg) ).to_msgpack
+    end
+  end
+
+  def suppress_sec
+    if (sz = @connect_error_history.size) < RECONNECT_WAIT_MAX_COUNT
+      RECONNECT_WAIT * (RECONNECT_WAIT_INCR_RATE ** (sz - 1))
+    else
+      RECONNECT_WAIT_MAX
+    end
+  end
+
+  def write(msg)
+    data = prepare_msg(msg)
+
+    return false if data === false
+
+    raw_write data
+  end
+
+  def prepare_msg(msg)
+    begin
+      to_msgpack(msg)
+    rescue => e
+      set_last_error(e)
+      @logger.error("FluentLogger: Can't convert to msgpack: #{msg.inspect}: #{$!}")
+      return false
+    end
+  end
+
+  def raw_write(data)
+    @mon.synchronize {
+      if @pending
+        @pending << data
+      else
+        @pending = data
+      end
+
+      # suppress reconnection burst
+      if !@connect_error_history.empty? && @pending.bytesize <= @limit
+        if Time.now.to_i - @connect_error_history.last < suppress_sec
+          return false
+        end
+      end
+
+      begin
+        send_data(@pending)
+        @pending = nil
+        true
+      rescue => e
+        set_last_error(e)
+        if @pending.bytesize > @limit
+          @logger.error("FluentLogger: Can't send logs to #{@host}:#{@port}: #{$!}")
+          @pending = nil
+        end
+        @con.close if connect?
+        @con = nil
+        false
+      end
+    }
+  end
+
+  def send_data(data)
+    unless connect?
+      connect!
+    end
+    @con.write data
+    #while true
+    #  puts "sending #{data.length} bytes"
+    #  if data.length > 32*1024
+    #    n = @con.syswrite(data[0..32*1024])
+    #  else
+    #    n = @con.syswrite(data)
+    #  end
+    #  puts "sent #{n}"
+    #  if n >= data.bytesize
+    #    break
+    #  end
+    #  data = data[n..-1]
+    #end
+    true
+  end
+
+  def connect!
+    @con = TCPSocket.new(@host, @port)
+    @con.sync = true
+    @connect_error_history.clear
+    @logged_reconnect_error = false
+  rescue => e
+    @connect_error_history << Time.now.to_i
+    if @connect_error_history.size > RECONNECT_WAIT_MAX_COUNT
+      @connect_error_history.shift
+    end
+
+    if @connect_error_history.size >= @log_reconnect_error_threshold && !@logged_reconnect_error
+      log_reconnect_error
+      @logged_reconnect_error = true
+    end
+
+    raise e
+  end
+
+  def log_reconnect_error
+    @logger.error("FluentLogger: Can't connect to #{@host}:#{@port}(#{@connect_error_history.size} retried): #{$!}")
+  end
+
+  def set_last_error(e)
+    # TODO: Check non GVL env
+    @last_error[Thread.current.object_id] = e
+  end
+end
+
+
+end
+end