td-logger 0.2.3 → 0.2.4

data/ChangeLog

@@ -1,4 +1,14 @@
 
+== 2011-09-13 version 0.2.4
+
+* Increase flush_interval from 10 sec to 5 mins gradually
+* Flush buffered logs when Rails stops
+* Adds 'elapsed' column to access logs
+* Use TREASURE_DATA_API_KEY environment variable by default
+* Enable event logger if apikey is available with default database name
+  and access log table name
+
+
 == 2011-09-08 version 0.2.3
 
 * Loggers use STDERR instead of STDOUT

data/lib/td/logger/agent/access_log.rb

@@ -35,10 +35,20 @@ module Agent
       data = {}
       Thread.current['td.access_log'] = data
       env['td.access_log'] = data
+      env['td.access_time'] = Time.now
     end
 
     Middleware.after do |env,result|
       data = env['td.access_log'] || {}
+      access_time = env['td.access_time']
+
+      # add 'elapsed' column
+      if access_time
+        elapsed = Time.now - access_time
+        data['elapsed'] = elapsed
+        # set 'time' column to access time
+        data['time'] = access_time
+      end
 
       ACCESS_LOG_PRESET_ENV_KEYS.each_pair {|key,val|
         data[key] ||= env[val] if env[val]

data/lib/td/logger/agent/rails.rb

@@ -86,14 +86,31 @@ EOF
       require 'yaml'
       require 'erb'
       logger = ::Rails.logger || ::Logger.new(STDERR)
+
+      unless File.exist?(CONFIG_PATH)
+        apikey = ENV['TREASURE_DATA_API_KEY'] || ENV['TD_API_KEY']
+        unless apikey
+          logger.warn "TREASURE_DATA_API_KEY environment variable is not set"
+          logger.warn "#{CONFIG_PATH} does not exist."
+          logger.warn "Disabling Treasure Data logger."
+          return
+        end
+        return Config.new({
+          'apikey' => apikey,
+          'database' => ENV['TREASURE_DATA_DB'] || "rails_#{rails_env}",
+          'access_log_table' => ENV['TREASURE_DATA_TABLE'] || 'web_access',
+          'auto_create_table' => true
+        })
+      end
+
       begin
         src = File.read("#{rails_root}/#{CONFIG_PATH}")
         yaml = ERB.new(src).result
         env_conf = YAML.load(yaml)
       rescue
-        logger.warn "Can't load #{CONFIG_PATH} file."
-        logger.warn "  #{$!}"
-        logger.warn "Put the following file:"
+        logger.warn "Can't load #{CONFIG_PATH} file: #{$!}"
+        logger.warn "Disabling Treasure Data logger."
+        logger.warn "Example:"
         logger.warn CONFIG_SAMPLE
         return
       end

data/lib/td/logger/tdlog.rb

@@ -2,9 +2,27 @@
 module TreasureData
 module Logger
 
-# TODO shutdown handler  (deadlock)
 
 class TreasureDataLogger < Fluent::Logger::LoggerBase
+  module Finalizable
+    require 'delegate'
+    def new(*args, &block)
+      obj = allocate
+      obj.instance_eval { initialize(*args, &block) }
+      dc = DelegateClass(obj.class).new(obj)
+      ObjectSpace.define_finalizer(dc, finalizer(obj))
+      dc
+    end
+
+    def finalizer(obj)
+      fin = obj.method(:finalize)
+      proc {|id|
+        fin.call
+      }
+    end
+  end
+  extend Finalizable
+
   def initialize(apikey, tag, auto_create_table)
     require 'thread'
     require 'stringio'
@@ -18,7 +36,9 @@ class TreasureDataLogger < Fluent::Logger::LoggerBase
 
     @tag = tag
     @auto_create_table = auto_create_table
+
     @logger = ::Logger.new(STDERR)
+    @logger.level = ::Logger::INFO
 
     @client = TreasureData::Client.new(apikey)
 
@@ -28,7 +48,8 @@ class TreasureDataLogger < Fluent::Logger::LoggerBase
     @queue = []
 
     @chunk_limit = 8*1024*1024
-    @flush_interval = 300
+    @flush_interval = 10
+    @max_flush_interval = 300
     @retry_wait = 1.0
     @retry_limit = 8
 
@@ -41,11 +62,21 @@ class TreasureDataLogger < Fluent::Logger::LoggerBase
   attr_accessor :logger
 
   def close
-    @finish = true
-    @mutex.synchronize {
-      @flush_now = true
-      @cond.signal
-    }
+    unless @finish
+      @finish = true
+      @mutex.synchronize {
+        @flush_now = true
+        @cond.signal
+      }
+      @upload_thread.join
+
+      @map.each {|(db,table),buffer|
+        upload(db, table, buffer)
+      }
+      @queue.each {|tuple|
+        upload(*tuple)
+      }
+    end
   end
 
   def post(tag, record)
@@ -70,29 +101,32 @@ class TreasureDataLogger < Fluent::Logger::LoggerBase
   end
 
   def upload_main
-  @mutex.lock
-  until @finish
-    now = Time.now.to_i
+    @mutex.lock
+    until @finish
+      now = Time.now.to_i
+
+      if @next_time <= now || (@flush_now && @error_count == 0)
+        @mutex.unlock
+        begin
+          flushed = try_flush
+        ensure
+          @mutex.lock
+        end
+        @flush_now = false
+      end
 
-    if @next_time <= now || (@flush_now && @error_count == 0)
-      @mutex.unlock
-      begin
-        try_flush
-      ensure
-        @mutex.lock
+      if @error_count == 0
+        if flushed && @flush_interval < @max_flush_interval
+          @flush_interval = [@flush_interval + 60, @max_flush_interval].min
+        end
+        next_wait = @flush_interval
+      else
+        next_wait = @retry_wait * (2 ** (@error_count-1))
       end
-      @flush_now = false
-    end
+      @next_time = next_wait + now
 
-    if @error_count == 0
-      next_wait = @flush_interval
-    else
-      next_wait = @retry_wait * (2 ** (@error_count-1))
+      cond_wait(next_wait)
     end
-    @next_time = next_wait + now
-
-    cond_wait(next_wait)
-  end
 
   rescue
     @logger.error "Unexpected error: #{$!}"
@@ -113,6 +147,8 @@ class TreasureDataLogger < Fluent::Logger::LoggerBase
       end
     end
 
+    flushed = false
+
     until @queue.empty?
       tuple = @queue.first
 
@@ -120,12 +156,13 @@ class TreasureDataLogger < Fluent::Logger::LoggerBase
         upload(*tuple)
         @queue.shift
         @error_count = 0
+        flushed = true
       rescue
         if @error_count < @retry_limit
-          @logger.error "Failed to upload logs to Treasure Data, retrying: #{$!}"
+          @logger.error "Failed to import logs to Treasure Data, retrying: #{$!}"
           @error_count += 1
         else
-          @logger.error "Failed to upload logs to Treasure Data, trashed: #{$!}"
+          @logger.error "Failed to import logs to Treasure Data, trashed: #{$!}"
           $!.backtrace.each {|bt|
             @logger.info bt
           }
@@ -135,14 +172,17 @@ class TreasureDataLogger < Fluent::Logger::LoggerBase
         return
       end
     end
+
+    flushed
   end
 
   def upload(db, table, buffer)
-    out = StringIO.new
-    Zlib::GzipWriter.wrap(out) {|gz| gz.write buffer }
-    stream = StringIO.new(out.string)
-
+    @logger.debug "Importing logs to #{db}.#{table} table on TreasureData"
     begin
+      out = StringIO.new
+      Zlib::GzipWriter.wrap(out) {|gz| gz.write buffer }
+      stream = StringIO.new(out.string)
+
       @client.import(db, table, "msgpack.gz", stream, stream.size)
     rescue TreasureData::NotFoundError
       unless @auto_create_table
@@ -159,8 +199,8 @@ class TreasureDataLogger < Fluent::Logger::LoggerBase
     end
   end
 
-  def e(s)
-    CGI.escape(s.to_s)
+  def finalize
+    close
   end
 
   if ConditionVariable.new.method(:wait).arity == 1

data/lib/td/logger/version.rb

@@ -1,7 +1,7 @@
 module TreasureData
 module Logger
 
-VERSION = '0.2.3'
+VERSION = '0.2.4'
 
 end
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: td-logger
 version: !ruby/object:Gem::Version 
-  hash: 17
+  hash: 31
   prerelease: false
   segments: 
   - 0
   - 2
-  - 3
-  version: 0.2.3
+  - 4
+  version: 0.2.4
 platform: ruby
 authors: 
 - Sadayuki Furuhashi
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-09-09 00:00:00 +09:00
+date: 2011-09-13 00:00:00 +09:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency