td-logger 0.3.11 → 0.3.12

data/ChangeLog

@@ -1,4 +1,12 @@
 
+== 2012-03-31 version 0.3.12
+
+* TreasureDataLogger#close shows a error message if upload failed
+  instead of raising errors
+* TreasureDataLogger#close deletes uploaded queue entry
+* Changed increment storategy of flush_interval from += 10 to ** 2
+
+
 == 2012-03-12 version 0.3.11
 
 * Added TD.event.post_with_time(action, record, time) method

data/VERSION

@@ -1 +1 @@
-0.3.11
+0.3.12

data/lib/td/logger/td_logger.rb

@@ -9,7 +9,7 @@ class TreasureDataLogger < Fluent::Logger::LoggerBase
       obj = allocate
       obj.instance_eval { initialize(*args, &block) }
       dc = DelegateClass(obj.class).new(obj)
-      ObjectSpace.define_finalizer(dc, finalizer(obj))
+      ObjectSpace.define_finalizer(obj, finalizer(obj))
       dc
     end
 
@@ -66,7 +66,7 @@ class TreasureDataLogger < Fluent::Logger::LoggerBase
     @chunk_limit = 8*1024*1024
     @queue_limit = 50
 
-    @flush_interval = 10
+    @flush_interval = 2
     @max_flush_interval = 300
     @retry_wait = 1.0
     @retry_limit = 12
@@ -91,12 +91,24 @@ class TreasureDataLogger < Fluent::Logger::LoggerBase
       }
       @upload_thread.join if @upload_thread
 
-      @map.each {|(db,table),buffer|
-        data = buffer.flush!
-        upload(db, table, data)
+      @queue.reject! {|db,table,data|
+        begin
+          upload(db, table, data)
+          true
+        rescue
+          @logger.error "Failed to upload event logs to Treasure Data, trashed: #{$!}"
+          false
+        end
       }
-      @queue.each {|db,table,data|
-        upload(ddb, data, table)
+      @map.reject! {|(db,table),buffer|
+        data = buffer.flush!
+        begin
+          upload(db, table, data)
+          true
+        rescue
+          @logger.error "Failed to upload event logs to Treasure Data, trashed: #{$!}"
+          false
+        end
       }
     end
   end
@@ -118,18 +130,13 @@ class TreasureDataLogger < Fluent::Logger::LoggerBase
       now = Time.now.to_i
 
       if @next_time <= now || (@flush_now && @error_count == 0)
-        @mutex.unlock
-        begin
-          flushed = try_flush
-        ensure
-          @mutex.lock
-        end
+        flushed = try_flush
         @flush_now = false
       end
 
       if @error_count == 0
         if flushed && @flush_interval < @max_flush_interval
-          @flush_interval = [@flush_interval + 60, @max_flush_interval].min
+          @flush_interval = [@flush_interval ** 2, @max_flush_interval].min
         end
         next_wait = @flush_interval
       else
@@ -173,7 +180,6 @@ class TreasureDataLogger < Fluent::Logger::LoggerBase
 
     def append(data)
       @gz << data
-      map = MessagePack.unpack(data)
     end
 
     def size
@@ -249,6 +255,7 @@ class TreasureDataLogger < Fluent::Logger::LoggerBase
       buffer.append(data)
 
       if buffer.size > @chunk_limit
+        # flush this buffer
         data = buffer.flush!
         @queue << [db, table, data]
         @map.delete(key)
@@ -264,43 +271,55 @@ class TreasureDataLogger < Fluent::Logger::LoggerBase
     true
   end
 
+  # assume @mutex is locked
   def try_flush
-    @mutex.synchronize do
-      if @queue.empty?
-        @map.reject! {|(db,table),buffer|
-          data = buffer.flush!
-          @queue << [db, table, data]
-        }
-      end
+    # force flush small buffers if queue is empty
+    if @queue.empty?
+      @map.reject! {|(db,table),buffer|
+        data = buffer.flush!
+        @queue << [db, table, data]
+      }
+    end
+
+    if @queue.empty?
+      return false
     end
 
     flushed = false
 
-    until @queue.empty?
-      db, table, data = @queue.first
+    @mutex.unlock
+    begin
+      until @queue.empty?
+        db, table, data = @queue.first
 
-      begin
-        upload(db, table, data)
-        @queue.shift
-        @error_count = 0
-        flushed = true
-      rescue
-        if @error_count < @retry_limit
-          @logger.error "Failed to upload event logs to Treasure Data, retrying: #{$!}"
-          @error_count += 1
-        else
-          @logger.error "Failed to upload event logs to Treasure Data, trashed: #{$!}"
-          $!.backtrace.each {|bt|
-            @logger.info bt
-          }
+        begin
+          upload(db, table, data)
+          @queue.shift
           @error_count = 0
-          @queue.clear
+          flushed = true
+
+        rescue
+          if @error_count < @retry_limit
+            @logger.error "Failed to upload event logs to Treasure Data, retrying: #{$!}"
+            @error_count += 1
+          else
+            @logger.error "Failed to upload event logs to Treasure Data, trashed: #{$!}"
+            $!.backtrace.each {|bt|
+              @logger.info bt
+            }
+            @error_count = 0
+            @queue.clear
+          end
+          return nil
+
         end
-        return
       end
+
+    ensure
+      @mutex.lock
     end
 
-    flushed
+    return flushed
   end
 
   def upload(db, table, data)

data/lib/td/logger/version.rb

@@ -1,7 +1,7 @@
 module TreasureData
 module Logger
 
-VERSION = '0.3.11'
+VERSION = '0.3.12'
 
 end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: td-logger
 version: !ruby/object:Gem::Version
-  version: 0.3.11
+  version: 0.3.12
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-03-12 00:00:00.000000000Z
+date: 2012-05-31 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
-  requirement: &70348710334220 !ruby/object:Gem::Requirement
+  requirement: &70152935365260 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: 0.4.4
   type: :runtime
   prerelease: false
-  version_requirements: *70348710334220
+  version_requirements: *70152935365260
 - !ruby/object:Gem::Dependency
   name: td-client
-  requirement: &70348710333700 !ruby/object:Gem::Requirement
+  requirement: &70152935363700 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -32,10 +32,10 @@ dependencies:
         version: 0.8.4
   type: :runtime
   prerelease: false
-  version_requirements: *70348710333700
+  version_requirements: *70152935363700
 - !ruby/object:Gem::Dependency
   name: fluent-logger
-  requirement: &70348710333180 !ruby/object:Gem::Requirement
+  requirement: &70152935337300 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,10 +43,10 @@ dependencies:
         version: 0.4.1
   type: :runtime
   prerelease: false
-  version_requirements: *70348710333180
+  version_requirements: *70152935337300
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &70348710332340 !ruby/object:Gem::Requirement
+  requirement: &70152935327220 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: 0.9.2
   type: :development
   prerelease: false
-  version_requirements: *70348710332340
+  version_requirements: *70152935327220
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70348710331800 !ruby/object:Gem::Requirement
+  requirement: &70152935313920 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,7 +65,7 @@ dependencies:
         version: 2.7.0
   type: :development
   prerelease: false
-  version_requirements: *70348710331800
+  version_requirements: *70152935313920
 description: Treasure Data logging library for Rails
 email: 
 executables: []