td-logger 0.3.9 → 0.3.10

data/ChangeLog

@@ -1,7 +1,12 @@
 
-== 2012-03-05 version 0.3.9
+== 2012-03-05 version 0.3.10
 
 * Fixed 'ConditionVariable' is not defined problem
+* Compress buffer chunk in memory
+
+
+== 2012-03-05 version 0.3.9
+
 * Fixed Event.use feature
 
 

data/VERSION

@@ -1 +1 @@
-0.3.9
+0.3.10

data/lib/td/logger/event.rb

@@ -1,12 +1,6 @@
 module TreasureData
 module Logger
 
-  module Event
-    def self.use(mod)
-      send(:include, mod)
-    end
-  end
-
   module EventPreset
     def action(name, record, uid=TD.event.attribute[:uid])
       unless uid
@@ -37,9 +31,7 @@ module Logger
     end
   end
 
-  Event.use EventPreset
-
-  class EventCollector
+  class Event
     def initialize
       @attribute = {}
     end
@@ -50,11 +42,15 @@ module Logger
       TreasureData::Logger.post(action, @attribute.merge(record))
     end
 
-    include Event
+    def self.use(mod)
+      send(:include, mod)
+    end
   end
 
+  Event.use EventPreset
+
   def self.event
-    Thread.current[:td_event_collector] ||= EventCollector.new
+    Thread.current[:td_event] ||= Event.new
   end
 
 end

data/lib/td/logger/td_logger.rb

@@ -92,10 +92,11 @@ class TreasureDataLogger < Fluent::Logger::LoggerBase
       @upload_thread.join if @upload_thread
 
       @map.each {|(db,table),buffer|
-        upload(db, table, buffer)
+        data = buffer.flush!
+        upload(db, table, data)
       }
-      @queue.each {|tuple|
-        upload(*tuple)
+      @queue.each {|db,table,data|
+        upload(ddb, data, table)
       }
     end
   end
@@ -149,6 +150,46 @@ class TreasureDataLogger < Fluent::Logger::LoggerBase
   end
 
   private
+  MAX_KEY_CARDINALITY = 512
+  WARN_KEY_CARDINALITY = 256
+
+  class Buffer
+    def initialize
+      @key_set = {}
+      @data = StringIO.new
+      @gz = Zlib::GzipWriter.new(@data)
+    end
+
+    def key_set_size
+      @key_set.size
+    end
+
+    def update_key_set(record)
+      record.each_key {|key|
+        @key_set[key] = true
+      }
+      @key_set.size
+    end
+
+    def append(data)
+      @gz << data
+      map = MessagePack.unpack(data)
+    end
+
+    def size
+      @data.size
+    end
+
+    def flush!
+      close
+      @data.string
+    end
+
+    def close
+      @gz.close unless @gz.closed?
+    end
+  end
+
   def to_msgpack(msg)
     begin
       msg.to_msgpack
@@ -186,12 +227,30 @@ class TreasureDataLogger < Fluent::Logger::LoggerBase
         return false
       end
 
-      buffer = (@map[key] ||= '')
+      buffer = (@map[key] ||= Buffer.new)
+
+      record = MessagePack.unpack(data)
+      unless record.is_a?(Hash)
+        @logger.error("TreasureDataLogger: record must be a Hash: #{msg.inspect}")
+        return false
+      end
+
+      before = buffer.key_set_size
+      after = buffer.update_key_set(record)
+      if after > MAX_KEY_CARDINALITY
+        @logger.error("TreasureDataLogger: kind of keys in a buffer exceeds #{MAX_KEY_CARDINALITY}.")
+        @map.delete(key)
+        return false
+      end
+      if before <= WARN_KEY_CARDINALITY && after > WARN_KEY_CARDINALITY
+        @logger.warn("TreasureDataLogger: kind of keys in a buffer exceeds #{WARN_KEY_CARDINALITY} which is too large. please check the schema design.")
+      end
 
-      buffer << data
+      buffer.append(data)
 
       if buffer.size > @chunk_limit
-        @queue << [db, table, buffer]
+        data = buffer.flush!
+        @queue << [db, table, data]
         @map.delete(key)
         @cond.signal
       end
@@ -209,7 +268,8 @@ class TreasureDataLogger < Fluent::Logger::LoggerBase
     @mutex.synchronize do
       if @queue.empty?
         @map.reject! {|(db,table),buffer|
-          @queue << [db, table, buffer]
+          data = buffer.flush!
+          @queue << [db, table, data]
         }
       end
     end
@@ -217,10 +277,10 @@ class TreasureDataLogger < Fluent::Logger::LoggerBase
     flushed = false
 
     until @queue.empty?
-      tuple = @queue.first
+      db, table, data = @queue.first
 
       begin
-        upload(*tuple)
+        upload(db, table, data)
         @queue.shift
         @error_count = 0
         flushed = true
@@ -243,11 +303,9 @@ class TreasureDataLogger < Fluent::Logger::LoggerBase
     flushed
   end
 
-  def upload(db, table, buffer)
+  def upload(db, table, data)
     begin
-      out = StringIO.new
-      Zlib::GzipWriter.wrap(out) {|gz| gz.write buffer }
-      stream = StringIO.new(out.string)
+      stream = StringIO.new(data)
 
       @logger.debug "Uploading event logs to #{db}.#{table} table on Treasure Data (#{stream.size} bytes)"
 

data/lib/td/logger/version.rb

@@ -1,7 +1,7 @@
 module TreasureData
 module Logger
 
-VERSION = '0.3.9'
+VERSION = '0.3.10'
 
 end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: td-logger
 version: !ruby/object:Gem::Version
-  version: 0.3.9
+  version: 0.3.10
   prerelease: 
 platform: ruby
 authors:
@@ -13,7 +13,7 @@ date: 2012-03-05 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
-  requirement: &70209063366040 !ruby/object:Gem::Requirement
+  requirement: &70312343520780 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: 0.4.4
   type: :runtime
   prerelease: false
-  version_requirements: *70209063366040
+  version_requirements: *70312343520780
 - !ruby/object:Gem::Dependency
   name: td-client
-  requirement: &70209063365440 !ruby/object:Gem::Requirement
+  requirement: &70312343518300 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -32,10 +32,10 @@ dependencies:
         version: 0.8.4
   type: :runtime
   prerelease: false
-  version_requirements: *70209063365440
+  version_requirements: *70312343518300
 - !ruby/object:Gem::Dependency
   name: fluent-logger
-  requirement: &70209063364980 !ruby/object:Gem::Requirement
+  requirement: &70312343515480 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,10 +43,10 @@ dependencies:
         version: 0.4.1
   type: :runtime
   prerelease: false
-  version_requirements: *70209063364980
+  version_requirements: *70312343515480
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &70209063364500 !ruby/object:Gem::Requirement
+  requirement: &70312343514500 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: 0.9.2
   type: :development
   prerelease: false
-  version_requirements: *70209063364500
+  version_requirements: *70312343514500
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70209063363940 !ruby/object:Gem::Requirement
+  requirement: &70312343511440 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,7 +65,7 @@ dependencies:
         version: 2.7.0
   type: :development
   prerelease: false
-  version_requirements: *70209063363940
+  version_requirements: *70312343511440
 description: Treasure Data logging library for Rails
 email: 
 executables: []