fluent-plugin-td 0.10.20 → 0.10.21

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 45aa1dc756f40504543010be6622b6af518fa46c
+  data.tar.gz: 72c08e7060e8e6e1fb3af3144a746f92e78f4cf0
+SHA512:
+  metadata.gz: 3d494ed57200fb989b24db7552a917f690ce5e96b83a8a108e56d03d2d41274c1a2b37ee610494790ea4f21be41cefcbcd783fad74cd1df03fa5fbd84caa079e
+  data.tar.gz: e62a326f85674fe4acdb78f7f9b7df0c27cc45e569b5af171b1754e99204222e8ff0cb286d0a4667ae6c64d83611b4aec742c98a5c95fdc77bf6f3fc5c45b884

data/.travis.yml

@@ -6,6 +6,5 @@ rvm:
 
 gemfile:
  - Gemfile
- - Gemfile.fluentd.lt.0.10.43
 
 script: bundle exec rake test

data/ChangeLog

@@ -1,3 +1,9 @@
+Release 0.10.21 - 2014/07/04
+
+  * Use Yajl for summarizing json
+  * Change key_num_limit to 512
+
+
 Release 0.10.20 - 2014/06/05
 
   * Fix require order bug for td-client-ruby

data/VERSION

@@ -1 +1 @@
-0.10.20
+0.10.21

data/lib/fluent/plugin/out_tditem.rb

@@ -37,7 +37,7 @@ module Fluent
       @key_num_limit = 1024  # Item table default limitation
       @record_size_limit = 32 * 1024 * 1024  # TODO
       @empty_gz_data = TreasureData::API.create_empty_gz_data
-      @user_agent = "fluent-plugin-td-item: 0.10.20".freeze  # TODO: automatic increment version
+      @user_agent = "fluent-plugin-td-item: 0.10.21".freeze  # TODO: automatic increment version
     end
 
     def configure(conf)

data/lib/fluent/plugin/out_tdlog.rb

@@ -1,358 +1,347 @@
 require 'td-client'
 
 module Fluent
+  class TreasureDataLogOutput < BufferedOutput
+    Plugin.register_output('tdlog', self)
 
+    IMPORT_SIZE_LIMIT = 32 * 1024 * 1024
 
-class TreasureDataLogOutput < BufferedOutput
-  Plugin.register_output('tdlog', self)
-
-  IMPORT_SIZE_LIMIT = 32*1024*1024
-
-  class Anonymizer
-    include Configurable
-  end
+    class Anonymizer
+      include Configurable
+    end
 
-  class RawAnonymizer < Anonymizer
-    def anonymize(obj)
-      if obj == nil
-        nil
-      elsif obj.is_a?(String)
-        anonymize_raw obj
-      elsif obj.is_a?(Numeric)
-        anonymize_raw obj.to_s
-      else
-        # boolean, array, map
-        anonymize_raw MessagePack.pack(obj)
+    class RawAnonymizer < Anonymizer
+      def anonymize(obj)
+        if obj.nil?
+          nil
+        elsif obj.is_a?(String)
+          anonymize_raw obj
+        elsif obj.is_a?(Numeric)
+          anonymize_raw obj.to_s
+        else
+          # boolean, array, map
+          anonymize_raw MessagePack.pack(obj)
+        end
       end
     end
-  end
 
-  class MD5Anonymizer < RawAnonymizer
-    def anonymize_raw(raw)
-      Digest::MD5.hexdigest(raw)
+    class MD5Anonymizer < RawAnonymizer
+      def anonymize_raw(raw)
+        Digest::MD5.hexdigest(raw)
+      end
     end
-  end
 
-  class IPXORAnonymizer < RawAnonymizer
-    config_param :xor_key, :string
+    class IPXORAnonymizer < RawAnonymizer
+      config_param :xor_key, :string
 
-    def configure(conf)
-      super
+      def configure(conf)
+        super
 
-      a1, a2, a3, a4 = @xor_key.split('.')
-      @xor_keys = [a1.to_i, a2.to_i, a3.to_i, a4.to_i]
+        a1, a2, a3, a4 = @xor_key.split('.')
+        @xor_keys = [a1.to_i, a2.to_i, a3.to_i, a4.to_i]
 
-      if @xor_keys == [0, 0, 0, 0]
-        raise ConfigError, "'xor_key' must be IPv4 address"
+        if @xor_keys == [0, 0, 0, 0]
+          raise ConfigError, "'xor_key' must be IPv4 address"
+        end
       end
-    end
 
-    def anonymize_raw(raw)
-      m = /\A(\d+)\.(\d+)\.(\d+)\.(\d+)/.match(raw)
-      return nil unless m
+      def anonymize_raw(raw)
+        m = /\A(\d+)\.(\d+)\.(\d+)\.(\d+)/.match(raw)
+        return nil unless m
 
-      k1, k2, k3, k4 = @xor_keys
+        k1, k2, k3, k4 = @xor_keys
 
-      o1 = m[1].to_i ^ k1
-      o2 = m[2].to_i ^ k2
-      o3 = m[3].to_i ^ k3
-      o4 = m[4].to_i ^ k4
+        o1 = m[1].to_i ^ k1
+        o2 = m[2].to_i ^ k2
+        o3 = m[3].to_i ^ k3
+        o4 = m[4].to_i ^ k4
 
-      "#{o1}.#{o2}.#{o3}.#{o4}"
+        "#{o1}.#{o2}.#{o3}.#{o4}"
+      end
     end
-  end
 
-  # To support log_level option since Fluentd v0.10.43
-  unless method_defined?(:log)
-    define_method(:log) { $log }
-  end
+    # To support log_level option since Fluentd v0.10.43
+    unless method_defined?(:log)
+      define_method(:log) { $log }
+    end
 
-  config_param :endpoint, :string, :default => TreasureData::API::NEW_DEFAULT_ENDPOINT
-
-  config_param :connect_timeout, :integer, :default => nil
-  config_param :read_timeout, :integer, :default => nil
-  config_param :send_timeout, :integer, :default => nil
-  config_set_default :buffer_type, 'file'
-  config_set_default :flush_interval, 300
-
-  def initialize
-    require 'fileutils'
-    require 'tempfile'
-    require 'zlib'
-    require 'net/http'
-    require 'json'
-    require 'cgi' # CGI.escape
-    require 'time' # Time#rfc2822
-    require 'digest/md5'
-    require 'stringio'
-    super
-    @tmpdir = nil
-    @apikey = nil
-    @key = nil
-    @key_num_limit = 5120  # TODO
-    @record_size_limit = 32*1024*1024  # TODO
-    @table_list = {}
-    @auto_create_table = true
-    @use_ssl = true
-    @empty_gz_data = create_empty_gz_data
-  end
+    config_param :endpoint, :string, :default => TreasureData::API::NEW_DEFAULT_ENDPOINT
+
+    config_param :connect_timeout, :integer, :default => nil
+    config_param :read_timeout, :integer, :default => nil
+    config_param :send_timeout, :integer, :default => nil
+    config_set_default :buffer_type, 'file'
+    config_set_default :flush_interval, 300
+
+    def initialize
+      require 'fileutils'
+      require 'tempfile'
+      require 'zlib'
+      require 'net/http'
+      require 'json'
+      require 'cgi' # CGI.escape
+      require 'time' # Time#rfc2822
+      require 'digest/md5'
+      require 'stringio'
+      super
+      @tmpdir = nil
+      @apikey = nil
+      @key = nil
+      @key_num_limit = 512  # TODO: Our one-time import has the restriction about the number of record keys.
+      @record_size_limit = 32 * 1024 * 1024  # TODO
+      @table_list = {}
+      @auto_create_table = true
+      @use_ssl = true
+      @empty_gz_data = TreasureData::API.create_empty_gz_data
+    end
 
-  def configure(conf)
-    super
+    def configure(conf)
+      super
 
-    # overwrite default value of buffer_chunk_limit
-    if @buffer.respond_to?(:buffer_chunk_limit=) && !conf['buffer_chunk_limit']
-      @buffer.buffer_chunk_limit = IMPORT_SIZE_LIMIT
-    end
+      # overwrite default value of buffer_chunk_limit
+      if @buffer.respond_to?(:buffer_chunk_limit=) && !conf['buffer_chunk_limit']
+        @buffer.buffer_chunk_limit = IMPORT_SIZE_LIMIT
+      end
 
-    if conf.has_key?('tmpdir')
-      @tmpdir = conf['tmpdir']
-      FileUtils.mkdir_p(@tmpdir)
-    end
+      if conf.has_key?('tmpdir')
+        @tmpdir = conf['tmpdir']
+        FileUtils.mkdir_p(@tmpdir)
+      end
 
-    @apikey = conf['apikey']
-    unless @apikey
-      raise ConfigError, "'apikey' parameter is required on tdlog output"
-    end
+      @apikey = conf['apikey']
+      unless @apikey
+        raise ConfigError, "'apikey' parameter is required on tdlog output"
+      end
 
-    if auto_create_table = conf['auto_create_table']
-      if auto_create_table.empty?
-        @auto_create_table = true
-      else
-        @auto_create_table = Config.bool_value(auto_create_table)
-        if @auto_create_table == nil
-          raise ConfigError, "'true' or 'false' is required for auto_create_table option on tdlog output"
+      if auto_create_table = conf['auto_create_table']
+        if auto_create_table.empty?
+          @auto_create_table = true
+        else
+          @auto_create_table = Config.bool_value(auto_create_table)
+          if @auto_create_table == nil
+            raise ConfigError, "'true' or 'false' is required for auto_create_table option on tdlog output"
+          end
         end
       end
-    end
 
-    if use_ssl = conf['use_ssl']
-      if use_ssl.empty?
-        @use_ssl = true
-      else
-        @use_ssl = Config.bool_value(use_ssl)
-        if @use_ssl == nil
-          raise ConfigError, "'true' or 'false' is required for use_ssl option on tdlog output"
+      if use_ssl = conf['use_ssl']
+        if use_ssl.empty?
+          @use_ssl = true
+        else
+          @use_ssl = Config.bool_value(use_ssl)
+          if @use_ssl == nil
+            raise ConfigError, "'true' or 'false' is required for use_ssl option on tdlog output"
+          end
         end
       end
-    end
 
-    database = conf['database']
-    table = conf['table']
-    if database && table
-      validate_database_and_table_name(database, table, conf)
-      @key = "#{database}.#{table}"
-    end
-
-    @anonymizes = {}
-    conf.elements.select {|e|
-      e.name == 'anonymize'
-    }.each {|e|
-      key = e['key']
-      method = e['method']
-
-      case method
-      when 'md5'
-        scr = MD5Anonymizer.new
-      when 'ip_xor'
-        scr = IPXORAnonymizer.new
-      else
-        raise ConfigError, "Unknown anonymize method: #{method}"
+      database = conf['database']
+      table = conf['table']
+      if database && table
+        validate_database_and_table_name(database, table, conf)
+        @key = "#{database}.#{table}"
       end
 
-      scr.configure(e)
+      @anonymizes = {}
+      conf.elements.select { |e|
+        e.name == 'anonymize'
+      }.each { |e|
+        key = e['key']
+        method = e['method']
+
+        case method
+        when 'md5'
+          scr = MD5Anonymizer.new
+        when 'ip_xor'
+          scr = IPXORAnonymizer.new
+        else
+          raise ConfigError, "Unknown anonymize method: #{method}"
+        end
 
-      @anonymizes[key] = scr
-    }
-    @anonymizes = nil if @anonymizes.empty?
+        scr.configure(e)
 
-    @http_proxy = conf['http_proxy']
-    @user_agent = "fluent-plugin-td: 0.10.20"  # TODO: automatic increment version
-  end
+        @anonymizes[key] = scr
+      }
+      @anonymizes = nil if @anonymizes.empty?
 
-  def start
-    super
+      @http_proxy = conf['http_proxy']
+      @user_agent = "fluent-plugin-td: 0.10.21"  # TODO: automatic increment version
+    end
 
-    client_opts = {
-      :ssl => @use_ssl, :http_proxy => @http_proxy, :user_agent => @user_agent, :endpoint => @endpoint,
-      :connect_timeout => @connect_timeout, :read_timeout => @read_timeout, :send_timeout => @send_timeout
-    }
-    @client = TreasureData::Client.new(@apikey, client_opts)
+    def start
+      super
 
-    if @key
-      if @auto_create_table
-        database, table = @key.split('.',2)
-        ensure_database_and_table(database, table)
-      else
-        check_table_exists(@key)
+      client_opts = {
+        :ssl => @use_ssl, :http_proxy => @http_proxy, :user_agent => @user_agent, :endpoint => @endpoint,
+        :connect_timeout => @connect_timeout, :read_timeout => @read_timeout, :send_timeout => @send_timeout
+      }
+      @client = TreasureData::Client.new(@apikey, client_opts)
+
+      if @key
+        if @auto_create_table
+          database, table = @key.split('.',2)
+          ensure_database_and_table(database, table)
+        else
+          check_table_exists(@key)
+        end
       end
     end
-  end
 
-  def emit(tag, es, chain)
-    if @key
-      key = @key
-    else
-      database, table = tag.split('.')[-2,2]
-      database = TreasureData::API.normalize_database_name(database)
-      table = TreasureData::API.normalize_table_name(table)
-      key = "#{database}.#{table}"
-    end
+    def emit(tag, es, chain)
+      if @key
+        key = @key
+      else
+        database, table = tag.split('.')[-2,2]
+        database = TreasureData::API.normalize_database_name(database)
+        table = TreasureData::API.normalize_table_name(table)
+        key = "#{database}.#{table}"
+      end
 
-    unless @auto_create_table
-      check_table_exists(key)
-    end
+      unless @auto_create_table
+        check_table_exists(key)
+      end
 
-    super(tag, es, chain, key)
-  end
+      super(tag, es, chain, key)
+    end
 
-  def format_stream(tag, es)
-    out = ''
-    off = out.bytesize
-    es.each {|time,record|
-      begin
-        if @anonymizes
-          @anonymizes.each_pair {|key,scr|
-            if value = record[key]
-              record[key] = scr.anonymize(value)
-            end
-          }
+    def format_stream(tag, es)
+      out = ''
+      off = out.bytesize
+      es.each { |time, record|
+        begin
+          if @anonymizes
+            @anonymizes.each_pair { |key, scr|
+              if value = record[key]
+                record[key] = scr.anonymize(value)
+              end
+            }
+          end
+
+          record['time'] = time
+
+          if record.size > @key_num_limit
+            raise "Too many number of keys (#{record.size} keys)"  # TODO include summary of the record
+          end
+        rescue => e
+          # TODO (a) Remove the transaction mechanism of fluentd
+          #      or (b) keep transaction boundaries in in/out_forward.
+          #      This code disables the transaction mechanism (a).
+          log.error "#{e}: #{summarize_record(record)}"
+          log.error_backtrace e.backtrace
+          next
         end
 
-        record['time'] = time
-
-        if record.size > @key_num_limit
-          raise "Too many number of keys (#{record.size} keys)"  # TODO include summary of the record
+        begin
+          record.to_msgpack(out)
+        rescue RangeError
+          TreasureData::API.normalized_msgpack(record, out)
         end
 
-      rescue
-        # TODO (a) Remove the transaction mechanism of fluentd
-        #      or (b) keep transaction boundaries in in/out_forward.
-        #      This code disables the transaction mechanism (a).
-        log.error "#{$!}: #{summarize_record(record)}"
-        log.error_backtrace $!.backtrace
-        next
-      end
-
-      begin
-        record.to_msgpack(out)
-      rescue RangeError
-        TreasureData::API.normalized_msgpack(record, out)
-      end
+        noff = out.bytesize
+        sz = noff - off
+        if sz > @record_size_limit
+          # TODO don't raise error
+          #raise "Size of a record too large (#{sz} bytes)"  # TODO include summary of the record
+          log.warn "Size of a record too large (#{sz} bytes): #{summarize_record(record)}"
+        end
+        off = noff
+      }
+      out
+    end
 
-      noff = out.bytesize
-      sz = noff - off
-      if sz > @record_size_limit
-        # TODO don't raise error
-        #raise "Size of a record too large (#{sz} bytes)"  # TODO include summary of the record
-        log.warn "Size of a record too large (#{sz} bytes): #{summarize_record(record)}"
+    def summarize_record(record)
+      json = Yajl.dump(record)
+      if json.size > 100
+        json[0..97] + "..."
+      else
+        json
       end
-      off = noff
-    }
-    out
-  end
-
-  def summarize_record(record)
-    json = record.to_json
-    if json.size > 100
-      json[0..97]+"..."
-    else
-      json
     end
-  end
 
-  def write(chunk)
-    unique_id = chunk.unique_id
-    database, table = chunk.key.split('.',2)
+    def write(chunk)
+      unique_id = chunk.unique_id
+      database, table = chunk.key.split('.', 2)
 
-    FileUtils.mkdir_p(@tmpdir) unless @tmpdir.nil?
-    f = Tempfile.new("tdlog-", @tmpdir)
-    w = Zlib::GzipWriter.new(f)
+      FileUtils.mkdir_p(@tmpdir) unless @tmpdir.nil?
+      f = Tempfile.new("tdlog-", @tmpdir)
+      w = Zlib::GzipWriter.new(f)
 
-    chunk.write_to(w)
-    w.finish
-    w = nil
+      chunk.write_to(w)
+      w.finish
+      w = nil
 
-    size = f.pos
-    f.pos = 0
-    upload(database, table, f, size, unique_id)
-
-  ensure
-    w.close if w
-    f.close if f
-  end
+      size = f.pos
+      f.pos = 0
+      upload(database, table, f, size, unique_id)
 
-  def create_empty_gz_data
-    io = StringIO.new
-    Zlib::GzipWriter.new(io).close
-    io.string
-  end
+    ensure
+      w.close if w
+      f.close if f
+    end
 
-  def upload(database, table, io, size, unique_id)
-    unique_str = unique_id.unpack('C*').map {|x| "%02x" % x }.join
-    log.trace { "uploading logs to Treasure Data database=#{database} table=#{table} (#{size}bytes)" }
+    def upload(database, table, io, size, unique_id)
+      unique_str = unique_id.unpack('C*').map { |x| "%02x" % x }.join
+      log.trace { "uploading logs to Treasure Data database=#{database} table=#{table} (#{size}bytes)" }
 
-    begin
       begin
-        start = Time.now
-        @client.import(database, table, "msgpack.gz", io, size, unique_str)
-      rescue TreasureData::NotFoundError
-        unless @auto_create_table
-          raise $!
+        begin
+          start = Time.now
+          @client.import(database, table, "msgpack.gz", io, size, unique_str)
+        rescue TreasureData::NotFoundError => e
+          unless @auto_create_table
+            raise e
+          end
+          ensure_database_and_table(database, table)
+          io.pos = 0
+          retry
         end
-        ensure_database_and_table(database, table)
-        io.pos = 0
-        retry
+      rescue => e
+        elapsed = Time.now - start
+        ne = RuntimeError.new("Failed to upload to Treasure Data '#{database}.#{table}' table: #{$!} (#{size} bytes; #{elapsed} seconds)")
+        ne.set_backtrace(e.backtrace)
+        raise ne
       end
-    rescue => e
-      elapsed = Time.now - start
-      ne = RuntimeError.new("Failed to upload to Treasure Data '#{database}.#{table}' table: #{$!} (#{size} bytes; #{elapsed} seconds)")
-      ne.set_backtrace(e.backtrace)
-      raise ne
     end
-  end
 
-  def check_table_exists(key)
-    unless @table_list.has_key?(key)
-      database, table = key.split('.',2)
-      log.debug "checking whether table '#{database}.#{table}' exists on Treasure Data"
-      io = StringIO.new(@empty_gz_data)
-      begin
-        @client.import(database, table, "msgpack.gz", io, io.size)
-        @table_list[key] = true
-      rescue TreasureData::NotFoundError
-        raise "Table #{key.inspect} does not exist on Treasure Data. Use 'td table:create #{database} #{table}' to create it."
-      rescue
-        log.warn "failed to check existence of '#{database}.#{table}' table on Treasure Data", :error=>$!.to_s
-        log.debug_backtrace $!
+    def check_table_exists(key)
+      unless @table_list.has_key?(key)
+        database, table = key.split('.', 2)
+        log.debug "checking whether table '#{database}.#{table}' exists on Treasure Data"
+        io = StringIO.new(@empty_gz_data)
+        begin
+          @client.import(database, table, "msgpack.gz", io, io.size)
+          @table_list[key] = true
+        rescue TreasureData::NotFoundError
+          raise "Table #{key.inspect} does not exist on Treasure Data. Use 'td table:create #{database} #{table}' to create it."
+        rescue => e
+          log.warn "failed to check existence of '#{database}.#{table}' table on Treasure Data", :error => e.to_s
+          log.debug_backtrace e.backtrace
+        end
       end
     end
-  end
 
-  def validate_database_and_table_name(database, table, conf)
-    begin
-      TreasureData::API.validate_database_name(database)
-    rescue => e
-      raise ConfigError, "Invalid database name #{database.inspect}: #{e}: #{conf}"
-    end
-    begin
-      TreasureData::API.validate_table_name(table)
-    rescue => e
-      raise ConfigError, "Invalid table name #{table.inspect}: #{e}: #{conf}"
+    def validate_database_and_table_name(database, table, conf)
+      begin
+        TreasureData::API.validate_database_name(database)
+      rescue => e
+        raise ConfigError, "Invalid database name #{database.inspect}: #{e}: #{conf}"
+      end
+      begin
+        TreasureData::API.validate_table_name(table)
+      rescue => e
+        raise ConfigError, "Invalid table name #{table.inspect}: #{e}: #{conf}"
+      end
     end
-  end
 
-  def ensure_database_and_table(database, table)
-    log.info "Creating table #{database}.#{table} on TreasureData"
-    begin
-      @client.create_log_table(database, table)
-    rescue TreasureData::NotFoundError
-      @client.create_database(database)
-      @client.create_log_table(database, table)
-    rescue TreasureData::AlreadyExistsError
+    def ensure_database_and_table(database, table)
+      log.info "Creating table #{database}.#{table} on TreasureData"
+      begin
+        @client.create_log_table(database, table)
+      rescue TreasureData::NotFoundError
+        @client.create_database(database)
+        @client.create_log_table(database, table)
+      rescue TreasureData::AlreadyExistsError
+      end
     end
   end
 end
-
-
-end

data/test/plugin/test_out_tditem.rb

@@ -57,7 +57,6 @@ class TreasureDataItemOutputTest < Test::Unit::TestCase
 
   def test_emit
     d = create_driver
-
     time, records = stub_seed_values
     stub_td_import_request(stub_request_body(records), d.instance.database, d.instance.table)
 
@@ -69,10 +68,9 @@ class TreasureDataItemOutputTest < Test::Unit::TestCase
     assert_equal('TD1 testkey', @auth_header)
   end
 
-  def test_emit
+  def test_emit_with_endpoint
     d = create_driver(DEFAULT_CONFIG + "endpoint foo.bar.baz")
     opts = {:endpoint => 'foo.bar.baz'}
-
     time, records = stub_seed_values
     stub_td_import_request(stub_request_body(records), d.instance.database, d.instance.table, opts)
 
@@ -83,4 +81,19 @@ class TreasureDataItemOutputTest < Test::Unit::TestCase
 
     assert_equal('TD1 testkey', @auth_header)
   end
+
+  def test_emit_with_too_many_keys
+    d = create_driver(DEFAULT_CONFIG + "endpoint foo.bar.baz")
+    opts = {:endpoint => 'foo.bar.baz'}
+    time, _ = stub_seed_values
+    stub_td_import_request(stub_request_body([]), d.instance.database, d.instance.table, opts)
+
+    d.emit(create_too_many_keys_record, time)
+    d.run
+
+    assert_equal 0, d.emits.size
+    assert d.instance.log.logs.select{ |line|
+      line =~ / \[error\]: Too many number of keys/
+    }.size == 1, "too many keys error is not logged"
+  end
 end

data/test/plugin/test_out_tdlog.rb

@@ -43,7 +43,6 @@ class TreasureDataLogOutputTest < Test::Unit::TestCase
 
   def test_emit
     d = create_driver
-
     time, records = stub_seed_values
     database, table = d.instance.instance_variable_get(:@key).split(".", 2)
     stub_td_table_create_request(database, table)
@@ -71,7 +70,24 @@ class TreasureDataLogOutputTest < Test::Unit::TestCase
     d.run
   end
 
-  # TODO: add normalized_msgpack / key_num_limit / tag split test
+  def test_emit_with_too_many_keys
+    d = create_driver(DEFAULT_CONFIG + "endpoint foo.bar.baz")
+    opts = {:endpoint => 'foo.bar.baz'}
+    time, records = stub_seed_values
+    database, table = d.instance.instance_variable_get(:@key).split(".", 2)
+    stub_td_table_create_request(database, table, opts)
+    stub_td_import_request(stub_request_body([], time), database, table, opts)
+
+    d.emit(create_too_many_keys_record, time)
+    d.run
+
+    assert_equal 0, d.emits.size
+    assert d.instance.log.logs.select{ |line|
+      line =~ / \[error\]: Too many number of keys/
+    }.size == 1, "too many keys error is not logged"
+  end
+
+  # TODO: add normalized_msgpack / tag split test
 
 ## TODO invalid names are normalized
 #  def test_invalid_name

data/test/test_helper.rb

@@ -12,6 +12,12 @@ def e(s)
 end
 
 class Test::Unit::TestCase
+  def create_too_many_keys_record
+    record = {}
+    5012.times { |i| record["k#{i}"] = i }
+    record
+  end
+
   def stub_seed_values
     time = Time.parse("2014-01-01 00:00:00 UTC").to_i
     records = [{"a" => 1}, {"a" => 2}]

metadata

@@ -1,78 +1,69 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-td
 version: !ruby/object:Gem::Version
-  version: 0.10.20
-  prerelease: 
+  version: 0.10.21
 platform: ruby
 authors:
 - Treasure Data, Inc.
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-06-05 00:00:00.000000000 Z
+date: 2014-07-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.10.27
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.10.27
 - !ruby/object:Gem::Dependency
   name: td-client
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.8.61
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.8.61
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.9.2
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.9.2
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.16'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.16'
 description: Treasure Data Cloud Data Service plugin for Fluentd
@@ -81,12 +72,11 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
-- .travis.yml
+- ".gitignore"
+- ".travis.yml"
 - AUTHORS
 - ChangeLog
 - Gemfile
-- Gemfile.fluentd.lt.0.10.43
 - README.rdoc
 - Rakefile
 - VERSION
@@ -100,33 +90,26 @@ files:
 - test/test_helper.rb
 homepage: http://www.treasuredata.com/
 licenses: []
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: 4219119697860923650
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: 4219119697860923650
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.23
+rubygems_version: 2.2.2
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Treasure Data Cloud Data Service plugin for Fluentd
 test_files:
 - test/plugin/test_out_tditem.rb

data/Gemfile.fluentd.lt.0.10.43

@@ -1,4 +0,0 @@
-source "http://rubygems.org"
-
-gem 'fluentd', '= 0.10.42'
-gemspec