fluent-plugin-td 0.10.17 → 0.10.18

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 04dd7b7e2f098cf317a600be1125138cc6d9a440
+  data.tar.gz: 3b211b46e282e9e94189dcc4a388223b45bbc72d
+SHA512:
+  metadata.gz: 3c65e15dcaf7ab967da39f5479088d30a22c37de275e2e50a6fb17577e3eb6202167f0bec453eb5729cd7eab745d3c2a926533aef020cb4093ee779ef3e4e77e
+  data.tar.gz: dc56feaad2c8a2c554a03fc95a64cf8e9dd84941101efd44f43b5af5258da74f3308af54687ad42ed936359770ac5b8b9ee7a7feab4dedc387621dabfec8fe3b

data/.travis.yml

@@ -0,0 +1,11 @@
+rvm:
+ - 1.9.2
+ - 1.9.3
+ - 2.0.0
+ - 2.1.0
+
+gemfile:
+ - Gemfile
+ - Gemfile.fluentd.lt.0.10.43
+
+script: bundle exec rake test

data/AUTHORS

@@ -1 +1,2 @@
 FURUHASHI Sadayuki <frsyuki _at_ gmail.com>
+MASAHIRO Nakagawa <repeatedly _at_ gmail.com>

data/ChangeLog

@@ -1,3 +1,12 @@
+Release 0.10.18 - 2014/03/31
+
+  * use_ssl is true by default
+  * Support log_level option
+  * Add endpoint option to set treasuredata.com for the future
+  * Add experimental out_tditem plugin for item table
+  * Update td-client to v0.8.58
+
+
 Release 0.10.17 - 2013/12/05
 
   * Change table create / check mechanizm

data/Gemfile.fluentd.lt.0.10.43

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+gem 'fluentd', '= 0.10.42'
+gemspec

data/README.rdoc

@@ -1,4 +1,6 @@
-= Treasure Data output plugin for Fluent
+= Treasure Data output plugin for Fluentd
+
+This plugin is used to upload logs to Treasure Data using Treasure Data REST APIs.
 
 == Copyright
 

data/Rakefile

@@ -6,9 +6,8 @@ require 'rake/testtask'
 
 Rake::TestTask.new(:test) do |test|
   test.libs << 'lib' << 'test'
-  test.test_files = FileList['test/*.rb']
+  test.test_files = FileList['test/**/test_*.rb']
   test.verbose = true
 end
 
 task :default => [:build]
-

data/VERSION

@@ -1 +1 @@
-0.10.17
+0.10.18

data/fluent-plugin-td.gemspec

@@ -17,6 +17,7 @@ Gem::Specification.new do |gem|
   gem.require_paths = ['lib']
 
   gem.add_dependency "fluentd", "~> 0.10.27"
-  gem.add_dependency "td-client", "~> 0.8.56"
+  gem.add_dependency "td-client", "~> 0.8.58"
   gem.add_development_dependency "rake", ">= 0.9.2"
+  gem.add_development_dependency "webmock", "~> 1.16"
 end

data/lib/fluent/plugin/out_tditem.rb

@@ -0,0 +1,105 @@
+module Fluent
+  class TreasureDataItemOutput < BufferedOutput
+    Plugin.register_output('tditem', self)
+
+    require_relative 'td_plugin_util'
+    include TDPluginUtil
+
+    IMPORT_SIZE_LIMIT = 32 * 1024 * 1024
+
+    # To support log_level option since Fluentd v0.10.43
+    unless method_defined?(:log)
+      define_method(:log) { $log }
+    end
+
+    config_param :apikey, :string
+    config_param :database, :string
+    config_param :table, :string
+    config_param :tmpdir, :string, :default => nil
+    #config_param :auto_create_table, :bool, :default => true # TODO: implement if user wants this feature
+
+    config_param :endpoint, :string, :default => nil
+    config_param :use_ssl, :bool, :default => true
+    config_param :http_proxy, :string, :default => nil
+    config_param :connect_timeout, :integer, :default => nil
+    config_param :read_timeout, :integer, :default => nil
+    config_param :send_timeout, :integer, :default => nil
+
+    def initialize
+      super
+
+      @auto_create_table = false
+      @tmpdir_prefix = 'tditem-'.freeze
+      @key_num_limit = 1024  # Item table default limitation
+      @record_size_limit = 32 * 1024 * 1024  # TODO
+      @empty_gz_data = TreasureData::API.create_empty_gz_data
+      @user_agent = "fluent-plugin-td-item: 0.10.18".freeze  # TODO: automatic increment version
+
+      # overwrite default configurations
+      @buffer_type = 'file'
+      @flush_interval = 300
+    end
+
+    def configure(conf)
+      super
+
+      # overwrite default value of buffer_chunk_limit
+      if @buffer.respond_to?(:buffer_chunk_limit=) && !conf.has_key?('buffer_chunk_limit')
+        @buffer.buffer_chunk_limit = IMPORT_SIZE_LIMIT
+      end
+
+      validate_database_and_table_name(@database, @table, conf)
+      @key = "#{@database}.#{@table}".freeze
+      @use_ssl = parse_bool_parameter(@use_ssl) if @use_ssl.instance_of?(String)
+      FileUtils.mkdir_p(@tmpdir) unless @tmpdir.nil?
+
+      if @endpoint.nil?
+        $log.warn "tditem plugin will change the API endpoint from api.treasure-data.com to api.treasuredata.com"
+        $log.warn "If want to keep api.treasure-data.com, please set 'endpoint api.treasure-data.com' in tditem configuration"
+      end
+    end
+
+    def start
+      super
+
+      client_opts = {
+        :ssl => @use_ssl, :http_proxy => @http_proxy, :user_agent => @user_agent, :endpoint => @endpoint,
+        :connect_timeout => @connect_timeout, :read_timeout => @read_timeout, :send_timeout => @send_timeout
+      }
+      @client = TreasureData::Client.new(@apikey, client_opts)
+
+      check_table_existence(@database, @table)
+    end
+
+    def emit(tag, es, chain)
+      super(tag, es, chain, @key)
+    end
+
+    def format_stream(tag, es)
+      out = ''
+      off = out.bytesize
+      es.each { |time, record|
+        if record.size > @key_num_limit
+          log.error "Too many number of keys (#{record.size} keys)" # TODO include summary of the record
+          next
+        end
+
+        begin
+          record.to_msgpack(out)
+        rescue RangeError
+          TreasureData::API.normalized_msgpack(record, out)
+        end
+
+        noff = out.bytesize
+        sz = noff - off
+        if sz > @record_size_limit
+          # TODO don't raise error
+          #raise "Size of a record too large (#{sz} bytes)"  # TODO include summary of the record
+          log.warn "Size of a record too large (#{sz} bytes): #{summarize_record(record)}"
+        end
+        off = noff
+      }
+      out
+    end
+  end
+end

data/lib/fluent/plugin/out_tdlog.rb

@@ -60,6 +60,13 @@ class TreasureDataLogOutput < BufferedOutput
     end
   end
 
+  # To support log_level option since Fluentd v0.10.43
+  unless method_defined?(:log)
+    define_method(:log) { $log }
+  end
+
+  config_param :endpoint, :string, :default => nil
+
   config_param :connect_timeout, :integer, :default => nil
   config_param :read_timeout, :integer, :default => nil
   config_param :send_timeout, :integer, :default => nil
@@ -83,7 +90,7 @@ class TreasureDataLogOutput < BufferedOutput
     @record_size_limit = 32*1024*1024  # TODO
     @table_list = {}
     @auto_create_table = true
-    @use_ssl = false
+    @use_ssl = true
     @buffer_type = 'file'  # overwrite default buffer_type
     @flush_interval = 300  # overwrite default flush_interval to 5mins
     @empty_gz_data = create_empty_gz_data
@@ -159,14 +166,22 @@ class TreasureDataLogOutput < BufferedOutput
     @anonymizes = nil if @anonymizes.empty?
 
     @http_proxy = conf['http_proxy']
-    @user_agent = "fluent-plugin-td: 0.10.17"  # TODO: automatic increment version
+    @user_agent = "fluent-plugin-td: 0.10.18"  # TODO: automatic increment version
+
+    if @endpoint.nil?
+      $log.warn "tdlog plugin will change the API endpoint from api.treasure-data.com to api.treasuredata.com"
+      $log.warn "If want to keep api.treasure-data.com, please set 'endpoint api.treasure-data.com' in tdlog configuration"
+    end
   end
 
   def start
     super
 
-    @client = TreasureData::Client.new(@apikey, :ssl => @use_ssl, :http_proxy => @http_proxy, :user_agent => @user_agent,
-      :connect_timeout => @connect_timeout, :read_timeout => @read_timeout, :send_timeout => @send_timeout)
+    client_opts = {
+      :ssl => @use_ssl, :http_proxy => @http_proxy, :user_agent => @user_agent, :endpoint => @endpoint,
+      :connect_timeout => @connect_timeout, :read_timeout => @read_timeout, :send_timeout => @send_timeout
+    }
+    @client = TreasureData::Client.new(@apikey, client_opts)
 
     if @key
       if @auto_create_table
@@ -218,8 +233,8 @@ class TreasureDataLogOutput < BufferedOutput
         # TODO (a) Remove the transaction mechanism of fluentd
         #      or (b) keep transaction boundaries in in/out_forward.
         #      This code disables the transaction mechanism (a).
-        $log.error "#{$!}: #{summarize_record(record)}"
-        $log.error_backtrace $!.backtrace
+        log.error "#{$!}: #{summarize_record(record)}"
+        log.error_backtrace $!.backtrace
         next
       end
 
@@ -234,7 +249,7 @@ class TreasureDataLogOutput < BufferedOutput
       if sz > @record_size_limit
         # TODO don't raise error
         #raise "Size of a record too large (#{sz} bytes)"  # TODO include summary of the record
-        $log.warn "Size of a record too large (#{sz} bytes): #{summarize_record(record)}"
+        log.warn "Size of a record too large (#{sz} bytes): #{summarize_record(record)}"
       end
       off = noff
     }
@@ -279,7 +294,7 @@ class TreasureDataLogOutput < BufferedOutput
 
   def upload(database, table, io, size, unique_id)
     unique_str = unique_id.unpack('C*').map {|x| "%02x" % x }.join
-    $log.trace { "uploading logs to Treasure Data database=#{database} table=#{table} (#{size}bytes)" }
+    log.trace { "uploading logs to Treasure Data database=#{database} table=#{table} (#{size}bytes)" }
 
     begin
       begin
@@ -295,7 +310,7 @@ class TreasureDataLogOutput < BufferedOutput
       end
     rescue => e
       elapsed = Time.now - start
-      ne = RuntimeError.new("Failed to upload to TreasureData: #{$!} (#{size} bytes; #{elapsed} seconds)")
+      ne = RuntimeError.new("Failed to upload to Treasure Data '#{database}.#{table}' table: #{$!} (#{size} bytes; #{elapsed} seconds)")
       ne.set_backtrace(e.backtrace)
       raise ne
     end
@@ -304,7 +319,7 @@ class TreasureDataLogOutput < BufferedOutput
   def check_table_exists(key)
     unless @table_list.has_key?(key)
       database, table = key.split('.',2)
-      $log.debug "checking whether table '#{database}.#{table}' exists on Treasure Data"
+      log.debug "checking whether table '#{database}.#{table}' exists on Treasure Data"
       io = StringIO.new(@empty_gz_data)
       begin
         @client.import(database, table, "msgpack.gz", io, io.size)
@@ -312,8 +327,8 @@ class TreasureDataLogOutput < BufferedOutput
       rescue TreasureData::NotFoundError
         raise "Table #{key.inspect} does not exist on Treasure Data. Use 'td table:create #{database} #{table}' to create it."
       rescue
-        $log.warn "failed to check table existence on Treasure Data", :error=>$!.to_s
-        $log.debug_backtrace $!
+        log.warn "failed to check existence of '#{database}.#{table}' table on Treasure Data", :error=>$!.to_s
+        log.debug_backtrace $!
       end
     end
   end
@@ -332,7 +347,7 @@ class TreasureDataLogOutput < BufferedOutput
   end
 
   def ensure_database_and_table(database, table)
-    $log.info "Creating table #{database}.#{table} on TreasureData"
+    log.info "Creating table #{database}.#{table} on TreasureData"
     begin
       @client.create_log_table(database, table)
     rescue TreasureData::NotFoundError

data/lib/fluent/plugin/td_plugin_util.rb

@@ -0,0 +1,107 @@
+module Fluent
+  module TDPluginUtil
+    require 'fileutils'
+    require 'stringio'
+    require 'tempfile'
+    require 'zlib'
+    require 'td-client'
+
+    def validate_database_and_table_name(database, table, conf)
+      begin
+        TreasureData::API.validate_database_name(database)
+      rescue => e
+        raise ConfigError, "Invalid database name #{database.inspect}: #{e}: #{conf}"
+      end
+      begin
+        TreasureData::API.validate_table_name(table)
+      rescue => e
+        raise ConfigError, "Invalid table name #{table.inspect}: #{e}: #{conf}"
+      end
+    end
+
+    def parse_bool_parameter(param)
+      if param.empty?
+        true
+      else
+        param = Config.bool_value(param)
+        raise ConfigError, "'true' or 'false' is required for #{key} option on tdlog output" if param.nil?
+        param
+      end
+    end
+
+    def summarize_record(record)
+      json = record.to_json
+      if json.size > 100
+        json[0..97] + "..."
+      else
+        json
+      end
+    end
+
+    def check_table_existence(database, table)
+      @table_list ||= {}
+      key = "#{database}.#{table}"
+      unless @table_list.has_key?(key)
+        log.debug "checking whether table '#{key}' exists on Treasure Data"
+        io = StringIO.new(@empty_gz_data)
+        begin
+          # here doesn't check whether target table is item table or not because import-only user can't read the table status.
+          # So I use empty import request to check table existence.
+          @client.import(database, table, "msgpack.gz", io, io.size)
+          @table_list[key] = true
+        rescue TreasureData::NotFoundError
+          args = self.class == TreasureDataItemOutput ? ' -t item' : ''
+          raise "Table #{key.inspect} does not exist on Treasure Data. Use 'td table:create #{database} #{table}#{args}' to create it."
+        rescue => e
+          log.warn "failed to check table existence on Treasure Data", :error => e.to_s
+          log.debug_backtrace e
+        end
+      end
+    end
+
+    def write(chunk)
+      unique_id = chunk.unique_id
+      database, table = chunk.key.split('.', 2)
+
+      FileUtils.mkdir_p(@tmpdir) unless @tmpdir.nil?
+      f = Tempfile.new(@tmpdir_prefix, @tmpdir)
+      w = Zlib::GzipWriter.new(f)
+
+      chunk.write_to(w)
+      w.finish
+      w = nil
+
+      size = f.pos
+      f.pos = 0
+      upload(database, table, f, size, unique_id)
+    ensure
+      w.close if w
+      f.close if f
+    end
+
+    # assume @client and @auto_create_table variable exist
+    def upload(database, table, io, size, unique_id)
+      unique_str = unique_id.unpack('C*').map {|x| "%02x" % x }.join
+      log.trace { "uploading logs to Treasure Data database=#{database} table=#{table} (#{size}bytes)" }
+
+      begin
+        begin
+          start = Time.now
+          @client.import(database, table, "msgpack.gz", io, size, unique_str)
+        rescue TreasureData::NotFoundError => e
+          unless @auto_create_table
+            raise e
+          end
+          ensure_database_and_table(database, table)
+          io.pos = 0
+          retry
+        end
+      rescue => e
+        elapsed = Time.now - start
+        ne = RuntimeError.new("Failed to upload to TreasureData: #{e} (#{size} bytes; #{elapsed} seconds)")
+        ne.set_backtrace(e.backtrace)
+        raise ne
+      end
+    end
+  end
+end

data/test/plugin/test_out_tditem.rb

@@ -0,0 +1,84 @@
+require 'test_helper'
+require 'fluent/plugin/out_tditem'
+
+class TreasureDataItemOutputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  DEFAULT_CONFIG = %[
+    database test
+    table table
+  ]
+
+  def create_driver(conf = DEFAULT_CONFIG)
+    config = %[
+      apikey testkey
+      buffer_type memory
+    ] + conf
+
+    Fluent::Test::BufferedOutputTestDriver.new(Fluent::TreasureDataItemOutput) do 
+      def write(chunk)
+        # TestDriver doesn't call acutual Output#emit so set key to get database and table in this place.
+        chunk.instance_variable_set(:@key, @key)
+        super(chunk)
+      end
+    end.configure(config)
+  end
+
+  def test_configure
+    d = create_driver
+
+    assert_equal(d.instance.apikey, 'testkey')
+    assert_equal(d.instance.database, 'test')
+    assert_equal(d.instance.table, 'table')
+    assert_equal(d.instance.use_ssl, true)
+  end
+
+  def test_configure_with_invalid_database
+    assert_raise(Fluent::ConfigError) {
+      create_driver(%[
+        database a
+        table table
+      ])
+    }
+  end
+
+  def test_configure_with_invalid_table
+    assert_raise(Fluent::ConfigError) {
+      create_driver(%[
+        database test
+        table 1
+      ])
+    }
+  end
+
+  def test_emit
+    d = create_driver
+
+    time, records = stub_seed_values
+    stub_td_import_request(stub_request_body(records), d.instance.database, d.instance.table)
+
+    records.each { |record|
+      d.emit(record, time)
+    }
+    d.run
+
+    assert_equal(@auth_header, 'TD1 testkey')
+  end
+
+  def test_emit
+    d = create_driver(DEFAULT_CONFIG + "endpoint foo.bar.baz")
+    opts = {:endpoint => 'foo.bar.baz'}
+
+    time, records = stub_seed_values
+    stub_td_import_request(stub_request_body(records), d.instance.database, d.instance.table, opts)
+
+    records.each { |record|
+      d.emit(record, time)
+    }
+    d.run
+
+    assert_equal(@auth_header, 'TD1 testkey')
+  end
+end

data/test/plugin/test_out_tdlog.rb

@@ -0,0 +1,106 @@
+require 'fluent/test'
+require 'fluent/plugin/out_tdlog'
+
+class TreasureDataLogOutputTest < Test::Unit::TestCase
+  # BufferedOutputTestDriver uses module_eval, not inheritance.
+  # This DummyOutput is for testing actual write method with webmock
+  class TreasureDataLogDummyOutput < Fluent::TreasureDataLogOutput
+  end
+
+  def setup
+    Fluent::Test.setup
+  end
+
+  TMP_DIR = File.dirname(__FILE__) + "/tmp"
+
+  DEFAULT_CONFIG = %[
+    database test
+    table table
+  ]
+
+  def create_driver(conf = DEFAULT_CONFIG)
+    config = %[
+      apikey testkey
+      buffer_path #{TMP_DIR}/buffer
+    ] + conf
+
+    Fluent::Test::BufferedOutputTestDriver.new(TreasureDataLogDummyOutput) do
+      def write(chunk)
+        chunk.instance_variable_set(:@key, @key)
+        super(chunk)
+      end
+    end.configure(config)
+  end
+
+  def test_configure
+    d = create_driver
+
+    {:@apikey => 'testkey', :@use_ssl => true, :@auto_create_table => true,
+     :@buffer_type => 'file', :@flush_interval => 300}.each { |k, v|
+      assert_equal(d.instance.instance_variable_get(k), v)
+    }
+  end
+
+  def test_emit
+    d = create_driver
+
+    time, records = stub_seed_values
+    database, table = d.instance.instance_variable_get(:@key).split(".", 2)
+    stub_td_table_create_request(database, table)
+    stub_td_import_request(stub_request_body(records, time), database, table)
+
+    records.each { |record|
+      d.emit(record, time)
+    }
+    d.run
+
+    assert_equal(@auth_header, 'TD1 testkey')
+  end
+
+  def test_emit_with_endpoint
+    d = create_driver(DEFAULT_CONFIG + "endpoint foo.bar.baz")
+    opts = {:endpoint => 'foo.bar.baz'}
+    time, records = stub_seed_values
+    database, table = d.instance.instance_variable_get(:@key).split(".", 2)
+    stub_td_table_create_request(database, table, opts)
+    stub_td_import_request(stub_request_body(records, time), database, table, opts)
+
+    records.each { |record|
+      d.emit(record, time)
+    }
+    d.run
+  end
+
+  # TODO: add normalized_msgpack / key_num_limit / tag split test
+
+## TODO invalid names are normalized
+#  def test_invalid_name
+#    d = create_driver
+#    d.instance.start
+#
+#    es = Fluent::OneEventStream.new(Time.now.to_i, {})
+#    chain = Fluent::NullOutputChain.instance
+#    assert_raise(RuntimeError) do
+#      d.instance.emit("test.invalid-name", es, chain)
+#    end
+#    assert_raise(RuntimeError) do
+#      d.instance.emit("empty", es, chain)
+#    end
+#    assert_raise(RuntimeError) do
+#      d.instance.emit("", es, chain)
+#    end
+#  end
+
+## TODO invalid data is ignored
+#  def test_invalid_data
+#    d = create_driver
+#    d.instance.start
+#
+#    es = Fluent::OneEventStream.new(Time.now.to_i, "invalid")
+#    chain = Fluent::NullOutputChain.instance
+#    assert_nothing_raised do
+#      d.instance.emit("test.name", es, chain)
+#    end
+#  end
+end
+

data/test/test_helper.rb

@@ -0,0 +1,71 @@
+require 'json'
+require 'msgpack'
+require 'fluent/test'
+require 'webmock/test_unit'
+require 'stringio'
+require 'td-client'
+require 'zlib'
+
+def e(s)
+  require 'cgi'
+  CGI.escape(s.to_s)
+end
+
+class Test::Unit::TestCase
+  def stub_seed_values
+    time = Time.parse("2014-01-01 00:00:00 UTC").to_i
+    records = [{"a" => 1}, {"a" => 2}]
+    return time, records
+  end
+
+  def stub_request_body(records, time = nil)
+    out = ''
+    records.each { |record|
+      r = record.dup
+      if time
+        r['time'] = time
+      end
+      r.to_msgpack(out)
+    }
+
+    io = StringIO.new
+    gz = Zlib::GzipWriter.new(io)
+    FileUtils.copy_stream(StringIO.new(out), gz)
+    gz.finish
+    io.string
+  end
+
+  def stub_gzip_unwrap(body)
+    io = StringIO.new(body)
+    gz = Zlib::GzipReader.new(io)
+    gz.read
+  end
+
+  def stub_td_table_create_request(database, table, opts = {})
+    opts[:use_ssl] = true unless opts.has_key?(:use_ssl)
+    schema = opts[:use_ssl] ? 'https' : 'http'
+    response = {"database" => database, "table" => table}.to_json
+    endpoint = opts[:endpoint] ? opts[:endpoint] : TreasureData::API::DEFAULT_ENDPOINT
+
+    url = "#{schema}://#{endpoint}/v3/table/create/#{e(database)}/#{e(table)}/log"
+    stub_request(:post, url).to_return(:status => 200, :body => response)
+  end
+
+  def stub_td_import_request(body, db, table, opts = {})
+    opts[:use_ssl] = true unless opts.has_key?(:use_ssl)
+    format = opts[:format] || 'msgpack.gz'
+    schema = opts[:use_ssl] ? 'https' : 'http'
+    response = {"database" => db, "table" => table, "elapsed_time" => 0}.to_json
+    endpoint = opts[:endpoint] ? opts[:endpoint] : TreasureData::API::DEFAULT_IMPORT_ENDPOINT
+
+    # for check_table_existence
+    url_with_empty = "#{schema}://#{endpoint}//v3/table/import/#{e(db)}/#{e(table)}/#{format}"
+    stub_request(:put, url_with_empty).to_return(:status => 200, :body => response)
+
+    url_with_unique = Regexp.compile("#{schema}://#{endpoint}//v3/table/import_with_id/#{e(db)}/#{e(table)}/.*/#{format}")
+    stub_request(:put, url_with_unique).with(:headers => {'Content-Type' => 'application/octet-stream'}) { |req|
+      @auth_header = req.headers["Authorization"]
+      stub_gzip_unwrap(req.body) == stub_gzip_unwrap(body)
+    }.to_return(:status => 200, :body => response)
+  end
+end

metadata

@@ -1,20 +1,18 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-td
 version: !ruby/object:Gem::Version
-  version: 0.10.17
-  prerelease: 
+  version: 0.10.18
 platform: ruby
 authors:
 - Treasure Data, Inc.
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-12-05 00:00:00.000000000 Z
+date: 2014-03-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -22,7 +20,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -30,35 +27,45 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: td-client
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.8.56
+        version: 0.8.58
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.8.56
+        version: 0.8.58
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 0.9.2
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 0.9.2
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.16'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.16'
 description: Treasure Data Cloud Data Service plugin for Fluentd
 email: support@treasure-data.com
 executables: []
@@ -66,45 +73,46 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- .travis.yml
 - AUTHORS
 - ChangeLog
 - Gemfile
+- Gemfile.fluentd.lt.0.10.43
 - README.rdoc
 - Rakefile
 - VERSION
 - example.conf
 - fluent-plugin-td.gemspec
+- lib/fluent/plugin/out_tditem.rb
 - lib/fluent/plugin/out_tdlog.rb
-- test/out_tdlog.rb
+- lib/fluent/plugin/td_plugin_util.rb
+- test/plugin/test_out_tditem.rb
+- test/plugin/test_out_tdlog.rb
+- test/test_helper.rb
 homepage: http://www.treasuredata.com/
 licenses: []
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: 79461157441894892
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: 79461157441894892
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.23
+rubygems_version: 2.0.2
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Treasure Data Cloud Data Service plugin for Fluentd
 test_files:
-- test/out_tdlog.rb
+- test/plugin/test_out_tditem.rb
+- test/plugin/test_out_tdlog.rb
+- test/test_helper.rb

data/test/out_tdlog.rb

@@ -1,67 +0,0 @@
-require 'fluent/test'
-require 'fluent/plugin/out_tdlog'
-
-class TreasureDataLogOutputTest < Test::Unit::TestCase
-  def setup
-    Fluent::Test.setup
-  end
-
-  TMP_DIR = File.dirname(__FILE__) + "/tmp"
-
-  CONFIG = %[
-    apikey testkey
-    buffer_path #{TMP_DIR}/buffer
-  ]
-
-  def create_driver(conf = CONFIG)
-    Fluent::Test::BufferedOutputTestDriver.new(Fluent::TreasureDataLogOutput) do
-      def start
-        super
-      end
-
-      def write(chunk)
-        chunk.read
-      end
-    end.configure(conf)
-  end
-
-  def test_emit
-    d = create_driver
-
-    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
-    d.emit({"a"=>1}, time)
-    d.emit({"a"=>2}, time)
-    d.run
-  end
-
-## TODO invalid names are normalized
-#  def test_invalid_name
-#    d = create_driver
-#    d.instance.start
-#
-#    es = Fluent::OneEventStream.new(Time.now.to_i, {})
-#    chain = Fluent::NullOutputChain.instance
-#    assert_raise(RuntimeError) do
-#      d.instance.emit("test.invalid-name", es, chain)
-#    end
-#    assert_raise(RuntimeError) do
-#      d.instance.emit("empty", es, chain)
-#    end
-#    assert_raise(RuntimeError) do
-#      d.instance.emit("", es, chain)
-#    end
-#  end
-
-## TODO invalid data is ignored
-#  def test_invalid_data
-#    d = create_driver
-#    d.instance.start
-#
-#    es = Fluent::OneEventStream.new(Time.now.to_i, "invalid")
-#    chain = Fluent::NullOutputChain.instance
-#    assert_nothing_raised do
-#      d.instance.emit("test.name", es, chain)
-#    end
-#  end
-end
-