fluent-plugin-td 0.10.21 → 0.10.22

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 45aa1dc756f40504543010be6622b6af518fa46c
-  data.tar.gz: 72c08e7060e8e6e1fb3af3144a746f92e78f4cf0
+  metadata.gz: 7e4943818798256c0ec07e3db957ccd6edeb0de9
+  data.tar.gz: 50ca2274ebc97ad068189ca892fde5d7676d23af
 SHA512:
-  metadata.gz: 3d494ed57200fb989b24db7552a917f690ce5e96b83a8a108e56d03d2d41274c1a2b37ee610494790ea4f21be41cefcbcd783fad74cd1df03fa5fbd84caa079e
-  data.tar.gz: e62a326f85674fe4acdb78f7f9b7df0c27cc45e569b5af171b1754e99204222e8ff0cb286d0a4667ae6c64d83611b4aec742c98a5c95fdc77bf6f3fc5c45b884
+  metadata.gz: 45e450f1c6f593bf2a16607d07ed3d378cdb1f29c3b5e4f8058a319ae814f385b40692151b8e6a26344307f257baf310240421fa536885a78a90725a2a7fa407
+  data.tar.gz: 97abcbc88c894038e0f137c689030dffcae5d4a680560f382fe24a77177cfd709d9cbd44893c3f330389bbd30ffe0e973b0f0210c92c97fc1b878d542e1e45cd

data/ChangeLog

@@ -1,3 +1,11 @@
+Release 0.10.22 - 2014/10/17
+
+  * Remove :time key from a record
+  * Show deperecated warning for <anonymize>
+  * Use Error#inspect to get error class in error log
+  * Update td-client to v0.8.66
+
+
 Release 0.10.21 - 2014/07/04
 
   * Use Yajl for summarizing json

data/README.rdoc

@@ -1,6 +1,88 @@
 = Treasure Data output plugin for Fluentd
 
-This plugin is used to upload logs to Treasure Data using Treasure Data REST APIs.
+This Fluentd output plugin is used to upload logs to Treasure Data using
+Treasure Data's REST APIs.
+
+This plugin relies on the Treasure Data's {Ruby client library}[https://github.com/treasure-data/td-client-ruby] to communicate with the Treasure Data's REST APIs.
+
+For more information, please visit the official {Fluentd TD Output plugin}[http://docs.fluentd.org/articles/http-to-td] page at http://docs.fluentd.org.
+
+== Configuration
+
+Please refer to the +example.conf+ example configuration file contained in the
+root of this repository for some examples. The configuration file allows the user to specify various settings for this output plugin.
+
+The configuration options currently supported are:
+
++apikey+::
+  Sets the configuration parameter to specify the Treasure Data API key.
+  
+  Requires the API key as argument.
+  
+  This parameter is *required*.
+  
++auto_create_table+::
+  Specifies whether to create a database and/or table if the destination
+  database and/or table do not exist.
+  
+  Requires no argument to enable; one optional boolean (true or false) argument accepted.
+  When this option is enabled, the destination database and table will be
+  determined based on the *event* *tag*: e.g. an even with tag 'td.mydb.mytable'
+  will require the plugin to upload data to the 'mydb' database and 'mytable'
+  table within it.
+  
+  If this option is not used or set to false, the user is *required* to specify
+  the +database+ and +table+ configuration parameters as explained below.
+  
+  *NOTE* 
+  
+  depending on the access control permissions associated to the API key, a database can or cannot be created if not already available.
+  See the {Treasure Data Access Control documentation}[http://docs.treasuredata.com/articles/access-control] page for details.
+  
++database+::
+  Specifies the destination database in the Treasure Data cloud.
+  
+  Requires the database name as argument.
+  
+  This parameter is *required* unless the +auto_create_table+ option is used.
+  
++table+::
+  Specifies the destination table in the above mentioned database within the Treasure Data cloud.
+  
+  Requires the table name as argument.
+  
+  This parameter is *required* unless the +auto_create_table+ option is used.
+  
++endpoint+::
+  Specifies the Treasure Data's REST API endpoint.
+  
+  Requires the endpoint as argument.
+  
+  If not specified, a default is used; please refer to the {td-client-ruby's :endpoint}[https://github.com/treasure-data/td-client-ruby#endpoint] option for more details.
+  
++use_ssl+::
+  Specifies whether to communicate using SSL encryption over HTTPS.
+  
+  Requires no argument to enable; one optional boolean (true or false) argument accepted.
+  
+  This option corresponds to {td-client-ruby's :ssl}[https://github.com/treasure-data/td-client-ruby#ssl] option: please refer to that page for important information concerning it.
+  
++http_proxy+::
+  If communication with the Treasure Data API endpoint has to occur through a proxy, you can use this option to configure it.
+  
+  Requires the proxy URL as argument. Please refer to the {td-client-ruby's :http_proxy}[https://github.com/treasure-data/td-client-ruby#proxy] option for details.
+  
++connect_timeout+ +read_timeout+ +send_timeout+::
+  All three options specify the timeout used for every connection, when reading data from the Treasure Data API endpoints, and when sending data to it.
+  
+  They require the number of seconds for the timeout as argument.
+  
+  Please refer to the {td-client-ruby's :connect_timeout, :read_timeout, :send_timeout}[https://github.com/treasure-data/td-client-ruby#connection-read-and-send-timeouts] options for details.
+  
++tmpdir+::
+  Specifies the temporary folder to use for storing the temporarily converted and compressed inputs.
+  
+  Requires the path to the temporary folder as argument. If not specified, the system default as specified by Ruby's +Dir.tmpdir+ is used (the +TMPDIR+ environment variable is honored).
 
 == Copyright
 

data/fluent-plugin-td.gemspec

@@ -1,12 +1,13 @@
 # encoding: utf-8
 $:.push File.expand_path('../lib', __FILE__)
+require 'fluent/plugin/td_plugin_version'
 
 Gem::Specification.new do |gem|
   gem.name        = "fluent-plugin-td"
   gem.description = "Treasure Data Cloud Data Service plugin for Fluentd"
   gem.homepage    = "http://www.treasuredata.com/"
   gem.summary     = gem.description
-  gem.version     = File.read("VERSION").strip
+  gem.version     = Fluent::TreasureDataPlugin::VERSION
   gem.authors     = ["Treasure Data, Inc."]
   gem.email       = "support@treasure-data.com"
   gem.has_rdoc    = false
@@ -17,7 +18,7 @@ Gem::Specification.new do |gem|
   gem.require_paths = ['lib']
 
   gem.add_dependency "fluentd", "~> 0.10.27"
-  gem.add_dependency "td-client", "~> 0.8.61"
+  gem.add_dependency "td-client", "~> 0.8.66"
   gem.add_development_dependency "rake", ">= 0.9.2"
   gem.add_development_dependency "webmock", "~> 1.16"
 end

data/lib/fluent/plugin/out_tditem.rb

@@ -1,4 +1,5 @@
 require 'td-client'
+require 'fluent/plugin/td_plugin_version'
 
 module Fluent
   class TreasureDataItemOutput < BufferedOutput
@@ -37,7 +38,7 @@ module Fluent
       @key_num_limit = 1024  # Item table default limitation
       @record_size_limit = 32 * 1024 * 1024  # TODO
       @empty_gz_data = TreasureData::API.create_empty_gz_data
-      @user_agent = "fluent-plugin-td-item: 0.10.21".freeze  # TODO: automatic increment version
+      @user_agent = "fluent-plugin-td-item: #{TreasureDataPlugin::VERSION}".freeze
     end
 
     def configure(conf)

data/lib/fluent/plugin/out_tdlog.rb

@@ -1,4 +1,5 @@
 require 'td-client'
+require 'fluent/plugin/td_plugin_version'
 
 module Fluent
   class TreasureDataLogOutput < BufferedOutput
@@ -93,6 +94,7 @@ module Fluent
       @auto_create_table = true
       @use_ssl = true
       @empty_gz_data = TreasureData::API.create_empty_gz_data
+      @user_agent = "fluent-plugin-td: #{TreasureDataPlugin::VERSION}".freeze
     end
 
     def configure(conf)
@@ -162,10 +164,13 @@ module Fluent
 
         @anonymizes[key] = scr
       }
-      @anonymizes = nil if @anonymizes.empty?
+      if @anonymizes.empty?
+        @anonymizes = nil
+      else
+        log.warn "<anonymize> feature is deprecated and will be removed. Use fluent-plugin-anonymizer instead."
+      end
 
       @http_proxy = conf['http_proxy']
-      @user_agent = "fluent-plugin-td: 0.10.21"  # TODO: automatic increment version
     end
 
     def start
@@ -218,6 +223,7 @@ module Fluent
           end
 
           record['time'] = time
+          record.delete(:time) if record.has_key?(:time)
 
           if record.size > @key_num_limit
             raise "Too many number of keys (#{record.size} keys)"  # TODO include summary of the record
@@ -297,7 +303,7 @@ module Fluent
         end
       rescue => e
         elapsed = Time.now - start
-        ne = RuntimeError.new("Failed to upload to Treasure Data '#{database}.#{table}' table: #{$!} (#{size} bytes; #{elapsed} seconds)")
+        ne = RuntimeError.new("Failed to upload to Treasure Data '#{database}.#{table}' table: #{e.inspect} (#{size} bytes; #{elapsed} seconds)")
         ne.set_backtrace(e.backtrace)
         raise ne
       end
@@ -314,7 +320,7 @@ module Fluent
         rescue TreasureData::NotFoundError
           raise "Table #{key.inspect} does not exist on Treasure Data. Use 'td table:create #{database} #{table}' to create it."
         rescue => e
-          log.warn "failed to check existence of '#{database}.#{table}' table on Treasure Data", :error => e.to_s
+          log.warn "failed to check existence of '#{database}.#{table}' table on Treasure Data", :error => e.inspect
           log.debug_backtrace e.backtrace
         end
       end

data/lib/fluent/plugin/td_plugin_util.rb

@@ -53,7 +53,7 @@ module Fluent
           args = self.class == TreasureDataItemOutput ? ' -t item' : ''
           raise "Table #{key.inspect} does not exist on Treasure Data. Use 'td table:create #{database} #{table}#{args}' to create it."
         rescue => e
-          log.warn "failed to check table existence on Treasure Data", :error => e.to_s
+          log.warn "failed to check table existence on Treasure Data", :error => e.inspect
           log.debug_backtrace e
         end
       end

data/lib/fluent/plugin/td_plugin_version.rb

@@ -0,0 +1,5 @@
+module Fluent
+  module TreasureDataPlugin
+    VERSION = '0.10.22'
+  end
+end

data/test/plugin/test_out_tdlog.rb

@@ -56,6 +56,22 @@ class TreasureDataLogOutputTest < Test::Unit::TestCase
     assert_equal('TD1 testkey', @auth_header)
   end
 
+  def test_emit_with_time_symbole
+    d = create_driver
+    time, records = stub_seed_values
+    database, table = d.instance.instance_variable_get(:@key).split(".", 2)
+    stub_td_table_create_request(database, table)
+    stub_td_import_request(stub_request_body(records, time), database, table)
+
+    records.each { |record|
+      record[:time] = Time.now.to_i  # emit removes this :time key
+      d.emit(record, time)
+    }
+    d.run
+
+    assert_equal('TD1 testkey', @auth_header)
+  end
+
   def test_emit_with_endpoint
     d = create_driver(DEFAULT_CONFIG + "endpoint foo.bar.baz")
     opts = {:endpoint => 'foo.bar.baz'}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-td
 version: !ruby/object:Gem::Version
-  version: 0.10.21
+  version: 0.10.22
 platform: ruby
 authors:
 - Treasure Data, Inc.
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-07-03 00:00:00.000000000 Z
+date: 2014-10-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -30,14 +30,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.8.61
+        version: 0.8.66
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.8.61
+        version: 0.8.66
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -79,12 +79,12 @@ files:
 - Gemfile
 - README.rdoc
 - Rakefile
-- VERSION
 - example.conf
 - fluent-plugin-td.gemspec
 - lib/fluent/plugin/out_tditem.rb
 - lib/fluent/plugin/out_tdlog.rb
 - lib/fluent/plugin/td_plugin_util.rb
+- lib/fluent/plugin/td_plugin_version.rb
 - test/plugin/test_out_tditem.rb
 - test/plugin/test_out_tdlog.rb
 - test/test_helper.rb

data/VERSION

@@ -1 +0,0 @@
-0.10.21