fluent-plugin-td 0.10.16 → 0.10.17

data/ChangeLog

@@ -1,3 +1,9 @@
+Release 0.10.17 - 2013/12/05
+
+  * Change table create / check mechanizm
+  * Update td-client to v0.8.56
+
+
 Release 0.10.16 - 2013/09/25
 
   * Add connect_timeout / read_timeout / send_timeout parameters

data/VERSION

@@ -1 +1 @@
-0.10.16
+0.10.17

data/fluent-plugin-td.gemspec

@@ -3,8 +3,8 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |gem|
   gem.name        = "fluent-plugin-td"
-  gem.description = "Treasure Data Big Data as a Service plugin for Fluentd"
-  gem.homepage    = "http://treasure-data.com/"
+  gem.description = "Treasure Data Cloud Data Service plugin for Fluentd"
+  gem.homepage    = "http://www.treasuredata.com/"
   gem.summary     = gem.description
   gem.version     = File.read("VERSION").strip
   gem.authors     = ["Treasure Data, Inc."]
@@ -17,6 +17,6 @@ Gem::Specification.new do |gem|
   gem.require_paths = ['lib']
 
   gem.add_dependency "fluentd", "~> 0.10.27"
-  gem.add_dependency "td-client", "~> 0.8.55"
+  gem.add_dependency "td-client", "~> 0.8.56"
   gem.add_development_dependency "rake", ">= 0.9.2"
 end

data/lib/fluent/plugin/out_tdlog.rb

@@ -74,17 +74,19 @@ class TreasureDataLogOutput < BufferedOutput
     require 'time' # Time#rfc2822
     require 'td-client'
     require 'digest/md5'
+    require 'stringio'
     super
     @tmpdir = nil
     @apikey = nil
     @key = nil
     @key_num_limit = 5120  # TODO
     @record_size_limit = 32*1024*1024  # TODO
-    @table_list = []
+    @table_list = {}
     @auto_create_table = true
     @use_ssl = false
     @buffer_type = 'file'  # overwrite default buffer_type
     @flush_interval = 300  # overwrite default flush_interval to 5mins
+    @empty_gz_data = create_empty_gz_data
   end
 
   def configure(conf)
@@ -132,8 +134,6 @@ class TreasureDataLogOutput < BufferedOutput
     if database && table
       validate_database_and_table_name(database, table, conf)
       @key = "#{database}.#{table}"
-    else
-      raise ConfigError, "'database' and 'table' parameter are required on tdlog output without auto_create_table" unless @auto_create_table
     end
 
     @anonymizes = {}
@@ -159,7 +159,7 @@ class TreasureDataLogOutput < BufferedOutput
     @anonymizes = nil if @anonymizes.empty?
 
     @http_proxy = conf['http_proxy']
-    @user_agent = "fluent-plugin-td: 0.10.15"  # TODO: automatic increment version
+    @user_agent = "fluent-plugin-td: 0.10.17"  # TODO: automatic increment version
   end
 
   def start
@@ -167,14 +167,13 @@ class TreasureDataLogOutput < BufferedOutput
 
     @client = TreasureData::Client.new(@apikey, :ssl => @use_ssl, :http_proxy => @http_proxy, :user_agent => @user_agent,
       :connect_timeout => @connect_timeout, :read_timeout => @read_timeout, :send_timeout => @send_timeout)
-    begin
-      check_table_exists(@key) if @key
-    rescue => e
+
+    if @key
       if @auto_create_table
-        database, table = @key.split('.', 2)
+        database, table = @key.split('.',2)
         ensure_database_and_table(database, table)
       else
-        raise e
+        check_table_exists(@key)
       end
     end
   end
@@ -272,6 +271,12 @@ class TreasureDataLogOutput < BufferedOutput
     f.close if f
   end
 
+  def create_empty_gz_data
+    io = StringIO.new
+    Zlib::GzipWriter.new(io).close
+    io.string
+  end
+
   def upload(database, table, io, size, unique_id)
     unique_str = unique_id.unpack('C*').map {|x| "%02x" % x }.join
     $log.trace { "uploading logs to Treasure Data database=#{database} table=#{table} (#{size}bytes)" }
@@ -297,31 +302,22 @@ class TreasureDataLogOutput < BufferedOutput
   end
 
   def check_table_exists(key)
-    unless @table_list.include?(key)
+    unless @table_list.has_key?(key)
+      database, table = key.split('.',2)
+      $log.debug "checking whether table '#{database}.#{table}' exists on Treasure Data"
+      io = StringIO.new(@empty_gz_data)
       begin
-        @table_list = get_table_list
+        @client.import(database, table, "msgpack.gz", io, io.size)
+        @table_list[key] = true
+      rescue TreasureData::NotFoundError
+        raise "Table #{key.inspect} does not exist on Treasure Data. Use 'td table:create #{database} #{table}' to create it."
       rescue
-        $log.warn "failed to update table list on Treasure Data", :error=>$!.to_s
+        $log.warn "failed to check table existence on Treasure Data", :error=>$!.to_s
         $log.debug_backtrace $!
       end
-      unless @table_list.include?(key)
-        database, table = key.split('.',2)
-        raise "Table #{key.inspect} does not exist on Treasure Data. Use 'td create-log-table #{database} #{table}' to create it."
-      end
     end
   end
 
-  def get_table_list
-    $log.info "updating table list from Treasure Data"
-    list = []
-    @client.databases.each {|db|
-      db.tables.each {|tbl|
-        list << "#{db.name}.#{tbl.name}"
-      }
-    }
-    list
-  end
-
   def validate_database_and_table_name(database, table, conf)
     begin
       TreasureData::API.validate_database_name(database)
@@ -342,6 +338,7 @@ class TreasureDataLogOutput < BufferedOutput
     rescue TreasureData::NotFoundError
       @client.create_database(database)
       @client.create_log_table(database, table)
+    rescue TreasureData::AlreadyExistsError
     end
   end
 end

metadata

@@ -1,18 +1,20 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-td
 version: !ruby/object:Gem::Version
-  version: 0.10.16
+  version: 0.10.17
+  prerelease: 
 platform: ruby
 authors:
 - Treasure Data, Inc.
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-09-24 00:00:00.000000000 Z
+date: 2013-12-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -20,6 +22,7 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -27,32 +30,36 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: td-client
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.8.55
+        version: 0.8.56
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.8.55
+        version: 0.8.56
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: 0.9.2
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: 0.9.2
-description: Treasure Data Big Data as a Service plugin for Fluentd
+description: Treasure Data Cloud Data Service plugin for Fluentd
 email: support@treasure-data.com
 executables: []
 extensions: []
@@ -69,28 +76,35 @@ files:
 - fluent-plugin-td.gemspec
 - lib/fluent/plugin/out_tdlog.rb
 - test/out_tdlog.rb
-homepage: http://treasure-data.com/
+homepage: http://www.treasuredata.com/
 licenses: []
-metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
-  - - '>='
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
+      segments:
+      - 0
+      hash: 79461157441894892
 required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
-  - - '>='
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
+      segments:
+      - 0
+      hash: 79461157441894892
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.3
+rubygems_version: 1.8.23
 signing_key: 
-specification_version: 4
-summary: Treasure Data Big Data as a Service plugin for Fluentd
+specification_version: 3
+summary: Treasure Data Cloud Data Service plugin for Fluentd
 test_files:
 - test/out_tdlog.rb

checksums.yaml

@@ -1,7 +0,0 @@
----
-SHA1:
-  metadata.gz: 313c5a37b90313d1a1e805285da719c35951184b
-  data.tar.gz: c0f9975f641434550a2d18a9374bb84fc117b37f
-SHA512:
-  metadata.gz: 912b12246d9650b39137bb1ebd098b45ef9d6ecb4460c8a4667bf1daaaa40ec58ee5a5479a5d7a580b709a5aa5ecfd4b258f5394fbf47b98d1fff709f5d839c0
-  data.tar.gz: 6d7ac160d26c13d97c49930cb8b9fd045f3a04bfac1a05066ff05877ecf12a82e5562b3a37c7da0acc3c39f6649bedc1b8aecdff984a2fd4df3d9b528b62179c