fluent-plugin-td 0.9.7 → 0.9.8

data/ChangeLog

@@ -0,0 +1,6 @@
+
+Release 0.9.8 - 2011/09/03
+
+  * supports auto_create_table option
+  * uses td-client gem
+

data/Rakefile

@@ -13,6 +13,7 @@ begin
     gemspec.has_rdoc = false
     gemspec.require_paths = ["lib"]
     gemspec.add_dependency "fluent", "~> 0.9.7"
+    gemspec.add_dependency "td-client", "~> 0.8.0"
     gemspec.test_files = Dir["test/**/*.rb"]
     gemspec.files = Dir["bin/**/*", "lib/**/*", "test/**/*.rb"] +
       %w[example.conf VERSION AUTHORS Rakefile fluent-plugin-td.gemspec]

data/VERSION

@@ -1 +1 @@
-0.9.7
+0.9.8

data/fluent-plugin-td.gemspec

@@ -5,13 +5,14 @@
 
 Gem::Specification.new do |s|
   s.name = %q{fluent-plugin-td}
-  s.version = "0.9.7"
+  s.version = "0.9.8"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Sadayuki Furuhashi"]
-  s.date = %q{2011-08-06}
+  s.date = %q{2011-09-03}
   s.extra_rdoc_files = [
-    "README.rdoc"
+    "ChangeLog",
+     "README.rdoc"
   ]
   s.files = [
     "AUTHORS",
@@ -32,11 +33,14 @@ Gem::Specification.new do |s|
 
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
       s.add_runtime_dependency(%q<fluent>, ["~> 0.9.7"])
+      s.add_runtime_dependency(%q<td-client>, ["~> 0.8.0"])
     else
       s.add_dependency(%q<fluent>, ["~> 0.9.7"])
+      s.add_dependency(%q<td-client>, ["~> 0.8.0"])
     end
   else
     s.add_dependency(%q<fluent>, ["~> 0.9.7"])
+    s.add_dependency(%q<td-client>, ["~> 0.8.0"])
   end
 end
 

data/lib/fluent/plugin/out_tdlog.rb

@@ -16,11 +16,15 @@ class TreasureDataLogOutput < BufferedOutput
     require 'json'
     require 'cgi' # CGI.escape
     require 'time' # Time#rfc2822
+    require 'td-client'
     super
     @tmpdir = '/tmp/fluent/tdlog'
     @apikey = nil
     @key = nil
+    @key_num_limit = 5120  # TODO
+    @record_size_limit = 32*1024*1024  # TODO
     @table_list = []
+    @auto_create_table = false
   end
 
   def configure(conf)
@@ -48,10 +52,16 @@ class TreasureDataLogOutput < BufferedOutput
       raise ConfigError, "'database' and 'table' parameter are required on tdlog output"
     end
 
-    @table_list = get_table_list
+    if conf['auto_create_table']
+      @auto_create_table = true
+    end
+  end
 
-    if @key && !@table_list.include?(@key)
-      raise ConfigError, "Table #{@key.inspect} does not exist on Treasure Data. Use 'td create-log-table #{database} #{table}' to create it."
+  def start
+    super
+    @client = TreasureData::Client.new(@apikey)
+    unless @auto_create_table
+      check_table_exists(@key)
     end
   end
 
@@ -67,18 +77,8 @@ class TreasureDataLogOutput < BufferedOutput
       key = "#{database}.#{table}"
     end
 
-    # check the table exists
-    unless @table_list.include?(key)
-      begin
-        @table_list = get_table_list
-      rescue
-        $log.warn "failed to update table list on Treasure Data", :error=>$!.to_s
-        $log.debug_backtrace $!
-      end
-      unless @table_list.include?(key)
-        database, table = key.split('.',2)
-        raise "Table #{key.inspect} does not exist on Treasure Data. Use 'td create-log-table #{database} #{table}' to create it."
-      end
+    unless @auto_create_table
+      check_table_exists(key)
     end
 
     super(tag, es, chain, key)
@@ -90,10 +90,23 @@ class TreasureDataLogOutput < BufferedOutput
 
   def format_stream(tag, es)
     out = ''
+    off = out.bytesize
     es.each {|event|
       record = event.record
       record['time'] = event.time
+
+      if record.size > @key_num_limit
+        raise "Too many number of keys (#{record.size} keys)"  # TODO include summary of the record
+      end
+
       record.to_msgpack(out)
+
+      noff = out.bytesize
+      sz = noff - off
+      if sz > @record_size_limit
+        raise "Size of a record too large (#{sz} bytes)"  # TODO include summary of the record
+      end
+      off = noff
     }
     out
   end
@@ -122,87 +135,51 @@ class TreasureDataLogOutput < BufferedOutput
   end
 
   def upload(database, table, io, size)
-    http, header = new_http
-    header['Content-Length'] = size.to_s
-    header['Content-Type'] = 'application/octet-stream'
-
-    url = "/v3/table/import/#{e database}/#{e table}/msgpack.gz"
-
-    req = Net::HTTP::Put.new(url, header)
-    if req.respond_to?(:body_stream=)
-      req.body_stream = io
-    else  # Ruby 1.8
-      req.body = io.read
-    end
-
     $log.trace { "uploading logs to Treasure Data database=#{database} table=#{table} (#{size}bytes)" }
 
-    response = http.request(req)
+    begin
+      @client.import(database, table, "msgpack.gz", io, size)
+    rescue TreasureData::NotFoundError
+      unless @auto_create_table
+        raise $!
+      end
+      $log.info "Creating table #{database}.#{table} on TreasureData"
+      begin
+        @client.create_log_table(database, table)
+      rescue TreasureData::NotFoundError
+        @client.create_database(database)
+        @client.create_log_table(database, table)
+      end
+      io.pos = 0
+      retry
+    end
+  end
 
-    if response.code[0] != ?2
-      raise "Treasure Data upload failed: #{response.body}"
+  def check_table_exists(key)
+    unless @table_list.include?(key)
+      begin
+        @table_list = get_table_list
+      rescue
+        $log.warn "failed to update table list on Treasure Data", :error=>$!.to_s
+        $log.debug_backtrace $!
+      end
+      unless @table_list.include?(key)
+        database, table = key.split('.',2)
+        raise "Table #{key.inspect} does not exist on Treasure Data. Use 'td create-log-table #{database} #{table}' to create it."
+      end
     end
   end
 
   def get_table_list
     $log.info "updating table list from Treasure Data"
     list = []
-    api_list_database.each {|db|
-      api_list_table(db).each {|t|
-        list << "#{db}.#{t}"
+    @client.databases.each {|db|
+      db.tables.each {|tbl|
+        list << "#{db.name}.#{tbl.name}"
       }
     }
     list
   end
-
-  def api_list_database
-    body = get("/v3/database/list")
-    js = JSON.load(body)
-    return js["databases"].map {|m| m['name'] }
-  end
-
-  def api_list_table(db)
-    body = get("/v3/table/list/#{e db}")
-    js = JSON.load(body)
-    return js["tables"].map {|m| m['name'] }
-  end
-
-  def get(path)
-    http, header = new_http
-
-    request = Net::HTTP::Get.new(path, header)
-
-    response = http.request(request)
-
-    if response.code[0] != ?2
-      raise "Treasure Data API failed: #{response.body}"
-    end
-
-    return response.body
-  end
-
-  def new_http
-    http = Net::HTTP.new(HOST, PORT)
-    if USE_SSL
-      http.use_ssl = true
-      http.verify_mode = OpenSSL::SSL::VERIFY_PEER
-      store = OpenSSL::X509::Store.new
-      http.cert_store = store
-    end
-
-    # TODO read_timeout
-    #http.read_timeout = options[:read_timeout]
-
-    header = {}
-    header['Authorization'] = "TD1 #{@apikey}"
-    header['Date'] = Time.now.rfc2822
-
-    return http, header
-  end
-
-  def e(s)
-    CGI.escape(s.to_s)
-  end
 end
 
 

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: fluent-plugin-td
 version: !ruby/object:Gem::Version 
-  hash: 53
+  hash: 43
   prerelease: false
   segments: 
   - 0
   - 9
-  - 7
-  version: 0.9.7
+  - 8
+  version: 0.9.8
 platform: ruby
 authors: 
 - Sadayuki Furuhashi
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-08-06 00:00:00 +09:00
+date: 2011-09-03 00:00:00 +09:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -34,6 +34,22 @@ dependencies:
         version: 0.9.7
   type: :runtime
   version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: td-client
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 63
+        segments: 
+        - 0
+        - 8
+        - 0
+        version: 0.8.0
+  type: :runtime
+  version_requirements: *id002
 description: 
 email: 
 executables: []
@@ -41,6 +57,7 @@ executables: []
 extensions: []
 
 extra_rdoc_files: 
+- ChangeLog
 - README.rdoc
 files: 
 - AUTHORS
@@ -49,6 +66,7 @@ files:
 - example.conf
 - fluent-plugin-td.gemspec
 - lib/fluent/plugin/out_tdlog.rb
+- ChangeLog
 - README.rdoc
 has_rdoc: true
 homepage: