fluent-plugin-td 0.10.0 → 0.10.1

data/.gitignore

@@ -0,0 +1,2 @@
+Gemfile.lock
+pkg/*

data/ChangeLog

@@ -1,4 +1,10 @@
 
+Release 0.10.1 - 2011/11/11
+
+  * Validate database name and table name on emit
+  * Include more information on error message
+
+
 Release 0.10.0 - 2011/10/16
 
   * Use buf_file by default

data/Gemfile

@@ -0,0 +1,3 @@
+source "http://rubygems.org"
+
+gemspec

data/Rakefile

@@ -1,58 +1,14 @@
-require 'rake'
-require 'rake/testtask'
-require 'rake/clean'
 
-begin
-  require 'jeweler'
-  Jeweler::Tasks.new do |gemspec|
-    gemspec.name = "fluent-plugin-td"
-    gemspec.summary = "Treasure Data plugin for Fluent event collector"
-    gemspec.author = "Sadayuki Furuhashi"
-    #gemspec.email = "frsyuki@gmail.com"
-    #gemspec.homepage = "http://fluent.github.com/"
-    gemspec.has_rdoc = false
-    gemspec.require_paths = ["lib"]
-    gemspec.add_dependency "fluentd", "~> 0.10.0"
-    gemspec.add_dependency "td-client", "~> 0.8.0"
-    gemspec.test_files = Dir["test/**/*.rb"]
-    gemspec.files = Dir["bin/**/*", "lib/**/*", "test/**/*.rb"] +
-      %w[example.conf VERSION AUTHORS Rakefile fluent-plugin-td.gemspec]
-    gemspec.executables = []
-  end
-  Jeweler::GemcutterTasks.new
-rescue LoadError
-  puts "Jeweler not available. Install it with: gem install jeweler"
-end
+require 'bundler'
+Bundler::GemHelper.install_tasks
 
-Rake::TestTask.new(:test) do |t|
-  t.test_files = Dir['test/*_test.rb']
-  t.ruby_opts = ['-rubygems'] if defined? Gem
-  t.ruby_opts << '-I.'
-end
-
-#VERSION_FILE = "lib/fluent/version.rb"
-#
-#file VERSION_FILE => ["VERSION"] do |t|
-#  version = File.read("VERSION").strip
-#  File.open(VERSION_FILE, "w") {|f|
-#    f.write <<EOF
-#module Fluent
-#
-#VERSION = '#{version}'
-#
-#end
-#EOF
-#  }
-#end
-#
-#task :default => [VERSION_FILE, :build]
+require 'rake/testtask'
 
-Rake::TestTask.new(:test) do |t|
-  t.libs << "test"
-  t.test_files = Dir["test/*.rb"].sort
-  t.verbose = true
-  #t.warning = true
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.test_files = FileList['test/*.rb']
+  test.verbose = true
 end
 
-task :default => [:build, :gemspec]
+task :default => [:build]
 

data/VERSION

@@ -1 +1 @@
-0.10.0
+0.10.1

data/fluent-plugin-td.gemspec

@@ -1,50 +1,21 @@
-# Generated by jeweler
-# DO NOT EDIT THIS FILE DIRECTLY
-# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
-# -*- encoding: utf-8 -*-
+# encoding: utf-8
+$:.push File.expand_path('../lib', __FILE__)
 
-Gem::Specification.new do |s|
-  s.name = %q{fluent-plugin-td}
-  s.version = "0.10.0"
+Gem::Specification.new do |gem|
+  gem.name        = "fluent-plugin-td"
+  gem.description = "Treasure Data output plugin for Fluent event collector"
+  #gem.homepage    = "http://fluentd.org/"
+  gem.summary     = gem.description
+  gem.version     = File.read("VERSION").strip
+  gem.authors     = ["Sadayuki Furuhashi"]
+  #gem.email       = "frsyuki@gmail.com"
+  gem.has_rdoc    = false
+  #gem.platform    = Gem::Platform::RUBY
+  gem.files       = `git ls-files`.split("\n")
+  gem.test_files  = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.require_paths = ['lib']
 
-  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
-  s.authors = ["Sadayuki Furuhashi"]
-  s.date = %q{2011-10-16}
-  s.extra_rdoc_files = [
-    "ChangeLog",
-     "README.rdoc"
-  ]
-  s.files = [
-    "AUTHORS",
-     "Rakefile",
-     "VERSION",
-     "example.conf",
-     "fluent-plugin-td.gemspec",
-     "lib/fluent/plugin/out_tdlog.rb",
-     "test/out_tdlog.rb"
-  ]
-  s.rdoc_options = ["--charset=UTF-8"]
-  s.require_paths = ["lib"]
-  s.rubygems_version = %q{1.3.7}
-  s.summary = %q{Treasure Data plugin for Fluent event collector}
-  s.test_files = [
-    "test/out_tdlog.rb"
-  ]
-
-  if s.respond_to? :specification_version then
-    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
-    s.specification_version = 3
-
-    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
-      s.add_runtime_dependency(%q<fluentd>, ["~> 0.10.0"])
-      s.add_runtime_dependency(%q<td-client>, ["~> 0.8.0"])
-    else
-      s.add_dependency(%q<fluentd>, ["~> 0.10.0"])
-      s.add_dependency(%q<td-client>, ["~> 0.8.0"])
-    end
-  else
-    s.add_dependency(%q<fluentd>, ["~> 0.10.0"])
-    s.add_dependency(%q<td-client>, ["~> 0.8.0"])
-  end
+  gem.add_dependency "fluentd", "~> 0.10.0"
+  gem.add_dependency "td-client", "~> 0.8.4"
 end
-

data/lib/fluent/plugin/out_tdlog.rb

@@ -67,11 +67,15 @@ class TreasureDataLogOutput < BufferedOutput
       if !database || !table
         raise ConfigError, "'database' and 'table' parameter are required on tdlog output"
       end
-      if !validate_name(database)
-        raise ConfigError, "Invalid database name #{database.inspect}: #{conf}"
+      begin
+        TreasureData::API.validate_database_name(database)
+      rescue
+        raise ConfigError, "Invalid database name #{database.inspect}: #{$!}: #{conf}"
       end
-      if !validate_name(table)
-        raise ConfigError, "Invalid table name #{table.inspect}: #{conf}"
+      begin
+        TreasureData::API.validate_table_name(table)
+      rescue
+        raise ConfigError, "Invalid table name #{table.inspect}: #{$!}: #{conf}"
       end
       @key = "#{database}.#{table}"
     end
@@ -90,10 +94,8 @@ class TreasureDataLogOutput < BufferedOutput
       key = @key
     else
       database, table = tag.split('.')[-2,2]
-      if !validate_name(database) || !validate_name(table)
-        $log.debug { "Invalid tag #{tag.inspect}" }
-        return
-      end
+      TreasureData::API.validate_database_name(database)
+      TreasureData::API.validate_table_name(table)
       key = "#{database}.#{table}"
     end
 
@@ -104,10 +106,6 @@ class TreasureDataLogOutput < BufferedOutput
     super(tag, es, chain, key)
   end
 
-  def validate_name(name)
-    true
-  end
-
   def format_stream(tag, es)
     out = ''
     off = out.bytesize
@@ -132,10 +130,6 @@ class TreasureDataLogOutput < BufferedOutput
 
   def write(chunk)
     database, table = chunk.key.split('.',2)
-    if !validate_name(database) || !validate_name(table)
-      $log.error "Invalid key name #{chunk.key.inspect}"
-      return
-    end
 
     f = Tempfile.new("tdlog-", @tmpdir)
     w = Zlib::GzipWriter.new(f)
@@ -157,20 +151,28 @@ class TreasureDataLogOutput < BufferedOutput
     $log.trace { "uploading logs to Treasure Data database=#{database} table=#{table} (#{size}bytes)" }
 
     begin
-      @client.import(database, table, "msgpack.gz", io, size)
-    rescue TreasureData::NotFoundError
-      unless @auto_create_table
-        raise $!
-      end
-      $log.info "Creating table #{database}.#{table} on TreasureData"
       begin
-        @client.create_log_table(database, table)
+        start = Time.now
+        @client.import(database, table, "msgpack.gz", io, size)
       rescue TreasureData::NotFoundError
-        @client.create_database(database)
-        @client.create_log_table(database, table)
+        unless @auto_create_table
+          raise $!
+        end
+        $log.info "Creating table #{database}.#{table} on TreasureData"
+        begin
+          @client.create_log_table(database, table)
+        rescue TreasureData::NotFoundError
+          @client.create_database(database)
+          @client.create_log_table(database, table)
+        end
+        io.pos = 0
+        retry
       end
-      io.pos = 0
-      retry
+    rescue => e
+      elapsed = Time.now - start
+      ne = RuntimeError.new("Failed to upload to TreasureData: #{$!} (#{size} bytes; #{elapsed} seconds)")
+      ne.set_backtrace(e.backtrace)
+      raise ne
     end
   end
 

data/test/out_tdlog.rb

@@ -33,5 +33,22 @@ class TreasureDataLogOutputTest < Test::Unit::TestCase
     d.emit({"a"=>2}, time)
     d.run
   end
+
+  def test_invalid_name
+    d = create_driver
+    d.instance.start
+
+    es = Fluent::OneEventStream.new(Time.now.to_i, {})
+    chain = Fluent::NullOutputChain.instance
+    assert_raise(RuntimeError) do
+      d.instance.emit("test.invalid-name", es, chain)
+    end
+    assert_raise(RuntimeError) do
+      d.instance.emit("empty", es, chain)
+    end
+    assert_raise(RuntimeError) do
+      d.instance.emit("", es, chain)
+    end
+  end
 end
 

metadata

@@ -1,107 +1,84 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: fluent-plugin-td
-version: !ruby/object:Gem::Version 
-  hash: 55
-  prerelease: false
-  segments: 
-  - 0
-  - 10
-  - 0
-  version: 0.10.0
+version: !ruby/object:Gem::Version
+  version: 0.10.1
+  prerelease: 
 platform: ruby
-authors: 
+authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2011-10-16 00:00:00 +09:00
-default_executable: 
-dependencies: 
-- !ruby/object:Gem::Dependency 
+date: 2011-11-10 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
   name: fluentd
-  prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
+  requirement: &70114372869080 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 55
-        segments: 
-        - 0
-        - 10
-        - 0
+      - !ruby/object:Gem::Version
         version: 0.10.0
   type: :runtime
-  version_requirements: *id001
-- !ruby/object:Gem::Dependency 
-  name: td-client
   prerelease: false
-  requirement: &id002 !ruby/object:Gem::Requirement 
+  version_requirements: *70114372869080
+- !ruby/object:Gem::Dependency
+  name: td-client
+  requirement: &70114372868600 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 63
-        segments: 
-        - 0
-        - 8
-        - 0
-        version: 0.8.0
+      - !ruby/object:Gem::Version
+        version: 0.8.4
   type: :runtime
-  version_requirements: *id002
-description: 
+  prerelease: false
+  version_requirements: *70114372868600
+description: Treasure Data output plugin for Fluent event collector
 email: 
 executables: []
-
 extensions: []
-
-extra_rdoc_files: 
+extra_rdoc_files: []
+files:
+- .gitignore
+- AUTHORS
 - ChangeLog
+- Gemfile
 - README.rdoc
-files: 
-- AUTHORS
 - Rakefile
 - VERSION
 - example.conf
 - fluent-plugin-td.gemspec
 - lib/fluent/plugin/out_tdlog.rb
 - test/out_tdlog.rb
-- ChangeLog
-- README.rdoc
-has_rdoc: true
 homepage: 
 licenses: []
-
 post_install_message: 
-rdoc_options: 
-- --charset=UTF-8
-require_paths: 
+rdoc_options: []
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
+required_ruby_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
       - 0
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
+      hash: -747294659517590458
+required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
       - 0
-      version: "0"
+      hash: -747294659517590458
 requirements: []
-
 rubyforge_project: 
-rubygems_version: 1.3.7
+rubygems_version: 1.8.10
 signing_key: 
 specification_version: 3
-summary: Treasure Data plugin for Fluent event collector
-test_files: 
+summary: Treasure Data output plugin for Fluent event collector
+test_files:
 - test/out_tdlog.rb