fluent-plugin-td 0.10.27 → 0.10.28

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 668a4eb674a2ea968b1fe87b2f18a48f65c500f4
-  data.tar.gz: efbee27ae80e7cd6091425ed2609e2f8ba2f3b1f
+  metadata.gz: 94a57a8ecc7cd03af85dfacdce2623dabf6862c7
+  data.tar.gz: 7e5046ff0e73947ce69d7d3daa40875f3c74be83
 SHA512:
-  metadata.gz: edc948913205c4bee2b87fc41824701e6d2c45a8569379ee98a41a8e866f67d4eeb4aa9a6dc7fc48d71190e549768cf4fe6251d120a8bbc6f9bb82ebcb60057b
-  data.tar.gz: 6c7b7d9e19501bbd357de9da7168fb4aa09d10636e5f3a617a3b201ded1d91fc8ba8f331866c65c7d312b9c131dfb81289af74e79f546855548267e0a96333fb
+  metadata.gz: 07b585a15dad7100ab91e7316c3248772233c70268139b6f762d02bdb8f356b9dbbe06b4c04dac015a92e483e2e4143c77d79ce7a66d6c0c5e4d489101de7645
+  data.tar.gz: fd1d7f3e57dad3acce19569a1e575b7c167c06209a9757a6517e3c8450b78e7e89eef026ab5562d47bd127c5925c3ed284d8982678e1a7283e8a08aeb94dd7c1

data/ChangeLog

@@ -1,3 +1,9 @@
+Release 0.10.28 - 2015/10/28
+
+  * Remove tditem output plugin
+  * Fix temporary file handling on windows
+
+
 Release 0.10.27 - 2015/06/10
 
   * Add secret option to apikey parameter

data/fluent-plugin-td.gemspec

@@ -16,6 +16,7 @@ Gem::Specification.new do |gem|
   gem.test_files  = `git ls-files -- {test,spec,features}/*`.split("\n")
   gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   gem.require_paths = ['lib']
+  gem.license       = "Apache-2.0"
 
   gem.add_dependency "fluentd", [">= 0.10.27", "< 2"]
   gem.add_dependency "td-client", "~> 0.8.66"

data/lib/fluent/plugin/out_tdlog.rb

@@ -171,6 +171,7 @@ module Fluent
 
       FileUtils.mkdir_p(@tmpdir) unless @tmpdir.nil?
       f = Tempfile.new("tdlog-#{chunk.key}-", @tmpdir)
+      f.binmode
 
       size = if @use_gzip_command
                gzip_by_command(chunk, f)
@@ -190,6 +191,7 @@ module Fluent
                chunk.path
              else
                w = Tempfile.new("gzip-tdlog-#{chunk.key}-", @tmpdir)
+               w.binmode
                chunk.write_to(w)
                w.close
                w.path

data/lib/fluent/plugin/td_plugin_version.rb

@@ -1,5 +1,5 @@
 module Fluent
   module TreasureDataPlugin
-    VERSION = '0.10.27'
+    VERSION = '0.10.28'
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-td
 version: !ruby/object:Gem::Version
-  version: 0.10.27
+  version: 0.10.28
 platform: ruby
 authors:
 - Treasure Data, Inc.
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-06-10 00:00:00.000000000 Z
+date: 2015-10-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -116,15 +116,14 @@ files:
 - Rakefile
 - example.conf
 - fluent-plugin-td.gemspec
-- lib/fluent/plugin/out_tditem.rb
 - lib/fluent/plugin/out_tdlog.rb
 - lib/fluent/plugin/td_plugin_util.rb
 - lib/fluent/plugin/td_plugin_version.rb
-- test/plugin/test_out_tditem.rb
 - test/plugin/test_out_tdlog.rb
 - test/test_helper.rb
 homepage: http://www.treasuredata.com/
-licenses: []
+licenses:
+- Apache-2.0
 metadata: {}
 post_install_message: 
 rdoc_options: []
@@ -142,11 +141,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.4.5.1
 signing_key: 
 specification_version: 4
 summary: Treasure Data Cloud Data Service plugin for Fluentd
 test_files:
-- test/plugin/test_out_tditem.rb
 - test/plugin/test_out_tdlog.rb
 - test/test_helper.rb

data/lib/fluent/plugin/out_tditem.rb

@@ -1,101 +0,0 @@
-require 'td-client'
-require 'fluent/plugin/td_plugin_version'
-
-module Fluent
-  class TreasureDataItemOutput < BufferedOutput
-    Plugin.register_output('tditem', self)
-
-    require_relative 'td_plugin_util'
-    include TDPluginUtil
-
-    IMPORT_SIZE_LIMIT = 32 * 1024 * 1024
-
-    # To support log_level option since Fluentd v0.10.43
-    unless method_defined?(:log)
-      define_method(:log) { $log }
-    end
-
-    config_param :apikey, :string, :secret => true
-    config_param :database, :string
-    config_param :table, :string
-    config_param :tmpdir, :string, :default => nil
-    #config_param :auto_create_table, :bool, :default => true # TODO: implement if user wants this feature
-
-    config_param :endpoint, :string, :default => TreasureData::API::NEW_DEFAULT_ENDPOINT
-    config_param :use_ssl, :bool, :default => true
-    config_param :http_proxy, :string, :default => nil
-    config_param :connect_timeout, :integer, :default => nil
-    config_param :read_timeout, :integer, :default => nil
-    config_param :send_timeout, :integer, :default => nil
-    config_set_default :buffer_type, 'file'
-    config_set_default :flush_interval, 300
-
-    def initialize
-      super
-
-      @auto_create_table = false
-      @tmpdir_prefix = 'tditem-'.freeze
-      @key_num_limit = 1024  # Item table default limitation
-      @record_size_limit = 32 * 1024 * 1024  # TODO
-      @empty_gz_data = TreasureData::API.create_empty_gz_data
-      @user_agent = "fluent-plugin-td-item: #{TreasureDataPlugin::VERSION}".freeze
-    end
-
-    def configure(conf)
-      super
-
-      # overwrite default value of buffer_chunk_limit
-      if @buffer.respond_to?(:buffer_chunk_limit=) && !conf.has_key?('buffer_chunk_limit')
-        @buffer.buffer_chunk_limit = IMPORT_SIZE_LIMIT
-      end
-
-      validate_database_and_table_name(@database, @table, conf)
-      @key = "#{@database}.#{@table}".freeze
-      @use_ssl = parse_bool_parameter(@use_ssl) if @use_ssl.instance_of?(String)
-      FileUtils.mkdir_p(@tmpdir) unless @tmpdir.nil?
-    end
-
-    def start
-      super
-
-      client_opts = {
-        :ssl => @use_ssl, :http_proxy => @http_proxy, :user_agent => @user_agent, :endpoint => @endpoint,
-        :connect_timeout => @connect_timeout, :read_timeout => @read_timeout, :send_timeout => @send_timeout
-      }
-      @client = TreasureData::Client.new(@apikey, client_opts)
-
-      check_table_existence(@database, @table)
-    end
-
-    def emit(tag, es, chain)
-      super(tag, es, chain, @key)
-    end
-
-    def format_stream(tag, es)
-      out = ''
-      off = out.bytesize
-      es.each { |time, record|
-        if record.size > @key_num_limit
-          log.error "Too many number of keys (#{record.size} keys)" # TODO include summary of the record
-          next
-        end
-
-        begin
-          record.to_msgpack(out)
-        rescue RangeError
-          TreasureData::API.normalized_msgpack(record, out)
-        end
-
-        noff = out.bytesize
-        sz = noff - off
-        if sz > @record_size_limit
-          # TODO don't raise error
-          #raise "Size of a record too large (#{sz} bytes)"  # TODO include summary of the record
-          log.warn "Size of a record too large (#{sz} bytes): #{summarize_record(record)}"
-        end
-        off = noff
-      }
-      out
-    end
-  end
-end

data/test/plugin/test_out_tditem.rb

@@ -1,99 +0,0 @@
-require 'test_helper'
-require 'fluent/plugin/out_tditem'
-
-class TreasureDataItemOutputTest < Test::Unit::TestCase
-  def setup
-    Fluent::Test.setup
-  end
-
-  DEFAULT_CONFIG = %[
-    database test
-    table table
-  ]
-
-  def create_driver(conf = DEFAULT_CONFIG)
-    config = %[
-      apikey testkey
-      buffer_type memory
-    ] + conf
-
-    Fluent::Test::BufferedOutputTestDriver.new(Fluent::TreasureDataItemOutput) do 
-      def write(chunk)
-        # TestDriver doesn't call acutual Output#emit so set key to get database and table in this place.
-        chunk.instance_variable_set(:@key, @key)
-        super(chunk)
-      end
-    end.configure(config)
-  end
-
-  def test_configure
-    d = create_driver
-
-    assert_equal('testkey', d.instance.apikey)
-    assert_equal('test', d.instance.database)
-    assert_equal('table', d.instance.table)
-    assert_equal(true, d.instance.use_ssl)
-    assert_equal('memory', d.instance.buffer_type)
-    assert_equal(300, d.instance.flush_interval)
-  end
-
-  def test_configure_with_invalid_database
-    assert_raise(Fluent::ConfigError) {
-      create_driver(%[
-        database a
-        table table
-      ])
-    }
-  end
-
-  def test_configure_with_invalid_table
-    assert_raise(Fluent::ConfigError) {
-      create_driver(%[
-        database test
-        table 1
-      ])
-    }
-  end
-
-  def test_emit
-    d = create_driver
-    time, records = stub_seed_values
-    stub_td_import_request(stub_request_body(records), d.instance.database, d.instance.table)
-
-    records.each { |record|
-      d.emit(record, time)
-    }
-    d.run
-
-    assert_equal('TD1 testkey', @auth_header)
-  end
-
-  def test_emit_with_endpoint
-    d = create_driver(DEFAULT_CONFIG + "endpoint foo.bar.baz")
-    opts = {:endpoint => 'foo.bar.baz'}
-    time, records = stub_seed_values
-    stub_td_import_request(stub_request_body(records), d.instance.database, d.instance.table, opts)
-
-    records.each { |record|
-      d.emit(record, time)
-    }
-    d.run
-
-    assert_equal('TD1 testkey', @auth_header)
-  end
-
-  def test_emit_with_too_many_keys
-    d = create_driver(DEFAULT_CONFIG + "endpoint foo.bar.baz")
-    opts = {:endpoint => 'foo.bar.baz'}
-    time, _ = stub_seed_values
-    stub_td_import_request(stub_request_body([]), d.instance.database, d.instance.table, opts)
-
-    d.emit(create_too_many_keys_record, time)
-    d.run
-
-    assert_equal 0, d.emits.size
-    assert d.instance.log.logs.select{ |line|
-      line =~ / \[error\]: Too many number of keys/
-    }.size == 1, "too many keys error is not logged"
-  end
-end