fluent-plugin-td 0.10.25 → 0.10.26

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a1db60dd2b0ce33a5752790d6f586c86ecf5d5d1
-  data.tar.gz: 860714c30461e7bd0904b91355eba1c35571e212
+  metadata.gz: a3b96e1e6043112fc55402699b53ff5874f91f15
+  data.tar.gz: 4a62e347a754636e4e44e633639b9fb4fac3fbbc
 SHA512:
-  metadata.gz: 0b7a4e3c95a2180751bde7ae165f4e278aeb961907c16081fc6fba460b92841810dc1c3f5f1fe4b8b1a3c82e71bfc82558eb78121c0c98bc63bdf397549f4a40
-  data.tar.gz: cc08c575896bc7eb8361ee283391ae71db9e1a6dc3b6820d0327098c2c0347ad0e977a0d5044c7ce428d1dfac0d9865c83aae7828535cbef70e9e1016243f662
+  metadata.gz: 4d1f260c0b0f30bd7fbcbbc41dd5ac9bfbb8f0efc42efc2691d10a08d699aaa1514a0d382c7327ea4ff134a3336d4df4230a32e056c86acaf1622da1bc01d5cf
+  data.tar.gz: bb9a87d4389f5eef5223842ab59b1ee054e2a6b8ddc06e74f54a75794671be9455da2493696aacc657039bbd2e2b148f6a69207a1b5cb8e975452ae631264b6d

data/ChangeLog

@@ -1,3 +1,9 @@
+Release 0.10.26 - 2015/03/03
+
+  * Add use_gzip_command option to reduce CPU usage
+  * Remove useless anonymizer feature
+
+
 Release 0.10.25 - 2015/02/06
 
   * Improve tempfile handling

data/fluent-plugin-td.gemspec

@@ -21,4 +21,6 @@ Gem::Specification.new do |gem|
   gem.add_dependency "td-client", "~> 0.8.66"
   gem.add_development_dependency "rake", ">= 0.9.2"
   gem.add_development_dependency "webmock", "~> 1.16"
+  gem.add_development_dependency "test-unit", "~> 3.0.8"
+  gem.add_development_dependency "test-unit-rr", "~> 1.0.3"
 end

data/lib/fluent/plugin/out_tdlog.rb

@@ -7,60 +7,6 @@ module Fluent
 
     IMPORT_SIZE_LIMIT = 32 * 1024 * 1024
 
-    class Anonymizer
-      include Configurable
-    end
-
-    class RawAnonymizer < Anonymizer
-      def anonymize(obj)
-        if obj.nil?
-          nil
-        elsif obj.is_a?(String)
-          anonymize_raw obj
-        elsif obj.is_a?(Numeric)
-          anonymize_raw obj.to_s
-        else
-          # boolean, array, map
-          anonymize_raw MessagePack.pack(obj)
-        end
-      end
-    end
-
-    class MD5Anonymizer < RawAnonymizer
-      def anonymize_raw(raw)
-        Digest::MD5.hexdigest(raw)
-      end
-    end
-
-    class IPXORAnonymizer < RawAnonymizer
-      config_param :xor_key, :string
-
-      def configure(conf)
-        super
-
-        a1, a2, a3, a4 = @xor_key.split('.')
-        @xor_keys = [a1.to_i, a2.to_i, a3.to_i, a4.to_i]
-
-        if @xor_keys == [0, 0, 0, 0]
-          raise ConfigError, "'xor_key' must be IPv4 address"
-        end
-      end
-
-      def anonymize_raw(raw)
-        m = /\A(\d+)\.(\d+)\.(\d+)\.(\d+)/.match(raw)
-        return nil unless m
-
-        k1, k2, k3, k4 = @xor_keys
-
-        o1 = m[1].to_i ^ k1
-        o2 = m[2].to_i ^ k2
-        o3 = m[3].to_i ^ k3
-        o4 = m[4].to_i ^ k4
-
-        "#{o1}.#{o2}.#{o3}.#{o4}"
-      end
-    end
-
     # To support log_level option since Fluentd v0.10.43
     unless method_defined?(:log)
       define_method(:log) { $log }
@@ -68,6 +14,7 @@ module Fluent
 
     config_param :apikey, :string
     config_param :auto_create_table, :bool, :default => true
+    config_param :use_gzip_command, :bool, :default => false
 
     config_param :endpoint, :string, :default => TreasureData::API::NEW_DEFAULT_ENDPOINT
     config_param :use_ssl, :bool, :default => true
@@ -100,6 +47,16 @@ module Fluent
     def configure(conf)
       super
 
+      if @use_gzip_command
+        require 'open3'
+
+        begin
+          Open3.capture3("gzip -V")
+        rescue Errno::ENOENT
+          raise ConfigError, "'gzip' utility must be in PATH for use_gzip_command parameter"
+        end
+      end
+
       # overwrite default value of buffer_chunk_limit
       if !conf['buffer_chunk_limit']
         @buffer.buffer_chunk_limit = IMPORT_SIZE_LIMIT
@@ -117,32 +74,6 @@ module Fluent
         @key = "#{database}.#{table}"
       end
 
-      @anonymizes = {}
-      conf.elements.select { |e|
-        e.name == 'anonymize'
-      }.each { |e|
-        key = e['key']
-        method = e['method']
-
-        case method
-        when 'md5'
-          scr = MD5Anonymizer.new
-        when 'ip_xor'
-          scr = IPXORAnonymizer.new
-        else
-          raise ConfigError, "Unknown anonymize method: #{method}"
-        end
-
-        scr.configure(e)
-
-        @anonymizes[key] = scr
-      }
-      if @anonymizes.empty?
-        @anonymizes = nil
-      else
-        log.warn "<anonymize> feature is deprecated and will be removed. Use fluent-plugin-anonymizer instead."
-      end
-
       @http_proxy = conf['http_proxy']
     end
 
@@ -190,14 +121,6 @@ module Fluent
         next unless record.is_a?(Hash)
 
         begin
-          if @anonymizes
-            @anonymizes.each_pair { |key, scr|
-              if value = record[key]
-                record[key] = scr.anonymize(value)
-              end
-            }
-          end
-
           record['time'] = time
           record.delete(:time) if record.has_key?(:time)
 
@@ -248,21 +171,56 @@ module Fluent
 
       FileUtils.mkdir_p(@tmpdir) unless @tmpdir.nil?
       f = Tempfile.new("tdlog-#{chunk.key}-", @tmpdir)
-      w = Zlib::GzipWriter.new(f)
 
-      chunk.write_to(w)
-      w.finish
-      w = nil
-
-      size = f.pos
+      size = if @use_gzip_command
+               gzip_by_command(chunk, f)
+             else
+               gzip_by_writer(chunk, f)
+             end
       f.pos = 0
       upload(database, table, f, size, unique_id)
-
     ensure
-      w.close if w
       f.close(true) if f
     end
 
+    # TODO: Share this routine with s3 compressors
+    def gzip_by_command(chunk, tmp)
+      chunk_is_file = @buffer_type == 'file'
+      path = if chunk_is_file
+               chunk.path
+             else
+               w = Tempfile.new("gzip-tdlog-#{chunk.key}-", @tmpdir)
+               chunk.write_to(w)
+               w.close
+               w.path
+             end
+      res = system "gzip -c #{path} > #{tmp.path}"
+      unless res
+        log.warn "failed to execute gzip command. Fallback to GzipWriter. status = #{$?}"
+        begin
+          tmp.truncate(0)
+          return gzip_by_writer(chunk, tmp)
+        end
+      end
+      File.size(tmp.path)
+    ensure
+      unless chunk_is_file
+        w.close(true) rescue nil
+      end
+    end
+
+    def gzip_by_writer(chunk, tmp)
+      w = Zlib::GzipWriter.new(tmp)
+      chunk.write_to(w)
+      w.finish
+      w = nil
+      tmp.pos
+    ensure
+      if w
+        w.close rescue nil
+      end
+    end
+
     def upload(database, table, io, size, unique_id)
       unique_str = unique_id.unpack('C*').map { |x| "%02x" % x }.join
       log.trace { "uploading logs to Treasure Data database=#{database} table=#{table} (#{size}bytes)" }

data/lib/fluent/plugin/td_plugin_version.rb

@@ -1,5 +1,5 @@
 module Fluent
   module TreasureDataPlugin
-    VERSION = '0.10.25'
+    VERSION = '0.10.26'
   end
 end

data/test/plugin/test_out_tdlog.rb

@@ -37,7 +37,7 @@ class TreasureDataLogOutputTest < Test::Unit::TestCase
     d = create_driver
 
     {:@apikey => 'testkey', :@use_ssl => true, :@auto_create_table => true,
-     :@buffer_type => 'file', :@flush_interval => 300}.each { |k, v|
+     :@buffer_type => 'file', :@flush_interval => 300, :@use_gzip_command => false}.each { |k, v|
       assert_equal(d.instance.instance_variable_get(k), v)
     }
   end
@@ -48,11 +48,35 @@ class TreasureDataLogOutputTest < Test::Unit::TestCase
     database, table = d.instance.instance_variable_get(:@key).split(".", 2)
     stub_td_table_create_request(database, table)
     stub_td_import_request(stub_request_body(records, time), database, table)
+    assert_rr {
+      # mock(d.instance).gzip_by_writer(is_a(Fluent::BufferChunk), is_a(Tempfile)) causes empty request body so using dont_allow instead to check calling method
+      # We need actual gzipped content to verify compressed body is correct or not.
+      dont_allow(d.instance).gzip_by_command(is_a(Fluent::BufferChunk), is_a(Tempfile))
+
+      records.each { |record|
+        d.emit(record, time)
+      }
+      d.run
+    }
 
-    records.each { |record|
-      d.emit(record, time)
+    assert_equal('TD1 testkey', @auth_header)
+  end
+
+  def test_emit_with_gzip_command
+    d = create_driver(DEFAULT_CONFIG + "use_gzip_command true")
+    time, records = stub_seed_values
+    database, table = d.instance.instance_variable_get(:@key).split(".", 2)
+    stub_td_table_create_request(database, table)
+    stub_td_import_request(stub_request_body(records, time), database, table)
+    assert_rr {
+      # same as test_emit
+      dont_allow(d.instance).gzip_by_writer(is_a(Fluent::BufferChunk), is_a(Tempfile))
+
+      records.each { |record|
+        d.emit(record, time)
+      }
+      d.run
     }
-    d.run
 
     assert_equal('TD1 testkey', @auth_header)
   end

data/test/test_helper.rb

@@ -5,6 +5,7 @@ require 'webmock/test_unit'
 require 'stringio'
 require 'td-client'
 require 'zlib'
+require 'test/unit/rr'
 
 def e(s)
   require 'cgi'
@@ -67,10 +68,10 @@ class Test::Unit::TestCase
     endpoint = opts[:endpoint] ? opts[:endpoint] : TreasureData::API::NEW_DEFAULT_IMPORT_ENDPOINT
 
     # for check_table_existence
-    url_with_empty = "#{schema}://#{endpoint}//v3/table/import/#{e(db)}/#{e(table)}/#{format}"
+    url_with_empty = "#{schema}://#{endpoint}/v3/table/import/#{e(db)}/#{e(table)}/#{format}"
     stub_request(:put, url_with_empty).to_return(:status => 200, :body => response)
 
-    url_with_unique = Regexp.compile("#{schema}://#{endpoint}//v3/table/import_with_id/#{e(db)}/#{e(table)}/.*/#{format}")
+    url_with_unique = Regexp.compile("#{schema}://#{endpoint}/v3/table/import_with_id/#{e(db)}/#{e(table)}/.*/#{format}")
     stub_request(:put, url_with_unique).with(:headers => {'Content-Type' => 'application/octet-stream'}) { |req|
       @auth_header = req.headers["Authorization"]
       stub_gzip_unwrap(req.body) == stub_gzip_unwrap(body)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-td
 version: !ruby/object:Gem::Version
-  version: 0.10.25
+  version: 0.10.26
 platform: ruby
 authors:
 - Treasure Data, Inc.
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-02-05 00:00:00.000000000 Z
+date: 2015-03-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -72,6 +72,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.16'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.0.8
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.0.8
+- !ruby/object:Gem::Dependency
+  name: test-unit-rr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.0.3
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.0.3
 description: Treasure Data Cloud Data Service plugin for Fluentd
 email: support@treasure-data.com
 executables: []