fluent-plugin-webhdfs 0.4.1 → 0.4.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5e43e061acd7552d5a0e482bf7ced1e319d1d107
-  data.tar.gz: 934ac37c1a6d985261ae573ce0724546481269ff
+  metadata.gz: 61f1146fdfa68de12937082f2321f097e4aa5858
+  data.tar.gz: 502673646e6db2db20973079a4c2056515b7de11
 SHA512:
-  metadata.gz: be9c06348692d4f0506f0e7d8caa1fc4d6bfa4d683c1fdb0daa3fd87d4657ddbebaff4380c3fc03405030836cabc2fb856e02754aa8bccff1ddd5f7c7b49980f
-  data.tar.gz: fee1d87a57a64ea65ca14a2f3f7765eb115bd362d504f0ccade232132a384b4d8f599ccd264c8cdcb2f326c4a8062b7c5e456e4cf11c8cc7fab5d78dadf09e3a
+  metadata.gz: b634dd49ac2d1e3d552e7dbda396cc5aca47c0cd090219c5464a3a834ec9ffd5c0030d013474e4b4ac83870252f45af567788d97e0671113278d41bd168b969a
+  data.tar.gz: 90dfe3fc5e45a3667df905874607875cba4cd18e49e22cc05318dd4141ec1e1cec3d36d933ac15634d780ef46ab75565d63fea6ffcc5c3678b901ced071042c4

data/.travis.yml

@@ -1,12 +1,15 @@
 language: ruby
 
 rvm:
-  - 1.9.3
   - 2.0.0
-  - 2.1.1
+  - 2.1
+  - 2.2
 
 branches:
   only:
     - master
 
+before_install:
+  - gem update bundler
+
 script: bundle exec rake test

data/fluent-plugin-webhdfs.gemspec

@@ -2,13 +2,13 @@
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-webhdfs"
-  gem.version       = "0.4.1"
+  gem.version       = "0.4.2"
   gem.authors       = ["TAGOMORI Satoshi"]
   gem.email         = ["tagomoris@gmail.com"]
   gem.summary       = %q{Fluentd plugin to write data on HDFS over WebHDFS, with flexible formatting}
   gem.description   = %q{For WebHDFS and HttpFs of Hadoop HDFS}
   gem.homepage      = "https://github.com/fluent/fluent-plugin-webhdfs"
-  gem.license       = "APLv2"
+  gem.license       = "Apache-2.0"
 
   gem.files         = `git ls-files`.split($\)
   gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
@@ -16,8 +16,10 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
 
   gem.add_development_dependency "rake"
+  gem.add_development_dependency "test-unit"
   gem.add_runtime_dependency "fluentd", '>= 0.10.53'
   gem.add_runtime_dependency "fluent-mixin-plaintextformatter", '>= 0.2.1'
   gem.add_runtime_dependency "fluent-mixin-config-placeholders", ">= 0.3.0"
   gem.add_runtime_dependency "webhdfs", '>= 0.6.0'
+  gem.add_runtime_dependency "bzip2-ffi"
 end

data/lib/fluent/plugin/out_webhdfs.rb

@@ -1,5 +1,7 @@
 # -*- coding: utf-8 -*-
 
+require 'tempfile'
+
 require 'fluent/mixin/config_placeholders'
 require 'fluent/mixin/plaintextformatter'
 
@@ -9,40 +11,66 @@ class Fluent::WebHDFSOutput < Fluent::TimeSlicedOutput
   config_set_default :buffer_type, 'memory'
   config_set_default :time_slice_format, '%Y%m%d'
 
+  # For fluentd v0.12.16 or earlier
+  class << self
+    unless method_defined?(:desc)
+      def desc(description)
+      end
+    end
+  end
+
+  desc 'WebHDFS/HttpFs host'
   config_param :host, :string, :default => nil
+  desc 'WebHDFS/HttpFs port'
   config_param :port, :integer, :default => 50070
+  desc 'Namenode (host:port)'
   config_param :namenode, :string, :default => nil # host:port
+  desc 'Standby namenode for Namenode HA (host:port)'
   config_param :standby_namenode, :string, :default => nil # host:port
 
+  desc 'Ignore errors on start up'
   config_param :ignore_start_check_error, :bool, :default => false
 
   include Fluent::Mixin::ConfigPlaceholders
 
+  desc 'Output file path on HDFS'
   config_param :path, :string
+  desc 'User name for pseudo authentication'
   config_param :username, :string, :default => nil
 
+  desc 'Store data over HttpFs instead of WebHDFS'
   config_param :httpfs, :bool, :default => false
 
+  desc 'Number of seconds to wait for the connection to open'
   config_param :open_timeout, :integer, :default => 30 # from ruby net/http default
+  desc 'Number of seconds to wait for one block to be read'
   config_param :read_timeout, :integer, :default => 60 # from ruby net/http default
 
+  desc 'Retry automatically when known errors of HDFS are occurred'
   config_param :retry_known_errors, :bool, :default => false
+  desc 'Retry interval'
   config_param :retry_interval, :integer, :default => nil
+  desc 'The number of retries'
   config_param :retry_times, :integer, :default => nil
 
   # how many times of write failure before switch to standby namenode
   # by default it's 11 times that costs 1023 seconds inside fluentd,
   # which is considered enough to exclude the scenes that caused by temporary network fail or single datanode fail
+  desc 'How many times of write failure before switch to standby namenode'
   config_param :failures_before_use_standby, :integer, :default => 11
 
   include Fluent::Mixin::PlainTextFormatter
 
   config_param :default_tag, :string, :default => 'tag_missing'
 
+  desc 'Append data or not'
   config_param :append, :bool, :default => true
 
+  desc 'Use SSL or not'
   config_param :ssl, :bool, :default => false
+  desc 'OpenSSL certificate authority file'
   config_param :ssl_ca_file, :string, :default => nil
+  desc 'OpenSSL verify mode (none,peer)'
   config_param :ssl_verify_mode, :default => nil do |val|
     case val
     when 'none'
@@ -54,9 +82,11 @@ class Fluent::WebHDFSOutput < Fluent::TimeSlicedOutput
     end
   end
 
+  desc 'Use kerberos authentication or not'
   config_param :kerberos, :bool, :default => false
 
-  SUPPORTED_COMPRESS = ['gzip']
+  SUPPORTED_COMPRESS = ['gzip', 'bzip2']
+  desc "Compress method (#{SUPPORTED_COMPRESS.join(',')})"
   config_param :compress, :default => nil do |val|
     unless SUPPORTED_COMPRESS.include? val
       raise Fluent::ConfigError, "unsupported compress: #{val}"
@@ -66,11 +96,15 @@ class Fluent::WebHDFSOutput < Fluent::TimeSlicedOutput
 
   CHUNK_ID_PLACE_HOLDER = '${chunk_id}'
 
+  attr_reader :compressor
+
   def initialize
     super
     require 'net/http'
     require 'time'
     require 'webhdfs'
+
+    @compressor = nil
   end
 
   # Define `log` method for v0.10.42 or earlier
@@ -91,6 +125,13 @@ class Fluent::WebHDFSOutput < Fluent::TimeSlicedOutput
 
     super
 
+    begin
+      @compressor = COMPRESSOR_REGISTRY.lookup(@compress || 'text').new
+    rescue
+      $log.warn "#{@comress} not found. Use 'text' instead"
+      @compressor = COMPRESSOR_REGISTRY.lookup('text').new
+    end
+
     if @host
       @namenode_host = @host
       @namenode_port = @port
@@ -230,31 +271,18 @@ class Fluent::WebHDFSOutput < Fluent::TimeSlicedOutput
                 else
                   path_format(chunk.key).gsub(CHUNK_ID_PLACE_HOLDER, chunk_unique_id_to_str(chunk.unique_id))
                 end
-    if @compress
-      case @compress
-      when 'gzip'
-        hdfs_path = "#{hdfs_path}.gz"
-      end
-    end
+    hdfs_path = "#{hdfs_path}#{@compressor.ext}"
     hdfs_path
   end
 
   def compress_context(chunk, &block)
-    case @compress
-    when 'gzip'
-      require 'zlib'
-      require 'tempfile'
+    begin
       tmp = Tempfile.new("webhdfs-")
-      begin
-        w = Zlib::GzipWriter.new(tmp)
-        chunk.write_to(w)
-        w.close
-        tmp.close
-        tmp.open
-        yield tmp
-      ensure
-        tmp.close(true) rescue nil
-      end
+      @compressor.compress(chunk, tmp)
+      tmp.rewind
+      yield tmp
+    ensure
+      tmp.close(true) rescue nil
     end
   end
 
@@ -263,12 +291,8 @@ class Fluent::WebHDFSOutput < Fluent::TimeSlicedOutput
 
     failovered = false
     begin
-      if @compress
-        compress_context(chunk) do |data|
-          send_data(hdfs_path, data)
-        end
-      else
-        send_data(hdfs_path, chunk.read)
+      compress_context(chunk) do |data|
+        send_data(hdfs_path, data)
       end
     rescue => e
       log.warn "failed to communicate hdfs cluster, path: #{hdfs_path}"
@@ -291,4 +315,45 @@ class Fluent::WebHDFSOutput < Fluent::TimeSlicedOutput
     end
     hdfs_path
   end
+
+  class Compressor
+    include Fluent::Configurable
+
+    def initialise(options = {})
+      super()
+    end
+
+    def configure(conf)
+      super
+    end
+
+    def ext
+    end
+
+    def compress(chunk)
+    end
+
+    private
+
+    def check_command(command, algo = nil)
+      require 'open3'
+
+      algo = command if algo.nil?
+      begin
+        Open3.capture3("#{command} -V")
+      rescue Errno::ENOENT
+        raise ConfigError, "'#{command}' utility must be in PATH for #{algo} compression"
+      end
+    end
+  end
+
+  COMPRESSOR_REGISTRY = Fluent::Registry.new(:webhdfs_compressor_type, 'fluent/plugin/webhdfs_compressor_')
+
+  def self.register_compressor(name, compressor)
+    COMPRESSOR_REGISTRY.register(name, compressor)
+  end
 end
+
+require 'fluent/plugin/webhdfs_compressor_text'
+require 'fluent/plugin/webhdfs_compressor_gzip'
+require 'fluent/plugin/webhdfs_compressor_bzip2'

data/lib/fluent/plugin/webhdfs_compressor_bzip2.rb

@@ -0,0 +1,21 @@
+module Fluent
+  class WebHDFSOutput < Fluent::TimeSlicedOutput
+    class Bzip2Compressor < Compressor
+      WebHDFSOutput.register_compressor('bzip2', self)
+
+      def initialize(options = {})
+        require "bzip2/ffi"
+      end
+
+      def ext
+        ".bz2"
+      end
+
+      def compress(chunk, tmp)
+        Bzip2::FFI::Writer.open(tmp) do |writer|
+          chunk.write_to(writer)
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/webhdfs_compressor_gzip.rb

@@ -0,0 +1,21 @@
+module Fluent
+  class WebHDFSOutput < Fluent::TimeSlicedOutput
+    class GzipCompressor < Compressor
+      WebHDFSOutput.register_compressor('gzip', self)
+
+      def initialize(options = {})
+        require "zlib"
+      end
+
+      def ext
+        ".gz"
+      end
+
+      def compress(chunk, tmp)
+        w = Zlib::GzipWriter.new(tmp)
+        chunk.write_to(w)
+        w.close
+      end
+    end
+  end
+end

data/lib/fluent/plugin/webhdfs_compressor_text.rb

@@ -0,0 +1,15 @@
+module Fluent
+  class WebHDFSOutput < Fluent::TimeSlicedOutput
+    class TextCompressor < Compressor
+      WebHDFSOutput.register_compressor('text', self)
+
+      def ext
+        ""
+      end
+
+      def compress(chunk, tmp)
+        chunk.write_to(tmp)
+      end
+    end
+  end
+end

data/test/plugin/test_out_webhdfs.rb

@@ -5,43 +5,51 @@ class WebHDFSOutputTest < Test::Unit::TestCase
 host namenode.local
 path /hdfs/path/file.%Y%m%d.log
   ]
+  def setup
+    Fluent::Test.setup
+  end
 
   def create_driver(conf=CONFIG,tag='test')
     Fluent::Test::OutputTestDriver.new(Fluent::WebHDFSOutput, tag).configure(conf)
   end
 
-  def test_configure
-    d = create_driver
-    assert_equal 'namenode.local', d.instance.instance_eval{ @namenode_host }
-    assert_equal 50070, d.instance.instance_eval{ @namenode_port }
-    assert_equal '/hdfs/path/file.%Y%m%d.log', d.instance.path
-    assert_equal '%Y%m%d', d.instance.time_slice_format
-    assert_equal false, d.instance.httpfs
-    assert_nil d.instance.username
-    assert_equal false, d.instance.ignore_start_check_error
-    
-    assert_equal true, d.instance.output_include_time
-    assert_equal true, d.instance.output_include_tag
-    assert_equal 'json', d.instance.output_data_type
-    assert_nil d.instance.remove_prefix
-    assert_equal 'TAB', d.instance.field_separator
-    assert_equal true, d.instance.add_newline
-    assert_equal 'tag_missing', d.instance.default_tag
+  class ConfigureTest < self
+    def test_default
+      d = create_driver
+      assert_equal 'namenode.local', d.instance.instance_eval{ @namenode_host }
+      assert_equal 50070, d.instance.instance_eval{ @namenode_port }
+      assert_equal '/hdfs/path/file.%Y%m%d.log', d.instance.path
+      assert_equal '%Y%m%d', d.instance.time_slice_format
+      assert_equal false, d.instance.httpfs
+      assert_nil d.instance.username
+      assert_equal false, d.instance.ignore_start_check_error
+
+      assert_equal true, d.instance.output_include_time
+      assert_equal true, d.instance.output_include_tag
+      assert_equal 'json', d.instance.output_data_type
+      assert_nil d.instance.remove_prefix
+      assert_equal 'TAB', d.instance.field_separator
+      assert_equal true, d.instance.add_newline
+      assert_equal 'tag_missing', d.instance.default_tag
+    end
 
-    d = create_driver %[
+    def test_httpfs
+      d = create_driver %[
 namenode server.local:14000
 path /hdfs/path/file.%Y%m%d.%H%M.log
 httpfs yes
 username hdfs_user
 ]
-    assert_equal 'server.local', d.instance.instance_eval{ @namenode_host }
-    assert_equal 14000, d.instance.instance_eval{ @namenode_port }
-    assert_equal '/hdfs/path/file.%Y%m%d.%H%M.log', d.instance.path
-    assert_equal '%Y%m%d%H%M', d.instance.time_slice_format
-    assert_equal true, d.instance.httpfs
-    assert_equal 'hdfs_user', d.instance.username
+      assert_equal 'server.local', d.instance.instance_eval{ @namenode_host }
+      assert_equal 14000, d.instance.instance_eval{ @namenode_port }
+      assert_equal '/hdfs/path/file.%Y%m%d.%H%M.log', d.instance.path
+      assert_equal '%Y%m%d%H%M', d.instance.time_slice_format
+      assert_equal true, d.instance.httpfs
+      assert_equal 'hdfs_user', d.instance.username
+    end
 
-    d = create_driver %[
+    def test_ssl
+      d = create_driver %[
 namenode server.local:14000
 path /hdfs/path/file.%Y%m%d.%H%M.log
 ssl true
@@ -49,75 +57,92 @@ ssl_ca_file /path/to/ca_file.pem
 ssl_verify_mode peer
 kerberos true
 ]
-    assert_equal 'server.local', d.instance.instance_eval{ @namenode_host }
-    assert_equal 14000, d.instance.instance_eval{ @namenode_port }
-    assert_equal '/hdfs/path/file.%Y%m%d.%H%M.log', d.instance.path
-    assert_equal '%Y%m%d%H%M', d.instance.time_slice_format
-    assert_equal true, d.instance.ssl
-    assert_equal '/path/to/ca_file.pem', d.instance.ssl_ca_file
-    assert_equal :peer, d.instance.ssl_verify_mode
-    assert_equal true, d.instance.kerberos
+      assert_equal 'server.local', d.instance.instance_eval{ @namenode_host }
+      assert_equal 14000, d.instance.instance_eval{ @namenode_port }
+      assert_equal '/hdfs/path/file.%Y%m%d.%H%M.log', d.instance.path
+      assert_equal '%Y%m%d%H%M', d.instance.time_slice_format
+      assert_equal true, d.instance.ssl
+      assert_equal '/path/to/ca_file.pem', d.instance.ssl_ca_file
+      assert_equal :peer, d.instance.ssl_verify_mode
+      assert_equal true, d.instance.kerberos
+    end
 
-    d = create_driver %[
+    data(gzip: ['gzip', Fluent::WebHDFSOutput::GzipCompressor],
+         bzip2: ['bzip2', Fluent::WebHDFSOutput::Bzip2Compressor])
+    def test_compress(data)
+      compress_type, compressor_class = data
+      d = create_driver %[
 namenode server.local:14000
 path /hdfs/path/file.%Y%m%d.%H%M.log
-compress gzip
+compress #{compress_type}
 ]
-    assert_equal 'server.local', d.instance.instance_eval{ @namenode_host }
-    assert_equal 14000, d.instance.instance_eval{ @namenode_port }
-    assert_equal '/hdfs/path/file.%Y%m%d.%H%M.log', d.instance.path
-    assert_equal '%Y%m%d%H%M', d.instance.time_slice_format
-    assert_equal 'gzip', d.instance.compress
-  end
+      assert_equal 'server.local', d.instance.instance_eval{ @namenode_host }
+      assert_equal 14000, d.instance.instance_eval{ @namenode_port }
+      assert_equal '/hdfs/path/file.%Y%m%d.%H%M.log', d.instance.path
+      assert_equal '%Y%m%d%H%M', d.instance.time_slice_format
+      assert_equal compress_type, d.instance.compress
+      assert_equal compressor_class, d.instance.compressor.class
+    end
 
-  def test_configure_placeholders
-    d = create_driver %[
+    def test_placeholders
+      d = create_driver %[
 hostname testing.node.local
 namenode server.local:50070
 path /hdfs/${hostname}/file.%Y%m%d%H.log
 ]
-    assert_equal '/hdfs/testing.node.local/file.%Y%m%d%H.log', d.instance.path
-  end
+      assert_equal '/hdfs/testing.node.local/file.%Y%m%d%H.log', d.instance.path
+    end
 
-  def test_path_format
-    d = create_driver
-    assert_equal '/hdfs/path/file.%Y%m%d.log', d.instance.path
-    assert_equal '%Y%m%d', d.instance.time_slice_format
-    assert_equal '/hdfs/path/file.20120718.log', d.instance.path_format('20120718')
+    class PathFormatTest < self
+      def test_default
+        d = create_driver
+        assert_equal '/hdfs/path/file.%Y%m%d.log', d.instance.path
+        assert_equal '%Y%m%d', d.instance.time_slice_format
+        assert_equal '/hdfs/path/file.20120718.log', d.instance.path_format('20120718')
+      end
 
-    d = create_driver %[
+      def test_time_slice_format
+        d = create_driver %[
 namenode server.local:14000
 path /hdfs/path/file.%Y%m%d.%H%M.log
 ]
-    assert_equal '/hdfs/path/file.%Y%m%d.%H%M.log', d.instance.path
-    assert_equal '%Y%m%d%H%M', d.instance.time_slice_format
-    assert_equal '/hdfs/path/file.20120718.1503.log', d.instance.path_format('201207181503')
+        assert_equal '/hdfs/path/file.%Y%m%d.%H%M.log', d.instance.path
+        assert_equal '%Y%m%d%H%M', d.instance.time_slice_format
+        assert_equal '/hdfs/path/file.20120718.1503.log', d.instance.path_format('201207181503')
+      end
+    end
 
-    assert_raise Fluent::ConfigError do
-      d = create_driver %[
+    class InvalidTest < self
+      def test_path
+        assert_raise Fluent::ConfigError do
+          d = create_driver %[
             namenode server.local:14000
             path /hdfs/path/file.%Y%m%d.%H%M.log
             append false
           ]
-    end
-  end
+        end
+      end
 
-  def test_invalid_configure
-    assert_raise Fluent::ConfigError do
-      create_driver %[
-        namenode server.local:14000
-        path /hdfs/path/file.%Y%m%d.%H%M.log
-        ssl true
-        ssl_verify_mode invalid
-      ]
-    end
-    assert_raise Fluent::ConfigError do
-      create_driver %[
-        namenode server.local:14000
-        path /hdfs/path/file.%Y%m%d.%H%M.log
-        compress invalid
-      ]
+      def test_ssl
+        assert_raise Fluent::ConfigError do
+          create_driver %[
+            namenode server.local:14000
+            path /hdfs/path/file.%Y%m%d.%H%M.log
+            ssl true
+            ssl_verify_mode invalid
+          ]
+        end
+      end
+
+      def test_invalid_compress
+        assert_raise Fluent::ConfigError do
+          create_driver %[
+            namenode server.local:14000
+            path /hdfs/path/file.%Y%m%d.%H%M.log
+            compress invalid
+          ]
+        end
+      end
     end
   end
-
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-webhdfs
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.4.2
 platform: ruby
 authors:
 - TAGOMORI Satoshi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-11-10 00:00:00.000000000 Z
+date: 2016-01-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -24,6 +24,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
@@ -80,6 +94,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.6.0
+- !ruby/object:Gem::Dependency
+  name: bzip2-ffi
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: For WebHDFS and HttpFs of Hadoop HDFS
 email:
 - tagomoris@gmail.com
@@ -95,11 +123,14 @@ files:
 - Rakefile
 - fluent-plugin-webhdfs.gemspec
 - lib/fluent/plugin/out_webhdfs.rb
+- lib/fluent/plugin/webhdfs_compressor_bzip2.rb
+- lib/fluent/plugin/webhdfs_compressor_gzip.rb
+- lib/fluent/plugin/webhdfs_compressor_text.rb
 - test/helper.rb
 - test/plugin/test_out_webhdfs.rb
 homepage: https://github.com/fluent/fluent-plugin-webhdfs
 licenses:
-- APLv2
+- Apache-2.0
 metadata: {}
 post_install_message: 
 rdoc_options: []
@@ -117,10 +148,11 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.4.5
 signing_key: 
 specification_version: 4
 summary: Fluentd plugin to write data on HDFS over WebHDFS, with flexible formatting
 test_files:
 - test/helper.rb
 - test/plugin/test_out_webhdfs.rb
+has_rdoc: