fluent-plugin-webhdfs 0.3.1 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5df26424bd649dd03e273297dc2f0d0a8c58a2d1
-  data.tar.gz: 06554c7a7da3c9a0a2ecaa1654212368ba0f196e
+  metadata.gz: e67f69a6f94c64ebe9b463b262111626a5cfb0e3
+  data.tar.gz: 9786c0456e296851caf6001898c8e30751724b04
 SHA512:
-  metadata.gz: 417ef686757c6fdc7b66128a1f7296050f821a7d42f658f2e933f14c5002254832a289935b7b99752014248331992baef818a7563660b2c524ef81ea4aa76231
-  data.tar.gz: d22224126c6324b21ea505159a2d8a2487270cead8a0ee39181b5fa3a23459b7296210cf54aa613fc64fc7ca02acf79d0efc98089c3db5169ac743e735ba7c6b
+  metadata.gz: 6766521d3edaf773135c7d948ba0f8989241535795f7a38d61961c3dd9823df83955c0e4f817b029600c8e83495e24443015d2b863f68dd911dc6ea7ca5f8274
+  data.tar.gz: 3834bcb691d9d5c86df91169f44f77a9aaad09754d849919138be5ce403271b3a168c3620a0bbafc80bd8f3a2553cc0b3fd97bc14e794c0637dd1cc06dc3c7ec

data/README.md

@@ -93,6 +93,33 @@ Store data as TSV (TAB separated values) of specified keys, without time, with t
 
 If message doesn't have specified attribute, fluent-plugin-webhdfs outputs 'NULL' instead of values.
 
+With ssl:
+
+    <match access.**>
+      type webhdfs
+      host namenode.your.cluster.local
+      port 50070
+      path /path/on/hdfs/access.log.%Y%m%d_%H.log
+      ssl true
+      ssl_ca_file /path/to/ca_file.pem   # if needed
+      ssl_verify_mode peer               # if needed (peer or none)
+    </match>
+
+Here `ssl_verify_mode peer` means to verify the server's certificate.
+You can turn off it by setting `ssl_verify_mode none`. The default is `peer`.
+See [net/http](http://www.ruby-doc.org/stdlib-2.1.3/libdoc/net/http/rdoc/Net/HTTP.html)
+and [openssl](http://www.ruby-doc.org/stdlib-2.1.3/libdoc/openssl/rdoc/OpenSSL.html) documentation for further details.
+
+With kerberos authentication:
+
+    <match access.**>
+      type webhdfs
+      host namenode.your.cluster.local
+      port 50070
+      path /path/on/hdfs/access.log.%Y%m%d_%H.log
+      kerberos true
+    </match>
+
 ### Namenode HA / Auto retry for WebHDFS known errors
 
 `fluent-plugin-webhdfs` (v0.2.0 or later) accepts 2 namenodes for Namenode HA (active/standby). Use `standby_namenode` like this:

data/fluent-plugin-webhdfs.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-webhdfs"
-  gem.version       = "0.3.1"
+  gem.version       = "0.4.0"
   gem.authors       = ["TAGOMORI Satoshi"]
   gem.email         = ["tagomoris@gmail.com"]
   gem.summary       = %q{Fluentd plugin to write data on HDFS over WebHDFS, with flexible formatting}
@@ -19,5 +19,5 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency "fluentd", '>= 0.10.53'
   gem.add_runtime_dependency "fluent-mixin-plaintextformatter", '>= 0.2.1'
   gem.add_runtime_dependency "fluent-mixin-config-placeholders", ">= 0.3.0"
-  gem.add_runtime_dependency "webhdfs", '>= 0.5.3'
+  gem.add_runtime_dependency "webhdfs", '>= 0.6.0'
 end

data/lib/fluent/plugin/out_webhdfs.rb

@@ -41,6 +41,21 @@ class Fluent::WebHDFSOutput < Fluent::TimeSlicedOutput
 
   config_param :append, :bool, :default => true
 
+  config_param :ssl, :bool, :default => false
+  config_param :ssl_ca_file, :string, :default => nil
+  config_param :ssl_verify_mode, :default => nil do |val|
+    case val
+    when 'none'
+      :none
+    when 'peer'
+      :peer
+    else
+      raise ConfigError, "unexpected parameter on ssl_verify_mode: #{val}"
+    end
+  end
+
+  config_param :kerberos, :bool, :default => false
+
   CHUNK_ID_PLACE_HOLDER = '${chunk_id}'
 
   def initialize
@@ -120,6 +135,14 @@ class Fluent::WebHDFSOutput < Fluent::TimeSlicedOutput
       client.retry_interval = @retry_interval if @retry_interval
       client.retry_times = @retry_times if @retry_times
     end
+    if @ssl
+      client.ssl = true
+      client.ssl_ca_file = @ssl_ca_file if @ssl_ca_file
+      client.ssl_verify_mode = @ssl_verify_mode if @ssl_verify_mode
+    end
+    if @kerberos
+      client.kerberos = true
+    end
 
     client
   end

data/test/plugin/test_out_webhdfs.rb

@@ -40,6 +40,23 @@ username hdfs_user
     assert_equal '%Y%m%d%H%M', d.instance.time_slice_format
     assert_equal true, d.instance.httpfs
     assert_equal 'hdfs_user', d.instance.username
+
+    d = create_driver %[
+namenode server.local:14000
+path /hdfs/path/file.%Y%m%d.%H%M.log
+ssl true
+ssl_ca_file /path/to/ca_file.pem
+ssl_verify_mode peer
+kerberos true
+]
+    assert_equal 'server.local', d.instance.instance_eval{ @namenode_host }
+    assert_equal 14000, d.instance.instance_eval{ @namenode_port }
+    assert_equal '/hdfs/path/file.%Y%m%d.%H%M.log', d.instance.path
+    assert_equal '%Y%m%d%H%M', d.instance.time_slice_format
+    assert_equal true, d.instance.ssl
+    assert_equal '/path/to/ca_file.pem', d.instance.ssl_ca_file
+    assert_equal :peer, d.instance.ssl_verify_mode
+    assert_equal true, d.instance.kerberos
   end
 
   def test_configure_placeholders

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-webhdfs
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.4.0
 platform: ruby
 authors:
 - TAGOMORI Satoshi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-09-29 00:00:00.000000000 Z
+date: 2014-10-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -72,14 +72,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.5.3
+        version: 0.6.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.5.3
+        version: 0.6.0
 description: For WebHDFS and HttpFs of Hadoop HDFS
 email:
 - tagomoris@gmail.com
@@ -124,4 +124,3 @@ summary: Fluentd plugin to write data on HDFS over WebHDFS, with flexible format
 test_files:
 - test/helper.rb
 - test/plugin/test_out_webhdfs.rb
-has_rdoc: