fluent-plugin-webhdfs 1.2.0 → 1.2.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: a68b2c521a6b09397a3ec62b31a839a746b968d6
-  data.tar.gz: d41eda6c2024f91215d13af9a3bfe606c3f5a984
+SHA256:
+  metadata.gz: 7a0c203ea54e21d95aea500fbb547ea965d5a5c07c1b277b69df2bbc44fa50fe
+  data.tar.gz: 6c93680eeba4575e2ef817e58772d4eb64a2e205696ad87b4b6decc1ed300e90
 SHA512:
-  metadata.gz: 0d90a9def013f28eaf3e710aa039b2a40f7ac79215ee8c3b0bf88936f53b4a27fe12c6681ad89da2cd2be8a14026f4194022aa2f9ee860b658f26b38e4ca03ad
-  data.tar.gz: ee3ee7ff6c0524e1b5b14fd5bd168d44094cfd84e36f9d872f3ad28f3f5dfc6e4bd76b14c4ba60bd472bd40f34b2f63bcd9f53c4c62a2290e204402e8bfcb444
+  metadata.gz: 91be9ef776972c9d149221b2de2b765890e43e6df1e2a2408876b353aae629552a64d8029cfda84d8c03510e77ac0e164c7a8ed379d8d878505764df6181ca7e
+  data.tar.gz: 4b82ec9889bbd1a4e4fdd03150d082477a71ac03bacf5a72510e0dd816f8cfdb7e16dc5dc40ed02fdd2118f3a6e35919cf7032b0c7194bbdf76965dfc61c587e

data/README.md

@@ -1,5 +1,7 @@
 # fluent-plugin-webhdfs
 
+[![Build Status](https://travis-ci.org/fluent/fluent-plugin-webhdfs.svg?branch=master)](https://travis-ci.org/fluent/fluent-plugin-webhdfs)
+
 [Fluentd](http://fluentd.org/) output plugin to write data into Hadoop HDFS over WebHDFS/HttpFs.
 
 "webhdfs" output plugin formats data into plain text, and store it as files on HDFS. This plugin supports:
@@ -143,8 +145,11 @@ With kerberos authentication:
       port 50070
       path /path/on/hdfs/access.log.%Y%m%d_%H.log
       kerberos true
+      kerberos_keytab /path/to/keytab # if needed
     </match>
 
+NOTE: You need to install `gssapi` gem for kerberos. See https://github.com/kzk/webhdfs#for-kerberos-authentication
+
 If you want to compress data before storing it:
 
     <match access.**>
@@ -222,7 +227,7 @@ For high load cluster nodes, you can specify timeouts for HTTP requests.
 
 With default configuration, fluent-plugin-webhdfs checks HDFS filesystem status and raise error for inacive NameNodes.
 
-If you were usging unstable NameNodes and have wanted to ignore NameNode errors on startup of fluentd, enable `ignore_start_check_error` option like below:
+If you were using unstable NameNodes and have wanted to ignore NameNode errors on startup of fluentd, enable `ignore_start_check_error` option like below:
 
     <match access.**>
       @type webhdfs

data/fluent-plugin-webhdfs.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-webhdfs"
-  gem.version       = "1.2.0"
+  gem.version       = "1.2.5"
   gem.authors       = ["TAGOMORI Satoshi"]
   gem.email         = ["tagomoris@gmail.com"]
   gem.summary       = %q{Fluentd plugin to write data on HDFS over WebHDFS, with flexible formatting}
@@ -21,6 +21,6 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency "appraisal"
   gem.add_development_dependency "snappy", '>= 0.0.13'
   gem.add_development_dependency "bzip2-ffi"
-  gem.add_runtime_dependency "fluentd", '>= 0.14.4'
+  gem.add_runtime_dependency "fluentd", '>= 0.14.22'
   gem.add_runtime_dependency "webhdfs", '>= 0.6.0'
 end

data/lib/fluent/plugin/out_webhdfs.rb

@@ -64,6 +64,8 @@ class Fluent::Plugin::WebHDFSOutput < Fluent::Plugin::Output
 
   desc 'Use kerberos authentication or not'
   config_param :kerberos, :bool, default: false
+  desc 'kerberos keytab file'
+  config_param :kerberos_keytab, :string, default: nil
 
   SUPPORTED_COMPRESS = [:gzip, :bzip2, :snappy, :lzo_command, :text]
   desc "Compress method (#{SUPPORTED_COMPRESS.join(',')})"
@@ -96,20 +98,27 @@ class Fluent::Plugin::WebHDFSOutput < Fluent::Plugin::Output
   end
 
   def configure(conf)
-    compat_parameters_convert(conf, :buffer, default_chunk_key: "time")
-
+    # #compat_parameters_convert ignore time format in conf["path"],
+    # so check conf["path"] and overwrite the default value later if needed
     timekey = case conf["path"]
               when /%S/ then 1
               when /%M/ then 60
               when /%H/ then 3600
               else 86400
               end
+    if buffer_config = conf.elements(name: "buffer").first
+      timekey = buffer_config["timekey"] || timekey 
+    end
+
+    compat_parameters_convert(conf, :buffer, default_chunk_key: "time")
+
     if conf.elements(name: "buffer").empty?
       e = Fluent::Config::Element.new("buffer", "time", {}, [])
       conf.elements << e
     end
     buffer_config = conf.elements(name: "buffer").first
-    buffer_config["timekey"] = timekey unless buffer_config["timekey"]
+    # explicitly set timekey
+    buffer_config["timekey"] = timekey
 
     compat_parameters_convert_plaintextformatter(conf)
     verify_config_placeholders_in_path!(conf)
@@ -208,6 +217,7 @@ class Fluent::Plugin::WebHDFSOutput < Fluent::Plugin::Output
     end
     if @kerberos
       client.kerberos = true
+      client.kerberos_keytab = @kerberos_keytab if @kerberos_keytab
     end
 
     client
@@ -305,9 +315,9 @@ class Fluent::Plugin::WebHDFSOutput < Fluent::Plugin::Output
 
   def generate_path(chunk)
     hdfs_path = if @append
-                  extract_placeholders(@path, chunk.metadata)
+                  extract_placeholders(@path, chunk)
                 else
-                  extract_placeholders(@path, chunk.metadata).gsub(CHUNK_ID_PLACE_HOLDER, dump_unique_id_hex(chunk.unique_id))
+                  extract_placeholders(@path.gsub(CHUNK_ID_PLACE_HOLDER, dump_unique_id_hex(chunk.unique_id)), chunk)
                 end
     hdfs_path = "#{hdfs_path}#{@compressor.ext}"
     if @replace_random_uuid

data/lib/fluent/plugin/webhdfs_compressor_snappy.rb

@@ -16,9 +16,10 @@ module Fluent::Plugin
       end
 
       def compress(chunk, tmp)
-        w = Snappy::Writer.new(tmp)
-        chunk.write_to(w)
-        w.close
+        Snappy::Writer.new(tmp) do |w|
+          w << chunk.read
+          w.flush
+        end
       end
     end
   end

data/test/plugin/test_out_webhdfs.rb

@@ -89,7 +89,8 @@ class WebHDFSOutputTest < Test::Unit::TestCase
           "ssl" => true,
           "ssl_ca_file" => "/path/to/ca_file.pem",
           "ssl_verify_mode" => "peer",
-          "kerberos" => true
+          "kerberos" => true,
+          "kerberos_keytab" => "/path/to/kerberos.keytab"
         })
       d = create_driver(conf)
 
@@ -100,6 +101,7 @@ class WebHDFSOutputTest < Test::Unit::TestCase
       assert_equal '/path/to/ca_file.pem', d.instance.ssl_ca_file
       assert_equal :peer, d.instance.ssl_verify_mode
       assert_equal true, d.instance.kerberos
+      assert_equal '/path/to/kerberos.keytab', d.instance.kerberos_keytab
     end
 
     data(gzip: [:gzip, Fluent::Plugin::WebHDFSOutput::GzipCompressor],
@@ -166,6 +168,7 @@ class WebHDFSOutputTest < Test::Unit::TestCase
       metadata = d.instance.metadata("test", nil, {})
       chunk = d.instance.buffer.generate_chunk(metadata)
       assert_equal "/hdfs/path/file.#{dump_unique_id_hex(chunk.unique_id)}.log", d.instance.generate_path(chunk)
+      assert_empty d.instance.log.out.logs
     end
 
     data(path: { "append" => false },
@@ -204,6 +207,21 @@ class WebHDFSOutputTest < Test::Unit::TestCase
       assert_equal 1, d.instance.buffer_config.timekey
       assert_equal "/hdfs/path/file.20120718.log", d.instance.generate_path(chunk)
     end
+
+    def test_time_key_without_buffer_section
+      conf = config_element(
+        "ROOT", "", {
+          "host" => "namenode.local",
+          "path" => "/hdfs/path/file.%Y%m%d-%M.log"
+        }
+      )
+      d = create_driver(conf)
+      time = event_time("2012-07-18 15:03:00 +0900")
+      metadata = d.instance.metadata("test", time, {})
+      chunk = d.instance.buffer.generate_chunk(metadata)
+      assert_equal 60, d.instance.buffer_config.timekey
+      assert_equal "/hdfs/path/file.20120718-03.log", d.instance.generate_path(chunk)
+    end
   end
 
   sub_test_case "using format subsection" do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-webhdfs
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 1.2.5
 platform: ruby
 authors:
 - TAGOMORI Satoshi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-05-19 00:00:00.000000000 Z
+date: 2020-06-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -100,14 +100,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.14.4
+        version: 0.14.22
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.14.4
+        version: 0.14.22
 - !ruby/object:Gem::Dependency
   name: webhdfs
   requirement: !ruby/object:Gem::Requirement
@@ -166,8 +166,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.11
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Fluentd plugin to write data on HDFS over WebHDFS, with flexible formatting