fluent-plugin-webhdfs 1.4.0 → 1.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2d96f9304470f4d3409a1209b96c56722a62acfefc55633d81525e21103ec7e9
-  data.tar.gz: 8c6982670e30e112815a3abec28a2865609e4ca2cccbe097a42b6abb9080af21
+  metadata.gz: bc42357da759e1c34ec12b3994bdd96b9f56cc1b093bb890f0ec4bccf929362d
+  data.tar.gz: e63cb6a5df15e5cf2fe8228d9e0e21ff5adf6db9cc5c4e11138aaac77429dc85
 SHA512:
-  metadata.gz: a56a3b8ac2e7bf279ddb23d5a4fafb187289883442b01355f86cf3d626332aff35a78f4fee38c32767a7162741b550143fbfc50f9d953db0ce8a8220d022d35f
-  data.tar.gz: 7110d25391fc90d0e0aa8042014596994b3fe700737a08febd1f1d69485fd9eef212f0882e51832e2a3f95d64a7e21e0091f8f8e37444151404104230645029d
+  metadata.gz: e613ca241b2624ac77c1b1651de28aad3b4e7060086067d1eefc4874241e0a60437bddf61f3775c834a94fdaf1cd374fad1cb5b60e16909db49cf9dc7663770b
+  data.tar.gz: 7eb4b39ab4763f661e1e213d736eb3544d88e1a73b24a0b0d59b6a715fa3fdbeceb62f5a47d14be6d06126747d4751f0b73fa7267591c6d2713d2dac35901f5e

data/.github/workflows/linux.yml

@@ -0,0 +1,35 @@
+name: Testing on Ubuntu
+on:
+  - push
+  - pull_request
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    continue-on-error: ${{ matrix.experimental }}
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: [ '2.5', '2.6', '2.7', '3.0' ]
+        os:
+          - ubuntu-latest
+        experimental: [false]
+        include:
+          - ruby: head
+            os: ubuntu-latest
+            experimental: true
+
+    name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v2
+    - name: Install dependencies
+      run: sudo apt-get install libsnappy-dev libzstd-dev
+    - uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby }}
+    - name: unit testing
+      env:
+        CI: true
+      run: |
+        gem install bundler rake
+        bundle install --jobs 4 --retry 3
+        bundle exec rake test

data/README.md

@@ -146,6 +146,7 @@ With kerberos authentication:
       path /path/on/hdfs/access.log.%Y%m%d_%H.log
       kerberos true
       kerberos_keytab /path/to/keytab # if needed
+      renew_kerberos_delegation_token true # if needed
     </match>
 
 NOTE: You need to install `gssapi` gem for kerberos. See https://github.com/kzk/webhdfs#for-kerberos-authentication
@@ -254,7 +255,7 @@ For high load cluster nodes, you can specify timeouts for HTTP requests.
 
 ### For unstable Namenodes
 
-With default configuration, fluent-plugin-webhdfs checks HDFS filesystem status and raise error for inacive NameNodes.
+With default configuration, fluent-plugin-webhdfs checks HDFS filesystem status and raise error for inactive NameNodes.
 
 If you were using unstable NameNodes and have wanted to ignore NameNode errors on startup of fluentd, enable `ignore_start_check_error` option like below:
 

data/fluent-plugin-webhdfs.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-webhdfs"
-  gem.version       = "1.4.0"
+  gem.version       = "1.5.0"
   gem.authors       = ["TAGOMORI Satoshi"]
   gem.email         = ["tagomoris@gmail.com"]
   gem.summary       = %q{Fluentd plugin to write data on HDFS over WebHDFS, with flexible formatting}
@@ -23,5 +23,5 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency "bzip2-ffi"
   gem.add_development_dependency "zstandard"
   gem.add_runtime_dependency "fluentd", '>= 0.14.22'
-  gem.add_runtime_dependency "webhdfs", '>= 0.6.0'
+  gem.add_runtime_dependency "webhdfs", '>= 0.10.0'
 end

data/lib/fluent/plugin/out_webhdfs.rb

@@ -66,6 +66,10 @@ class Fluent::Plugin::WebHDFSOutput < Fluent::Plugin::Output
   config_param :kerberos, :bool, default: false
   desc 'kerberos keytab file'
   config_param :kerberos_keytab, :string, default: nil
+  desc 'Use delegation token while upload webhdfs or not'
+  config_param :renew_kerberos_delegation_token, :bool, default: false
+  desc 'delegation token reuse timer (default 8h)'
+  config_param :renew_kerberos_delegation_token_interval, :time, default: 8 * 60 * 60
 
   SUPPORTED_COMPRESS = [:gzip, :bzip2, :snappy, :hadoop_snappy, :lzo_command, :zstd, :text]
   desc "Compression method (#{SUPPORTED_COMPRESS.join(',')})"
@@ -184,6 +188,14 @@ class Fluent::Plugin::WebHDFSOutput < Fluent::Plugin::Output
       raise Fluent::ConfigError, "Path on hdfs MUST starts with '/', but '#{@path}'"
     end
 
+    @renew_kerberos_delegation_token_interval_hour = nil
+    if @renew_kerberos_delegation_token
+      unless @username
+        raise Fluent::ConfigError, "username is missing. If you want to reuse delegation token, follow with kerberos accounts"
+      end
+      @renew_kerberos_delegation_token_interval_hour = @renew_kerberos_delegation_token_interval / 60 / 60
+    end
+
     @client = prepare_client(@namenode_host, @namenode_port, @username)
     if @standby_namenode_host
       @client_standby = prepare_client(@standby_namenode_host, @standby_namenode_port, @username)
@@ -203,7 +215,7 @@ class Fluent::Plugin::WebHDFSOutput < Fluent::Plugin::Output
   end
 
   def prepare_client(host, port, username)
-    client = WebHDFS::Client.new(host, port, username)
+    client = WebHDFS::Client.new(host, port, username, nil, nil, nil, {}, @renew_kerberos_delegation_token_interval_hour)
     if @httpfs
       client.httpfs_mode = true
     end

data/test/plugin/test_out_webhdfs.rb

@@ -117,7 +117,13 @@ class WebHDFSOutputTest < Test::Unit::TestCase
             "namenode" => "server.local:14000",
             "path" => "/hdfs/path/file.%Y%m%d.%H%M.log",
             "compress" => compress_type
-          })
+          }, [
+            config_element("buffer", "tag, time", {
+                             "@type" => "memory",
+                             "timekey_zone" => "+0300",
+                             "timekey" => 60
+                           })
+          ])
         d = create_driver(conf)
       rescue Fluent::ConfigError => ex
         omit ex.message
@@ -310,4 +316,99 @@ class WebHDFSOutputTest < Test::Unit::TestCase
       assert_equal "2017-01-24T20:10:30Z\ttest.now\t{\"message\":\"yay\",\"name\":\"tagomoris\"}\n", line
     end
   end
+
+  sub_test_case "kerberos config" do
+    CONFIG_KERBEROS = config_element(
+      "ROOT", "", {
+        "namenode" => "server.local:14000",
+        "path" => "/hdfs/path/file.%Y%m%d.%H%M.log",
+        "username" => "hdfs_user",
+        "kerberos" => true,
+        "kerberos_keytab" => "/path/to/kerberos.keytab",
+      })
+
+    test "renew_kerberos_delegation_token default" do
+      mock.proxy(WebHDFS::Client).new("server.local", 14000, "hdfs_user", nil, nil, nil, {}, nil).once
+
+      d = create_driver(CONFIG_KERBEROS)
+
+      assert_equal(
+        {
+          kerberos: true,
+          renew_kerberos_delegation_token: false,
+          renew_kerberos_delegation_token_interval_hour: nil,
+        },
+        {
+          kerberos: d.instance.kerberos,
+          renew_kerberos_delegation_token: d.instance.instance_eval("@renew_kerberos_delegation_token"),
+          renew_kerberos_delegation_token_interval_hour: d.instance.instance_eval("@renew_kerberos_delegation_token_interval_hour"),
+        })
+    end
+
+    test "default renew_kerberos_delegation_token_interval" do
+      expected_hour = 8
+
+      mock.proxy(WebHDFS::Client).new("server.local", 14000, "hdfs_user", nil, nil, nil, {}, expected_hour).once
+
+      d = create_driver(CONFIG_KERBEROS +
+                        config_element("", "", { "renew_kerberos_delegation_token" => true }))
+
+      assert_equal(
+        {
+          kerberos: true,
+          renew_kerberos_delegation_token: true,
+          renew_kerberos_delegation_token_interval: expected_hour * 60 * 60,
+          renew_kerberos_delegation_token_interval_hour: expected_hour,
+        },
+        {
+          kerberos: d.instance.kerberos,
+          renew_kerberos_delegation_token: d.instance.instance_eval("@renew_kerberos_delegation_token"),
+          renew_kerberos_delegation_token_interval: d.instance.instance_eval("@renew_kerberos_delegation_token_interval"),
+          renew_kerberos_delegation_token_interval_hour: d.instance.instance_eval("@renew_kerberos_delegation_token_interval_hour"),
+        })
+    end
+
+    test "renew_kerberos_delegation_token_interval" do
+      expected_hour = 10
+
+      mock.proxy(WebHDFS::Client).new("server.local", 14000, "hdfs_user", nil, nil, nil, {}, expected_hour).once
+
+      d = create_driver(
+        CONFIG_KERBEROS +
+        config_element(
+          "", "",
+          {
+            "renew_kerberos_delegation_token" => true,
+            "renew_kerberos_delegation_token_interval" => "#{expected_hour}h",
+          }))
+
+      assert_equal(
+        {
+          kerberos: true,
+          renew_kerberos_delegation_token: true,
+          renew_kerberos_delegation_token_interval: expected_hour * 60 * 60,
+          renew_kerberos_delegation_token_interval_hour: expected_hour,
+        },
+        {
+          kerberos: d.instance.kerberos,
+          renew_kerberos_delegation_token: d.instance.instance_eval("@renew_kerberos_delegation_token"),
+          renew_kerberos_delegation_token_interval: d.instance.instance_eval("@renew_kerberos_delegation_token_interval"),
+          renew_kerberos_delegation_token_interval_hour: d.instance.instance_eval("@renew_kerberos_delegation_token_interval_hour"),
+        })
+    end
+
+    test "username is required for renew_kerberos_delegation_token" do
+      conf = config_element(
+      "ROOT", "", {
+        "namenode" => "server.local:14000",
+        "path" => "/hdfs/path/file.%Y%m%d.%H%M.log",
+        "kerberos" => true,
+        "renew_kerberos_delegation_token" => true,
+      })
+
+      assert_raise(Fluent::ConfigError) do
+        create_driver(conf)
+      end
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-webhdfs
 version: !ruby/object:Gem::Version
-  version: 1.4.0
+  version: 1.5.0
 platform: ruby
 authors:
 - TAGOMORI Satoshi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-12-22 00:00:00.000000000 Z
+date: 2021-08-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -128,14 +128,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.6.0
+        version: 0.10.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.6.0
+        version: 0.10.0
 description: For WebHDFS and HttpFs of Hadoop HDFS
 email:
 - tagomoris@gmail.com
@@ -143,6 +143,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/linux.yml"
 - ".gitignore"
 - ".travis.yml"
 - Appraisals
@@ -183,7 +184,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.2.5
 signing_key: 
 specification_version: 4
 summary: Fluentd plugin to write data on HDFS over WebHDFS, with flexible formatting