fluent-plugin-webhdfs 1.5.0 → 1.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bc42357da759e1c34ec12b3994bdd96b9f56cc1b093bb890f0ec4bccf929362d
-  data.tar.gz: e63cb6a5df15e5cf2fe8228d9e0e21ff5adf6db9cc5c4e11138aaac77429dc85
+  metadata.gz: 73ccae52314663de476e1dd1de3a9e57e134b925be65b9d9fa093870c6c03371
+  data.tar.gz: 594b0eea372de63d9c8d64af590514b07dd2e01652ab1c1b37fa6303ad141f68
 SHA512:
-  metadata.gz: e613ca241b2624ac77c1b1651de28aad3b4e7060086067d1eefc4874241e0a60437bddf61f3775c834a94fdaf1cd374fad1cb5b60e16909db49cf9dc7663770b
-  data.tar.gz: 7eb4b39ab4763f661e1e213d736eb3544d88e1a73b24a0b0d59b6a715fa3fdbeceb62f5a47d14be6d06126747d4751f0b73fa7267591c6d2713d2dac35901f5e
+  metadata.gz: a73865d4ee2f052fbacda6d30a6fe24d1ea7c6d701358d3742bba04395daf4989f6b0538f7afc988c8c2cae70ddd94a0143151159231a6c96f479219dc43467f
+  data.tar.gz: 5483f28a5b3ba3e7b64726c6974ddb05c0900bf000021e0d92bfb1c45850db928be278d375a45a1dfbb0d8da74ee14e0855a34336cbae0531c585dc15b391643

data/.github/dependabot.yml

@@ -0,0 +1,6 @@
+version: 2
+updates:
+  - package-ecosystem: 'github-actions'
+    directory: '/'
+    schedule:
+      interval: 'weekly'

data/.github/workflows/test-ruby-head.yml

@@ -0,0 +1,28 @@
+name: Testing with Ruby head
+on:
+  schedule:
+    - cron: '32 14 * * 0'
+  workflow_dispatch:
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: ['ubuntu-latest']
+        ruby: [ 'head' ]
+
+    name: Ruby ${{ matrix.ruby }} on ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v4
+    - name: Install dependencies
+      run: sudo apt-get install libsnappy-dev libzstd-dev
+    - uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby }}
+    - name: unit testing
+      env:
+        CI: true
+      run: |
+        bundle install --jobs 4 --retry 3
+        bundle exec rake test

data/.github/workflows/test.yml

@@ -0,0 +1,29 @@
+name: Test
+on:
+  push:
+    branches: [master]
+  pull_request:
+    branches: [master]
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: ['ubuntu-latest']
+        ruby: [ '3.3', '3.2', '3.1', '3.0', '2.7' ]
+
+    name: Ruby ${{ matrix.ruby }} on ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v4
+    - name: Install dependencies
+      run: sudo apt-get install libsnappy-dev libzstd-dev
+    - uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby }}
+    - name: unit testing
+      env:
+        CI: true
+      run: |
+        bundle install --jobs 4 --retry 3
+        bundle exec rake test

data/fluent-plugin-webhdfs.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-webhdfs"
-  gem.version       = "1.5.0"
+  gem.version       = "1.6.0"
   gem.authors       = ["TAGOMORI Satoshi"]
   gem.email         = ["tagomoris@gmail.com"]
   gem.summary       = %q{Fluentd plugin to write data on HDFS over WebHDFS, with flexible formatting}
@@ -23,5 +23,5 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency "bzip2-ffi"
   gem.add_development_dependency "zstandard"
   gem.add_runtime_dependency "fluentd", '>= 0.14.22'
-  gem.add_runtime_dependency "webhdfs", '>= 0.10.0'
+  gem.add_runtime_dependency "webhdfs", '>= 0.11.0'
 end

data/lib/fluent/plugin/out_webhdfs.rb

@@ -70,6 +70,8 @@ class Fluent::Plugin::WebHDFSOutput < Fluent::Plugin::Output
   config_param :renew_kerberos_delegation_token, :bool, default: false
   desc 'delegation token reuse timer (default 8h)'
   config_param :renew_kerberos_delegation_token_interval, :time, default: 8 * 60 * 60
+  desc 'delegation token max-lifetime (default 7d)'
+  config_param :kerberos_delegation_token_max_lifetime, :time, default: 7 * 24 * 60 * 60
 
   SUPPORTED_COMPRESS = [:gzip, :bzip2, :snappy, :hadoop_snappy, :lzo_command, :zstd, :text]
   desc "Compression method (#{SUPPORTED_COMPRESS.join(',')})"
@@ -114,7 +116,7 @@ class Fluent::Plugin::WebHDFSOutput < Fluent::Plugin::Output
               else 86400
               end
     if buffer_config = conf.elements(name: "buffer").first
-      timekey = buffer_config["timekey"] || timekey 
+      timekey = buffer_config["timekey"] || timekey
     end
 
     compat_parameters_convert(conf, :buffer, default_chunk_key: "time")
@@ -189,7 +191,9 @@ class Fluent::Plugin::WebHDFSOutput < Fluent::Plugin::Output
     end
 
     @renew_kerberos_delegation_token_interval_hour = nil
+    @kerberos_delegation_token_max_lifetime_hour = nil
     if @renew_kerberos_delegation_token
+      @kerberos_delegation_token_max_lifetime_hour = @kerberos_delegation_token_max_lifetime / 60 / 60
       unless @username
         raise Fluent::ConfigError, "username is missing. If you want to reuse delegation token, follow with kerberos accounts"
       end
@@ -215,7 +219,7 @@ class Fluent::Plugin::WebHDFSOutput < Fluent::Plugin::Output
   end
 
   def prepare_client(host, port, username)
-    client = WebHDFS::Client.new(host, port, username, nil, nil, nil, {}, @renew_kerberos_delegation_token_interval_hour)
+    client = WebHDFS::Client.new(host, port, username, nil, nil, nil, {}, @renew_kerberos_delegation_token_interval_hour, @kerberos_delegation_token_max_lifetime_hour)
     if @httpfs
       client.httpfs_mode = true
     end
@@ -542,4 +546,4 @@ require 'fluent/plugin/webhdfs_compressor_bzip2'
 require 'fluent/plugin/webhdfs_compressor_snappy'
 require 'fluent/plugin/webhdfs_compressor_hadoop_snappy'
 require 'fluent/plugin/webhdfs_compressor_lzo_command'
-require 'fluent/plugin/webhdfs_compressor_zstd'
+require 'fluent/plugin/webhdfs_compressor_zstd'

data/test/plugin/test_out_webhdfs.rb

@@ -328,7 +328,7 @@ class WebHDFSOutputTest < Test::Unit::TestCase
       })
 
     test "renew_kerberos_delegation_token default" do
-      mock.proxy(WebHDFS::Client).new("server.local", 14000, "hdfs_user", nil, nil, nil, {}, nil).once
+      mock.proxy(WebHDFS::Client).new("server.local", 14000, "hdfs_user", nil, nil, nil, {}, nil, nil).once
 
       d = create_driver(CONFIG_KERBEROS)
 
@@ -337,18 +337,20 @@ class WebHDFSOutputTest < Test::Unit::TestCase
           kerberos: true,
           renew_kerberos_delegation_token: false,
           renew_kerberos_delegation_token_interval_hour: nil,
+          kerberos_delegation_token_max_lifetime_hour: nil,
         },
         {
           kerberos: d.instance.kerberos,
           renew_kerberos_delegation_token: d.instance.instance_eval("@renew_kerberos_delegation_token"),
           renew_kerberos_delegation_token_interval_hour: d.instance.instance_eval("@renew_kerberos_delegation_token_interval_hour"),
+          kerberos_delegation_token_max_lifetime_hour: d.instance.instance_eval("@kerberos_delegation_token_max_lifetime_hour"),
         })
     end
 
     test "default renew_kerberos_delegation_token_interval" do
       expected_hour = 8
-
-      mock.proxy(WebHDFS::Client).new("server.local", 14000, "hdfs_user", nil, nil, nil, {}, expected_hour).once
+      expected_delegation_token_max_lifetime_hour = 7 * 24
+      mock.proxy(WebHDFS::Client).new("server.local", 14000, "hdfs_user", nil, nil, nil, {}, expected_hour, expected_delegation_token_max_lifetime_hour).once
 
       d = create_driver(CONFIG_KERBEROS +
                         config_element("", "", { "renew_kerberos_delegation_token" => true }))
@@ -359,19 +361,24 @@ class WebHDFSOutputTest < Test::Unit::TestCase
           renew_kerberos_delegation_token: true,
           renew_kerberos_delegation_token_interval: expected_hour * 60 * 60,
           renew_kerberos_delegation_token_interval_hour: expected_hour,
+          kerberos_delegation_token_max_lifetime: expected_delegation_token_max_lifetime_hour * 60 * 60,
+          kerberos_delegation_token_max_lifetime_hour: expected_delegation_token_max_lifetime_hour,
         },
         {
           kerberos: d.instance.kerberos,
           renew_kerberos_delegation_token: d.instance.instance_eval("@renew_kerberos_delegation_token"),
           renew_kerberos_delegation_token_interval: d.instance.instance_eval("@renew_kerberos_delegation_token_interval"),
           renew_kerberos_delegation_token_interval_hour: d.instance.instance_eval("@renew_kerberos_delegation_token_interval_hour"),
+          kerberos_delegation_token_max_lifetime: d.instance.instance_eval("@kerberos_delegation_token_max_lifetime"),
+          kerberos_delegation_token_max_lifetime_hour: d.instance.instance_eval("@kerberos_delegation_token_max_lifetime_hour"),
         })
     end
 
     test "renew_kerberos_delegation_token_interval" do
       expected_hour = 10
+      expected_delegation_token_max_lifetime_hour = 24
 
-      mock.proxy(WebHDFS::Client).new("server.local", 14000, "hdfs_user", nil, nil, nil, {}, expected_hour).once
+      mock.proxy(WebHDFS::Client).new("server.local", 14000, "hdfs_user", nil, nil, nil, {}, expected_hour,expected_delegation_token_max_lifetime_hour).once
 
       d = create_driver(
         CONFIG_KERBEROS +
@@ -380,6 +387,7 @@ class WebHDFSOutputTest < Test::Unit::TestCase
           {
             "renew_kerberos_delegation_token" => true,
             "renew_kerberos_delegation_token_interval" => "#{expected_hour}h",
+            "kerberos_delegation_token_max_lifetime" => "#{expected_delegation_token_max_lifetime_hour}h"
           }))
 
       assert_equal(
@@ -388,12 +396,16 @@ class WebHDFSOutputTest < Test::Unit::TestCase
           renew_kerberos_delegation_token: true,
           renew_kerberos_delegation_token_interval: expected_hour * 60 * 60,
           renew_kerberos_delegation_token_interval_hour: expected_hour,
+          kerberos_delegation_token_max_lifetime: expected_delegation_token_max_lifetime_hour * 60 * 60,
+          kerberos_delegation_token_max_lifetime_hour: expected_delegation_token_max_lifetime_hour
         },
         {
           kerberos: d.instance.kerberos,
           renew_kerberos_delegation_token: d.instance.instance_eval("@renew_kerberos_delegation_token"),
           renew_kerberos_delegation_token_interval: d.instance.instance_eval("@renew_kerberos_delegation_token_interval"),
           renew_kerberos_delegation_token_interval_hour: d.instance.instance_eval("@renew_kerberos_delegation_token_interval_hour"),
+          kerberos_delegation_token_max_lifetime: d.instance.instance_eval("@kerberos_delegation_token_max_lifetime"),
+          kerberos_delegation_token_max_lifetime_hour: d.instance.instance_eval("@kerberos_delegation_token_max_lifetime_hour"),
         })
     end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-webhdfs
 version: !ruby/object:Gem::Version
-  version: 1.5.0
+  version: 1.6.0
 platform: ruby
 authors:
 - TAGOMORI Satoshi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-08-04 00:00:00.000000000 Z
+date: 2024-03-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -128,14 +128,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.10.0
+        version: 0.11.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.10.0
+        version: 0.11.0
 description: For WebHDFS and HttpFs of Hadoop HDFS
 email:
 - tagomoris@gmail.com
@@ -143,7 +143,9 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- ".github/workflows/linux.yml"
+- ".github/dependabot.yml"
+- ".github/workflows/test-ruby-head.yml"
+- ".github/workflows/test.yml"
 - ".gitignore"
 - ".travis.yml"
 - Appraisals
@@ -184,7 +186,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.5
+rubygems_version: 3.3.5
 signing_key: 
 specification_version: 4
 summary: Fluentd plugin to write data on HDFS over WebHDFS, with flexible formatting

data/.github/workflows/linux.yml

@@ -1,35 +0,0 @@
-name: Testing on Ubuntu
-on:
-  - push
-  - pull_request
-jobs:
-  build:
-    runs-on: ${{ matrix.os }}
-    continue-on-error: ${{ matrix.experimental }}
-    strategy:
-      fail-fast: false
-      matrix:
-        ruby: [ '2.5', '2.6', '2.7', '3.0' ]
-        os:
-          - ubuntu-latest
-        experimental: [false]
-        include:
-          - ruby: head
-            os: ubuntu-latest
-            experimental: true
-
-    name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
-    steps:
-    - uses: actions/checkout@v2
-    - name: Install dependencies
-      run: sudo apt-get install libsnappy-dev libzstd-dev
-    - uses: ruby/setup-ruby@v1
-      with:
-        ruby-version: ${{ matrix.ruby }}
-    - name: unit testing
-      env:
-        CI: true
-      run: |
-        gem install bundler rake
-        bundle install --jobs 4 --retry 3
-        bundle exec rake test