fluent-plugin-webhdfs 1.4.0 → 1.6.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/dependabot.yml +6 -0
- data/.github/workflows/test-ruby-head.yml +28 -0
- data/.github/workflows/test.yml +29 -0
- data/README.md +2 -1
- data/fluent-plugin-webhdfs.gemspec +2 -2
- data/lib/fluent/plugin/out_webhdfs.rb +19 -3
- data/test/plugin/test_out_webhdfs.rb +114 -1
- metadata +8 -5
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 73ccae52314663de476e1dd1de3a9e57e134b925be65b9d9fa093870c6c03371
|
4
|
+
data.tar.gz: 594b0eea372de63d9c8d64af590514b07dd2e01652ab1c1b37fa6303ad141f68
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a73865d4ee2f052fbacda6d30a6fe24d1ea7c6d701358d3742bba04395daf4989f6b0538f7afc988c8c2cae70ddd94a0143151159231a6c96f479219dc43467f
|
7
|
+
data.tar.gz: 5483f28a5b3ba3e7b64726c6974ddb05c0900bf000021e0d92bfb1c45850db928be278d375a45a1dfbb0d8da74ee14e0855a34336cbae0531c585dc15b391643
|
@@ -0,0 +1,28 @@
|
|
1
|
+
name: Testing with Ruby head
|
2
|
+
on:
|
3
|
+
schedule:
|
4
|
+
- cron: '32 14 * * 0'
|
5
|
+
workflow_dispatch:
|
6
|
+
jobs:
|
7
|
+
build:
|
8
|
+
runs-on: ${{ matrix.os }}
|
9
|
+
strategy:
|
10
|
+
fail-fast: false
|
11
|
+
matrix:
|
12
|
+
os: ['ubuntu-latest']
|
13
|
+
ruby: [ 'head' ]
|
14
|
+
|
15
|
+
name: Ruby ${{ matrix.ruby }} on ${{ matrix.os }}
|
16
|
+
steps:
|
17
|
+
- uses: actions/checkout@v4
|
18
|
+
- name: Install dependencies
|
19
|
+
run: sudo apt-get install libsnappy-dev libzstd-dev
|
20
|
+
- uses: ruby/setup-ruby@v1
|
21
|
+
with:
|
22
|
+
ruby-version: ${{ matrix.ruby }}
|
23
|
+
- name: unit testing
|
24
|
+
env:
|
25
|
+
CI: true
|
26
|
+
run: |
|
27
|
+
bundle install --jobs 4 --retry 3
|
28
|
+
bundle exec rake test
|
@@ -0,0 +1,29 @@
|
|
1
|
+
name: Test
|
2
|
+
on:
|
3
|
+
push:
|
4
|
+
branches: [master]
|
5
|
+
pull_request:
|
6
|
+
branches: [master]
|
7
|
+
jobs:
|
8
|
+
build:
|
9
|
+
runs-on: ${{ matrix.os }}
|
10
|
+
strategy:
|
11
|
+
fail-fast: false
|
12
|
+
matrix:
|
13
|
+
os: ['ubuntu-latest']
|
14
|
+
ruby: [ '3.3', '3.2', '3.1', '3.0', '2.7' ]
|
15
|
+
|
16
|
+
name: Ruby ${{ matrix.ruby }} on ${{ matrix.os }}
|
17
|
+
steps:
|
18
|
+
- uses: actions/checkout@v4
|
19
|
+
- name: Install dependencies
|
20
|
+
run: sudo apt-get install libsnappy-dev libzstd-dev
|
21
|
+
- uses: ruby/setup-ruby@v1
|
22
|
+
with:
|
23
|
+
ruby-version: ${{ matrix.ruby }}
|
24
|
+
- name: unit testing
|
25
|
+
env:
|
26
|
+
CI: true
|
27
|
+
run: |
|
28
|
+
bundle install --jobs 4 --retry 3
|
29
|
+
bundle exec rake test
|
data/README.md
CHANGED
@@ -146,6 +146,7 @@ With kerberos authentication:
|
|
146
146
|
path /path/on/hdfs/access.log.%Y%m%d_%H.log
|
147
147
|
kerberos true
|
148
148
|
kerberos_keytab /path/to/keytab # if needed
|
149
|
+
renew_kerberos_delegation_token true # if needed
|
149
150
|
</match>
|
150
151
|
|
151
152
|
NOTE: You need to install `gssapi` gem for kerberos. See https://github.com/kzk/webhdfs#for-kerberos-authentication
|
@@ -254,7 +255,7 @@ For high load cluster nodes, you can specify timeouts for HTTP requests.
|
|
254
255
|
|
255
256
|
### For unstable Namenodes
|
256
257
|
|
257
|
-
With default configuration, fluent-plugin-webhdfs checks HDFS filesystem status and raise error for
|
258
|
+
With default configuration, fluent-plugin-webhdfs checks HDFS filesystem status and raise error for inactive NameNodes.
|
258
259
|
|
259
260
|
If you were using unstable NameNodes and have wanted to ignore NameNode errors on startup of fluentd, enable `ignore_start_check_error` option like below:
|
260
261
|
|
@@ -2,7 +2,7 @@
|
|
2
2
|
|
3
3
|
Gem::Specification.new do |gem|
|
4
4
|
gem.name = "fluent-plugin-webhdfs"
|
5
|
-
gem.version = "1.
|
5
|
+
gem.version = "1.6.0"
|
6
6
|
gem.authors = ["TAGOMORI Satoshi"]
|
7
7
|
gem.email = ["tagomoris@gmail.com"]
|
8
8
|
gem.summary = %q{Fluentd plugin to write data on HDFS over WebHDFS, with flexible formatting}
|
@@ -23,5 +23,5 @@ Gem::Specification.new do |gem|
|
|
23
23
|
gem.add_development_dependency "bzip2-ffi"
|
24
24
|
gem.add_development_dependency "zstandard"
|
25
25
|
gem.add_runtime_dependency "fluentd", '>= 0.14.22'
|
26
|
-
gem.add_runtime_dependency "webhdfs", '>= 0.
|
26
|
+
gem.add_runtime_dependency "webhdfs", '>= 0.11.0'
|
27
27
|
end
|
@@ -66,6 +66,12 @@ class Fluent::Plugin::WebHDFSOutput < Fluent::Plugin::Output
|
|
66
66
|
config_param :kerberos, :bool, default: false
|
67
67
|
desc 'kerberos keytab file'
|
68
68
|
config_param :kerberos_keytab, :string, default: nil
|
69
|
+
desc 'Use delegation token while upload webhdfs or not'
|
70
|
+
config_param :renew_kerberos_delegation_token, :bool, default: false
|
71
|
+
desc 'delegation token reuse timer (default 8h)'
|
72
|
+
config_param :renew_kerberos_delegation_token_interval, :time, default: 8 * 60 * 60
|
73
|
+
desc 'delegation token max-lifetime (default 7d)'
|
74
|
+
config_param :kerberos_delegation_token_max_lifetime, :time, default: 7 * 24 * 60 * 60
|
69
75
|
|
70
76
|
SUPPORTED_COMPRESS = [:gzip, :bzip2, :snappy, :hadoop_snappy, :lzo_command, :zstd, :text]
|
71
77
|
desc "Compression method (#{SUPPORTED_COMPRESS.join(',')})"
|
@@ -110,7 +116,7 @@ class Fluent::Plugin::WebHDFSOutput < Fluent::Plugin::Output
|
|
110
116
|
else 86400
|
111
117
|
end
|
112
118
|
if buffer_config = conf.elements(name: "buffer").first
|
113
|
-
timekey = buffer_config["timekey"] || timekey
|
119
|
+
timekey = buffer_config["timekey"] || timekey
|
114
120
|
end
|
115
121
|
|
116
122
|
compat_parameters_convert(conf, :buffer, default_chunk_key: "time")
|
@@ -184,6 +190,16 @@ class Fluent::Plugin::WebHDFSOutput < Fluent::Plugin::Output
|
|
184
190
|
raise Fluent::ConfigError, "Path on hdfs MUST starts with '/', but '#{@path}'"
|
185
191
|
end
|
186
192
|
|
193
|
+
@renew_kerberos_delegation_token_interval_hour = nil
|
194
|
+
@kerberos_delegation_token_max_lifetime_hour = nil
|
195
|
+
if @renew_kerberos_delegation_token
|
196
|
+
@kerberos_delegation_token_max_lifetime_hour = @kerberos_delegation_token_max_lifetime / 60 / 60
|
197
|
+
unless @username
|
198
|
+
raise Fluent::ConfigError, "username is missing. If you want to reuse delegation token, follow with kerberos accounts"
|
199
|
+
end
|
200
|
+
@renew_kerberos_delegation_token_interval_hour = @renew_kerberos_delegation_token_interval / 60 / 60
|
201
|
+
end
|
202
|
+
|
187
203
|
@client = prepare_client(@namenode_host, @namenode_port, @username)
|
188
204
|
if @standby_namenode_host
|
189
205
|
@client_standby = prepare_client(@standby_namenode_host, @standby_namenode_port, @username)
|
@@ -203,7 +219,7 @@ class Fluent::Plugin::WebHDFSOutput < Fluent::Plugin::Output
|
|
203
219
|
end
|
204
220
|
|
205
221
|
def prepare_client(host, port, username)
|
206
|
-
client = WebHDFS::Client.new(host, port, username)
|
222
|
+
client = WebHDFS::Client.new(host, port, username, nil, nil, nil, {}, @renew_kerberos_delegation_token_interval_hour, @kerberos_delegation_token_max_lifetime_hour)
|
207
223
|
if @httpfs
|
208
224
|
client.httpfs_mode = true
|
209
225
|
end
|
@@ -530,4 +546,4 @@ require 'fluent/plugin/webhdfs_compressor_bzip2'
|
|
530
546
|
require 'fluent/plugin/webhdfs_compressor_snappy'
|
531
547
|
require 'fluent/plugin/webhdfs_compressor_hadoop_snappy'
|
532
548
|
require 'fluent/plugin/webhdfs_compressor_lzo_command'
|
533
|
-
require 'fluent/plugin/webhdfs_compressor_zstd'
|
549
|
+
require 'fluent/plugin/webhdfs_compressor_zstd'
|
@@ -117,7 +117,13 @@ class WebHDFSOutputTest < Test::Unit::TestCase
|
|
117
117
|
"namenode" => "server.local:14000",
|
118
118
|
"path" => "/hdfs/path/file.%Y%m%d.%H%M.log",
|
119
119
|
"compress" => compress_type
|
120
|
-
}
|
120
|
+
}, [
|
121
|
+
config_element("buffer", "tag, time", {
|
122
|
+
"@type" => "memory",
|
123
|
+
"timekey_zone" => "+0300",
|
124
|
+
"timekey" => 60
|
125
|
+
})
|
126
|
+
])
|
121
127
|
d = create_driver(conf)
|
122
128
|
rescue Fluent::ConfigError => ex
|
123
129
|
omit ex.message
|
@@ -310,4 +316,111 @@ class WebHDFSOutputTest < Test::Unit::TestCase
|
|
310
316
|
assert_equal "2017-01-24T20:10:30Z\ttest.now\t{\"message\":\"yay\",\"name\":\"tagomoris\"}\n", line
|
311
317
|
end
|
312
318
|
end
|
319
|
+
|
320
|
+
sub_test_case "kerberos config" do
|
321
|
+
CONFIG_KERBEROS = config_element(
|
322
|
+
"ROOT", "", {
|
323
|
+
"namenode" => "server.local:14000",
|
324
|
+
"path" => "/hdfs/path/file.%Y%m%d.%H%M.log",
|
325
|
+
"username" => "hdfs_user",
|
326
|
+
"kerberos" => true,
|
327
|
+
"kerberos_keytab" => "/path/to/kerberos.keytab",
|
328
|
+
})
|
329
|
+
|
330
|
+
test "renew_kerberos_delegation_token default" do
|
331
|
+
mock.proxy(WebHDFS::Client).new("server.local", 14000, "hdfs_user", nil, nil, nil, {}, nil, nil).once
|
332
|
+
|
333
|
+
d = create_driver(CONFIG_KERBEROS)
|
334
|
+
|
335
|
+
assert_equal(
|
336
|
+
{
|
337
|
+
kerberos: true,
|
338
|
+
renew_kerberos_delegation_token: false,
|
339
|
+
renew_kerberos_delegation_token_interval_hour: nil,
|
340
|
+
kerberos_delegation_token_max_lifetime_hour: nil,
|
341
|
+
},
|
342
|
+
{
|
343
|
+
kerberos: d.instance.kerberos,
|
344
|
+
renew_kerberos_delegation_token: d.instance.instance_eval("@renew_kerberos_delegation_token"),
|
345
|
+
renew_kerberos_delegation_token_interval_hour: d.instance.instance_eval("@renew_kerberos_delegation_token_interval_hour"),
|
346
|
+
kerberos_delegation_token_max_lifetime_hour: d.instance.instance_eval("@kerberos_delegation_token_max_lifetime_hour"),
|
347
|
+
})
|
348
|
+
end
|
349
|
+
|
350
|
+
test "default renew_kerberos_delegation_token_interval" do
|
351
|
+
expected_hour = 8
|
352
|
+
expected_delegation_token_max_lifetime_hour = 7 * 24
|
353
|
+
mock.proxy(WebHDFS::Client).new("server.local", 14000, "hdfs_user", nil, nil, nil, {}, expected_hour, expected_delegation_token_max_lifetime_hour).once
|
354
|
+
|
355
|
+
d = create_driver(CONFIG_KERBEROS +
|
356
|
+
config_element("", "", { "renew_kerberos_delegation_token" => true }))
|
357
|
+
|
358
|
+
assert_equal(
|
359
|
+
{
|
360
|
+
kerberos: true,
|
361
|
+
renew_kerberos_delegation_token: true,
|
362
|
+
renew_kerberos_delegation_token_interval: expected_hour * 60 * 60,
|
363
|
+
renew_kerberos_delegation_token_interval_hour: expected_hour,
|
364
|
+
kerberos_delegation_token_max_lifetime: expected_delegation_token_max_lifetime_hour * 60 * 60,
|
365
|
+
kerberos_delegation_token_max_lifetime_hour: expected_delegation_token_max_lifetime_hour,
|
366
|
+
},
|
367
|
+
{
|
368
|
+
kerberos: d.instance.kerberos,
|
369
|
+
renew_kerberos_delegation_token: d.instance.instance_eval("@renew_kerberos_delegation_token"),
|
370
|
+
renew_kerberos_delegation_token_interval: d.instance.instance_eval("@renew_kerberos_delegation_token_interval"),
|
371
|
+
renew_kerberos_delegation_token_interval_hour: d.instance.instance_eval("@renew_kerberos_delegation_token_interval_hour"),
|
372
|
+
kerberos_delegation_token_max_lifetime: d.instance.instance_eval("@kerberos_delegation_token_max_lifetime"),
|
373
|
+
kerberos_delegation_token_max_lifetime_hour: d.instance.instance_eval("@kerberos_delegation_token_max_lifetime_hour"),
|
374
|
+
})
|
375
|
+
end
|
376
|
+
|
377
|
+
test "renew_kerberos_delegation_token_interval" do
|
378
|
+
expected_hour = 10
|
379
|
+
expected_delegation_token_max_lifetime_hour = 24
|
380
|
+
|
381
|
+
mock.proxy(WebHDFS::Client).new("server.local", 14000, "hdfs_user", nil, nil, nil, {}, expected_hour,expected_delegation_token_max_lifetime_hour).once
|
382
|
+
|
383
|
+
d = create_driver(
|
384
|
+
CONFIG_KERBEROS +
|
385
|
+
config_element(
|
386
|
+
"", "",
|
387
|
+
{
|
388
|
+
"renew_kerberos_delegation_token" => true,
|
389
|
+
"renew_kerberos_delegation_token_interval" => "#{expected_hour}h",
|
390
|
+
"kerberos_delegation_token_max_lifetime" => "#{expected_delegation_token_max_lifetime_hour}h"
|
391
|
+
}))
|
392
|
+
|
393
|
+
assert_equal(
|
394
|
+
{
|
395
|
+
kerberos: true,
|
396
|
+
renew_kerberos_delegation_token: true,
|
397
|
+
renew_kerberos_delegation_token_interval: expected_hour * 60 * 60,
|
398
|
+
renew_kerberos_delegation_token_interval_hour: expected_hour,
|
399
|
+
kerberos_delegation_token_max_lifetime: expected_delegation_token_max_lifetime_hour * 60 * 60,
|
400
|
+
kerberos_delegation_token_max_lifetime_hour: expected_delegation_token_max_lifetime_hour
|
401
|
+
},
|
402
|
+
{
|
403
|
+
kerberos: d.instance.kerberos,
|
404
|
+
renew_kerberos_delegation_token: d.instance.instance_eval("@renew_kerberos_delegation_token"),
|
405
|
+
renew_kerberos_delegation_token_interval: d.instance.instance_eval("@renew_kerberos_delegation_token_interval"),
|
406
|
+
renew_kerberos_delegation_token_interval_hour: d.instance.instance_eval("@renew_kerberos_delegation_token_interval_hour"),
|
407
|
+
kerberos_delegation_token_max_lifetime: d.instance.instance_eval("@kerberos_delegation_token_max_lifetime"),
|
408
|
+
kerberos_delegation_token_max_lifetime_hour: d.instance.instance_eval("@kerberos_delegation_token_max_lifetime_hour"),
|
409
|
+
})
|
410
|
+
end
|
411
|
+
|
412
|
+
test "username is required for renew_kerberos_delegation_token" do
|
413
|
+
conf = config_element(
|
414
|
+
"ROOT", "", {
|
415
|
+
"namenode" => "server.local:14000",
|
416
|
+
"path" => "/hdfs/path/file.%Y%m%d.%H%M.log",
|
417
|
+
"kerberos" => true,
|
418
|
+
"renew_kerberos_delegation_token" => true,
|
419
|
+
})
|
420
|
+
|
421
|
+
assert_raise(Fluent::ConfigError) do
|
422
|
+
create_driver(conf)
|
423
|
+
end
|
424
|
+
end
|
425
|
+
end
|
313
426
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-webhdfs
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.6.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- TAGOMORI Satoshi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2024-03-19 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake
|
@@ -128,14 +128,14 @@ dependencies:
|
|
128
128
|
requirements:
|
129
129
|
- - ">="
|
130
130
|
- !ruby/object:Gem::Version
|
131
|
-
version: 0.
|
131
|
+
version: 0.11.0
|
132
132
|
type: :runtime
|
133
133
|
prerelease: false
|
134
134
|
version_requirements: !ruby/object:Gem::Requirement
|
135
135
|
requirements:
|
136
136
|
- - ">="
|
137
137
|
- !ruby/object:Gem::Version
|
138
|
-
version: 0.
|
138
|
+
version: 0.11.0
|
139
139
|
description: For WebHDFS and HttpFs of Hadoop HDFS
|
140
140
|
email:
|
141
141
|
- tagomoris@gmail.com
|
@@ -143,6 +143,9 @@ executables: []
|
|
143
143
|
extensions: []
|
144
144
|
extra_rdoc_files: []
|
145
145
|
files:
|
146
|
+
- ".github/dependabot.yml"
|
147
|
+
- ".github/workflows/test-ruby-head.yml"
|
148
|
+
- ".github/workflows/test.yml"
|
146
149
|
- ".gitignore"
|
147
150
|
- ".travis.yml"
|
148
151
|
- Appraisals
|
@@ -183,7 +186,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
183
186
|
- !ruby/object:Gem::Version
|
184
187
|
version: '0'
|
185
188
|
requirements: []
|
186
|
-
rubygems_version: 3.
|
189
|
+
rubygems_version: 3.3.5
|
187
190
|
signing_key:
|
188
191
|
specification_version: 4
|
189
192
|
summary: Fluentd plugin to write data on HDFS over WebHDFS, with flexible formatting
|