fluent-plugin-webhdfs 1.3.0 → 1.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +14 -0
- data/fluent-plugin-webhdfs.gemspec +1 -1
- data/lib/fluent/plugin/out_webhdfs.rb +5 -1
- data/test/plugin/test_out_webhdfs.rb +20 -0
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 64bdfcd21e62ec4f83f7807b7b7a60e51f0ed68f2d015f39424dd47b76473be3
|
4
|
+
data.tar.gz: cbf248448c4eaa9b35689076ac0010753af7eb138d24cd377c43bcf5dfd46595
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d76806177dc1ff5e33dbf4fcc0e1fd0b3a47bd1a4fa1e51287a033ba4ccb94c750a540f735e1144be0bd502629c05ee19d74462ab2fe6831979d378b35686d20
|
7
|
+
data.tar.gz: e7d195870652a72c6d3edea039336742e3564a01ac9871036ddc4dcb3d0eb58bb4b9c73ac6598ea2712ec1ec17870e9ee6d091a2a89204688fb3935aa5ccc34b
|
data/README.md
CHANGED
@@ -169,6 +169,20 @@ Note that you have to install additional gem for several compress algorithms:
|
|
169
169
|
|
170
170
|
Note that zstd will require installation of the libzstd native library. See the [zstandard-ruby](https://github.com/msievers/zstandard-ruby#examples-for-installing-libzstd) repo for infomration on the required packages for your operating system.
|
171
171
|
|
172
|
+
If you want to explicitly specify file extensions in HDFS (override default compressor extensions):
|
173
|
+
|
174
|
+
<match access.**>
|
175
|
+
@type webhdfs
|
176
|
+
host namenode.your.cluster.local
|
177
|
+
port 50070
|
178
|
+
path /path/on/hdfs/access.log.%Y%m%d_%H
|
179
|
+
compress snappy
|
180
|
+
extension ".snappy"
|
181
|
+
</match>
|
182
|
+
|
183
|
+
With this configuration paths in HDFS will be like `/path/on/hdfs/access.log.20201003_12.snappy`.
|
184
|
+
This one may be useful when (for example) you need to use snappy codec but `.sz` files are not recognized as snappy files in HDFS.
|
185
|
+
|
172
186
|
### Namenode HA / Auto retry for WebHDFS known errors
|
173
187
|
|
174
188
|
`fluent-plugin-webhdfs` (v0.2.0 or later) accepts 2 namenodes for Namenode HA (active/standby). Use `standby_namenode` like this:
|
@@ -2,7 +2,7 @@
|
|
2
2
|
|
3
3
|
Gem::Specification.new do |gem|
|
4
4
|
gem.name = "fluent-plugin-webhdfs"
|
5
|
-
gem.version = "1.3.
|
5
|
+
gem.version = "1.3.1"
|
6
6
|
gem.authors = ["TAGOMORI Satoshi"]
|
7
7
|
gem.email = ["tagomoris@gmail.com"]
|
8
8
|
gem.summary = %q{Fluentd plugin to write data on HDFS over WebHDFS, with flexible formatting}
|
@@ -71,6 +71,9 @@ class Fluent::Plugin::WebHDFSOutput < Fluent::Plugin::Output
|
|
71
71
|
desc "Compress method (#{SUPPORTED_COMPRESS.join(',')})"
|
72
72
|
config_param :compress, :enum, list: SUPPORTED_COMPRESS, default: :text
|
73
73
|
|
74
|
+
desc 'HDFS file extensions (overrides default compressor extensions)'
|
75
|
+
config_param :extension, :string, default: nil
|
76
|
+
|
74
77
|
config_param :remove_prefix, :string, default: nil, deprecated: "use @label for routing"
|
75
78
|
config_param :default_tag, :string, default: nil, deprecated: "use @label for routing"
|
76
79
|
config_param :null_value, :string, default: nil, deprecated: "use filter plugins to convert null values into any specified string"
|
@@ -319,7 +322,8 @@ class Fluent::Plugin::WebHDFSOutput < Fluent::Plugin::Output
|
|
319
322
|
else
|
320
323
|
extract_placeholders(@path.gsub(CHUNK_ID_PLACE_HOLDER, dump_unique_id_hex(chunk.unique_id)), chunk)
|
321
324
|
end
|
322
|
-
|
325
|
+
hdfs_ext = @extension || @compressor.ext
|
326
|
+
hdfs_path = "#{hdfs_path}#{hdfs_ext}"
|
323
327
|
if @replace_random_uuid
|
324
328
|
uuid_random = SecureRandom.uuid
|
325
329
|
hdfs_path = hdfs_path.gsub('%{uuid}', uuid_random).gsub('%{uuid_flush}', uuid_random)
|
@@ -126,6 +126,26 @@ class WebHDFSOutputTest < Test::Unit::TestCase
|
|
126
126
|
assert_equal '/hdfs/path/file.%Y%m%d.%H%M.log', d.instance.path
|
127
127
|
assert_equal compress_type, d.instance.compress
|
128
128
|
assert_equal compressor_class, d.instance.compressor.class
|
129
|
+
|
130
|
+
time = event_time("2020-10-03 15:07:00 +0300")
|
131
|
+
metadata = d.instance.metadata("test", time, {})
|
132
|
+
chunk = d.instance.buffer.generate_chunk(metadata)
|
133
|
+
assert_equal "/hdfs/path/file.20201003.1507.log#{d.instance.compressor.ext}", d.instance.generate_path(chunk)
|
134
|
+
end
|
135
|
+
|
136
|
+
def test_explicit_extensions
|
137
|
+
conf = config_element(
|
138
|
+
"ROOT", "", {
|
139
|
+
"host" => "namenode.local",
|
140
|
+
"path" => "/hdfs/path/file.%Y%m%d.log",
|
141
|
+
"compress" => "snappy",
|
142
|
+
"extension" => ".snappy"
|
143
|
+
})
|
144
|
+
d = create_driver(conf)
|
145
|
+
time = event_time("2020-10-07 15:15:00 +0300")
|
146
|
+
metadata = d.instance.metadata("test", time, {})
|
147
|
+
chunk = d.instance.buffer.generate_chunk(metadata)
|
148
|
+
assert_equal "/hdfs/path/file.20201007.log.snappy", d.instance.generate_path(chunk)
|
129
149
|
end
|
130
150
|
|
131
151
|
def test_placeholders_old_style
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-webhdfs
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.3.
|
4
|
+
version: 1.3.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- TAGOMORI Satoshi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-
|
11
|
+
date: 2020-11-09 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake
|