fluent-plugin-webhdfs 1.0.0 → 1.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.travis.yml +2 -2
- data/fluent-plugin-webhdfs.gemspec +1 -1
- data/lib/fluent/plugin/out_webhdfs.rb +6 -2
- data/test/helper.rb +1 -0
- data/test/plugin/test_out_webhdfs.rb +14 -0
- metadata +2 -4
- data/gemfiles/fluentd_v0.12.gemfile +0 -7
- data/gemfiles/fluentd_v0.14.gemfile +0 -7
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7ebfc106d895f4cb137cf632a3680312d6e6ed84
|
4
|
+
data.tar.gz: 84b238ea1329df8a871555c9aae43f0dc4659141
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a4a83f5591129c2e8649bf39d5a8ea1368f2389729a4362efb14ebefb6afebc9f1a3e89f8d90c0c5cb59e6238f9c2b1cde0727023d2c6e16967dfe104b34235c
|
7
|
+
data.tar.gz: de25df285af31e3aec52a2fcd0b04c55085a5a03dcdeaf0ba8862635e0b01f7a05b589f8364fa0842b44b9f7d48eaa90f017585e72313e86b58bbc52e5c384fe
|
data/.travis.yml
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
|
3
3
|
Gem::Specification.new do |gem|
|
4
4
|
gem.name = "fluent-plugin-webhdfs"
|
5
|
-
gem.version = "1.
|
5
|
+
gem.version = "1.1.0"
|
6
6
|
gem.authors = ["TAGOMORI Satoshi"]
|
7
7
|
gem.email = ["tagomoris@gmail.com"]
|
8
8
|
gem.summary = %q{Fluentd plugin to write data on HDFS over WebHDFS, with flexible formatting}
|
@@ -112,6 +112,7 @@ class Fluent::Plugin::WebHDFSOutput < Fluent::Plugin::Output
|
|
112
112
|
buffer_config["timekey"] = timekey unless buffer_config["timekey"]
|
113
113
|
|
114
114
|
compat_parameters_convert_plaintextformatter(conf)
|
115
|
+
verify_config_placeholders_in_path!(conf)
|
115
116
|
|
116
117
|
super
|
117
118
|
|
@@ -132,7 +133,6 @@ class Fluent::Plugin::WebHDFSOutput < Fluent::Plugin::Output
|
|
132
133
|
@remove_prefix_actual_length = @remove_prefix_actual.length
|
133
134
|
end
|
134
135
|
|
135
|
-
verify_config_placeholders_in_path!(conf)
|
136
136
|
@replace_random_uuid = @path.include?('%{uuid}') || @path.include?('%{uuid_flush}')
|
137
137
|
if @replace_random_uuid
|
138
138
|
# to check SecureRandom.uuid is available or not (NotImplementedError raised in such environment)
|
@@ -185,6 +185,10 @@ class Fluent::Plugin::WebHDFSOutput < Fluent::Plugin::Output
|
|
185
185
|
end
|
186
186
|
end
|
187
187
|
|
188
|
+
def multi_workers_ready?
|
189
|
+
true
|
190
|
+
end
|
191
|
+
|
188
192
|
def prepare_client(host, port, username)
|
189
193
|
client = WebHDFS::Client.new(host, port, username)
|
190
194
|
if @httpfs
|
@@ -303,7 +307,7 @@ class Fluent::Plugin::WebHDFSOutput < Fluent::Plugin::Output
|
|
303
307
|
hdfs_path = if @append
|
304
308
|
extract_placeholders(@path, chunk.metadata)
|
305
309
|
else
|
306
|
-
extract_placeholders(@path, chunk.metadata).gsub(CHUNK_ID_PLACE_HOLDER,
|
310
|
+
extract_placeholders(@path, chunk.metadata).gsub(CHUNK_ID_PLACE_HOLDER, dump_unique_id_hex(chunk.unique_id))
|
307
311
|
end
|
308
312
|
hdfs_path = "#{hdfs_path}#{@compressor.ext}"
|
309
313
|
if @replace_random_uuid
|
data/test/helper.rb
CHANGED
@@ -154,6 +154,20 @@ class WebHDFSOutputTest < Test::Unit::TestCase
|
|
154
154
|
assert_equal expected, d.instance.generate_path(chunk)
|
155
155
|
end
|
156
156
|
|
157
|
+
data(path: "/hdfs/path/file.${chunk_id}.log")
|
158
|
+
test "generate_path without append" do |path|
|
159
|
+
conf = config_element(
|
160
|
+
"ROOT", "", {
|
161
|
+
"namenode" => "server.local:14000",
|
162
|
+
"path" => path,
|
163
|
+
"append" => false
|
164
|
+
})
|
165
|
+
d = create_driver(conf)
|
166
|
+
metadata = d.instance.metadata("test", nil, {})
|
167
|
+
chunk = d.instance.buffer.generate_chunk(metadata)
|
168
|
+
assert_equal "/hdfs/path/file.#{dump_unique_id_hex(chunk.unique_id)}.log", d.instance.generate_path(chunk)
|
169
|
+
end
|
170
|
+
|
157
171
|
data(path: { "append" => false },
|
158
172
|
ssl: { "ssl" => true, "ssl_verify_mode" => "invalid" },
|
159
173
|
compress: { "compress" => "invalid" })
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-webhdfs
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- TAGOMORI Satoshi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-01-
|
11
|
+
date: 2017-01-31 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake
|
@@ -137,8 +137,6 @@ files:
|
|
137
137
|
- README.md
|
138
138
|
- Rakefile
|
139
139
|
- fluent-plugin-webhdfs.gemspec
|
140
|
-
- gemfiles/fluentd_v0.12.gemfile
|
141
|
-
- gemfiles/fluentd_v0.14.gemfile
|
142
140
|
- lib/fluent/plugin/out_webhdfs.rb
|
143
141
|
- lib/fluent/plugin/webhdfs_compressor_bzip2.rb
|
144
142
|
- lib/fluent/plugin/webhdfs_compressor_gzip.rb
|