fluent-plugin-webhdfs 1.2.2 → 1.2.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +2 -0
- data/fluent-plugin-webhdfs.gemspec +1 -1
- data/lib/fluent/plugin/out_webhdfs.rb +10 -3
- data/test/plugin/test_out_webhdfs.rb +15 -0
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 39cdb79659d5d95293f47ef4c3ae99e8827a3af9
|
4
|
+
data.tar.gz: 91674f2e77c578f1f7761c41e971495541c30043
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 6d72ec81812d8fb264ccde08e0dfb4c4d06605f488509cff5046539e070e840fe2ca9b7e0caa481778f325b1dbf3a4bda2fe632cc007d1be6053c9dceb939de0
|
7
|
+
data.tar.gz: 3c258d6362c05f6778bbf5d6b9e18d96565ed4914d15c8988bfd0334d14ccbdb1d4d3c26073b8d8700931a64cb9667b7c3413db2d87aaae693b78be3f3341a23
|
data/README.md
CHANGED
@@ -1,5 +1,7 @@
|
|
1
1
|
# fluent-plugin-webhdfs
|
2
2
|
|
3
|
+
[![Build Status](https://travis-ci.org/fluent/fluent-plugin-webhdfs.svg?branch=master)](https://travis-ci.org/fluent/fluent-plugin-webhdfs)
|
4
|
+
|
3
5
|
[Fluentd](http://fluentd.org/) output plugin to write data into Hadoop HDFS over WebHDFS/HttpFs.
|
4
6
|
|
5
7
|
"webhdfs" output plugin formats data into plain text, and store it as files on HDFS. This plugin supports:
|
@@ -2,7 +2,7 @@
|
|
2
2
|
|
3
3
|
Gem::Specification.new do |gem|
|
4
4
|
gem.name = "fluent-plugin-webhdfs"
|
5
|
-
gem.version = "1.2.
|
5
|
+
gem.version = "1.2.3"
|
6
6
|
gem.authors = ["TAGOMORI Satoshi"]
|
7
7
|
gem.email = ["tagomoris@gmail.com"]
|
8
8
|
gem.summary = %q{Fluentd plugin to write data on HDFS over WebHDFS, with flexible formatting}
|
@@ -98,20 +98,27 @@ class Fluent::Plugin::WebHDFSOutput < Fluent::Plugin::Output
|
|
98
98
|
end
|
99
99
|
|
100
100
|
def configure(conf)
|
101
|
-
compat_parameters_convert
|
102
|
-
|
101
|
+
# #compat_parameters_convert ignore time format in conf["path"],
|
102
|
+
# so check conf["path"] and overwrite the default value later if needed
|
103
103
|
timekey = case conf["path"]
|
104
104
|
when /%S/ then 1
|
105
105
|
when /%M/ then 60
|
106
106
|
when /%H/ then 3600
|
107
107
|
else 86400
|
108
108
|
end
|
109
|
+
if buffer_config = conf.elements(name: "buffer").first
|
110
|
+
timekey = buffer_config["timekey"] || timekey
|
111
|
+
end
|
112
|
+
|
113
|
+
compat_parameters_convert(conf, :buffer, default_chunk_key: "time")
|
114
|
+
|
109
115
|
if conf.elements(name: "buffer").empty?
|
110
116
|
e = Fluent::Config::Element.new("buffer", "time", {}, [])
|
111
117
|
conf.elements << e
|
112
118
|
end
|
113
119
|
buffer_config = conf.elements(name: "buffer").first
|
114
|
-
|
120
|
+
# explicitly set timekey
|
121
|
+
buffer_config["timekey"] = timekey
|
115
122
|
|
116
123
|
compat_parameters_convert_plaintextformatter(conf)
|
117
124
|
verify_config_placeholders_in_path!(conf)
|
@@ -207,6 +207,21 @@ class WebHDFSOutputTest < Test::Unit::TestCase
|
|
207
207
|
assert_equal 1, d.instance.buffer_config.timekey
|
208
208
|
assert_equal "/hdfs/path/file.20120718.log", d.instance.generate_path(chunk)
|
209
209
|
end
|
210
|
+
|
211
|
+
def test_time_key_without_buffer_section
|
212
|
+
conf = config_element(
|
213
|
+
"ROOT", "", {
|
214
|
+
"host" => "namenode.local",
|
215
|
+
"path" => "/hdfs/path/file.%Y%m%d-%M.log"
|
216
|
+
}
|
217
|
+
)
|
218
|
+
d = create_driver(conf)
|
219
|
+
time = event_time("2012-07-18 15:03:00 +0900")
|
220
|
+
metadata = d.instance.metadata("test", time, {})
|
221
|
+
chunk = d.instance.buffer.generate_chunk(metadata)
|
222
|
+
assert_equal 60, d.instance.buffer_config.timekey
|
223
|
+
assert_equal "/hdfs/path/file.20120718-03.log", d.instance.generate_path(chunk)
|
224
|
+
end
|
210
225
|
end
|
211
226
|
|
212
227
|
sub_test_case "using format subsection" do
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-webhdfs
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.2.
|
4
|
+
version: 1.2.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- TAGOMORI Satoshi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-
|
11
|
+
date: 2017-12-21 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake
|
@@ -167,7 +167,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
167
167
|
version: '0'
|
168
168
|
requirements: []
|
169
169
|
rubyforge_project:
|
170
|
-
rubygems_version: 2.6.
|
170
|
+
rubygems_version: 2.6.13
|
171
171
|
signing_key:
|
172
172
|
specification_version: 4
|
173
173
|
summary: Fluentd plugin to write data on HDFS over WebHDFS, with flexible formatting
|