fluent-plugin-webhdfs 0.5.0 → 0.5.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.travis.yml +5 -0
- data/README.md +2 -2
- data/fluent-plugin-webhdfs.gemspec +2 -1
- data/lib/fluent/plugin/out_webhdfs.rb +2 -1
- data/lib/fluent/plugin/webhdfs_compressor_snappy.rb +21 -0
- data/test/plugin/test_compressor.rb +29 -0
- data/test/plugin/test_out_webhdfs.rb +1 -0
- metadata +19 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: a23088f567c4fe47907a5e411a3af7eb0d380e38
|
4
|
+
data.tar.gz: b33cc9662b3b56734bb0a82681e0925c41167be3
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 0611d377387664c3de86ba93ac030450e593ffaf52cac32ea819191efa2e8544e3348cf6715b2df8d776a439ca54d2cdba40fef784237f1a78fb8038c40d131a
|
7
|
+
data.tar.gz: ab4b691ae3ed374d5fbec3f9f6ab8605f9a7563f520ac40fc66c7c7d842fabc784f04e54a92eaccc9cb6a7e6eedc6ed474359e9695d11cfedf1e5da4514f92d8
|
data/.travis.yml
CHANGED
data/README.md
CHANGED
@@ -127,10 +127,10 @@ If you want to compress data before storing it:
|
|
127
127
|
host namenode.your.cluster.local
|
128
128
|
port 50070
|
129
129
|
path /path/on/hdfs/access.log.%Y%m%d_%H
|
130
|
-
compress gzip # or 'bzip2', 'lzo_command'
|
130
|
+
compress gzip # or 'bzip2', 'snappy', 'lzo_command'
|
131
131
|
</match>
|
132
132
|
|
133
|
-
Note that if you set `compress gzip`, then the suffix `.gz` will be added to path (or `.bz2`, `.lzo`).
|
133
|
+
Note that if you set `compress gzip`, then the suffix `.gz` will be added to path (or `.bz2`, `sz`, `.lzo`).
|
134
134
|
|
135
135
|
### Namenode HA / Auto retry for WebHDFS known errors
|
136
136
|
|
@@ -2,7 +2,7 @@
|
|
2
2
|
|
3
3
|
Gem::Specification.new do |gem|
|
4
4
|
gem.name = "fluent-plugin-webhdfs"
|
5
|
-
gem.version = "0.5.
|
5
|
+
gem.version = "0.5.1"
|
6
6
|
gem.authors = ["TAGOMORI Satoshi"]
|
7
7
|
gem.email = ["tagomoris@gmail.com"]
|
8
8
|
gem.summary = %q{Fluentd plugin to write data on HDFS over WebHDFS, with flexible formatting}
|
@@ -23,4 +23,5 @@ Gem::Specification.new do |gem|
|
|
23
23
|
gem.add_runtime_dependency "fluent-mixin-config-placeholders", ">= 0.3.0"
|
24
24
|
gem.add_runtime_dependency "webhdfs", '>= 0.6.0'
|
25
25
|
gem.add_runtime_dependency "bzip2-ffi"
|
26
|
+
gem.add_runtime_dependency "snappy", '>= 0.0.13'
|
26
27
|
end
|
@@ -85,7 +85,7 @@ class Fluent::WebHDFSOutput < Fluent::TimeSlicedOutput
|
|
85
85
|
desc 'Use kerberos authentication or not'
|
86
86
|
config_param :kerberos, :bool, :default => false
|
87
87
|
|
88
|
-
SUPPORTED_COMPRESS = ['gzip', 'bzip2', 'lzo_command']
|
88
|
+
SUPPORTED_COMPRESS = ['gzip', 'bzip2', 'snappy', 'lzo_command', 'text']
|
89
89
|
desc "Compress method (#{SUPPORTED_COMPRESS.join(',')})"
|
90
90
|
config_param :compress, :default => nil do |val|
|
91
91
|
unless SUPPORTED_COMPRESS.include? val
|
@@ -357,4 +357,5 @@ end
|
|
357
357
|
require 'fluent/plugin/webhdfs_compressor_text'
|
358
358
|
require 'fluent/plugin/webhdfs_compressor_gzip'
|
359
359
|
require 'fluent/plugin/webhdfs_compressor_bzip2'
|
360
|
+
require 'fluent/plugin/webhdfs_compressor_snappy'
|
360
361
|
require 'fluent/plugin/webhdfs_compressor_lzo_command'
|
@@ -0,0 +1,21 @@
|
|
1
|
+
module Fluent
|
2
|
+
class WebHDFSOutput < Fluent::TimeSlicedOutput
|
3
|
+
class SnappyCompressor < Compressor
|
4
|
+
WebHDFSOutput.register_compressor('snappy', self)
|
5
|
+
|
6
|
+
def initialize(options = {})
|
7
|
+
require "snappy"
|
8
|
+
end
|
9
|
+
|
10
|
+
def ext
|
11
|
+
".sz"
|
12
|
+
end
|
13
|
+
|
14
|
+
def compress(chunk, tmp)
|
15
|
+
w = Snappy::Writer.new(tmp)
|
16
|
+
chunk.write_to(w)
|
17
|
+
w.close
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
require "helper"
|
2
|
+
require "fluent/plugin/buf_memory"
|
3
|
+
require "snappy"
|
4
|
+
|
5
|
+
class CompressorTest < Test::Unit::TestCase
|
6
|
+
class Snappy < self
|
7
|
+
def setup
|
8
|
+
@compressor = Fluent::WebHDFSOutput::SnappyCompressor.new
|
9
|
+
end
|
10
|
+
|
11
|
+
def test_ext
|
12
|
+
assert_equal(".sz", @compressor.ext)
|
13
|
+
end
|
14
|
+
|
15
|
+
def test_compress
|
16
|
+
chunk = Fluent::MemoryBufferChunk.new("test")
|
17
|
+
chunk << "hello snappy\n" * 32 * 1024
|
18
|
+
io = Tempfile.new("snappy-")
|
19
|
+
@compressor.compress(chunk, io)
|
20
|
+
io.open
|
21
|
+
assert(chunk.size > io.read.bytesize)
|
22
|
+
io.rewind
|
23
|
+
reader = ::Snappy::Reader.new(io)
|
24
|
+
assert_equal(chunk.read, reader.read)
|
25
|
+
io.close
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
@@ -69,6 +69,7 @@ kerberos true
|
|
69
69
|
|
70
70
|
data(gzip: ['gzip', Fluent::WebHDFSOutput::GzipCompressor],
|
71
71
|
bzip2: ['bzip2', Fluent::WebHDFSOutput::Bzip2Compressor],
|
72
|
+
snappy: ['snappy', Fluent::WebHDFSOutput::SnappyCompressor],
|
72
73
|
lzo: ['lzo_command', Fluent::WebHDFSOutput::LZOCommandCompressor])
|
73
74
|
def test_compress(data)
|
74
75
|
compress_type, compressor_class = data
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-webhdfs
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.5.
|
4
|
+
version: 0.5.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- TAGOMORI Satoshi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-02-
|
11
|
+
date: 2016-02-05 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake
|
@@ -122,6 +122,20 @@ dependencies:
|
|
122
122
|
- - ">="
|
123
123
|
- !ruby/object:Gem::Version
|
124
124
|
version: '0'
|
125
|
+
- !ruby/object:Gem::Dependency
|
126
|
+
name: snappy
|
127
|
+
requirement: !ruby/object:Gem::Requirement
|
128
|
+
requirements:
|
129
|
+
- - ">="
|
130
|
+
- !ruby/object:Gem::Version
|
131
|
+
version: 0.0.13
|
132
|
+
type: :runtime
|
133
|
+
prerelease: false
|
134
|
+
version_requirements: !ruby/object:Gem::Requirement
|
135
|
+
requirements:
|
136
|
+
- - ">="
|
137
|
+
- !ruby/object:Gem::Version
|
138
|
+
version: 0.0.13
|
125
139
|
description: For WebHDFS and HttpFs of Hadoop HDFS
|
126
140
|
email:
|
127
141
|
- tagomoris@gmail.com
|
@@ -142,8 +156,10 @@ files:
|
|
142
156
|
- lib/fluent/plugin/webhdfs_compressor_bzip2.rb
|
143
157
|
- lib/fluent/plugin/webhdfs_compressor_gzip.rb
|
144
158
|
- lib/fluent/plugin/webhdfs_compressor_lzo_command.rb
|
159
|
+
- lib/fluent/plugin/webhdfs_compressor_snappy.rb
|
145
160
|
- lib/fluent/plugin/webhdfs_compressor_text.rb
|
146
161
|
- test/helper.rb
|
162
|
+
- test/plugin/test_compressor.rb
|
147
163
|
- test/plugin/test_out_webhdfs.rb
|
148
164
|
homepage: https://github.com/fluent/fluent-plugin-webhdfs
|
149
165
|
licenses:
|
@@ -171,5 +187,6 @@ specification_version: 4
|
|
171
187
|
summary: Fluentd plugin to write data on HDFS over WebHDFS, with flexible formatting
|
172
188
|
test_files:
|
173
189
|
- test/helper.rb
|
190
|
+
- test/plugin/test_compressor.rb
|
174
191
|
- test/plugin/test_out_webhdfs.rb
|
175
192
|
has_rdoc:
|