logstash-codec-compress_spooler 0.1.6-java → 0.2.0-java
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +5 -0
- data/CONTRIBUTORS +1 -0
- data/Gemfile +1 -1
- data/NOTICE.TXT +5 -0
- data/README.md +1 -1
- data/lib/logstash/codecs/compress_spooler.rb +27 -20
- data/logstash-codec-compress_spooler.gemspec +5 -5
- data/spec/codecs/compress_spooler_spec.rb +102 -0
- metadata +18 -16
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 27a52170cd8af6d61263363855f549a4c385aa8f
|
4
|
+
data.tar.gz: 7858feb1f93f7b899a29232d9475a4a1f6ffbdfc
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 58be29508e461f884e0dce405b3d6935d6d4c8c359233b43f359589a7195847668d34ebd56635b9f05e1559573bdba00032da163eecc5628275f740808602949
|
7
|
+
data.tar.gz: 4aa77b6f4806fb2143ce511c52f4eee57e5e3f1bacc5883c3cd43f12d93140fecd8ad6c8942e9a813e50c34f90dcd3edfb7f891158a8a492c6dcc04bd06e7ea3
|
data/CHANGELOG.md
ADDED
data/CONTRIBUTORS
CHANGED
@@ -8,6 +8,7 @@ Contributors:
|
|
8
8
|
* Pier-Hugues Pellerin (ph)
|
9
9
|
* Richard Pijnenburg (electrical)
|
10
10
|
* avleen
|
11
|
+
* Pere Urbon-Bayes (purbon)
|
11
12
|
|
12
13
|
Note: If you've sent us patches, bug reports, or otherwise contributed to
|
13
14
|
Logstash, and you aren't on the list above and want to be, please let us know
|
data/Gemfile
CHANGED
@@ -1,2 +1,2 @@
|
|
1
1
|
source 'https://rubygems.org'
|
2
|
-
gemspec
|
2
|
+
gemspec
|
data/NOTICE.TXT
ADDED
data/README.md
CHANGED
@@ -13,7 +13,7 @@ Logstash provides infrastructure to automatically generate documentation for thi
|
|
13
13
|
|
14
14
|
## Need Help?
|
15
15
|
|
16
|
-
Need help? Try #logstash on freenode IRC or the logstash
|
16
|
+
Need help? Try #logstash on freenode IRC or the https://discuss.elastic.co/c/logstash discussion forum.
|
17
17
|
|
18
18
|
## Developing
|
19
19
|
|
@@ -7,43 +7,50 @@ class LogStash::Codecs::CompressSpooler < LogStash::Codecs::Base
|
|
7
7
|
config :compress_level, :validate => :number, :default => 6
|
8
8
|
|
9
9
|
public
|
10
|
+
|
10
11
|
def register
|
11
12
|
require "msgpack"
|
12
13
|
require "zlib"
|
13
14
|
@buffer = []
|
14
15
|
end
|
15
16
|
|
16
|
-
public
|
17
17
|
def decode(data)
|
18
|
-
|
19
|
-
|
20
|
-
z.finish
|
21
|
-
z.close
|
22
|
-
data.each do |event|
|
23
|
-
event = LogStash::Event.new(event)
|
24
|
-
event["@timestamp"] = Time.at(event["@timestamp"]).utc if event["@timestamp"].is_a? Float
|
25
|
-
yield event
|
18
|
+
decompress(data).each do |event|
|
19
|
+
yield(LogStash::Event.new(event))
|
26
20
|
end
|
27
21
|
end # def decode
|
28
22
|
|
29
|
-
|
30
|
-
def encode(data)
|
23
|
+
def encode(event)
|
31
24
|
if @buffer.length >= @spool_size
|
32
|
-
|
33
|
-
@on_event.call data, z.deflate(MessagePack.pack(@buffer), Zlib::FINISH)
|
34
|
-
z.close
|
25
|
+
@on_event.call(compress(@buffer, @compress_level))
|
35
26
|
@buffer.clear
|
36
27
|
else
|
37
|
-
|
38
|
-
|
28
|
+
# use normalize to make sure returned Hash is pure Ruby for
|
29
|
+
# MessagePack#pack which relies on pure Ruby object recognition
|
30
|
+
@buffer << LogStash::Util.normalize(event.to_hash).merge(LogStash::Event::TIMESTAMP => event.timestamp.to_iso8601)
|
39
31
|
end
|
40
32
|
end # def encode
|
41
33
|
|
42
|
-
public
|
43
34
|
def teardown
|
44
|
-
if
|
45
|
-
|
46
|
-
end
|
35
|
+
return if @buffer.empty?
|
36
|
+
@on_event.call(compress(@buffer, @compress_level))
|
47
37
|
@buffer.clear
|
48
38
|
end
|
39
|
+
|
40
|
+
private
|
41
|
+
|
42
|
+
def compress(data, level)
|
43
|
+
z = Zlib::Deflate.new(level)
|
44
|
+
result = z.deflate(MessagePack.pack(data), Zlib::FINISH)
|
45
|
+
z.close
|
46
|
+
result
|
47
|
+
end
|
48
|
+
|
49
|
+
def decompress(data)
|
50
|
+
z = Zlib::Inflate.new
|
51
|
+
result = MessagePack.unpack(z.inflate(data))
|
52
|
+
z.finish
|
53
|
+
z.close
|
54
|
+
result
|
55
|
+
end
|
49
56
|
end # class LogStash::Codecs::CompressSpooler
|
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-codec-compress_spooler'
|
4
|
-
s.version = '0.
|
4
|
+
s.version = '0.2.0'
|
5
5
|
s.licenses = ['Apache License (2.0)']
|
6
6
|
s.summary = "codec that processes compressed spooled data"
|
7
7
|
s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
|
@@ -21,12 +21,12 @@ Gem::Specification.new do |s|
|
|
21
21
|
|
22
22
|
# Gem dependencies
|
23
23
|
s.add_runtime_dependency "logstash-core", '>= 1.4.0', '< 2.0.0'
|
24
|
+
s.add_development_dependency 'logstash-devutils'
|
24
25
|
|
25
26
|
if RUBY_PLATFORM == 'java'
|
26
27
|
s.platform = RUBY_PLATFORM
|
28
|
+
s.add_runtime_dependency "msgpack-jruby" #(Apache 2.0 license)
|
29
|
+
else
|
30
|
+
s.add_runtime_dependency "msgpack" #(Apache 2.0 license)
|
27
31
|
end
|
28
|
-
s.add_runtime_dependency 'msgpack-jruby'
|
29
|
-
|
30
|
-
s.add_development_dependency 'logstash-devutils'
|
31
32
|
end
|
32
|
-
|
@@ -1 +1,103 @@
|
|
1
1
|
require "logstash/devutils/rspec/spec_helper"
|
2
|
+
require "logstash/codecs/compress_spooler"
|
3
|
+
|
4
|
+
describe LogStash::Codecs::CompressSpooler do
|
5
|
+
|
6
|
+
it "should work" do
|
7
|
+
expect(true).to be true
|
8
|
+
end
|
9
|
+
|
10
|
+
context "#decode" do
|
11
|
+
it "should return an event from msgpack data" do
|
12
|
+
codec = LogStash::Codecs::CompressSpooler.new
|
13
|
+
events = [{"foo" => "bar", "baz" => {"bah" => ["a", "b", "c"]}, "@timestamp" => "2014-05-30T02:52:17.929Z"}]
|
14
|
+
|
15
|
+
z = Zlib::Deflate.new
|
16
|
+
data = z.deflate(MessagePack.pack(events), Zlib::FINISH)
|
17
|
+
z.close
|
18
|
+
|
19
|
+
codec.decode(data) do |event|
|
20
|
+
insist { event.is_a? LogStash::Event }
|
21
|
+
insist { event["foo"] } == events.first["foo"]
|
22
|
+
insist { event["baz"] } == events.first["baz"]
|
23
|
+
insist { event["bah"] } == events.first["bah"]
|
24
|
+
insist { event["@timestamp"].to_iso8601 } == events.first["@timestamp"]
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
context "#encode" do
|
30
|
+
|
31
|
+
it "should return compressed data from pure ruby hash" do
|
32
|
+
codec = LogStash::Codecs::CompressSpooler.new("spool_size" => 1)
|
33
|
+
data = {"foo" => "bar", "baz" => {"bah" => ["a","b","c"]}, "@timestamp" => "2014-05-30T02:52:17.929Z"}
|
34
|
+
event = LogStash::Event.new(data)
|
35
|
+
results = []
|
36
|
+
codec.on_event{|data| results << data}
|
37
|
+
|
38
|
+
# spool_size is one so encode twice to trigger encoding on the 2nd one
|
39
|
+
codec.encode(event)
|
40
|
+
codec.encode(event)
|
41
|
+
|
42
|
+
insist { results.size } == 1
|
43
|
+
z = Zlib::Inflate.new
|
44
|
+
events = MessagePack.unpack(z.inflate(results.first))
|
45
|
+
z.finish
|
46
|
+
z.close
|
47
|
+
|
48
|
+
insist { events.first["foo"] } == data["foo"]
|
49
|
+
insist { events.first["baz"] } == data["baz"]
|
50
|
+
insist { events.first["bah"] } == data["bah"]
|
51
|
+
insist { events.first["@timestamp"] } == "2014-05-30T02:52:17.929Z"
|
52
|
+
insist { events.first["@timestamp"] } == event["@timestamp"].to_iso8601
|
53
|
+
end
|
54
|
+
|
55
|
+
it "should return compressed data from deserialized json with normalization" do
|
56
|
+
codec = LogStash::Codecs::CompressSpooler.new("spool_size" => 1)
|
57
|
+
data = LogStash::Json.load('{"foo": "bar", "baz": {"bah": ["a","b","c"]}, "@timestamp": "2014-05-30T02:52:17.929Z"}')
|
58
|
+
event = LogStash::Event.new(data)
|
59
|
+
results = []
|
60
|
+
codec.on_event{|data| results << data}
|
61
|
+
|
62
|
+
# spool_size is one so encode twice to trigger encoding on the 2nd one
|
63
|
+
codec.encode(event)
|
64
|
+
codec.encode(event)
|
65
|
+
|
66
|
+
insist { results.size } == 1
|
67
|
+
z = Zlib::Inflate.new
|
68
|
+
events = MessagePack.unpack(z.inflate(results.first))
|
69
|
+
z.finish
|
70
|
+
z.close
|
71
|
+
|
72
|
+
insist { events.first["foo"] } == data["foo"]
|
73
|
+
insist { events.first["baz"] } == data["baz"]
|
74
|
+
insist { events.first["bah"] } == data["bah"]
|
75
|
+
insist { events.first["@timestamp"] } == "2014-05-30T02:52:17.929Z"
|
76
|
+
insist { events.first["@timestamp"] } == event["@timestamp"].to_iso8601
|
77
|
+
end
|
78
|
+
|
79
|
+
it "should support teardown and flush pending data" do
|
80
|
+
codec = LogStash::Codecs::CompressSpooler.new("spool_size" => 1)
|
81
|
+
data = {"foo" => "bar", "baz" => {"bah" => ["a","b","c"]}, "@timestamp" => "2014-05-30T02:52:17.929Z"}
|
82
|
+
event = LogStash::Event.new(data)
|
83
|
+
results = []
|
84
|
+
codec.on_event{|data| results << data}
|
85
|
+
|
86
|
+
codec.encode(event)
|
87
|
+
codec.teardown
|
88
|
+
|
89
|
+
insist { results.size } == 1
|
90
|
+
z = Zlib::Inflate.new
|
91
|
+
events = MessagePack.unpack(z.inflate(results.first))
|
92
|
+
z.finish
|
93
|
+
z.close
|
94
|
+
|
95
|
+
insist { events.first["foo"] } == data["foo"]
|
96
|
+
insist { events.first["baz"] } == data["baz"]
|
97
|
+
insist { events.first["bah"] } == data["bah"]
|
98
|
+
insist { events.first["@timestamp"] } == "2014-05-30T02:52:17.929Z"
|
99
|
+
insist { events.first["@timestamp"] } == event["@timestamp"].to_iso8601
|
100
|
+
end
|
101
|
+
end
|
102
|
+
|
103
|
+
end
|
metadata
CHANGED
@@ -1,17 +1,18 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-codec-compress_spooler
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: java
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-
|
11
|
+
date: 2015-07-24 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
|
-
|
14
|
+
name: logstash-core
|
15
|
+
version_requirements: !ruby/object:Gem::Requirement
|
15
16
|
requirements:
|
16
17
|
- - '>='
|
17
18
|
- !ruby/object:Gem::Version
|
@@ -19,10 +20,7 @@ dependencies:
|
|
19
20
|
- - <
|
20
21
|
- !ruby/object:Gem::Version
|
21
22
|
version: 2.0.0
|
22
|
-
|
23
|
-
prerelease: false
|
24
|
-
type: :runtime
|
25
|
-
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirement: !ruby/object:Gem::Requirement
|
26
24
|
requirements:
|
27
25
|
- - '>='
|
28
26
|
- !ruby/object:Gem::Version
|
@@ -30,34 +28,36 @@ dependencies:
|
|
30
28
|
- - <
|
31
29
|
- !ruby/object:Gem::Version
|
32
30
|
version: 2.0.0
|
33
|
-
- !ruby/object:Gem::Dependency
|
34
|
-
requirement: !ruby/object:Gem::Requirement
|
35
|
-
requirements:
|
36
|
-
- - '>='
|
37
|
-
- !ruby/object:Gem::Version
|
38
|
-
version: '0'
|
39
|
-
name: msgpack-jruby
|
40
31
|
prerelease: false
|
41
32
|
type: :runtime
|
33
|
+
- !ruby/object:Gem::Dependency
|
34
|
+
name: logstash-devutils
|
42
35
|
version_requirements: !ruby/object:Gem::Requirement
|
43
36
|
requirements:
|
44
37
|
- - '>='
|
45
38
|
- !ruby/object:Gem::Version
|
46
39
|
version: '0'
|
47
|
-
- !ruby/object:Gem::Dependency
|
48
40
|
requirement: !ruby/object:Gem::Requirement
|
49
41
|
requirements:
|
50
42
|
- - '>='
|
51
43
|
- !ruby/object:Gem::Version
|
52
44
|
version: '0'
|
53
|
-
name: logstash-devutils
|
54
45
|
prerelease: false
|
55
46
|
type: :development
|
47
|
+
- !ruby/object:Gem::Dependency
|
48
|
+
name: msgpack-jruby
|
56
49
|
version_requirements: !ruby/object:Gem::Requirement
|
57
50
|
requirements:
|
58
51
|
- - '>='
|
59
52
|
- !ruby/object:Gem::Version
|
60
53
|
version: '0'
|
54
|
+
requirement: !ruby/object:Gem::Requirement
|
55
|
+
requirements:
|
56
|
+
- - '>='
|
57
|
+
- !ruby/object:Gem::Version
|
58
|
+
version: '0'
|
59
|
+
prerelease: false
|
60
|
+
type: :runtime
|
61
61
|
description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
|
62
62
|
email: info@elastic.co
|
63
63
|
executables: []
|
@@ -65,9 +65,11 @@ extensions: []
|
|
65
65
|
extra_rdoc_files: []
|
66
66
|
files:
|
67
67
|
- .gitignore
|
68
|
+
- CHANGELOG.md
|
68
69
|
- CONTRIBUTORS
|
69
70
|
- Gemfile
|
70
71
|
- LICENSE
|
72
|
+
- NOTICE.TXT
|
71
73
|
- README.md
|
72
74
|
- Rakefile
|
73
75
|
- lib/logstash/codecs/compress_spooler.rb
|