logstash-codec-compress_spooler 2.0.4-java → 2.0.5-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/Gemfile +9 -0
- data/LICENSE +1 -1
- data/README.md +12 -3
- data/docs/index.asciidoc +64 -0
- data/lib/logstash/codecs/compress_spooler.rb +13 -1
- data/logstash-codec-compress_spooler.gemspec +3 -3
- data/spec/codecs/compress_spooler_spec.rb +27 -0
- metadata +7 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 5e86db1e691c4ede63e18f698d253de72ef26a32aa655264121887bdb4905fe0
|
4
|
+
data.tar.gz: a244ba2bdb7aa08013534652cb6fe569b27290506cb3a881ce5db1a2a2f110f1
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 24e9c421fab67c92ffc33f90b93fe214e6545b083fccf6453ebfbfa31253b2a23137cb6d21f00458bf79ff5b9190eb84c68e560a4c73dcd7c40da4257f8c7a02
|
7
|
+
data.tar.gz: 0a25bb51dcfbac612cd99646edab34fa9ebec6c20b2e2ebb23a25aed22ffe1411d3b26b02f2d8bac706f4982c4ad7920f3ec24fd7ec1c527f066b351786e17d7
|
data/Gemfile
CHANGED
@@ -1,2 +1,11 @@
|
|
1
1
|
source 'https://rubygems.org'
|
2
|
+
|
2
3
|
gemspec
|
4
|
+
|
5
|
+
logstash_path = ENV["LOGSTASH_PATH"] || "../../logstash"
|
6
|
+
use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "1"
|
7
|
+
|
8
|
+
if Dir.exist?(logstash_path) && use_logstash_source
|
9
|
+
gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
|
10
|
+
gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
|
11
|
+
end
|
data/LICENSE
CHANGED
data/README.md
CHANGED
@@ -1,7 +1,6 @@
|
|
1
1
|
# Logstash Plugin
|
2
2
|
|
3
|
-
[](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Codecs/job/logstash-plugin-codec-compress_spooler-unit/)
|
3
|
+
[](https://travis-ci.org/logstash-plugins/logstash-codec-compress_spooler)
|
5
4
|
|
6
5
|
This is a plugin for [Logstash](https://github.com/elastic/logstash).
|
7
6
|
|
@@ -56,7 +55,12 @@ gem "logstash-filter-awesome", :path => "/your/local/logstash-filter-awesome"
|
|
56
55
|
```
|
57
56
|
- Install plugin
|
58
57
|
```sh
|
58
|
+
# Logstash 2.3 and higher
|
59
|
+
bin/logstash-plugin install --no-verify
|
60
|
+
|
61
|
+
# Prior to Logstash 2.3
|
59
62
|
bin/plugin install --no-verify
|
63
|
+
|
60
64
|
```
|
61
65
|
- Run Logstash with your plugin
|
62
66
|
```sh
|
@@ -74,7 +78,12 @@ gem build logstash-filter-awesome.gemspec
|
|
74
78
|
```
|
75
79
|
- Install the plugin from the Logstash home
|
76
80
|
```sh
|
77
|
-
|
81
|
+
# Logstash 2.3 and higher
|
82
|
+
bin/logstash-plugin install --no-verify
|
83
|
+
|
84
|
+
# Prior to Logstash 2.3
|
85
|
+
bin/plugin install --no-verify
|
86
|
+
|
78
87
|
```
|
79
88
|
- Start Logstash and proceed to test the plugin
|
80
89
|
|
data/docs/index.asciidoc
ADDED
@@ -0,0 +1,64 @@
|
|
1
|
+
:plugin: compress_spooler
|
2
|
+
:type: codec
|
3
|
+
|
4
|
+
///////////////////////////////////////////
|
5
|
+
START - GENERATED VARIABLES, DO NOT EDIT!
|
6
|
+
///////////////////////////////////////////
|
7
|
+
:version: %VERSION%
|
8
|
+
:release_date: %RELEASE_DATE%
|
9
|
+
:changelog_url: %CHANGELOG_URL%
|
10
|
+
:include_path: ../../../../logstash/docs/include
|
11
|
+
///////////////////////////////////////////
|
12
|
+
END - GENERATED VARIABLES, DO NOT EDIT!
|
13
|
+
///////////////////////////////////////////
|
14
|
+
|
15
|
+
[id="plugins-{type}-{plugin}"]
|
16
|
+
|
17
|
+
=== Compress_spooler codec plugin
|
18
|
+
|
19
|
+
include::{include_path}/plugin_header.asciidoc[]
|
20
|
+
|
21
|
+
==== Description
|
22
|
+
|
23
|
+
|
24
|
+
|
25
|
+
[id="plugins-{type}s-{plugin}-options"]
|
26
|
+
==== Compress_spooler Codec Configuration Options
|
27
|
+
|
28
|
+
[cols="<,<,<",options="header",]
|
29
|
+
|=======================================================================
|
30
|
+
|Setting |Input type|Required
|
31
|
+
| <<plugins-{type}s-{plugin}-compress_level>> |<<number,number>>|No
|
32
|
+
| <<plugins-{type}s-{plugin}-min_flush_time>> |<<number,number>>|No
|
33
|
+
| <<plugins-{type}s-{plugin}-spool_size>> |<<number,number>>|No
|
34
|
+
|=======================================================================
|
35
|
+
|
36
|
+
|
37
|
+
|
38
|
+
[id="plugins-{type}s-{plugin}-compress_level"]
|
39
|
+
===== `compress_level`
|
40
|
+
|
41
|
+
* Value type is <<number,number>>
|
42
|
+
* Default value is `6`
|
43
|
+
|
44
|
+
|
45
|
+
|
46
|
+
[id="plugins-{type}s-{plugin}-min_flush_time"]
|
47
|
+
===== `min_flush_time`
|
48
|
+
|
49
|
+
* Value type is <<number,number>>
|
50
|
+
* Default value is `0`
|
51
|
+
|
52
|
+
The amount of time in seconds since last flush before a flush is forced,
|
53
|
+
on the next event.
|
54
|
+
Values smaller than 0 disables time based flushing.
|
55
|
+
|
56
|
+
[id="plugins-{type}s-{plugin}-spool_size"]
|
57
|
+
===== `spool_size`
|
58
|
+
|
59
|
+
* Value type is <<number,number>>
|
60
|
+
* Default value is `50`
|
61
|
+
|
62
|
+
|
63
|
+
|
64
|
+
|
@@ -6,12 +6,18 @@ class LogStash::Codecs::CompressSpooler < LogStash::Codecs::Base
|
|
6
6
|
config :spool_size, :validate => :number, :default => 50
|
7
7
|
config :compress_level, :validate => :number, :default => 6
|
8
8
|
|
9
|
+
# The amount of time in seconds since last flush before a flush is forced,
|
10
|
+
# on the next event.
|
11
|
+
# Values smaller than 0 disables time based flushing.
|
12
|
+
config :min_flush_time, :validate => :number, :default => 0
|
13
|
+
|
9
14
|
public
|
10
15
|
|
11
16
|
def register
|
12
17
|
require "msgpack"
|
13
18
|
require "zlib"
|
14
19
|
@buffer = []
|
20
|
+
@last_flush = Time.now.to_i
|
15
21
|
end
|
16
22
|
|
17
23
|
def decode(data)
|
@@ -25,9 +31,10 @@ class LogStash::Codecs::CompressSpooler < LogStash::Codecs::Base
|
|
25
31
|
# MessagePack#pack which relies on pure Ruby object recognition
|
26
32
|
@buffer << LogStash::Util.normalize(event.to_hash).merge(LogStash::Event::TIMESTAMP => event.timestamp.to_iso8601)
|
27
33
|
# If necessary, we flush the buffer and get the data compressed
|
28
|
-
if @buffer.length >= @spool_size
|
34
|
+
if @buffer.length >= @spool_size || time_to_flush?
|
29
35
|
@on_event.call(compress(@buffer, @compress_level))
|
30
36
|
@buffer.clear
|
37
|
+
@last_flush = Time.now.to_i
|
31
38
|
end
|
32
39
|
end # def encode
|
33
40
|
|
@@ -53,4 +60,9 @@ class LogStash::Codecs::CompressSpooler < LogStash::Codecs::Base
|
|
53
60
|
z.close
|
54
61
|
result
|
55
62
|
end
|
63
|
+
|
64
|
+
def time_to_flush?
|
65
|
+
return @min_flush_time > 0 && (@last_flush + @min_flush_time < Time.now.to_i)
|
66
|
+
end
|
67
|
+
|
56
68
|
end # class LogStash::Codecs::CompressSpooler
|
@@ -1,17 +1,17 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-codec-compress_spooler'
|
4
|
-
s.version = '2.0.
|
4
|
+
s.version = '2.0.5'
|
5
5
|
s.licenses = ['Apache License (2.0)']
|
6
6
|
s.summary = "codec that processes compressed spooled data"
|
7
|
-
s.description = "This gem is a
|
7
|
+
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
8
8
|
s.authors = ["Elastic"]
|
9
9
|
s.email = 'info@elastic.co'
|
10
10
|
s.homepage = "http://www.elastic.co/guide/en/logstash/current/index.html"
|
11
11
|
s.require_paths = ["lib"]
|
12
12
|
|
13
13
|
# Files
|
14
|
-
s.files = Dir[
|
14
|
+
s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
|
15
15
|
|
16
16
|
# Tests
|
17
17
|
s.test_files = s.files.grep(%r{^(test|spec|features)/})
|
@@ -143,6 +143,33 @@ describe LogStash::Codecs::CompressSpooler do
|
|
143
143
|
expect(buffer.size).to eq(2)
|
144
144
|
end
|
145
145
|
end
|
146
|
+
|
147
|
+
context "message spooling if min flush time is set" do
|
148
|
+
let(:spool_size) { 10 }
|
149
|
+
let(:min_flush_time) { 2 }
|
150
|
+
let(:compress_spooler_config) { { "spool_size" => spool_size, "min_flush_time" => min_flush_time } }
|
151
|
+
subject(:codec) { LogStash::Codecs::CompressSpooler.new(compress_spooler_config) }
|
152
|
+
let(:data) { {"foo" => "bar"} }
|
153
|
+
|
154
|
+
it "dont'f flush the initial messages, but flush after min flush time on next message" do
|
155
|
+
codec.encode(event)
|
156
|
+
buffer= codec.instance_variable_get(:@buffer)
|
157
|
+
expect(buffer.size).to eq(1)
|
158
|
+
|
159
|
+
codec.encode(event)
|
160
|
+
buffer= codec.instance_variable_get(:@buffer)
|
161
|
+
expect(buffer.size).to eq(2)
|
162
|
+
|
163
|
+
sleep(min_flush_time + 1)
|
164
|
+
|
165
|
+
buffer= codec.instance_variable_get(:@buffer)
|
166
|
+
expect(buffer.size).to eq(2)
|
167
|
+
|
168
|
+
codec.encode(event)
|
169
|
+
buffer= codec.instance_variable_get(:@buffer)
|
170
|
+
expect(buffer.size).to eq(0)
|
171
|
+
end
|
172
|
+
end
|
146
173
|
end
|
147
174
|
end
|
148
175
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-codec-compress_spooler
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.0.
|
4
|
+
version: 2.0.5
|
5
5
|
platform: java
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2017-06-23 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -52,7 +52,9 @@ dependencies:
|
|
52
52
|
- - ">="
|
53
53
|
- !ruby/object:Gem::Version
|
54
54
|
version: '0'
|
55
|
-
description: This gem is a
|
55
|
+
description: This gem is a Logstash plugin required to be installed on top of the
|
56
|
+
Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
|
57
|
+
gem is not a stand-alone program
|
56
58
|
email: info@elastic.co
|
57
59
|
executables: []
|
58
60
|
extensions: []
|
@@ -64,6 +66,7 @@ files:
|
|
64
66
|
- LICENSE
|
65
67
|
- NOTICE.TXT
|
66
68
|
- README.md
|
69
|
+
- docs/index.asciidoc
|
67
70
|
- lib/logstash/codecs/compress_spooler.rb
|
68
71
|
- logstash-codec-compress_spooler.gemspec
|
69
72
|
- spec/codecs/compress_spooler_spec.rb
|
@@ -89,7 +92,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
89
92
|
version: '0'
|
90
93
|
requirements: []
|
91
94
|
rubyforge_project:
|
92
|
-
rubygems_version: 2.
|
95
|
+
rubygems_version: 2.6.11
|
93
96
|
signing_key:
|
94
97
|
specification_version: 4
|
95
98
|
summary: codec that processes compressed spooled data
|