logstash-input-dead_letter_queue 1.1.12 → 2.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +7 -0
- data/VERSION +1 -1
- data/docs/index.asciidoc +11 -0
- data/lib/logstash/inputs/dead_letter_queue.rb +22 -2
- data/lib/logstash-input-dead_letter_queue_jars.rb +1 -1
- data/logstash-input-dead_letter_queue.gemspec +1 -0
- data/spec/unit/inputs/dead_letter_queue_spec.rb +20 -2
- data/vendor/jar-dependencies/co/elastic/logstash/input/logstash-input-dead_letter_queue/2.0.1/logstash-input-dead_letter_queue-2.0.1.jar +0 -0
- metadata +21 -7
- data/vendor/jar-dependencies/co/elastic/logstash/input/logstash-input-dead_letter_queue/1.1.12/logstash-input-dead_letter_queue-1.1.12.jar +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ac404b1ce001f61569a36f0fb11f0b4ef7c407d016d6f495d88b8e36225452d2
|
4
|
+
data.tar.gz: 718ef622d210c128ec0ba102ebffb81d072d5315f1c6c524f82ed4d2bbfed2f3
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 7a552d511d22ad5e9096e33dfaaccf8c0d45b78be9eb53836354d58cfadfbcf947094c40c8b216d6da127234768614060c45264404d873f61843affdc9771234
|
7
|
+
data.tar.gz: 43630ce6b16a3548a3dd9cd5ffe7ac112dc04e56ced7c55541876aafb4c607ddbcf0e6e834a57dc5e5099b5e9a1035c6abceb1119401e5b36182def4b500828c
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,10 @@
|
|
1
|
+
## 2.0.1
|
2
|
+
- Fix NullPointerException when the plugin closes [#53](https://github.com/logstash-plugins/logstash-input-dead_letter_queue/pull/53)
|
3
|
+
|
4
|
+
## 2.0.0
|
5
|
+
- Introduce the boolean `clean_consumed` setting to enable the automatic removal of completely consumed segments. Requires Logstash 8.4.0 or above [#43](https://github.com/logstash-plugins/logstash-input-dead_letter_queue/pull/43)
|
6
|
+
- Exposes metrics about segments and events cleaned by this plugin [#45](https://github.com/logstash-plugins/logstash-input-dead_letter_queue/pull/45)
|
7
|
+
|
1
8
|
## 1.1.12
|
2
9
|
- Fix: Replace use of block with lambda to fix wrong number of arguments error on jruby-9.3.4.0 [#42](https://github.com/logstash-plugins/logstash-input-dead_letter_queue/pull/42)
|
3
10
|
- Refactor: separated sinceDb management is its separate class [#40](https://github.com/logstash-plugins/logstash-input-dead_letter_queue/pull/40)
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
|
1
|
+
2.0.1
|
data/docs/index.asciidoc
CHANGED
@@ -45,6 +45,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
|
|
45
45
|
[cols="<,<,<",options="header",]
|
46
46
|
|=======================================================================
|
47
47
|
|Setting |Input type|Required
|
48
|
+
| <<plugins-{type}s-{plugin}-clean_consumed>> |<<boolean,boolean>>|No
|
48
49
|
| <<plugins-{type}s-{plugin}-commit_offsets>> |<<boolean,boolean>>|No
|
49
50
|
| <<plugins-{type}s-{plugin}-path>> |a valid filesystem path|Yes
|
50
51
|
| <<plugins-{type}s-{plugin}-pipeline_id>> |<<string,string>>|No
|
@@ -57,6 +58,16 @@ input plugins.
|
|
57
58
|
|
58
59
|
|
59
60
|
|
61
|
+
[id="plugins-{type}s-{plugin}-clean_consumed"]
|
62
|
+
===== `clean_consumed`
|
63
|
+
|
64
|
+
* Value type is <<boolean,boolean>>
|
65
|
+
* Default value is `false`
|
66
|
+
|
67
|
+
When set to `true`, this option deletes the DLQ segments that have been read.
|
68
|
+
This feature requires that `commit_offsets` is set to `true`. If not, you'll get a configuration error.
|
69
|
+
This feature is available in Logstash 8.4.0 and later. If this setting is `true` and and you are using a Logstash version older than 8.4.0, then you'll get a configuration error.
|
70
|
+
|
60
71
|
[id="plugins-{type}s-{plugin}-commit_offsets"]
|
61
72
|
===== `commit_offsets`
|
62
73
|
|
@@ -21,6 +21,11 @@ class LogStash::Inputs::DeadLetterQueue < LogStash::Inputs::Base
|
|
21
21
|
|
22
22
|
default :codec, 'plain'
|
23
23
|
|
24
|
+
# If true deletes the DLQ segments that has been processed.
|
25
|
+
# Supported only Logstash >= 8.4.0
|
26
|
+
# If this setting is `true` and Logstash version doesn't provides this feature then result in a configuration error.
|
27
|
+
# This feature implicitly requires that the `commit_offsets` option is set to `true`. Iif it's not then you'll get a configuration error.
|
28
|
+
config :clean_consumed, :validate => :boolean, :default => false
|
24
29
|
# Path to the dead letter queue directory which was created by a Logstash instance.
|
25
30
|
# This is the path from where "dead" events are read from and is typically configured
|
26
31
|
# in the original Logstash instance with the setting path.dead_letter_queue.
|
@@ -53,10 +58,25 @@ class LogStash::Inputs::DeadLetterQueue < LogStash::Inputs::Base
|
|
53
58
|
dlq_path = java.nio.file.Paths.get(File.join(@path, @pipeline_id))
|
54
59
|
sincedb_path = @sincedb_path ? java.nio.file.Paths.get(@sincedb_path) : nil
|
55
60
|
start_timestamp = @start_timestamp ? org.logstash.Timestamp.new(@start_timestamp) : nil
|
56
|
-
|
61
|
+
logstash_version = Gem::Version.new(LOGSTASH_CORE_VERSION)
|
62
|
+
if clean_consumed && !Gem::Requirement.new('>= 8.4.0').satisfied_by?(logstash_version)
|
63
|
+
raise LogStash::ConfigurationError.new("clean_consumed can be used only with Logstash version 8.4.0 and above")
|
64
|
+
end
|
65
|
+
if clean_consumed && !commit_offsets
|
66
|
+
# clean_consumed requires the commit of offset
|
67
|
+
raise LogStash::ConfigurationError.new("enabling clean_consumed requires commit_offsets to also be enabled")
|
68
|
+
end
|
69
|
+
@cleaned_metrics = metric.namespace(@pipeline_id)
|
70
|
+
@inner_plugin = org.logstash.input.DeadLetterQueueInputPlugin.new(dlq_path, @commit_offsets, sincedb_path, start_timestamp, clean_consumed,
|
71
|
+
lambda do |segments, events|
|
72
|
+
# gauges is used instead of metric type because the updates that comes from the
|
73
|
+
# DLQ reader are already absolute values and not deltas.
|
74
|
+
@cleaned_metrics.gauge(:cleaned_segments, segments)
|
75
|
+
@cleaned_metrics.gauge(:cleaned_events, events)
|
76
|
+
end)
|
57
77
|
@inner_plugin.register
|
58
78
|
|
59
|
-
if Gem::Requirement.new('< 7.0').satisfied_by?(
|
79
|
+
if Gem::Requirement.new('< 7.0').satisfied_by?(logstash_version)
|
60
80
|
@event_creator = Proc.new do |entry|
|
61
81
|
clone = entry.event.clone
|
62
82
|
# LS 6 LogStash::Event.new accept Map not Event
|
@@ -21,6 +21,7 @@ Gem::Specification.new do |s|
|
|
21
21
|
s.metadata = { 'logstash_plugin' => 'true', 'group' => 'input'}
|
22
22
|
|
23
23
|
# Gem dependencies
|
24
|
+
s.add_runtime_dependency 'logstash-core', '>= 8.4.0'
|
24
25
|
s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
|
25
26
|
s.add_runtime_dependency 'logstash-codec-plain'
|
26
27
|
|
@@ -7,9 +7,12 @@ describe LogStash::Inputs::DeadLetterQueue do
|
|
7
7
|
let(:pipeline_id) { SecureRandom.hex(8)}
|
8
8
|
let(:path) { Dir.tmpdir }
|
9
9
|
let(:directory) { File.join(path, pipeline_id)}
|
10
|
+
let(:config) do
|
11
|
+
{ "path" => path,
|
12
|
+
"pipeline_id" => pipeline_id}
|
13
|
+
end
|
10
14
|
|
11
|
-
subject { LogStash::Inputs::DeadLetterQueue.new(
|
12
|
-
"pipeline_id" => pipeline_id}) }
|
15
|
+
subject { LogStash::Inputs::DeadLetterQueue.new(config) }
|
13
16
|
|
14
17
|
before(:each) do
|
15
18
|
Dir.mkdir(directory)
|
@@ -23,6 +26,21 @@ describe LogStash::Inputs::DeadLetterQueue do
|
|
23
26
|
FileUtils.remove_entry_secure directory
|
24
27
|
end
|
25
28
|
|
29
|
+
context "when clean_consumed is enabled" do
|
30
|
+
let(:config) { super().merge!("clean_consumed" => true) }
|
31
|
+
|
32
|
+
it "registers successfully" do
|
33
|
+
expect {subject.register}.to_not raise_error
|
34
|
+
end
|
35
|
+
|
36
|
+
context "and commit_offsets is not" do
|
37
|
+
let(:config) { super().merge!("commit_offsets" => false) }
|
38
|
+
it "raises a configuration error during register" do
|
39
|
+
expect {subject.register}.to raise_error(LogStash::ConfigurationError)
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
26
44
|
context 'test with real DLQ file' do
|
27
45
|
let(:dlq_dir) { Stud::Temporary.directory }
|
28
46
|
let(:fixture_dir) { File.expand_path(File.join(File.dirname(__FILE__),"fixtures", "main")) }
|
Binary file
|
metadata
CHANGED
@@ -1,15 +1,29 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-input-dead_letter_queue
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 2.0.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2024-09-04 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
requirement: !ruby/object:Gem::Requirement
|
15
|
+
requirements:
|
16
|
+
- - ">="
|
17
|
+
- !ruby/object:Gem::Version
|
18
|
+
version: 8.4.0
|
19
|
+
name: logstash-core
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ">="
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: 8.4.0
|
13
27
|
- !ruby/object:Gem::Dependency
|
14
28
|
requirement: !ruby/object:Gem::Requirement
|
15
29
|
requirements:
|
@@ -20,8 +34,8 @@ dependencies:
|
|
20
34
|
- !ruby/object:Gem::Version
|
21
35
|
version: '2.99'
|
22
36
|
name: logstash-core-plugin-api
|
23
|
-
prerelease: false
|
24
37
|
type: :runtime
|
38
|
+
prerelease: false
|
25
39
|
version_requirements: !ruby/object:Gem::Requirement
|
26
40
|
requirements:
|
27
41
|
- - ">="
|
@@ -37,8 +51,8 @@ dependencies:
|
|
37
51
|
- !ruby/object:Gem::Version
|
38
52
|
version: '0'
|
39
53
|
name: logstash-codec-plain
|
40
|
-
prerelease: false
|
41
54
|
type: :runtime
|
55
|
+
prerelease: false
|
42
56
|
version_requirements: !ruby/object:Gem::Requirement
|
43
57
|
requirements:
|
44
58
|
- - ">="
|
@@ -51,8 +65,8 @@ dependencies:
|
|
51
65
|
- !ruby/object:Gem::Version
|
52
66
|
version: '0'
|
53
67
|
name: logstash-devutils
|
54
|
-
prerelease: false
|
55
68
|
type: :development
|
69
|
+
prerelease: false
|
56
70
|
version_requirements: !ruby/object:Gem::Requirement
|
57
71
|
requirements:
|
58
72
|
- - ">="
|
@@ -80,7 +94,7 @@ files:
|
|
80
94
|
- logstash-input-dead_letter_queue.gemspec
|
81
95
|
- spec/unit/inputs/dead_letter_queue_spec.rb
|
82
96
|
- spec/unit/inputs/fixtures/main/1.log
|
83
|
-
- vendor/jar-dependencies/co/elastic/logstash/input/logstash-input-dead_letter_queue/
|
97
|
+
- vendor/jar-dependencies/co/elastic/logstash/input/logstash-input-dead_letter_queue/2.0.1/logstash-input-dead_letter_queue-2.0.1.jar
|
84
98
|
homepage: http://www.elastic.co/guide/en/logstash/current/index.html
|
85
99
|
licenses:
|
86
100
|
- Apache License (2.0)
|
@@ -103,7 +117,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
103
117
|
- !ruby/object:Gem::Version
|
104
118
|
version: '0'
|
105
119
|
requirements: []
|
106
|
-
rubygems_version: 3.
|
120
|
+
rubygems_version: 3.3.26
|
107
121
|
signing_key:
|
108
122
|
specification_version: 4
|
109
123
|
summary: read events from Logstash's dead letter queue
|