logstash-input-dead_letter_queue 1.1.12 → 2.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 4822656326d51f27c599ae21d09ed972c06983270b747303bf3725a219e46ecc
4
- data.tar.gz: bd3eb2dba2a634b84011df4c6ae2a039752cab829fb41620e29d98329f32b7de
3
+ metadata.gz: ac404b1ce001f61569a36f0fb11f0b4ef7c407d016d6f495d88b8e36225452d2
4
+ data.tar.gz: 718ef622d210c128ec0ba102ebffb81d072d5315f1c6c524f82ed4d2bbfed2f3
5
5
  SHA512:
6
- metadata.gz: cbe712e8376bd07c7da38ab5524a8c81f3bb55df28524a88d652a312caa8c4f4d6a5487e72979541d3f201d0e4f5009625040a9f63a58ecf795c62136b8bb80c
7
- data.tar.gz: 316038e10f17e8e4175d61d31fd107c690d6ad083e4e2678016e347c91bcf96739f73e9614fd67075f54a7f6ec66d680ba0fa85f7e88ef4d26078810b962a985
6
+ metadata.gz: 7a552d511d22ad5e9096e33dfaaccf8c0d45b78be9eb53836354d58cfadfbcf947094c40c8b216d6da127234768614060c45264404d873f61843affdc9771234
7
+ data.tar.gz: 43630ce6b16a3548a3dd9cd5ffe7ac112dc04e56ced7c55541876aafb4c607ddbcf0e6e834a57dc5e5099b5e9a1035c6abceb1119401e5b36182def4b500828c
data/CHANGELOG.md CHANGED
@@ -1,3 +1,10 @@
1
+ ## 2.0.1
2
+ - Fix NullPointerException when the plugin closes [#53](https://github.com/logstash-plugins/logstash-input-dead_letter_queue/pull/53)
3
+
4
+ ## 2.0.0
5
+ - Introduce the boolean `clean_consumed` setting to enable the automatic removal of completely consumed segments. Requires Logstash 8.4.0 or above [#43](https://github.com/logstash-plugins/logstash-input-dead_letter_queue/pull/43)
6
+ - Exposes metrics about segments and events cleaned by this plugin [#45](https://github.com/logstash-plugins/logstash-input-dead_letter_queue/pull/45)
7
+
1
8
  ## 1.1.12
2
9
  - Fix: Replace use of block with lambda to fix wrong number of arguments error on jruby-9.3.4.0 [#42](https://github.com/logstash-plugins/logstash-input-dead_letter_queue/pull/42)
3
10
  - Refactor: separated sinceDb management is its separate class [#40](https://github.com/logstash-plugins/logstash-input-dead_letter_queue/pull/40)
data/VERSION CHANGED
@@ -1 +1 @@
1
- 1.1.12
1
+ 2.0.1
data/docs/index.asciidoc CHANGED
@@ -45,6 +45,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
45
45
  [cols="<,<,<",options="header",]
46
46
  |=======================================================================
47
47
  |Setting |Input type|Required
48
+ | <<plugins-{type}s-{plugin}-clean_consumed>> |<<boolean,boolean>>|No
48
49
  | <<plugins-{type}s-{plugin}-commit_offsets>> |<<boolean,boolean>>|No
49
50
  | <<plugins-{type}s-{plugin}-path>> |a valid filesystem path|Yes
50
51
  | <<plugins-{type}s-{plugin}-pipeline_id>> |<<string,string>>|No
@@ -57,6 +58,16 @@ input plugins.
57
58
 
58
59
  &nbsp;
59
60
 
61
+ [id="plugins-{type}s-{plugin}-clean_consumed"]
62
+ ===== `clean_consumed`
63
+
64
+ * Value type is <<boolean,boolean>>
65
+ * Default value is `false`
66
+
67
+ When set to `true`, this option deletes the DLQ segments that have been read.
68
+ This feature requires that `commit_offsets` is set to `true`. If not, you'll get a configuration error.
69
+ This feature is available in Logstash 8.4.0 and later. If this setting is `true` and and you are using a Logstash version older than 8.4.0, then you'll get a configuration error.
70
+
60
71
  [id="plugins-{type}s-{plugin}-commit_offsets"]
61
72
  ===== `commit_offsets`
62
73
 
@@ -21,6 +21,11 @@ class LogStash::Inputs::DeadLetterQueue < LogStash::Inputs::Base
21
21
 
22
22
  default :codec, 'plain'
23
23
 
24
+ # If true deletes the DLQ segments that has been processed.
25
+ # Supported only Logstash >= 8.4.0
26
+ # If this setting is `true` and Logstash version doesn't provides this feature then result in a configuration error.
27
+ # This feature implicitly requires that the `commit_offsets` option is set to `true`. Iif it's not then you'll get a configuration error.
28
+ config :clean_consumed, :validate => :boolean, :default => false
24
29
  # Path to the dead letter queue directory which was created by a Logstash instance.
25
30
  # This is the path from where "dead" events are read from and is typically configured
26
31
  # in the original Logstash instance with the setting path.dead_letter_queue.
@@ -53,10 +58,25 @@ class LogStash::Inputs::DeadLetterQueue < LogStash::Inputs::Base
53
58
  dlq_path = java.nio.file.Paths.get(File.join(@path, @pipeline_id))
54
59
  sincedb_path = @sincedb_path ? java.nio.file.Paths.get(@sincedb_path) : nil
55
60
  start_timestamp = @start_timestamp ? org.logstash.Timestamp.new(@start_timestamp) : nil
56
- @inner_plugin = org.logstash.input.DeadLetterQueueInputPlugin.new(dlq_path, @commit_offsets, sincedb_path, start_timestamp)
61
+ logstash_version = Gem::Version.new(LOGSTASH_CORE_VERSION)
62
+ if clean_consumed && !Gem::Requirement.new('>= 8.4.0').satisfied_by?(logstash_version)
63
+ raise LogStash::ConfigurationError.new("clean_consumed can be used only with Logstash version 8.4.0 and above")
64
+ end
65
+ if clean_consumed && !commit_offsets
66
+ # clean_consumed requires the commit of offset
67
+ raise LogStash::ConfigurationError.new("enabling clean_consumed requires commit_offsets to also be enabled")
68
+ end
69
+ @cleaned_metrics = metric.namespace(@pipeline_id)
70
+ @inner_plugin = org.logstash.input.DeadLetterQueueInputPlugin.new(dlq_path, @commit_offsets, sincedb_path, start_timestamp, clean_consumed,
71
+ lambda do |segments, events|
72
+ # gauges is used instead of metric type because the updates that comes from the
73
+ # DLQ reader are already absolute values and not deltas.
74
+ @cleaned_metrics.gauge(:cleaned_segments, segments)
75
+ @cleaned_metrics.gauge(:cleaned_events, events)
76
+ end)
57
77
  @inner_plugin.register
58
78
 
59
- if Gem::Requirement.new('< 7.0').satisfied_by?(Gem::Version.new(LOGSTASH_CORE_VERSION))
79
+ if Gem::Requirement.new('< 7.0').satisfied_by?(logstash_version)
60
80
  @event_creator = Proc.new do |entry|
61
81
  clone = entry.event.clone
62
82
  # LS 6 LogStash::Event.new accept Map not Event
@@ -1,4 +1,4 @@
1
1
  # AUTOGENERATED BY THE GRADLE SCRIPT. DO NOT EDIT.
2
2
 
3
3
  require 'jar_dependencies'
4
- require_jar('co.elastic.logstash.input', 'logstash-input-dead_letter_queue', '1.1.12')
4
+ require_jar('co.elastic.logstash.input', 'logstash-input-dead_letter_queue', '2.0.1')
@@ -21,6 +21,7 @@ Gem::Specification.new do |s|
21
21
  s.metadata = { 'logstash_plugin' => 'true', 'group' => 'input'}
22
22
 
23
23
  # Gem dependencies
24
+ s.add_runtime_dependency 'logstash-core', '>= 8.4.0'
24
25
  s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
25
26
  s.add_runtime_dependency 'logstash-codec-plain'
26
27
 
@@ -7,9 +7,12 @@ describe LogStash::Inputs::DeadLetterQueue do
7
7
  let(:pipeline_id) { SecureRandom.hex(8)}
8
8
  let(:path) { Dir.tmpdir }
9
9
  let(:directory) { File.join(path, pipeline_id)}
10
+ let(:config) do
11
+ { "path" => path,
12
+ "pipeline_id" => pipeline_id}
13
+ end
10
14
 
11
- subject { LogStash::Inputs::DeadLetterQueue.new({ "path" => path,
12
- "pipeline_id" => pipeline_id}) }
15
+ subject { LogStash::Inputs::DeadLetterQueue.new(config) }
13
16
 
14
17
  before(:each) do
15
18
  Dir.mkdir(directory)
@@ -23,6 +26,21 @@ describe LogStash::Inputs::DeadLetterQueue do
23
26
  FileUtils.remove_entry_secure directory
24
27
  end
25
28
 
29
+ context "when clean_consumed is enabled" do
30
+ let(:config) { super().merge!("clean_consumed" => true) }
31
+
32
+ it "registers successfully" do
33
+ expect {subject.register}.to_not raise_error
34
+ end
35
+
36
+ context "and commit_offsets is not" do
37
+ let(:config) { super().merge!("commit_offsets" => false) }
38
+ it "raises a configuration error during register" do
39
+ expect {subject.register}.to raise_error(LogStash::ConfigurationError)
40
+ end
41
+ end
42
+ end
43
+
26
44
  context 'test with real DLQ file' do
27
45
  let(:dlq_dir) { Stud::Temporary.directory }
28
46
  let(:fixture_dir) { File.expand_path(File.join(File.dirname(__FILE__),"fixtures", "main")) }
metadata CHANGED
@@ -1,15 +1,29 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-input-dead_letter_queue
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.1.12
4
+ version: 2.0.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Elastic
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2022-06-16 00:00:00.000000000 Z
11
+ date: 2024-09-04 00:00:00.000000000 Z
12
12
  dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ requirement: !ruby/object:Gem::Requirement
15
+ requirements:
16
+ - - ">="
17
+ - !ruby/object:Gem::Version
18
+ version: 8.4.0
19
+ name: logstash-core
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - ">="
25
+ - !ruby/object:Gem::Version
26
+ version: 8.4.0
13
27
  - !ruby/object:Gem::Dependency
14
28
  requirement: !ruby/object:Gem::Requirement
15
29
  requirements:
@@ -20,8 +34,8 @@ dependencies:
20
34
  - !ruby/object:Gem::Version
21
35
  version: '2.99'
22
36
  name: logstash-core-plugin-api
23
- prerelease: false
24
37
  type: :runtime
38
+ prerelease: false
25
39
  version_requirements: !ruby/object:Gem::Requirement
26
40
  requirements:
27
41
  - - ">="
@@ -37,8 +51,8 @@ dependencies:
37
51
  - !ruby/object:Gem::Version
38
52
  version: '0'
39
53
  name: logstash-codec-plain
40
- prerelease: false
41
54
  type: :runtime
55
+ prerelease: false
42
56
  version_requirements: !ruby/object:Gem::Requirement
43
57
  requirements:
44
58
  - - ">="
@@ -51,8 +65,8 @@ dependencies:
51
65
  - !ruby/object:Gem::Version
52
66
  version: '0'
53
67
  name: logstash-devutils
54
- prerelease: false
55
68
  type: :development
69
+ prerelease: false
56
70
  version_requirements: !ruby/object:Gem::Requirement
57
71
  requirements:
58
72
  - - ">="
@@ -80,7 +94,7 @@ files:
80
94
  - logstash-input-dead_letter_queue.gemspec
81
95
  - spec/unit/inputs/dead_letter_queue_spec.rb
82
96
  - spec/unit/inputs/fixtures/main/1.log
83
- - vendor/jar-dependencies/co/elastic/logstash/input/logstash-input-dead_letter_queue/1.1.12/logstash-input-dead_letter_queue-1.1.12.jar
97
+ - vendor/jar-dependencies/co/elastic/logstash/input/logstash-input-dead_letter_queue/2.0.1/logstash-input-dead_letter_queue-2.0.1.jar
84
98
  homepage: http://www.elastic.co/guide/en/logstash/current/index.html
85
99
  licenses:
86
100
  - Apache License (2.0)
@@ -103,7 +117,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
103
117
  - !ruby/object:Gem::Version
104
118
  version: '0'
105
119
  requirements: []
106
- rubygems_version: 3.1.6
120
+ rubygems_version: 3.3.26
107
121
  signing_key:
108
122
  specification_version: 4
109
123
  summary: read events from Logstash's dead letter queue