logstash-input-dead_letter_queue 1.1.12 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4822656326d51f27c599ae21d09ed972c06983270b747303bf3725a219e46ecc
-  data.tar.gz: bd3eb2dba2a634b84011df4c6ae2a039752cab829fb41620e29d98329f32b7de
+  metadata.gz: 74a1df427003005bd0dd003bc442972a121506015a5af557cf4bd57abd50583f
+  data.tar.gz: 5b4b3938771f70157e5cf75e3a6bfafaf4a09de38870f406dfefc52384eb0908
 SHA512:
-  metadata.gz: cbe712e8376bd07c7da38ab5524a8c81f3bb55df28524a88d652a312caa8c4f4d6a5487e72979541d3f201d0e4f5009625040a9f63a58ecf795c62136b8bb80c
-  data.tar.gz: 316038e10f17e8e4175d61d31fd107c690d6ad083e4e2678016e347c91bcf96739f73e9614fd67075f54a7f6ec66d680ba0fa85f7e88ef4d26078810b962a985
+  metadata.gz: 726aebf62f60482362ee246e681be965b6f97c9721a66a514705687058e3223b89802d829e88e63e9a49c29920d705e40fcd4554757b442b4529290f241fb691
+  data.tar.gz: bada6c2af097fb986d684cdb5c5c8b336c131e4ec8fac09c53928430a3c28dbba91ab1efcf403c415f98529ecc907e841549b4c91b505dd86ee557d15fe7cc80

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 2.0.0
+  - Introduce the boolean `clean_consumed` setting to enable the automatic removal of completely consumed segments. Requires Logstash 8.4.0 or above [#43](https://github.com/logstash-plugins/logstash-input-dead_letter_queue/pull/43)
+  - Exposes metrics about segments and events cleaned by this plugin [#45](https://github.com/logstash-plugins/logstash-input-dead_letter_queue/pull/45)
+
 ## 1.1.12
   - Fix: Replace use of block with lambda to fix wrong number of arguments error on jruby-9.3.4.0 [#42](https://github.com/logstash-plugins/logstash-input-dead_letter_queue/pull/42)
   - Refactor: separated sinceDb management is its separate class [#40](https://github.com/logstash-plugins/logstash-input-dead_letter_queue/pull/40)

data/VERSION

@@ -1 +1 @@
-1.1.12
+2.0.0

data/docs/index.asciidoc

@@ -45,6 +45,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 [cols="<,<,<",options="header",]
 |=======================================================================
 |Setting |Input type|Required
+| <<plugins-{type}s-{plugin}-clean_consumed>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-commit_offsets>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-path>> |a valid filesystem path|Yes
 | <<plugins-{type}s-{plugin}-pipeline_id>> |<<string,string>>|No
@@ -57,6 +58,16 @@ input plugins.
 
 &nbsp;
 
+[id="plugins-{type}s-{plugin}-clean_consumed"]
+===== `clean_consumed`
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `false`
+
+When set to `true`, this option deletes the DLQ segments that have been read.
+This feature requires that `commit_offsets` is set to `true`. If not, you'll get a configuration error.
+This feature is available in Logstash 8.4.0 and later. If this setting is `true` and and you are using a Logstash version older than 8.4.0, then you'll get a configuration error.
+
 [id="plugins-{type}s-{plugin}-commit_offsets"]
 ===== `commit_offsets` 
 

data/lib/logstash/inputs/dead_letter_queue.rb

@@ -21,6 +21,11 @@ class LogStash::Inputs::DeadLetterQueue < LogStash::Inputs::Base
 
   default :codec, 'plain'
 
+  # If true deletes the DLQ segments that has been processed.
+  # Supported only Logstash >= 8.4.0
+  # If this setting is `true` and Logstash version doesn't provides this feature then result in a configuration error.
+  # This feature implicitly requires that the `commit_offsets` option is set to `true`. Iif it's not then you'll get a configuration error.
+  config :clean_consumed, :validate => :boolean, :default => false
   # Path to the dead letter queue directory which was created by a Logstash instance.
   # This is the path from where "dead" events are read from and is typically configured 
   # in the original Logstash instance with the setting path.dead_letter_queue.
@@ -53,10 +58,25 @@ class LogStash::Inputs::DeadLetterQueue < LogStash::Inputs::Base
     dlq_path = java.nio.file.Paths.get(File.join(@path, @pipeline_id))
     sincedb_path = @sincedb_path ? java.nio.file.Paths.get(@sincedb_path) : nil
     start_timestamp = @start_timestamp ? org.logstash.Timestamp.new(@start_timestamp) : nil
-    @inner_plugin = org.logstash.input.DeadLetterQueueInputPlugin.new(dlq_path, @commit_offsets, sincedb_path, start_timestamp)
+    logstash_version = Gem::Version.new(LOGSTASH_CORE_VERSION)
+    if clean_consumed && !Gem::Requirement.new('>= 8.4.0').satisfied_by?(logstash_version)
+      raise LogStash::ConfigurationError.new("clean_consumed can be used only with Logstash version 8.4.0 and above")
+    end
+    if clean_consumed && !commit_offsets
+      # clean_consumed requires the commit of offset
+      raise LogStash::ConfigurationError.new("enabling clean_consumed requires commit_offsets to also be enabled")
+    end
+    @cleaned_metrics = metric.namespace(@pipeline_id)
+    @inner_plugin = org.logstash.input.DeadLetterQueueInputPlugin.new(dlq_path, @commit_offsets, sincedb_path, start_timestamp, clean_consumed,
+            lambda do |segments, events|
+                # gauges is used instead of metric type because the updates that comes from the
+                # DLQ reader are already absolute values and not deltas.
+                @cleaned_metrics.gauge(:cleaned_segments, segments)
+                @cleaned_metrics.gauge(:cleaned_events, events)
+            end)
     @inner_plugin.register
 
-    if Gem::Requirement.new('< 7.0').satisfied_by?(Gem::Version.new(LOGSTASH_CORE_VERSION))
+    if Gem::Requirement.new('< 7.0').satisfied_by?(logstash_version)
       @event_creator = Proc.new do |entry|
         clone = entry.event.clone
         # LS 6 LogStash::Event.new accept Map not Event

data/lib/logstash-input-dead_letter_queue_jars.rb

@@ -1,4 +1,4 @@
 # AUTOGENERATED BY THE GRADLE SCRIPT. DO NOT EDIT.
 
 require 'jar_dependencies'
-require_jar('co.elastic.logstash.input', 'logstash-input-dead_letter_queue', '1.1.12')
+require_jar('co.elastic.logstash.input', 'logstash-input-dead_letter_queue', '2.0.0')

data/logstash-input-dead_letter_queue.gemspec

@@ -21,6 +21,7 @@ Gem::Specification.new do |s|
   s.metadata = { 'logstash_plugin' => 'true', 'group' => 'input'}
 
   # Gem dependencies
+  s.add_runtime_dependency 'logstash-core', '>= 8.4.0'
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
   s.add_runtime_dependency 'logstash-codec-plain'
 

data/spec/unit/inputs/dead_letter_queue_spec.rb

@@ -7,9 +7,12 @@ describe LogStash::Inputs::DeadLetterQueue do
   let(:pipeline_id) { SecureRandom.hex(8)}
   let(:path) { Dir.tmpdir }
   let(:directory) { File.join(path, pipeline_id)}
+  let(:config) do
+    { "path" => path,
+      "pipeline_id" => pipeline_id}
+  end
 
-  subject { LogStash::Inputs::DeadLetterQueue.new({ "path" => path,
-                                                    "pipeline_id" => pipeline_id}) }
+  subject { LogStash::Inputs::DeadLetterQueue.new(config) }
 
   before(:each) do
     Dir.mkdir(directory)
@@ -23,6 +26,21 @@ describe LogStash::Inputs::DeadLetterQueue do
     FileUtils.remove_entry_secure directory
   end
 
+  context "when clean_consumed is enabled" do
+    let(:config) { super().merge!("clean_consumed" => true) }
+
+    it "registers successfully" do
+      expect {subject.register}.to_not raise_error
+    end
+
+    context "and commit_offsets is not" do
+      let(:config) { super().merge!("commit_offsets" => false) }
+      it "raises a configuration error during register" do
+        expect {subject.register}.to raise_error(LogStash::ConfigurationError)
+      end
+    end
+  end
+
   context 'test with real DLQ file' do
     let(:dlq_dir) { Stud::Temporary.directory }
     let(:fixture_dir) { File.expand_path(File.join(File.dirname(__FILE__),"fixtures", "main")) }

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-dead_letter_queue
 version: !ruby/object:Gem::Version
-  version: 1.1.12
+  version: 2.0.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-06-16 00:00:00.000000000 Z
+date: 2022-08-02 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 8.4.0
+  name: logstash-core
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 8.4.0
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -80,7 +94,7 @@ files:
 - logstash-input-dead_letter_queue.gemspec
 - spec/unit/inputs/dead_letter_queue_spec.rb
 - spec/unit/inputs/fixtures/main/1.log
-- vendor/jar-dependencies/co/elastic/logstash/input/logstash-input-dead_letter_queue/1.1.12/logstash-input-dead_letter_queue-1.1.12.jar
+- vendor/jar-dependencies/co/elastic/logstash/input/logstash-input-dead_letter_queue/2.0.0/logstash-input-dead_letter_queue-2.0.0.jar
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
 - Apache License (2.0)