logstash-input-dead_letter_queue 1.1.10 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f72ab4a3d095208dd728b509cf547ef47c8951d5093027a72254674c52a79baa
-  data.tar.gz: 3c584acddee29a2f11327087a145b12cbee811ba844abc886df1c797cbf1cc21
+  metadata.gz: 74a1df427003005bd0dd003bc442972a121506015a5af557cf4bd57abd50583f
+  data.tar.gz: 5b4b3938771f70157e5cf75e3a6bfafaf4a09de38870f406dfefc52384eb0908
 SHA512:
-  metadata.gz: d66fc9550df26ec02d4f8872007131825e243c2429410698d4e94844ae71760831803b3beb7db8810de6698fb5e5b495d67763f96d79dfd076131a7d9aea92fd
-  data.tar.gz: 8a42ff2bc10cd95892a48bb6c38a6f95940cd0b3098bcc1b8084b2da4d12b6650dfa02d645e50d5c68a695f3b8fd2a590ee80f2d5f880bcad247a5815ed2d1e0
+  metadata.gz: 726aebf62f60482362ee246e681be965b6f97c9721a66a514705687058e3223b89802d829e88e63e9a49c29920d705e40fcd4554757b442b4529290f241fb691
+  data.tar.gz: bada6c2af097fb986d684cdb5c5c8b336c131e4ec8fac09c53928430a3c28dbba91ab1efcf403c415f98529ecc907e841549b4c91b505dd86ee557d15fe7cc80

data/CHANGELOG.md

@@ -1,5 +1,19 @@
+## 2.0.0
+  - Introduce the boolean `clean_consumed` setting to enable the automatic removal of completely consumed segments. Requires Logstash 8.4.0 or above [#43](https://github.com/logstash-plugins/logstash-input-dead_letter_queue/pull/43)
+  - Exposes metrics about segments and events cleaned by this plugin [#45](https://github.com/logstash-plugins/logstash-input-dead_letter_queue/pull/45)
+
+## 1.1.12
+  - Fix: Replace use of block with lambda to fix wrong number of arguments error on jruby-9.3.4.0 [#42](https://github.com/logstash-plugins/logstash-input-dead_letter_queue/pull/42)
+  - Refactor: separated sinceDb management is its separate class [#40](https://github.com/logstash-plugins/logstash-input-dead_letter_queue/pull/40)
+  - Build: cleanup/review (unused) dependencies [#36](https://github.com/logstash-plugins/logstash-input-dead_letter_queue/pull/36)
+  - Build: refactor tasks (runnable on windows) [#37](https://github.com/logstash-plugins/logstash-input-dead_letter_queue/pull/37)
+
+## 1.1.11
+  - Fix: pre-flight checks before creating DLQ reader [#35](https://github.com/logstash-plugins/logstash-input-dead_letter_queue/pull/35)
+
 ## 1.1.10
   - Fix, avoid Logstash crash on shutdown if DLQ files weren't created [#33](https://github.com/logstash-plugins/logstash-input-dead_letter_queue/pull/33)
+
 ## 1.1.9
   - Fix `@metadata` get overwritten by reestablishing metadata that stored in DLQ [#34](https://github.com/logstash-plugins/logstash-input-dead_letter_queue/pull/34)
 

data/VERSION

@@ -1 +1 @@
-1.1.10
+2.0.0

data/docs/index.asciidoc

@@ -45,6 +45,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 [cols="<,<,<",options="header",]
 |=======================================================================
 |Setting |Input type|Required
+| <<plugins-{type}s-{plugin}-clean_consumed>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-commit_offsets>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-path>> |a valid filesystem path|Yes
 | <<plugins-{type}s-{plugin}-pipeline_id>> |<<string,string>>|No
@@ -57,6 +58,16 @@ input plugins.
 
 &nbsp;
 
+[id="plugins-{type}s-{plugin}-clean_consumed"]
+===== `clean_consumed`
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `false`
+
+When set to `true`, this option deletes the DLQ segments that have been read.
+This feature requires that `commit_offsets` is set to `true`. If not, you'll get a configuration error.
+This feature is available in Logstash 8.4.0 and later. If this setting is `true` and and you are using a Logstash version older than 8.4.0, then you'll get a configuration error.
+
 [id="plugins-{type}s-{plugin}-commit_offsets"]
 ===== `commit_offsets` 
 

data/lib/logstash/inputs/dead_letter_queue.rb

@@ -2,6 +2,8 @@ require 'logstash/namespace'
 require 'logstash/inputs/base'
 require 'logstash-input-dead_letter_queue_jars'
 
+require 'fileutils'
+
 # Logstash input to read events from Logstash's dead letter queue
 # 
 # [source, sh]
@@ -19,6 +21,11 @@ class LogStash::Inputs::DeadLetterQueue < LogStash::Inputs::Base
 
   default :codec, 'plain'
 
+  # If true deletes the DLQ segments that has been processed.
+  # Supported only Logstash >= 8.4.0
+  # If this setting is `true` and Logstash version doesn't provides this feature then result in a configuration error.
+  # This feature implicitly requires that the `commit_offsets` option is set to `true`. Iif it's not then you'll get a configuration error.
+  config :clean_consumed, :validate => :boolean, :default => false
   # Path to the dead letter queue directory which was created by a Logstash instance.
   # This is the path from where "dead" events are read from and is typically configured 
   # in the original Logstash instance with the setting path.dead_letter_queue.
@@ -45,35 +52,54 @@ class LogStash::Inputs::DeadLetterQueue < LogStash::Inputs::Base
       FileUtils::mkdir_p datapath
       @sincedb_path = File.join(datapath, ".sincedb_" + Digest::MD5.hexdigest(@path))
     elsif File.directory?(@sincedb_path)
-        raise ArgumentError.new("The \"sincedb_path\" argument must point to a file, received a directory: \"#{@sincedb_path}\"")
+      raise ArgumentError.new("The \"sincedb_path\" argument must point to a file, received a directory: \"#{@sincedb_path}\"")
     end
 
     dlq_path = java.nio.file.Paths.get(File.join(@path, @pipeline_id))
     sincedb_path = @sincedb_path ? java.nio.file.Paths.get(@sincedb_path) : nil
     start_timestamp = @start_timestamp ? org.logstash.Timestamp.new(@start_timestamp) : nil
-    @inner_plugin = org.logstash.input.DeadLetterQueueInputPlugin.new(dlq_path, @commit_offsets, sincedb_path, start_timestamp)
+    logstash_version = Gem::Version.new(LOGSTASH_CORE_VERSION)
+    if clean_consumed && !Gem::Requirement.new('>= 8.4.0').satisfied_by?(logstash_version)
+      raise LogStash::ConfigurationError.new("clean_consumed can be used only with Logstash version 8.4.0 and above")
+    end
+    if clean_consumed && !commit_offsets
+      # clean_consumed requires the commit of offset
+      raise LogStash::ConfigurationError.new("enabling clean_consumed requires commit_offsets to also be enabled")
+    end
+    @cleaned_metrics = metric.namespace(@pipeline_id)
+    @inner_plugin = org.logstash.input.DeadLetterQueueInputPlugin.new(dlq_path, @commit_offsets, sincedb_path, start_timestamp, clean_consumed,
+            lambda do |segments, events|
+                # gauges is used instead of metric type because the updates that comes from the
+                # DLQ reader are already absolute values and not deltas.
+                @cleaned_metrics.gauge(:cleaned_segments, segments)
+                @cleaned_metrics.gauge(:cleaned_events, events)
+            end)
     @inner_plugin.register
+
+    if Gem::Requirement.new('< 7.0').satisfied_by?(logstash_version)
+      @event_creator = Proc.new do |entry|
+        clone = entry.event.clone
+        # LS 6 LogStash::Event.new accept Map not Event
+        event = LogStash::Event.new(clone.getData())
+        event.set("[@metadata]", clone.getMetadata())
+        event
+      end
+    else
+      @event_creator = -> (entry) { LogStash::Event.new(entry.event.clone) }
+    end
   end # def register
 
   public
   def run(logstash_queue)
-    @inner_plugin.run do |entry|
-      clone = entry.event.clone()
-      event = if Gem::Requirement.new('< 7.0').satisfied_by?(Gem::Version.new(LOGSTASH_CORE_VERSION))
-                # LS 6 LogStash::Event.new accept Map not Event
-                event = LogStash::Event.new(clone.getData())
-                event.set("[@metadata]", clone.getMetadata())
-                event
-              else
-                LogStash::Event.new(clone)
-              end
+    @inner_plugin.run(lambda do |entry|
+      event = @event_creator.(entry)
       event.set("[@metadata][dead_letter_queue][plugin_type]", entry.plugin_type)
       event.set("[@metadata][dead_letter_queue][plugin_id]", entry.plugin_id)
       event.set("[@metadata][dead_letter_queue][reason]", entry.reason)
       event.set("[@metadata][dead_letter_queue][entry_time]", entry.entry_time)
       decorate(event)
       logstash_queue << event
-    end
+    end)
   end # def run
 
   public

data/lib/logstash-input-dead_letter_queue_jars.rb

@@ -1,4 +1,4 @@
 # AUTOGENERATED BY THE GRADLE SCRIPT. DO NOT EDIT.
 
 require 'jar_dependencies'
-require_jar('co.elastic.logstash.input', 'logstash-input-dead_letter_queue', '1.1.10')
+require_jar('co.elastic.logstash.input', 'logstash-input-dead_letter_queue', '2.0.0')

data/logstash-input-dead_letter_queue.gemspec

@@ -21,6 +21,7 @@ Gem::Specification.new do |s|
   s.metadata = { 'logstash_plugin' => 'true', 'group' => 'input'}
 
   # Gem dependencies
+  s.add_runtime_dependency 'logstash-core', '>= 8.4.0'
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
   s.add_runtime_dependency 'logstash-codec-plain'
 

data/spec/unit/inputs/dead_letter_queue_spec.rb

@@ -7,9 +7,12 @@ describe LogStash::Inputs::DeadLetterQueue do
   let(:pipeline_id) { SecureRandom.hex(8)}
   let(:path) { Dir.tmpdir }
   let(:directory) { File.join(path, pipeline_id)}
+  let(:config) do
+    { "path" => path,
+      "pipeline_id" => pipeline_id}
+  end
 
-  subject { LogStash::Inputs::DeadLetterQueue.new({ "path" => path,
-                                                    "pipeline_id" => pipeline_id}) }
+  subject { LogStash::Inputs::DeadLetterQueue.new(config) }
 
   before(:each) do
     Dir.mkdir(directory)
@@ -23,6 +26,21 @@ describe LogStash::Inputs::DeadLetterQueue do
     FileUtils.remove_entry_secure directory
   end
 
+  context "when clean_consumed is enabled" do
+    let(:config) { super().merge!("clean_consumed" => true) }
+
+    it "registers successfully" do
+      expect {subject.register}.to_not raise_error
+    end
+
+    context "and commit_offsets is not" do
+      let(:config) { super().merge!("commit_offsets" => false) }
+      it "raises a configuration error during register" do
+        expect {subject.register}.to raise_error(LogStash::ConfigurationError)
+      end
+    end
+  end
+
   context 'test with real DLQ file' do
     let(:dlq_dir) { Stud::Temporary.directory }
     let(:fixture_dir) { File.expand_path(File.join(File.dirname(__FILE__),"fixtures", "main")) }

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-dead_letter_queue
 version: !ruby/object:Gem::Version
-  version: 1.1.10
+  version: 2.0.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-02-07 00:00:00.000000000 Z
+date: 2022-08-02 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 8.4.0
+  name: logstash-core
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 8.4.0
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -80,7 +94,7 @@ files:
 - logstash-input-dead_letter_queue.gemspec
 - spec/unit/inputs/dead_letter_queue_spec.rb
 - spec/unit/inputs/fixtures/main/1.log
-- vendor/jar-dependencies/co/elastic/logstash/input/logstash-input-dead_letter_queue/1.1.10/logstash-input-dead_letter_queue-1.1.10.jar
+- vendor/jar-dependencies/co/elastic/logstash/input/logstash-input-dead_letter_queue/2.0.0/logstash-input-dead_letter_queue-2.0.0.jar
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
 - Apache License (2.0)