logstash-input-dead_letter_queue 1.1.8 → 1.1.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 409103b04b24b573dff867b4a59f58dbfaad926f4fd8a390cbf08d4243b0976b
-  data.tar.gz: a911df00dc53b67bdf4bf9f2cf4d4f321a89dd9e7eb8e8862658deaddcc305d0
+  metadata.gz: 9a9f7c1ca29e4d2e0b409b3a7f56d69b800ca1321386e9402530605224fbc64e
+  data.tar.gz: efc30664c8fc92365eda9e20cadffdd67123c9a80d79a04a3160dae5fcf6b33e
 SHA512:
-  metadata.gz: 5b7a8789b7b9ff113a1aec644e4f8f4e32379d439dbc0ac555eb291e226ad81fc0b36aa5f14afe1ffeed3f7617a3d35a9a1123310f956cc71f7d14038c75f4eb
-  data.tar.gz: 845256cb793ca177652719dcd588ca294a70df4575234deb71d1af619c0ce58931ae6cd6cd0ab93ad852160bedbe8e7f2ce4d541656590a3b31568c6e5b02910
+  metadata.gz: 9da83407c607cd6977d988f3da9972ec6aba3e9383aabfef709f79f777b67a81c3d9f15f4a3fb7e7ab6e10e725bb7482b7689671851ba08e96e85b8a0337e8c9
+  data.tar.gz: 9bb864de1fe4c5d9d5c8a137e9977454dfcbf0d862454847ebd18cedc98451d4b3e3989a6e3bf68de65816600b424eb148e600beaea516aaea86d17a7c052f11

data/CHANGELOG.md

@@ -1,3 +1,12 @@
+## 1.1.11
+  - Fix: pre-flight checks before creating DLQ reader [#35](https://github.com/logstash-plugins/logstash-input-dead_letter_queue/pull/35)
+
+## 1.1.10
+  - Fix, avoid Logstash crash on shutdown if DLQ files weren't created [#33](https://github.com/logstash-plugins/logstash-input-dead_letter_queue/pull/33)
+
+## 1.1.9
+  - Fix `@metadata` get overwritten by reestablishing metadata that stored in DLQ [#34](https://github.com/logstash-plugins/logstash-input-dead_letter_queue/pull/34)
+
 ## 1.1.8
   - Update dependencies for log4j to 2.17.1
 

data/VERSION

@@ -1 +1 @@
-1.1.8
+1.1.11

data/lib/logstash/inputs/dead_letter_queue.rb

@@ -2,6 +2,8 @@ require 'logstash/namespace'
 require 'logstash/inputs/base'
 require 'logstash-input-dead_letter_queue_jars'
 
+require 'fileutils'
+
 # Logstash input to read events from Logstash's dead letter queue
 # 
 # [source, sh]
@@ -45,7 +47,7 @@ class LogStash::Inputs::DeadLetterQueue < LogStash::Inputs::Base
       FileUtils::mkdir_p datapath
       @sincedb_path = File.join(datapath, ".sincedb_" + Digest::MD5.hexdigest(@path))
     elsif File.directory?(@sincedb_path)
-        raise ArgumentError.new("The \"sincedb_path\" argument must point to a file, received a directory: \"#{@sincedb_path}\"")
+      raise ArgumentError.new("The \"sincedb_path\" argument must point to a file, received a directory: \"#{@sincedb_path}\"")
     end
 
     dlq_path = java.nio.file.Paths.get(File.join(@path, @pipeline_id))
@@ -53,12 +55,24 @@ class LogStash::Inputs::DeadLetterQueue < LogStash::Inputs::Base
     start_timestamp = @start_timestamp ? org.logstash.Timestamp.new(@start_timestamp) : nil
     @inner_plugin = org.logstash.input.DeadLetterQueueInputPlugin.new(dlq_path, @commit_offsets, sincedb_path, start_timestamp)
     @inner_plugin.register
+
+    if Gem::Requirement.new('< 7.0').satisfied_by?(Gem::Version.new(LOGSTASH_CORE_VERSION))
+      @event_creator = Proc.new do |entry|
+        clone = entry.event.clone
+        # LS 6 LogStash::Event.new accept Map not Event
+        event = LogStash::Event.new(clone.getData())
+        event.set("[@metadata]", clone.getMetadata())
+        event
+      end
+    else
+      @event_creator = -> (entry) { LogStash::Event.new(entry.event.clone) }
+    end
   end # def register
 
   public
   def run(logstash_queue)
     @inner_plugin.run do |entry|
-      event = LogStash::Event.new(entry.event.toMap())
+      event = @event_creator.(entry)
       event.set("[@metadata][dead_letter_queue][plugin_type]", entry.plugin_type)
       event.set("[@metadata][dead_letter_queue][plugin_id]", entry.plugin_id)
       event.set("[@metadata][dead_letter_queue][reason]", entry.reason)

data/lib/logstash-input-dead_letter_queue_jars.rb

@@ -1,4 +1,4 @@
 # AUTOGENERATED BY THE GRADLE SCRIPT. DO NOT EDIT.
 
 require 'jar_dependencies'
-require_jar('co.elastic.logstash.input', 'logstash-input-dead_letter_queue', '1.1.8')
+require_jar('co.elastic.logstash.input', 'logstash-input-dead_letter_queue', '1.1.11')

data/spec/unit/inputs/dead_letter_queue_spec.rb

@@ -19,8 +19,35 @@ describe LogStash::Inputs::DeadLetterQueue do
     expect {subject.register}.to_not raise_error
   end
 
-
   after(:each) do
     FileUtils.remove_entry_secure directory
   end
+
+  context 'test with real DLQ file' do
+    let(:dlq_dir) { Stud::Temporary.directory }
+    let(:fixture_dir) { File.expand_path(File.join(File.dirname(__FILE__),"fixtures", "main")) }
+    let(:plugin) { LogStash::Inputs::DeadLetterQueue.new({ "path" => dlq_dir, "commit_offsets" => true }) }
+    let(:queue) { Queue.new }
+
+    before do
+      FileUtils.cp_r fixture_dir, dlq_dir
+    end
+
+    it 'reserve original metadata' do
+      plugin.register
+      t = Thread.new {
+        begin
+          plugin.run(queue)
+        rescue java.nio.file.ClosedWatchServiceException => e
+        end
+      }
+
+      event = queue.pop
+      expect(event.get("[@metadata][some_key]")).to eq("some_value")
+      expect(event.get("[@metadata][dead_letter_queue][reason]")).to match(/Could not index event to Elasticsearch. status: 400/)
+
+      plugin.stop
+      t.join(1000)
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-dead_letter_queue
 version: !ruby/object:Gem::Version
-  version: 1.1.8
+  version: 1.1.11
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-01-05 00:00:00.000000000 Z
+date: 2022-03-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -79,7 +79,8 @@ files:
 - lib/logstash/inputs/dead_letter_queue.rb
 - logstash-input-dead_letter_queue.gemspec
 - spec/unit/inputs/dead_letter_queue_spec.rb
-- vendor/jar-dependencies/co/elastic/logstash/input/logstash-input-dead_letter_queue/1.1.8/logstash-input-dead_letter_queue-1.1.8.jar
+- spec/unit/inputs/fixtures/main/1.log
+- vendor/jar-dependencies/co/elastic/logstash/input/logstash-input-dead_letter_queue/1.1.11/logstash-input-dead_letter_queue-1.1.11.jar
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
 - Apache License (2.0)
@@ -108,3 +109,4 @@ specification_version: 4
 summary: read events from Logstash's dead letter queue
 test_files:
 - spec/unit/inputs/dead_letter_queue_spec.rb
+- spec/unit/inputs/fixtures/main/1.log