logstash-integration-rabbitmq 7.4.4-java → 7.4.5-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ee33feab4b89f243bac29940ce92eb8650a7bc18419e764e5f0388ce4111e30d
-  data.tar.gz: 25f6b19b9b7e15e2dfe9fce6efd096348c88669fe46fc8bedde51d9d4af7da6f
+  metadata.gz: 6bcb030b6451f96b859c628c1631a62083bfe171296d33e11555a908f59a90ef
+  data.tar.gz: 10edb4eb65e2ca490e60659025e414cb1e4fdd3dc708c3b9efeba19122643602
 SHA512:
-  metadata.gz: d3bf562cae12a69b5cbf4a8c87bddecc79df9a9fcb7c77bc04d1159f3159a0783c15d4f44b9370337c807a3f13bd093d6e858b12ba5fa94df95b8e297bbee153
-  data.tar.gz: 665169ebe7ad610dea3096e1aa323a95312089bea9f956bb7a7a610d607c301325e9c6322b0f81223a8e03431a261b44744a016b0a3db8304477d6572110cfe0
+  metadata.gz: 73847fec6a528310e5389e0da3036c6d019b926c3338d8d6bd8760099d692ebe36948f6080f908e4354a5e67bb390465bc6e6abbef6bbe3bbc6bda63a39c7bc5
+  data.tar.gz: 4219db729cc3fed03ff1e1bf6414a485218d4abaf00af4bd405cf230e97f545de85b6eb33bf015b21cb8279b3be4c209f589e33dc3b78bcde9108fa72b93ab71

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 7.4.5
+  - Recover from broker-closed output channels without leaking channels or losing events [#88](https://github.com/logstash-plugins/logstash-integration-rabbitmq/pull/88)
+
 ## 7.4.4
   - Upgrades march_hare to consume 4.10+ [#87](https://github.com/logstash-plugins/logstash-integration-rabbitmq/pull/87)
 

data/lib/logstash/outputs/rabbitmq.rb

@@ -17,6 +17,9 @@ module LogStash
 
       java_import java.util.concurrent.TimeoutException
       java_import com.rabbitmq.client.AlreadyClosedException
+      java_import com.rabbitmq.client.ShutdownSignalException
+
+      CONFIRM_TIMEOUT_MILLIS = 10_000
 
       include LogStash::PluginMixins::RabbitMQConnection
 
@@ -73,14 +76,26 @@ module LogStash
         raise ArgumentError, "No exchange set in HareInfo!!!" unless @hare_info.exchange
         routing_key = event.sprintf(@key)
         message_properties = @message_properties_template.build(event)
-        @gated_executor.execute do
-          local_exchange.publish(message, :routing_key => routing_key, :properties => message_properties)
-        end
-      rescue MarchHare::Exception, IOError, AlreadyClosedException, TimeoutException => e
-        @logger.error("Error while publishing, will retry", error_details(e, backtrace: true))
+        confirm = false
+        begin
+          @gated_executor.execute do
+            if confirm
+              publish_confirmed(message, routing_key, message_properties)
+            else
+              local_exchange.publish(message, :routing_key => routing_key, :properties => message_properties)
+            end
+          end
+        rescue MarchHare::Exception, IOError, AlreadyClosedException, TimeoutException => e
+          @logger.error("Error while publishing, will retry", error_details(e, backtrace: true))
 
-        sleep_for_retry
-        retry
+          # the broker closed only this channel; connection-level closes are left to automatic recovery
+          if channel_level_close?(e)
+            discard_local_channel
+            confirm = true
+          end
+          sleep_for_retry
+          retry
+        end
       end
 
       def local_exchange
@@ -107,6 +122,40 @@ module LogStash
 
       private
 
+      def channel_level_close?(e)
+        signal = e.is_a?(ShutdownSignalException) ? e : e.cause
+        signal.is_a?(ShutdownSignalException) && !signal.hard_error?
+      end
+
+      def discard_local_channel
+        channel = @thread_local_channel.get
+        @thread_local_exchange.remove
+        @thread_local_channel.remove
+        discard_channel(channel) if channel
+      end
+
+      # unregistering keeps march_hare's automatic recovery from reopening the channel
+      def discard_channel(channel)
+        channel.close if channel.open?
+      rescue StandardError, java.lang.Exception => e
+        @logger.debug("Ignoring error while closing channel", error_details(e))
+      ensure
+        @hare_info.connection.unregister_channel(channel)
+      end
+
+      # publishes on a temporary confirm-mode channel so a rejected message is retried instead of lost
+      def publish_confirmed(message, routing_key, properties)
+        channel = @hare_info.connection.create_channel
+        begin
+          channel.confirm_select
+          exchange = declare_exchange!(channel, @exchange, @exchange_type, @durable)
+          exchange.publish(message, :routing_key => routing_key, :properties => properties)
+          raise MarchHare::Exception, "message was nacked by the broker" unless channel.wait_for_confirms(CONFIRM_TIMEOUT_MILLIS)
+        ensure
+          discard_channel(channel)
+        end
+      end
+
       # When the other end of a RabbitMQ connection is either unwilling or unable to continue reading bytes from
       # its underlying TCP stream, the connection is flagged as "blocked", but attempts to publish onto exchanges
       # using the connection will not block in the client.

data/spec/outputs/rabbitmq_spec.rb

@@ -23,6 +23,16 @@ describe LogStash::Outputs::RabbitMQ do
   let(:instance) { klass.new(rabbitmq_settings) }
   let(:hare_info) { instance.instance_variable_get(:@hare_info) }
 
+  # march_hare converts AlreadyClosedException inside a rescue, so the Java exception becomes the `cause`
+  def channel_already_closed(hard:)
+    signal = com.rabbitmq.client.ShutdownSignalException.new(hard, false, nil, nil)
+    begin
+      raise com.rabbitmq.client.AlreadyClosedException.new(signal)
+    rescue java.lang.Exception
+      raise MarchHare::ChannelAlreadyClosed, "channel is already closed"
+    end
+  end
+
   shared_examples 'recovers from exception gracefully' do
     it 'should execute publish twice due to a retry' do
       expect(exchange).to have_received(:publish).twice
@@ -156,9 +166,12 @@ describe LogStash::Outputs::RabbitMQ do
           i = 0
           allow(instance).to receive(:connect!)
           allow(instance).to receive(:sleep_for_retry)
+          allow(connection).to receive(:unregister_channel)
           allow(exchange).to receive(:publish).with(any_args) do
             i += 1
-            raise exception if i == 1
+            if i == 1
+              exception.is_a?(Proc) ? exception.call : raise(exception)
+            end
           end
 
           instance.send(:publish, event, encoded_event)
@@ -167,18 +180,188 @@ describe LogStash::Outputs::RabbitMQ do
         context 'when it is a MarchHare exception' do
           let(:exception) { MarchHare::Exception }
           it_behaves_like 'recovers from exception gracefully'
+
+          it 'should reuse the existing channel' do
+            # once on register, once for the publishing thread
+            expect(connection).to have_received(:create_channel).twice
+          end
         end
 
-        context 'when it is a MarchHare::ChannelAlreadyClosed' do
+        context 'when it is a MarchHare::ChannelAlreadyClosed without a cause' do
           let(:exception) { MarchHare::ChannelAlreadyClosed }
           it_behaves_like 'recovers from exception gracefully'
         end
 
+        context 'when it is a MarchHare::ChannelAlreadyClosed caused by a connection close' do
+          let(:exception) { -> { channel_already_closed(hard: true) } }
+          it_behaves_like 'recovers from exception gracefully'
+
+          it 'should leave the channel to automatic recovery' do
+            expect(connection).not_to have_received(:unregister_channel)
+          end
+        end
+
         context 'when it is a TimeoutException' do
           let(:exception) { java.util.concurrent.TimeoutException.new }
           it_behaves_like 'recovers from exception gracefully'
         end
       end
+
+      context 'when the broker closed the channel' do
+        let(:stale_channel) { double("Stale channel") }
+        let(:stale_exchange) { double("Stale exchange") }
+        let(:stale_open) { false }
+        let(:stale_close_error) { nil }
+        let(:confirm_channel) { double("Confirm channel", confirm_select: nil, open?: true, close: nil) }
+        let(:confirm_exchange) { double("Confirm exchange", publish: nil) }
+        let(:fresh_channel) { double("Fresh channel") }
+        let(:fresh_exchange) { double("Fresh exchange", publish: nil) }
+        # outcome of each confirmed attempt: true (ack), false (nack) or a raised error
+        let(:confirm_results) { [proc { true }] }
+
+        before do
+          allow(instance).to receive(:sleep_for_retry)
+          allow(connection).to receive(:unregister_channel)
+          allow(stale_channel).to receive(:exchange).and_return(stale_exchange)
+          allow(stale_channel).to receive(:open?).and_return(stale_open)
+          allow(stale_channel).to receive(:close) { raise stale_close_error if stale_close_error }
+          allow(stale_exchange).to receive(:publish) { channel_already_closed(hard: false) }
+          allow(confirm_channel).to receive(:exchange).and_return(confirm_exchange)
+          allow(confirm_channel).to receive(:wait_for_confirms).and_invoke(*confirm_results)
+          allow(fresh_channel).to receive(:exchange).and_return(fresh_exchange)
+          confirm_channels = [confirm_channel] * confirm_results.size
+          allow(connection).to receive(:create_channel).and_return(stale_channel, *confirm_channels, fresh_channel)
+
+          instance.send(:publish, event, encoded_event)
+        end
+
+        it 'should not publish to the closed exchange again' do
+          expect(stale_exchange).to have_received(:publish).once
+        end
+
+        it 'should unregister the closed channel so recovery cannot reopen it' do
+          expect(connection).to have_received(:unregister_channel).with(stale_channel)
+        end
+
+        it 'should not close a channel that is already closed' do
+          expect(stale_channel).not_to have_received(:close)
+        end
+
+        it 'should retry the message on a channel in confirm mode' do
+          expect(confirm_channel).to have_received(:confirm_select).once
+          expect(confirm_channel).to have_received(:exchange).once
+          expect(confirm_exchange).to have_received(:publish).with(encoded_event, anything).once
+          expect(confirm_channel).to have_received(:wait_for_confirms).with(10_000)
+        end
+
+        it 'should close and unregister the confirm channel' do
+          expect(confirm_channel).to have_received(:close).once
+          expect(connection).to have_received(:unregister_channel).with(confirm_channel)
+        end
+
+        it 'should publish the next event on a new regular channel' do
+          instance.send(:publish, event, encoded_event)
+
+          expect(fresh_channel).to have_received(:exchange)
+          expect(fresh_exchange).to have_received(:publish).with(encoded_event, anything).once
+          expect(confirm_channel).to have_received(:confirm_select).once
+        end
+
+        context 'and the broker nacks the confirmed retry' do
+          let(:confirm_results) { [proc { false }, proc { true }] }
+
+          it 'should keep retrying with confirms until the message is acked' do
+            expect(confirm_exchange).to have_received(:publish).with(encoded_event, anything).twice
+            expect(instance).to have_received(:sleep_for_retry).twice
+          end
+
+          it 'should discard every confirm channel' do
+            expect(confirm_channel).to have_received(:close).twice
+            expect(connection).to have_received(:unregister_channel).with(confirm_channel).twice
+          end
+        end
+
+        context 'and the broker also closes the confirm channel' do
+          let(:confirm_results) { [proc { channel_already_closed(hard: false) }, proc { true }] }
+
+          it 'should keep retrying with confirms until the message is acked' do
+            expect(stale_exchange).to have_received(:publish).once
+            expect(confirm_exchange).to have_received(:publish).with(encoded_event, anything).twice
+            expect(instance).to have_received(:sleep_for_retry).twice
+          end
+        end
+
+        context 'and the closed channel still reports being open' do
+          let(:stale_open) { true }
+          let(:stale_close_error) { java.io.IOException.new("channel is already closed") }
+
+          it 'should ignore the error when closing it and still unregister it' do
+            expect(stale_channel).to have_received(:close)
+            expect(connection).to have_received(:unregister_channel).with(stale_channel)
+            expect(confirm_exchange).to have_received(:publish).with(encoded_event, anything).once
+          end
+        end
+      end
+
+      context 'when the connection was closed' do
+        let(:stale_exchange) { double("Stale exchange") }
+        let(:fresh_channel) { double("Fresh channel") }
+        let(:fresh_exchange) { double("Fresh exchange", publish: nil) }
+
+        before do
+          allow(instance).to receive(:sleep_for_retry)
+          allow(connection).to receive(:unregister_channel)
+          allow(fresh_channel).to receive(:exchange).and_return(fresh_exchange)
+        end
+
+        context 'while publishing' do
+          before do
+            i = 0
+            allow(exchange).to receive(:publish) do
+              i += 1
+              channel_already_closed(hard: true) if i == 1
+            end
+
+            instance.send(:publish, event, encoded_event)
+          end
+
+          it 'should retry on the same channel once it is recovered' do
+            expect(exchange).to have_received(:publish).with(encoded_event, anything).twice
+            # once on register, once for the publishing thread
+            expect(connection).to have_received(:create_channel).twice
+          end
+
+          it 'should not discard the channel' do
+            expect(connection).not_to have_received(:unregister_channel)
+          end
+        end
+
+        context 'while creating the channel' do
+          # create_channel is a plain Java call, so it raises the unconverted Java exception
+          let(:connection_closed) do
+            com.rabbitmq.client.AlreadyClosedException.new(
+              com.rabbitmq.client.ShutdownSignalException.new(true, false, nil, nil)
+            )
+          end
+
+          before do
+            allow(connection).to receive(:create_channel).and_invoke(
+              -> { raise connection_closed },
+              -> { fresh_channel }
+            )
+
+            instance.send(:publish, event, encoded_event)
+          end
+
+          it 'should retry until the channel can be created' do
+            expect(instance).to have_received(:sleep_for_retry).once
+          end
+
+          it 'should send the message once the connection is back' do
+            expect(fresh_exchange).to have_received(:publish).with(encoded_event, anything).once
+          end
+        end
+      end
     end
   end
 end

data/version

@@ -1 +1 @@
-7.4.4
+7.4.5

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: logstash-integration-rabbitmq
 version: !ruby/object:Gem::Version
-  version: 7.4.4
+  version: 7.4.5
 platform: java
 authors:
 - Elastic
 bindir: bin
 cert_chain: []
-date: 1980-01-02 00:00:00.000000000 Z
+date: 2026-11-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-core-plugin-api
@@ -216,7 +216,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.7.2
+rubygems_version: 3.6.3
 specification_version: 4
 summary: Integration with RabbitMQ - input and output plugins
 test_files: