logstash-codec-multiline 3.1.0 → 3.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a4ecbf8730e688adbacb1af57c98f7c5105b5677337b140a22b6ccb41e1468ea
-  data.tar.gz: 299167720ec0457af53b7f118b6e55bd6c078f6ac1b8e4978ca68872a04cbc7c
+  metadata.gz: 3e46243c32a2f87ede3a266ee117dd89086a5a48454813d8979fe04a25454692
+  data.tar.gz: fedc22e883fd7e842a4de3f8cf599268c57f7b1d7de06a40a5d5fb27cd86febb
 SHA512:
-  metadata.gz: 798b7fe3f963ade7578c655a00ba2ba7ea602dced4e463b170d8d7bf8b9e0585bedda761f9ee5bbbcb8026d99e0cbc3dc22bdfc19abe063c5d3716c2b8e0fd03
-  data.tar.gz: b4758f3f75e0ad1e470800aff9ccf353f8f4493cbcd6b30f687ef3fa74973e5baf0aba65a25450b83b3269879065ab5878141914dd3c35998e0bad4ca631b8bd
+  metadata.gz: faa8892c245b03438c1cd63f5812b8e5c80c145b55fd437e8eb3ab82baf4bb94f23754d720c40016fc8bfecf23a6eae7cbb1c1ab4e0a34f5a5a4e37efe1d8fec
+  data.tar.gz: 415515b5dc715d12df012febfc9a91cf986d8c05fd73f975bc284807bbd034e436168a6d624299bc53aea5417bae4d391f10b97b32a07b0b43be7ac197de8a7a

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 3.1.2
+  - Fix: periodic runner fails to stop and prevent pipeline shutdown [#72](https://github.com/logstash-plugins/logstash-codec-multiline/pull/72)
+
+## 3.1.1
+  - Fix: avoid reusing per-identity codec instances for differing identities. Removes a very minor optimization so that stateful codecs like CSV can work reliably [#70](https://github.com/logstash-plugins/logstash-codec-multiline/pull/70)
+
 ## 3.1.0
   - Feat: ECS compatibility [#69](https://github.com/logstash-plugins/logstash-codec-multiline/pull/69)
 

data/lib/logstash/codecs/identity_map_codec.rb

@@ -42,15 +42,20 @@ module LogStash module Codecs class IdentityMapCodec
     def initialize(listener, interval, method_symbol)
       @listener, @interval = listener, interval
       @method_symbol = method_symbol
-      @running = Concurrent::AtomicBoolean.new(false)
+      @running = Concurrent::AtomicBoolean.new(true)
     end
 
+    # The goal of periodic runner is to have a single thread to do clean up / auto flush
+    # This method is expected to access by a single thread,
+    # otherwise, multiple Thread.start could create more than one periodic runner
     def start
-      return self if running?
-      @running.make_true
+      return self unless running?
+      return self if running? && !@thread.nil?
+
       @thread = Thread.start do
         class_name = @listener.class.name.split('::').last # IdentityMapCodec
         LogStash::Util.set_thread_name("#{class_name}##{@method_symbol}")
+        @listener.logger.debug("Start periodic runner")
 
         while running? do
           sleep @interval
@@ -67,9 +72,11 @@ module LogStash module Codecs class IdentityMapCodec
     end
 
     def stop
-      return if !running?
+      return unless running?
+
+      @listener.logger.debug("Stop periodic runner")
       @running.make_false
-      while @thread.alive?
+      while @thread&.alive?
         begin
           @thread.wakeup
         rescue ThreadError
@@ -290,6 +297,7 @@ module LogStash module Codecs class IdentityMapCodec
   end
 
   def codec_without_usage_update(identity)
+    return base_codec if identity.nil? # mirror optimization in `stream_codec`
     find_codec_value(identity).codec
   end
 
@@ -312,7 +320,7 @@ module LogStash module Codecs class IdentityMapCodec
   def record_codec_usage(identity)
     check_map_limits
     # only start the cleaner if streams are in use
-    # continuous calls to start are OK
+    # continuous calls to start are OK if this codec method is accessed by a single thread
     cleaner.start
     auto_flusher.start
     compo = find_codec_value(identity)
@@ -336,7 +344,7 @@ module LogStash module Codecs class IdentityMapCodec
   end
 
   def codec_builder(hash, k)
-    codec = hash.empty? ? @base_codec : @base_codec.clone
+    codec = @base_codec.clone
     codec.use_mapper_auto_flush if using_mapped_auto_flush?
     compo = CodecValue.new(codec).tap do |o|
       now = Time.now

data/logstash-codec-multiline.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-codec-multiline'
-  s.version         = '3.1.0'
+  s.version         = '3.1.2'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Merges multiline messages into a single event"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/codecs/identity_map_codec_spec.rb

@@ -29,7 +29,8 @@ describe LogStash::Codecs::IdentityMapCodec do
     let(:stream1) { nil }
 
     it "transparently refers to the original codec" do
-      expect(codec).to eql(codec1)
+      resolved_codec = demuxer.codec_without_usage_update(stream1)
+      expect(resolved_codec).to eq(codec)
     end
   end
 
@@ -37,8 +38,8 @@ describe LogStash::Codecs::IdentityMapCodec do
 
     before { demuxer.decode(arg1, stream1) }
 
-    it "the first identity refers to the original codec" do
-      expect(codec).to eql(codec1)
+    it "the first identity refers to a copy of original codec" do
+      expect(codec1).to_not eql(codec)
     end
   end
 
@@ -212,8 +213,11 @@ describe LogStash::Codecs::IdentityMapCodec do
       let(:demuxer) { described_class.new(codec).evict_timeout(1).cleaner_interval(1) }
       it "the cleaner evicts the codec and flushes it first" do
         demuxer.decode(Object.new, "stream1"){|*| 42}
+        mapped_codec = demuxer.codec_without_usage_update("stream1") # get before eviction
+
         sleep(2.1)
-        expect(codec.trace_for(:flush)).to eq(42)
+
+        expect(mapped_codec.trace_for(:flush)).to eq(42)
         expect(demuxer.identity_map.keys).not_to include("stream1")
       end
     end
@@ -227,9 +231,12 @@ describe LogStash::Codecs::IdentityMapCodec do
       end
       it "the cleaner evicts the codec and flushes it first using the eviction_block" do
         demuxer.decode(Object.new, "stream1"){|*| 42}
+        mapped_codec = demuxer.codec_without_usage_update("stream1") # get before eviction
+
         sleep(2.1)
-        expect(codec.trace_for(:flush)).to eq(24)
-        expect(demuxer.identity_map.keys).not_to include("stream1")
+
+        expect(mapped_codec.trace_for(:flush)).to eq(24)
+        expect(demuxer.identity_map.keys).to_not include("stream1")
       end
     end
   end
@@ -251,7 +258,9 @@ describe LogStash::Codecs::IdentityMapCodec do
         it "no events are generated (the line is buffered)" do
           expect(imc.identity_count).to eq(1)
           expect(queue.size).to eq(0)
-          expect(mlc.internal_buffer[0]).to eq("foo")
+
+          mapped_codec = imc.codec_without_usage_update(identity)
+          expect(mapped_codec.internal_buffer[0]).to eq("foo")
         end
       end
 
@@ -290,3 +299,79 @@ describe LogStash::Codecs::IdentityMapCodec do
     end
   end
 end
+
+describe LogStash::Codecs::IdentityMapCodec::PeriodicRunner do
+  let(:listener) do
+    double('Listener', logger: double('Logger').as_null_object,
+           some_method: double('method_symbol').as_null_object)
+  end
+  let(:interval) { 1 }
+  let(:method_symbol) { :some_method }
+  subject(:runner) { described_class.new(listener, interval, method_symbol) }
+
+  before(:all) do
+    Thread.abort_on_exception = true
+  end
+
+  describe "normal shutdown" do
+    it "starts first and stops successfully" do
+      start_thread = Thread.start do
+        runner.start
+      end
+
+      stop_thread = Thread.start do
+        sleep(1)
+        runner.stop
+      end
+
+      start_thread.join
+      stop_thread.join
+
+      expect(runner.instance_variable_get('@listener')).to be_nil
+    end
+  end
+
+  describe "race condition" do
+    let(:atomic_bool) { Concurrent::AtomicBoolean.new(true) }
+    let(:running) { double('AtomicBooleanStub') }
+
+    before do
+      allow(running).to receive(:make_true) do
+        atomic_bool.make_true
+      end
+
+      allow(running).to receive(:make_false) do
+        atomic_bool.make_false
+        sleep(2)
+      end
+
+      allow(running).to receive(:true?) do
+        atomic_bool.true?
+      end
+    end
+
+    before :each do
+      runner.instance_variable_set('@running', running)
+    end
+
+    it "starts several times and stops successfully" do
+      start_thread = Thread.start do
+        5.times do
+          runner.start
+          sleep(1)
+        end
+      end
+
+      stop_thread = Thread.start do
+        sleep(1) # give time to runner start
+        runner.stop
+      end
+
+      start_thread.join
+      stop_thread.join
+
+      expect(runner.instance_variable_get('@listener')).to be_nil
+      expect(runner.instance_variable_get('@thread').alive?).to be_falsey
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-codec-multiline
 version: !ruby/object:Gem::Version
-  version: 3.1.0
+  version: 3.1.2
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-07-27 00:00:00.000000000 Z
+date: 2024-04-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -20,8 +20,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '2.99'
   name: logstash-core-plugin-api
-  type: :runtime
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -37,8 +37,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.3'
   name: logstash-mixin-ecs_compatibility_support
-  type: :runtime
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -51,8 +51,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.0'
   name: logstash-mixin-event_support
-  type: :runtime
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -65,8 +65,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
   name: logstash-patterns-core
-  type: :runtime
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -79,8 +79,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 0.11.1
   name: jls-grok
-  type: :runtime
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -93,8 +93,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
   name: concurrent-ruby
-  type: :runtime
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -107,8 +107,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
   name: logstash-devutils
-  type: :development
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -121,8 +121,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
   name: insist
-  type: :development
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -173,7 +173,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.6
+rubygems_version: 3.2.33
 signing_key:
 specification_version: 4
 summary: Merges multiline messages into a single event