logstash-output-tcp 6.1.0 → 6.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d30cf25cfbc7e1cc2e72d5e1eb8ab8179d42b31413a549801a012d5e1be7e640
-  data.tar.gz: 53632382865d4bade1e6d56e9872b1b5db690fd9b866cb4d0952dfa8d1156cee
+  metadata.gz: dd2acefd6c26058e3e2afc94abb73de74e2816ca57a6379229909434ad4c72e9
+  data.tar.gz: 38633ca4cf74d5360fa3644b9624cb21db997e9cda5ff8846f5ca510b811c133
 SHA512:
-  metadata.gz: e48f3511c25df04edb79dfd0425c6a84d18808bce5f8d4910cabe024955e6c16f8b856a8ac7fac01cb7721e7628934101e14e7180497a7c6cc1096169290963e
-  data.tar.gz: 94758e5adb6be529f4c70715aa0796d1a15bd8d17e91d35c63fdc1b29eccea58c0f36e4c29b0cf911a3691fade34c895922ca9b0384cd3df405f723a97ff642f
+  metadata.gz: ef785946ab821ca9347ef5fd04e17909ef573155ce790955640b875582a268363c436c3fa6a6c8c8cc19d086a6a9e6d5b8452448be59fbd80328481a7e0f292e
+  data.tar.gz: ac1c3b27079ecb0bb91eaa4a11eea17a1f79d4b6d935e72d82085b34f6731ee2ed6e6240250e4c18429b669d1430875d8ef958aa4c009ae71bb75d3e73f6a414

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 6.1.2
+  - Changed the client mode to write using the non-blocking method. [#52](https://github.com/logstash-plugins/logstash-output-tcp/pull/52)
+
+## 6.1.1
+  - Fixes an issue where payloads larger than a connection's current TCP window could be silently truncated [#49](https://github.com/logstash-plugins/logstash-output-tcp/pull/49)
+
 ## 6.1.0
   - Feat: ssl_supported_protocols (TLSv1.3) [#47](https://github.com/logstash-plugins/logstash-output-tcp/pull/47)
   - Fix: close server and client sockets on plugin close

data/lib/logstash/outputs/tcp.rb

@@ -56,18 +56,25 @@ class LogStash::Outputs::Tcp < LogStash::Outputs::Base
 
   class Client
 
-    def initialize(socket, logger)
+    ##
+    # @param socket [Socket]
+    # @param logger_context [#log_warn&#log_error]
+    def initialize(socket, logger_context)
       @socket = socket
-      @logger = logger
+      @logger_context = logger_context
       @queue  = Queue.new
     end
 
     def run
       loop do
         begin
-          @socket.write(@queue.pop)
+          remaining_payload = @queue.pop
+          while remaining_payload && remaining_payload.bytesize > 0
+            written_bytes_size = @socket.write(remaining_payload)
+            remaining_payload = remaining_payload.byteslice(written_bytes_size..-1)
+          end
         rescue => e
-          log_warn 'socket write failed:', e, socket: (@socket ? @socket.to_s : nil)
+          @logger_context.log_warn 'socket write failed:', e, socket: (@socket ? @socket.to_s : nil)
           break
         end
       end
@@ -80,7 +87,7 @@ class LogStash::Outputs::Tcp < LogStash::Outputs::Base
     def close
       @socket.close
     rescue => e
-      log_warn 'socket close failed:', e, socket: (@socket ? @socket.to_s : nil)
+      @logger_context.log_warn 'socket close failed:', e, socket: (@socket ? @socket.to_s : nil)
     end
   end # class Client
 
@@ -135,69 +142,82 @@ class LogStash::Outputs::Tcp < LogStash::Outputs::Base
     require "socket"
     require "stud/try"
     @closed = Concurrent::AtomicBoolean.new(false)
+    @thread_no = Concurrent::AtomicFixnum.new(0)
     setup_ssl if @ssl_enable
 
     if server?
-      @logger.info("Starting tcp output listener", :address => "#{@host}:#{@port}")
-      begin
-        @server_socket = TCPServer.new(@host, @port)
-      rescue Errno::EADDRINUSE
-        @logger.error("Could not start tcp server: Address in use", host: @host, port: @port)
-        raise
-      end
-      if @ssl_enable
-        @server_socket = OpenSSL::SSL::SSLServer.new(@server_socket, @ssl_context)
-      end # @ssl_enable
-      @client_threads = Concurrent::Array.new
-
-      @accept_thread = Thread.new(@server_socket) do |server_socket|
-        LogStash::Util.set_thread_name("[#{pipeline_id}]|output|tcp|server_accept")
-        loop do
-          break if @closed.value
-          client_socket = server_socket.accept_nonblock exception: false
-          if client_socket == :wait_readable
-            IO.select [ server_socket ]
-            next
-          end
-          Thread.start(client_socket) do |client_socket|
-            # monkeypatch a 'peer' method onto the socket.
-            client_socket.instance_eval { class << self; include ::LogStash::Util::SocketPeer end }
-            @logger.debug("accepted connection", client: client_socket.peer, server: "#{@host}:#{@port}")
-            client = Client.new(client_socket, @logger)
-            Thread.current[:client] = client
-            LogStash::Util.set_thread_name("[#{pipeline_id}]|output|tcp|client_socket-#{@client_threads.size}")
-            @client_threads << Thread.current
-            client.run unless @closed.value
-          end
+      run_as_server
+    else
+      run_as_client
+    end
+  end
+
+  def run_as_server
+    @logger.info("Starting tcp output listener", :address => "#{@host}:#{@port}")
+    begin
+      @server_socket = TCPServer.new(@host, @port)
+    rescue Errno::EADDRINUSE
+      @logger.error("Could not start tcp server: Address in use", host: @host, port: @port)
+      raise
+    end
+    if @ssl_enable
+      @server_socket = OpenSSL::SSL::SSLServer.new(@server_socket, @ssl_context)
+    end # @ssl_enable
+    @client_threads = Concurrent::Array.new
+
+    @accept_thread = Thread.new(@server_socket) do |server_socket|
+      LogStash::Util.set_thread_name("[#{pipeline_id}]|output|tcp|server_accept")
+      loop do
+        break if @closed.value
+        client_socket = server_socket.accept_nonblock exception: false
+        if client_socket == :wait_readable
+          IO.select [ server_socket ]
+          next
+        end
+        Thread.start(client_socket) do |client_socket|
+          # monkeypatch a 'peer' method onto the socket.
+          client_socket.extend(::LogStash::Util::SocketPeer)
+          @logger.debug("accepted connection", client: client_socket.peer, server: "#{@host}:#{@port}")
+          client = Client.new(client_socket, self)
+          Thread.current[:client] = client
+          LogStash::Util.set_thread_name("[#{pipeline_id}]|output|tcp|client_socket-#{@thread_no.increment}")
+          @client_threads << Thread.current
+          client.run unless @closed.value
         end
       end
+    end
 
-      @codec.on_event do |event, payload|
-        @client_threads.select!(&:alive?)
-        @client_threads.each do |client_thread|
-          client_thread[:client].write(payload)
-        end
+    @codec.on_event do |event, payload|
+      @client_threads.select!(&:alive?)
+      @client_threads.each do |client_thread|
+        client_thread[:client].write(payload)
       end
-    else
-      client_socket = nil
-      @codec.on_event do |event, payload|
-        begin
-          client_socket = connect unless client_socket
-          r,w,e = IO.select([client_socket], [client_socket], [client_socket], nil)
-          # don't expect any reads, but a readable socket might
-          # mean the remote end closed, so read it and throw it away.
-          # we'll get an EOFError if it happens.
-          client_socket.sysread(16384) if r.any?
-
-          # Now send the payload
-          client_socket.syswrite(payload) if w.any?
-        rescue => e
-          log_warn "client socket failed:", e, host: @host, port: @port, socket: (client_socket ? client_socket.to_s : nil)
-          client_socket.close rescue nil
-          client_socket = nil
-          sleep @reconnect_interval
-          retry
+    end
+  end
+
+  def run_as_client
+    client_socket = nil
+    @codec.on_event do |event, payload|
+      begin
+        client_socket = connect unless client_socket
+        while payload && payload.bytesize > 0
+          begin
+            written_bytes_size = client_socket.write_nonblock(payload)
+            payload = payload.byteslice(written_bytes_size..-1)
+          rescue IO::WaitReadable
+            IO.select([client_socket])
+            retry
+          rescue IO::WaitWritable
+            IO.select(nil, [client_socket])
+            retry
+          end
         end
+      rescue => e
+        log_warn "client socket failed:", e, host: @host, port: @port, socket: (client_socket ? client_socket.to_s : nil)
+        client_socket.close rescue nil
+        client_socket = nil
+        sleep @reconnect_interval
+        retry
       end
     end
   end
@@ -219,6 +239,18 @@ class LogStash::Outputs::Tcp < LogStash::Outputs::Base
     end
   end
 
+  def log_warn(msg, e, backtrace: @logger.debug?, **details)
+    details = details.merge message: e.message, exception: e.class
+    details[:backtrace] = e.backtrace if backtrace
+    @logger.warn(msg, details)
+  end
+
+  def log_error(msg, e, backtrace: @logger.info?, **details)
+    details = details.merge message: e.message, exception: e.class
+    details[:backtrace] = e.backtrace if backtrace
+    @logger.error(msg, details)
+  end
+
   private
 
   def connect
@@ -235,7 +267,7 @@ class LogStash::Outputs::Tcp < LogStash::Outputs::Base
           raise
         end
       end
-      client_socket.instance_eval { class << self; include ::LogStash::Util::SocketPeer end }
+      client_socket.extend(::LogStash::Util::SocketPeer)
       @logger.debug("opened connection", :client => client_socket.peer)
       return client_socket
     rescue => e
@@ -253,16 +285,4 @@ class LogStash::Outputs::Tcp < LogStash::Outputs::Base
     execution_context.pipeline_id || 'main'
   end
 
-  def log_warn(msg, e, backtrace: @logger.debug?, **details)
-    details = details.merge message: e.message, exception: e.class
-    details[:backtrace] = e.backtrace if backtrace
-    @logger.warn(msg, details)
-  end
-
-  def log_error(msg, e, backtrace: @logger.info?, **details)
-    details = details.merge message: e.message, exception: e.class
-    details[:backtrace] = e.backtrace if backtrace
-    @logger.error(msg, details)
-  end
-
 end # class LogStash::Outputs::Tcp

data/logstash-output-tcp.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-output-tcp'
-  s.version         = '6.1.0'
+  s.version         = '6.1.2'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Writes events over a TCP socket"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/outputs/tcp_spec.rb

@@ -108,6 +108,33 @@ describe LogStash::Outputs::Tcp do
     end
   end
 
+  context "client mode" do
+    before { subject.register }
+
+    let(:config) { super().merge 'mode' => 'client' }
+
+    it 'writes payload data' do
+      Thread.start { sleep 0.25; subject.receive event }
+
+      socket = server.accept
+      read = socket.sysread(100)
+
+      expect( read.size ).to be > 0
+      expect( read ).to eq(JSON.generate(event))
+    end
+
+    it 'writes payload data in multiple operations' do
+      full_payload = JSON.generate(event)
+      Thread.start { sleep 0.25; subject.receive event }
+
+      socket = server.accept
+      first_read = socket.sysread((full_payload.length / 2))
+      second_read = socket.sysread(((full_payload.length / 2) + 1))
+
+      expect( "#{first_read}#{second_read}" ).to eq(full_payload)
+    end
+  end
+
   context "when enabling SSL" do
     let(:config) { super().merge("ssl_enable" => true, 'codec' => 'plain') }
     context "and not providing a certificate/key pair" do
@@ -227,5 +254,58 @@ describe LogStash::Outputs::Tcp do
 
       end
     end
+
+    context "and protocol is TLSv1.3" do
+      let(:key_file) { File.join(FIXTURES_PATH, 'plaintext/instance.key') }
+      let(:crt_file) { File.join(FIXTURES_PATH, 'plaintext/instance.crt') }
+      let(:config) { super().merge("ssl_cert" => crt_file, "ssl_key" => key_file) }
+
+      let(:secure_server) do
+        ssl_context = OpenSSL::SSL::SSLContext.new
+        ssl_context.verify_mode = OpenSSL::SSL::VERIFY_NONE
+        ssl_context.cert = OpenSSL::X509::Certificate.new(File.read(crt_file))
+        ssl_context.key = OpenSSL::PKey::RSA.new(File.read(key_file), nil)
+        ssl_context.min_version = OpenSSL::SSL::TLS1_3_VERSION
+        OpenSSL::SSL::SSLServer.new(server, ssl_context)
+      end
+
+      before(:each) do
+        subject.register
+      end
+
+      after(:each) do
+        secure_server.close rescue nil
+      end
+
+      let(:message) { "a" }
+      let(:buffer) { "" }
+
+      # This test confirms that this plugin is able to write to a TLS socket
+      # multiple times.
+      # Previous implementation performed an IO.select first and called sysread
+      # if select signaled the socket was ready to read.
+      # For TLS1_3, due to control messages it may happen that the underlying
+      # socket is marked as readable but there is no new data available,
+      # causing a read to block forever.
+      # This test will raise a Timeout exception with the old implementation.
+      it 'successfully writes two messages' do
+        thread = Thread.start do
+          expect {
+            client = secure_server.accept
+            Timeout::timeout(5) do
+              buffer << client.sysread(1) # read first message
+              subject.receive(message)
+              buffer << client.sysread(1) # read second message
+              client.close
+            end
+          }.to_not raise_error
+        end
+        sleep 0.1 until thread.status == "sleep" # wait for TCP port to open
+        subject.receive(message) # send first message to unblock call to `accept`
+        thread.join(2)
+
+        expect(buffer).to eq(message * 2)
+      end
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-tcp
 version: !ruby/object:Gem::Version
-  version: 6.1.0
+  version: 6.1.2
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-06-06 00:00:00.000000000 Z
+date: 2023-06-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -171,7 +171,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.6
+rubygems_version: 3.2.33
 signing_key:
 specification_version: 4
 summary: Writes events over a TCP socket