logstash-output-tcp 6.0.2 → 6.0.3

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: f565b112e5a38f65bebf7c13061e0b7de65a460952afdc93bcf145998bd6ae30
4
- data.tar.gz: 97b30dfceff9b0d2187c408e3ac5fe4abc8def98e9fdd4e59580fa2f0d112ef9
3
+ metadata.gz: 5db164894be7c046e3dce9337af0ce5b684f19b1b35274a0d3b925e97fe402c8
4
+ data.tar.gz: 63dbb52d49285af564f83de8218c3af098b1d0f62d9deee019c15eae5cbd98e2
5
5
  SHA512:
6
- metadata.gz: ffec33897e55cd02b237b385dc4e6ffe1d90f2bb87b2fc4b1dbd4bb050f1db9787038885b7fbd77539b9e60fab1a44e1f9567925f7dc8a36eaab4be0d158b2a3
7
- data.tar.gz: cb2c4871d74fd661ee2d947a94535483fd54854fee574d4615a05634df74b97fedb6e01d73f645e8cc70e41de80fbf1eeceaa466855ff757cc320f19130f8362
6
+ metadata.gz: 10b22e722c22edce1fd3699e876ab898283d66ec0aa256f9d9c6159112bcf54f872b4e68d8bc7eb72b999c54442d5b65c69fa87194f6c6a58623f7c1939c9c09
7
+ data.tar.gz: eb6477d5f227184f46bac1c01453215fbd2f2ebf3c0fbb819413d5555d2ab7472ab61b11d737016487b3d1255bcc65fce008f350683ad28e9468fbb1f55ded49
data/CHANGELOG.md CHANGED
@@ -1,3 +1,9 @@
1
+ ## 6.0.3
2
+ - Pulled applicable back-ports from 6.1.0 [#50](https://github.com/logstash-plugins/logstash-output-tcp/pull/50)
3
+ - Fix: Ensure sockets are closed when this plugin is closed
4
+ - Fix: Fixes an issue in client mode where payloads larger than a connection's current TCP window could be silently truncated
5
+ - Fix: Fixes an issue in server mode where payloads larger than a connection's current TCP window could be silently truncated
6
+
1
7
  ## 6.0.2
2
8
  - Fix: unable to start with password protected key [#45](https://github.com/logstash-plugins/logstash-output-tcp/pull/45)
3
9
 
@@ -51,31 +51,44 @@ class LogStash::Outputs::Tcp < LogStash::Outputs::Base
51
51
  # SSL key passphrase
52
52
  config :ssl_key_passphrase, :validate => :password, :default => nil
53
53
 
54
+ ##
55
+ # @param socket [Socket]
56
+ # @param logger_context [#log_warn&#log_error&#logger]
54
57
  class Client
55
- public
56
- def initialize(socket, logger)
58
+ def initialize(socket, logger_context)
57
59
  @socket = socket
58
- @logger = logger
60
+ @peer_info = socket.peer
61
+ @logger_context = logger_context
59
62
  @queue = Queue.new
60
63
  end
61
64
 
62
- public
63
65
  def run
64
66
  loop do
65
67
  begin
66
- @socket.write(@queue.pop)
68
+ payload = @queue.pop
69
+
70
+ @logger_context.logger.trace("transmitting #{payload.bytesize} bytes", socket: @peer_info) if @logger_context.logger.trace? && payload && !payload.empty?
71
+ while payload && !payload.empty?
72
+ written_bytes_size = @socket.write(payload)
73
+ payload = payload.byteslice(written_bytes_size..-1)
74
+ @logger_context.logger.log_trace(">transmitted #{written_bytes_size} bytes; #{payload.bytesize} bytes remain", socket: @peer_info) if @logger_context.logger.trace?
75
+ end
67
76
  rescue => e
68
- @logger.warn("tcp output exception", :socket => @socket,
69
- :exception => e)
77
+ @logger_context.log_warn("tcp output exception: socket write failed", e, :socket => @peer_info)
70
78
  break
71
79
  end
72
80
  end
73
81
  end # def run
74
82
 
75
- public
76
83
  def write(msg)
77
84
  @queue.push(msg)
78
85
  end # def write
86
+
87
+ def close
88
+ @socket.close
89
+ rescue => e
90
+ @logger_context.log_warn 'socket close failed:', e, socket: @socket&.to_s
91
+ end
79
92
  end # class Client
80
93
 
81
94
  private
@@ -113,6 +126,8 @@ class LogStash::Outputs::Tcp < LogStash::Outputs::Base
113
126
  if @ssl_enable
114
127
  setup_ssl
115
128
  end # @ssl_enable
129
+ @closed = Concurrent::AtomicBoolean.new(false)
130
+ @thread_no = Concurrent::AtomicFixnum.new(0)
116
131
 
117
132
  if server?
118
133
  @logger.info("Starting tcp output listener", :address => "#{@host}:#{@port}")
@@ -129,44 +144,61 @@ class LogStash::Outputs::Tcp < LogStash::Outputs::Base
129
144
  @client_threads = []
130
145
 
131
146
  @accept_thread = Thread.new(@server_socket) do |server_socket|
147
+ LogStash::Util.set_thread_name("[#{pipeline_id}]|output|tcp|server_accept")
132
148
  loop do
149
+ break if @closed.value
133
150
  Thread.start(server_socket.accept) do |client_socket|
134
151
  # monkeypatch a 'peer' method onto the socket.
135
- client_socket.instance_eval { class << self; include ::LogStash::Util::SocketPeer end }
152
+ client_socket.extend(::LogStash::Util::SocketPeer)
136
153
  @logger.debug("Accepted connection", :client => client_socket.peer,
137
154
  :server => "#{@host}:#{@port}")
138
- client = Client.new(client_socket, @logger)
155
+ client = Client.new(client_socket, self)
139
156
  Thread.current[:client] = client
157
+ LogStash::Util.set_thread_name("[#{pipeline_id}]|output|tcp|client_socket-#{@thread_no.increment}")
140
158
  @client_threads << Thread.current
141
- client.run
159
+ client.run unless @closed.value
142
160
  end
143
161
  end
144
162
  end
145
163
 
146
164
  @codec.on_event do |event, payload|
165
+ @client_threads.select!(&:alive?)
147
166
  @client_threads.each do |client_thread|
148
167
  client_thread[:client].write(payload)
149
168
  end
150
- @client_threads.reject! {|t| !t.alive? }
151
169
  end
152
170
  else
153
- client_socket = nil
171
+ @client_socket = nil
172
+ peer_info = nil
154
173
  @codec.on_event do |event, payload|
155
174
  begin
156
- client_socket = connect unless client_socket
157
- r,w,e = IO.select([client_socket], [client_socket], [client_socket], nil)
158
- # don't expect any reads, but a readable socket might
159
- # mean the remote end closed, so read it and throw it away.
160
- # we'll get an EOFError if it happens.
161
- client_socket.sysread(16384) if r.any?
175
+ # not threadsafe; this is why we require `concurrency: single`
176
+ unless @client_socket
177
+ @client_socket = connect
178
+ peer_info = @client_socket.peer
179
+ end
180
+
181
+ writable_io = nil
182
+ while writable_io.nil? || writable_io.any? == false
183
+ readable_io, writable_io, _ = IO.select([@client_socket],[@client_socket])
184
+
185
+ # don't expect any reads, but a readable socket might
186
+ # mean the remote end closed, so read it and throw it away.
187
+ # we'll get an EOFError if it happens.
188
+ readable_io.each { |readable| readable.sysread(16384) }
189
+ end
162
190
 
163
191
  # Now send the payload
164
- client_socket.syswrite(payload) if w.any?
192
+ @logger.trace("transmitting #{payload.bytesize} bytes", socket: peer_info) if @logger.trace? && payload && !payload.empty?
193
+ while payload && payload.bytesize > 0
194
+ written_bytes_size = @client_socket.syswrite(payload)
195
+ payload = payload.byteslice(written_bytes_size..-1)
196
+ @logger.trace(">transmitted #{written_bytes_size} bytes; #{payload.bytesize} bytes remain", socket: peer_info) if @logger.trace?
197
+ end
165
198
  rescue => e
166
- @logger.warn("tcp output exception", :host => @host, :port => @port,
167
- :exception => e, :backtrace => e.backtrace)
168
- client_socket.close rescue nil
169
- client_socket = nil
199
+ log_warn "client socket failed:", e, host: @host, port: @port, socket: peer_info
200
+ @client_socket.close rescue nil
201
+ @client_socket = nil
170
202
  sleep @reconnect_interval
171
203
  retry
172
204
  end
@@ -174,6 +206,23 @@ class LogStash::Outputs::Tcp < LogStash::Outputs::Base
174
206
  end
175
207
  end # def register
176
208
 
209
+ # @overload Base#close
210
+ def close
211
+ if server?
212
+ # server-mode clean-up
213
+ @closed.make_true
214
+ @server_socket.shutdown rescue nil if @server_socket
215
+
216
+ @client_threads&.each do |thread|
217
+ client = thread[:client]
218
+ client.close rescue nil if client
219
+ end
220
+ else
221
+ # client-mode clean-up
222
+ @client_socket&.close
223
+ end
224
+ end
225
+
177
226
  private
178
227
  def connect
179
228
  begin
@@ -183,17 +232,17 @@ class LogStash::Outputs::Tcp < LogStash::Outputs::Base
183
232
  begin
184
233
  client_socket.connect
185
234
  rescue OpenSSL::SSL::SSLError => ssle
186
- @logger.error("SSL Error", :exception => ssle, :backtrace => ssle.backtrace)
235
+ log_error 'connect ssl failure:', ssle, backtrace: false
187
236
  # NOTE(mrichar1): Hack to prevent hammering peer
188
237
  sleep(5)
189
238
  raise
190
239
  end
191
240
  end
192
- client_socket.instance_eval { class << self; include ::LogStash::Util::SocketPeer end }
241
+ client_socket.extend(::LogStash::Util::SocketPeer)
193
242
  @logger.debug("Opened connection", :client => "#{client_socket.peer}")
194
243
  return client_socket
195
244
  rescue StandardError => e
196
- @logger.error("Failed to connect: #{e.message}", :exception => e.class, :backtrace => e.backtrace)
245
+ log_error 'failed to connect:', e
197
246
  sleep @reconnect_interval
198
247
  retry
199
248
  end
@@ -208,4 +257,20 @@ class LogStash::Outputs::Tcp < LogStash::Outputs::Base
208
257
  def receive(event)
209
258
  @codec.encode(event)
210
259
  end # def receive
260
+
261
+ def pipeline_id
262
+ execution_context.pipeline_id || 'main'
263
+ end
264
+
265
+ def log_warn(msg, e, backtrace: @logger.debug?, **details)
266
+ details = details.merge message: e.message, exception: e.class
267
+ details[:backtrace] = e.backtrace if backtrace
268
+ @logger.warn(msg, details)
269
+ end
270
+
271
+ def log_error(msg, e, backtrace: @logger.info?, **details)
272
+ details = details.merge message: e.message, exception: e.class
273
+ details[:backtrace] = e.backtrace if backtrace
274
+ @logger.error(msg, details)
275
+ end
211
276
  end # class LogStash::Outputs::Tcp
@@ -1,7 +1,7 @@
1
1
  Gem::Specification.new do |s|
2
2
 
3
3
  s.name = 'logstash-output-tcp'
4
- s.version = '6.0.2'
4
+ s.version = '6.0.3'
5
5
  s.licenses = ['Apache License (2.0)']
6
6
  s.summary = "Writes events over a TCP socket"
7
7
  s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -3,7 +3,7 @@ require "logstash/outputs/tcp"
3
3
  require "flores/pki"
4
4
 
5
5
  describe LogStash::Outputs::Tcp do
6
- subject { described_class.new(config) }
6
+ subject(:instance) { described_class.new(config) }
7
7
  let(:config) { {
8
8
  "host" => "localhost",
9
9
  "port" => 2000 + rand(3000),
@@ -73,4 +73,139 @@ describe LogStash::Outputs::Tcp do
73
73
  end
74
74
  end
75
75
  end
76
+
77
+ ##
78
+ # Reads `in_io` until EOF and writes the bytes
79
+ # it receives to `out_io`, tolerating partial writes.
80
+ def siphon_until_eof(in_io, out_io)
81
+ buffer = ""
82
+ while (retval = in_io.read_nonblock(32*1024, buffer, exception:false)) do
83
+ (IO.select([in_io], nil, nil, 5); next) if retval == :wait_readable
84
+
85
+ while (buffer && !buffer.empty?) do
86
+ bytes_written = out_io.write(buffer)
87
+ buffer.replace buffer.byteslice(bytes_written..-1)
88
+ end
89
+ end
90
+ end
91
+
92
+ context 'client mode' do
93
+ context 'transmitting data' do
94
+ let!(:io) { StringIO.new } # somewhere for our server to stash the data it receives
95
+
96
+ let(:server_host) { 'localhost' }
97
+ let(:server_port) { server.addr[1] } # get actual since we bind to port 0
98
+
99
+ let!(:server) { TCPServer.new(server_host, 0) }
100
+
101
+ let(:config) do
102
+ { 'host' => server_host, 'port' => server_port, 'mode' => 'client' }
103
+ end
104
+
105
+ let(:event) { LogStash::Event.new({"hello" => "world"})}
106
+
107
+ subject(:instance) { described_class.new(config) }
108
+
109
+ before(:each) do
110
+ # accepts ONE connection
111
+ @server_socket_thread = Thread.start do
112
+ client = server.accept
113
+ siphon_until_eof(client, io)
114
+ end
115
+ instance.register
116
+ end
117
+
118
+ after(:each) do
119
+ @server_socket_thread&.join
120
+ end
121
+
122
+ it 'encodes and transmits data' do
123
+ instance.receive(event)
124
+ sleep 1
125
+ instance.close # release the connection
126
+ @server_socket_thread.join(30) || fail('server failed to join')
127
+ expect(io.string).to include('"hello"','"world"')
128
+ end
129
+
130
+ context 'when payload is very large' do
131
+ let(:one_hundred_megabyte_message) { "a" * 1024 * 1024 * 100 }
132
+ let(:event) { LogStash::Event.new("message" => one_hundred_megabyte_message) }
133
+
134
+
135
+ it 'encodes and transmits data' do
136
+ instance.receive(event)
137
+ sleep 1
138
+ instance.close # release the connection
139
+ @server_socket_thread.join(30) || fail('server failed to join')
140
+ expect(io.string).to include('"message"',%Q("#{one_hundred_megabyte_message}"))
141
+ end
142
+ end
143
+ end
144
+ end
145
+
146
+ context 'server mode' do
147
+
148
+ def wait_for_condition(total_time_in_seconds, &block)
149
+ deadline = Time.now + total_time_in_seconds
150
+ until Time.now > deadline
151
+ return if yield
152
+ sleep(1)
153
+ end
154
+ fail('condition not met!')
155
+ end
156
+
157
+ context 'transmitting data' do
158
+ let(:server_host) { 'localhost' }
159
+ let(:server_port) { Random.rand(1024...5000) }
160
+
161
+ let(:config) do
162
+ { 'host' => server_host, 'port' => server_port, 'mode' => 'server' }
163
+ end
164
+
165
+ subject(:instance) { described_class.new(config) }
166
+
167
+ before(:each) { instance.register } # start listener
168
+ after(:each) { instance.close }
169
+
170
+ let(:event) { LogStash::Event.new({"hello" => "world"})}
171
+
172
+ context 'when one client is connected' do
173
+ let(:io) { StringIO.new }
174
+ let(:client_socket) { TCPSocket.new(server_host, server_port) }
175
+
176
+ before(:each) do
177
+ @client_socket_thread = Thread.start { siphon_until_eof(client_socket, io) }
178
+ sleep 1 # wait for it to actually connect
179
+ end
180
+
181
+ it 'encodes and transmits data' do
182
+ sleep 1
183
+ instance.receive(event)
184
+
185
+ wait_for_condition(30) { !io.size.zero? }
186
+ sleep 1 # wait for the event to get sent...
187
+ instance.close # release the connection
188
+
189
+ @client_socket_thread.join(30) || fail('client failed to join')
190
+ expect(io.string).to include('"hello"','"world"')
191
+ end
192
+
193
+ context 'when payload is very large' do
194
+ let(:one_hundred_megabyte_message) { "a" * 1024 * 1024 * 100 }
195
+ let(:event) { LogStash::Event.new("message" => one_hundred_megabyte_message) }
196
+
197
+ it 'encodes and transmits data' do
198
+ instance.receive(event)
199
+
200
+ wait_for_condition(30) { io.size >= one_hundred_megabyte_message.size }
201
+ sleep 1 # wait for the event to get sent...
202
+ instance.close # release the connection
203
+
204
+ @client_socket_thread.join(30) || fail('client failed to join')
205
+ expect(io.string).to include('"message"',%Q("#{one_hundred_megabyte_message}"))
206
+ end
207
+ end
208
+ end
209
+ end
210
+ end
76
211
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-output-tcp
3
3
  version: !ruby/object:Gem::Version
4
- version: 6.0.2
4
+ version: 6.0.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Elastic
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2022-03-21 00:00:00.000000000 Z
11
+ date: 2022-11-04 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement