logstash-input-gelf 3.1.1 → 3.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: d9832ba400d65d31c41c684fa931d708020f132d2595a90c1074bb7d68b23e8b
4
- data.tar.gz: 111607b646a20a8a55be158af487524b26a1b81e2a7c862db327c247179f64f7
3
+ metadata.gz: 998fc138bf6014e94c8a76e60c3e3cfebc544b73b83266d5e9e84ae23b3a5f1d
4
+ data.tar.gz: 83766fd3878863f1dda826243124f3a3b7358a5b55386456a314226cbfa4508e
5
5
  SHA512:
6
- metadata.gz: 1e7d4a04aac25dafad63af6b9a296c81ec7e7611202fe9bd94ca10c1e623e1e7a8900877834aa4c8ddc04f4546d83790f5f32c76e98f1847346549ae5e84ba98
7
- data.tar.gz: 73fa9493e0a5dc35ea13d4d937c1ea06a5ad4d6c102677f32f0f47118a5bd593a4d5a854daf789fb129e072933128f8ba9f874b4746b553a2a4a8520a183631f
6
+ metadata.gz: ee88a28f662343cd752f7294a5f37fa1e275e3e5c2a39d946b18d42dc4c8a3d0e77df04cc2b3251b451442789c51afabdd18483f1263f91a25ff1f83249bb2aa
7
+ data.tar.gz: 38b4765dd2304e76590edb04f37184f450f2bca548a09f45e82b3881f5b1dc2a899cdce08537898fe7dc6b1481ca657b560e21ef44745b9bb98ae6e1c9212ef8
data/CHANGELOG.md CHANGED
@@ -1,3 +1,6 @@
1
+ ## 3.2.0
2
+ - Fixed shutdown handling, robustness in socket closing and restarting, json parsing, code DRYing and cleanups [62](https://github.com/logstash-plugins/logstash-input-gelf/pull/62)
3
+
1
4
  ## 3.1.1
2
5
  - Docs: Set the default_codec doc attribute.
3
6
 
@@ -67,143 +67,136 @@ class LogStash::Inputs::Gelf < LogStash::Inputs::Base
67
67
  config :use_tcp, :validate => :boolean, :default => false
68
68
  config :use_udp, :validate => :boolean, :default => true
69
69
 
70
- public
71
70
  def initialize(params)
72
71
  super
73
72
  BasicSocket.do_not_reverse_lookup = true
74
- end # def initialize
73
+ end
75
74
 
76
- public
77
75
  def register
78
76
  require 'gelfd'
79
77
  @port_tcp ||= @port
80
78
  @port_udp ||= @port
81
79
  end
82
80
 
83
- public
84
81
  def run(output_queue)
85
82
  begin
86
83
  if @use_tcp
87
- tcp_thr = Thread.new(output_queue) do |output_queue|
88
- tcp_listener(output_queue)
89
- end
84
+ @tcp_thr = Thread.new(output_queue) { |output_queue| tcp_listener(output_queue) }
90
85
  end
91
86
  if @use_udp
92
- udp_thr = Thread.new(output_queue) do |output_queue|
93
- udp_listener(output_queue)
94
- end
87
+ @udp_thr = Thread.new(output_queue) { |output_queue| udp_listener(output_queue) }
95
88
  end
96
89
  rescue => e
97
90
  unless stop?
98
91
  @logger.warn("gelf listener died", :exception => e, :backtrace => e.backtrace)
99
92
  Stud.stoppable_sleep(RECONNECT_BACKOFF_SLEEP) { stop? }
100
- retry unless stop?
93
+ if !stop?
94
+ # before retrying make sure we close all sockets
95
+ stop
96
+ wait_server_thread
97
+ retry
98
+ end
101
99
  end
102
- end # begin
103
- if @use_tcp
104
- tcp_thr.join
105
- end
106
- if @use_udp
107
- udp_thr.join
108
100
  end
109
- end # def run
110
101
 
111
- public
102
+ wait_server_thread
103
+ end
104
+
112
105
  def stop
113
106
  begin
114
107
  @udp.close if @use_udp
115
- rescue IOError => e
116
- @logger.warn("Caugh exception while closing udp socket", :exception => e.inspect)
108
+ @udp = nil
109
+ rescue => e
110
+ @logger.debug("Caught exception while closing udp socket", :exception => e)
117
111
  end
118
112
  begin
119
113
  @tcp.close if @use_tcp
120
- rescue IOError => e
121
- @logger.warn("Caugh exception while closing tcp socket", :exception => e.inspect)
114
+ @tcp = nil
115
+ rescue => e
116
+ @logger.debug("Caught exception while closing tcp socket", :exception => e)
122
117
  end
123
118
  end
124
119
 
125
120
  private
126
- def tcp_listener(output_queue)
127
121
 
122
+ def wait_server_thread
123
+ @tcp_thr.join if @use_tcp
124
+ @udp_thr.join if @use_udp
125
+ end
126
+
127
+ def tcp_listener(output_queue)
128
128
  @logger.info("Starting gelf listener (tcp) ...", :address => "#{@host}:#{@port_tcp}")
129
129
 
130
130
  if @tcp.nil?
131
131
  @tcp = TCPServer.new(@host, @port_tcp)
132
132
  end
133
133
 
134
- while !@shutdown_requested
134
+ while !stop?
135
135
  Thread.new(@tcp.accept) do |client|
136
- @logger.debug? && @logger.debug("Gelf (tcp): Accepting connection from: #{client.peeraddr[2]}:#{client.peeraddr[1]}")
137
-
138
136
  begin
139
- while !client.nil? && !client.eof?
137
+ @logger.debug? && @logger.debug("Gelf (tcp): Accepting connection from: #{client.peeraddr[2]}:#{client.peeraddr[1]}")
138
+
139
+ while !client.nil? && !client.eof? && !stop?
140
140
 
141
141
  begin # Read from socket
142
142
  data_in = client.gets("\u0000")
143
143
  rescue => ex
144
- @logger.warn("Gelf (tcp): failed gets from client socket:", :exception => ex, :backtrace => ex.backtrace)
144
+ if !stop?
145
+ @logger.warn("Gelf (tcp): failed gets from client socket:", :exception => ex, :backtrace => ex.backtrace)
146
+ end
145
147
  end
146
148
 
147
149
  if data_in.nil?
148
- @logger.warn("Gelf (tcp): socket read succeeded, but data is nil. Skipping.")
150
+ @logger.debug("Gelf (tcp): socket read succeeded, but data is nil. Skipping.")
149
151
  next
150
- end
152
+ end
151
153
 
152
154
  # data received. Remove trailing \0
153
155
  data_in[-1] == "\u0000" && data_in = data_in[0...-1]
154
- begin # Parse JSON
155
- jsonObj = JSON.parse(data_in)
156
- rescue => ex
157
- @logger.warn("Gelf (tcp): failed to parse a message. Skipping: " + data_in, :exception => ex, :backtrace => ex.backtrace)
158
- next
159
- end
160
156
 
161
- begin # Create event
162
- event = LogStash::Event.new(jsonObj)
163
- event.set(SOURCE_HOST_FIELD, host.force_encoding("UTF-8"))
164
- if event.get("timestamp").is_a?(Numeric)
165
- event.set("timestamp", LogStash::Timestamp.at(event.get("timestamp")))
166
- event.remove("timestamp")
167
- end
168
- remap_gelf(event) if @remap
169
- strip_leading_underscore(event) if @strip_leading_underscore
170
- decorate(event)
171
- output_queue << event
172
- rescue => ex
173
- @logger.warn("Gelf (tcp): failed to create event from json object. Skipping: " + jsonObj.to_s, :exception => ex, :backtrace => ex.backtrace)
174
- end
157
+ event = self.class.new_event(data_in, client.peeraddr[3])
158
+ next if event.nil?
159
+
160
+ remap_gelf(event) if @remap
161
+ strip_leading_underscore(event) if @strip_leading_underscore
162
+ decorate(event)
163
+ output_queue << event
164
+ end
175
165
 
176
- end # while client
177
166
  @logger.debug? && @logger.debug("Gelf (tcp): Closing client connection")
178
- client.close
167
+ client.close rescue nil
179
168
  client = nil
180
169
  rescue => ex
181
- @logger.warn("Gelf (tcp): client socket failed.", :exception => ex, :backtrace => ex.backtrace)
170
+ if !stop?
171
+ @logger.warn("Gelf (tcp): client socket failed.", :exception => ex, :backtrace => ex.backtrace)
172
+ end
182
173
  ensure
183
174
  if !client.nil?
184
175
  @logger.debug? && @logger.debug("Gelf (tcp): Ensuring client is closed")
185
- client.close
176
+ client.close rescue nil
186
177
  client = nil
187
178
  end
188
- end # begin client
189
- end # Thread.new
190
- end # @shutdown_requested
191
-
179
+ end
180
+ end
181
+ end
182
+ ensure
183
+ @logger.debug? && @logger.debug("Gelf (tcp): Ensuring tcp server is closed")
184
+ @tcp.close rescue nil
185
+ @tcp = nil
192
186
  end
193
187
 
194
- private
195
188
  def udp_listener(output_queue)
196
189
  @logger.info("Starting gelf listener (udp) ...", :address => "#{@host}:#{@port_udp}")
197
190
 
198
191
  @udp = UDPSocket.new(Socket::AF_INET)
199
192
  @udp.bind(@host, @port_udp)
200
193
 
201
- while !@udp.closed?
194
+ while !stop?
202
195
  begin
203
196
  line, client = @udp.recvfrom(8192)
204
197
  rescue => e
205
- if !stop? # if we're shutting down there's no point in logging anything
206
- @logger.error("Caught exception while reading from UDP socket", :exception => e.inspect)
198
+ if !stop?
199
+ @logger.error("Caught exception while reading from UDP socket", :exception => e)
207
200
  end
208
201
  next
209
202
  end
@@ -227,7 +220,11 @@ class LogStash::Inputs::Gelf < LogStash::Inputs::Base
227
220
 
228
221
  output_queue << event
229
222
  end
230
- end # def udp_listener
223
+ ensure
224
+ @logger.debug? && @logger.debug("Gelf (udp): Ensuring udp socket is closed")
225
+ @udp.close rescue nil
226
+ @udp = nil
227
+ end
231
228
 
232
229
  # generate a new LogStash::Event from json input and assign host to source_host event field.
233
230
  # @param json_gelf [String] GELF json data
@@ -256,32 +253,14 @@ class LogStash::Inputs::Gelf < LogStash::Inputs::Base
256
253
  timestamp.is_a?(BigDecimal) ? LogStash::Timestamp.at(timestamp.to_i, timestamp.frac * 1000000) : LogStash::Timestamp.at(timestamp)
257
254
  end
258
255
 
259
- # from_json_parse uses the Event#from_json method to deserialize and directly produce events
260
- def self.from_json_parse(json)
261
- # from_json will always return an array of item.
262
- # in the context of gelf, the payload should be an array of 1
263
- LogStash::Event.from_json(json).first
264
- rescue LogStash::Json::ParserError => e
265
- logger.error(PARSE_FAILURE_LOG_MESSAGE, :error => e, :data => json)
266
- LogStash::Event.new(MESSAGE_FIELD => json, TAGS_FIELD => [PARSE_FAILURE_TAG, '_fromjsonparser'])
267
- end # def self.from_json_parse
268
-
269
- # legacy_parse uses the LogStash::Json class to deserialize json
270
- def self.legacy_parse(json)
256
+ def self.parse(json)
271
257
  o = LogStash::Json.load(json)
272
258
  LogStash::Event.new(o)
273
259
  rescue LogStash::Json::ParserError => e
274
- logger.error(PARSE_FAILURE_LOG_MESSAGE, :error => e, :data => json)
275
- LogStash::Event.new(MESSAGE_FIELD => json, TAGS_FIELD => [PARSE_FAILURE_TAG, '_legacyjsonparser'])
276
- end # def self.parse
277
-
278
- # keep compatibility with all v2.x distributions. only in 2.3 will the Event#from_json method be introduced
279
- # and we need to keep compatibility for all v2 releases.
280
- class << self
281
- alias_method :parse, LogStash::Event.respond_to?(:from_json) ? :from_json_parse : :legacy_parse
260
+ logger.error(PARSE_FAILURE_LOG_MESSAGE, :error => e, :data => json.inspect)
261
+ LogStash::Event.new(MESSAGE_FIELD => json.inspect, TAGS_FIELD => [PARSE_FAILURE_TAG])
282
262
  end
283
263
 
284
- private
285
264
  def remap_gelf(event)
286
265
  if event.get("full_message") && !event.get("full_message").empty?
287
266
  event.set("message", event.get("full_message").dup)
@@ -293,9 +272,8 @@ class LogStash::Inputs::Gelf < LogStash::Inputs::Base
293
272
  event.set("message", event.get("short_message").dup)
294
273
  event.remove("short_message")
295
274
  end
296
- end # def remap_gelf
275
+ end
297
276
 
298
- private
299
277
  def strip_leading_underscore(event)
300
278
  # Map all '_foo' fields to simply 'foo'
301
279
  event.to_hash.keys.each do |key|
@@ -303,5 +281,5 @@ class LogStash::Inputs::Gelf < LogStash::Inputs::Base
303
281
  event.set(key[1..-1], event.get(key))
304
282
  event.remove(key)
305
283
  end
306
- end # deef removing_leading_underscores
307
- end # class LogStash::Inputs::Gelf
284
+ end
285
+ end
@@ -1,7 +1,7 @@
1
1
  Gem::Specification.new do |s|
2
2
 
3
3
  s.name = 'logstash-input-gelf'
4
- s.version = '3.1.1'
4
+ s.version = '3.2.0'
5
5
  s.licenses = ['Apache License (2.0)']
6
6
  s.summary = "Reads GELF-format messages from Graylog2 as events"
7
7
  s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -24,7 +24,7 @@ Gem::Specification.new do |s|
24
24
 
25
25
  s.add_runtime_dependency "gelfd", ["0.2.0"] #(Apache 2.0 license)
26
26
  s.add_runtime_dependency 'logstash-codec-plain'
27
- s.add_runtime_dependency "stud", "~> 0.0.22"
27
+ s.add_runtime_dependency 'stud', '>= 0.0.22', '< 0.1.0'
28
28
 
29
29
  s.add_development_dependency 'logstash-devutils'
30
30
  s.add_development_dependency "gelf", ["3.0.0"] #(MIT license)
@@ -148,38 +148,15 @@ describe LogStash::Inputs::Gelf do
148
148
  context "when an invalid JSON is fed to the listener" do
149
149
  subject { LogStash::Inputs::Gelf.new_event(message, "host") }
150
150
  let(:message) { "Invalid JSON message" }
151
+ context "JSON parser output" do
152
+ it { should be_a(LogStash::Event) }
151
153
 
152
- if LogStash::Event.respond_to?(:from_json)
153
- context "default :from_json parser output" do
154
- it { should be_a(LogStash::Event) }
155
-
156
- it "falls back to plain-text" do
157
- expect(subject.get("message")).to eq(message)
158
- end
159
-
160
- it "tags message with _jsonparsefailure" do
161
- expect(subject.get("tags")).to include("_jsonparsefailure")
162
- end
163
-
164
- it "tags message with _fromjsonparser" do
165
- expect(subject.get("tags")).to include("_fromjsonparser")
166
- end
154
+ it "falls back to plain-text" do
155
+ expect(subject.get("message")).to eq(message.inspect)
167
156
  end
168
- else
169
- context "legacy JSON parser output" do
170
- it { should be_a(LogStash::Event) }
171
-
172
- it "falls back to plain-text" do
173
- expect(subject.get("message")).to eq(message)
174
- end
175
-
176
- it "tags message with _jsonparsefailure" do
177
- expect(subject.get("tags")).to include("_jsonparsefailure")
178
- end
179
157
 
180
- it "tags message with _legacyjsonparser" do
181
- expect(subject.get("tags")).to include("_legacyjsonparser")
182
- end
158
+ it "tags message with _jsonparsefailure" do
159
+ expect(subject.get("tags")).to include("_jsonparsefailure")
183
160
  end
184
161
  end
185
162
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-input-gelf
3
3
  version: !ruby/object:Gem::Version
4
- version: 3.1.1
4
+ version: 3.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Elastic
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2018-04-06 00:00:00.000000000 Z
11
+ date: 2018-12-12 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -61,17 +61,23 @@ dependencies:
61
61
  - !ruby/object:Gem::Dependency
62
62
  requirement: !ruby/object:Gem::Requirement
63
63
  requirements:
64
- - - "~>"
64
+ - - ">="
65
65
  - !ruby/object:Gem::Version
66
66
  version: 0.0.22
67
+ - - "<"
68
+ - !ruby/object:Gem::Version
69
+ version: 0.1.0
67
70
  name: stud
68
71
  prerelease: false
69
72
  type: :runtime
70
73
  version_requirements: !ruby/object:Gem::Requirement
71
74
  requirements:
72
- - - "~>"
75
+ - - ">="
73
76
  - !ruby/object:Gem::Version
74
77
  version: 0.0.22
78
+ - - "<"
79
+ - !ruby/object:Gem::Version
80
+ version: 0.1.0
75
81
  - !ruby/object:Gem::Dependency
76
82
  requirement: !ruby/object:Gem::Requirement
77
83
  requirements:
@@ -155,7 +161,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
155
161
  version: '0'
156
162
  requirements: []
157
163
  rubyforge_project:
158
- rubygems_version: 2.6.11
164
+ rubygems_version: 2.6.13
159
165
  signing_key:
160
166
  specification_version: 4
161
167
  summary: Reads GELF-format messages from Graylog2 as events