logstash-input-gelf 3.0.7 → 3.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +3 -0
- data/LICENSE +1 -1
- data/docs/index.asciidoc +37 -1
- data/lib/logstash/inputs/gelf.rb +119 -11
- data/logstash-input-gelf.gemspec +1 -1
- data/spec/inputs/gelf_spec.rb +23 -25
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b380005f437f84a9255e8583c3b3ea1fc6c54e7c7f4ab67aef8f5e674729887a
|
4
|
+
data.tar.gz: fe84ff257d1cffec9cf3c9281e08a9dd3b5d32388eb7c50682ba2ec38db7ce28
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 2a0aea419cecf15aef6c248b71c3c46098ed974cc534cb32746c416feae7e6c9de03126a399b23ef5540ab5c306aaac9429f629fdd03499387507583a58f9791
|
7
|
+
data.tar.gz: dd6dd7344a6facc9519e9beee80aff02771fea93023af8578dcf3593d97364576e97ff01ca95bb32124aa3a62bf3019615a13b10160c98aed308c0da2e41d6fe
|
data/CHANGELOG.md
CHANGED
data/LICENSE
CHANGED
data/docs/index.asciidoc
CHANGED
@@ -44,7 +44,11 @@ This plugin supports the following configuration options plus the <<plugins-{typ
|
|
44
44
|
|=======================================================================
|
45
45
|
|Setting |Input type|Required
|
46
46
|
| <<plugins-{type}s-{plugin}-host>> |<<string,string>>|No
|
47
|
+
| <<plugins-{type}s-{plugin}-use_udp>> |<<boolean,boolean>>|No
|
48
|
+
| <<plugins-{type}s-{plugin}-use_tcp>> |<<boolean,boolean>>|No
|
47
49
|
| <<plugins-{type}s-{plugin}-port>> |<<number,number>>|No
|
50
|
+
| <<plugins-{type}s-{plugin}-port_tcp>> |<<number,number>>|No
|
51
|
+
| <<plugins-{type}s-{plugin}-port_udp>> |<<number,number>>|No
|
48
52
|
| <<plugins-{type}s-{plugin}-remap>> |<<boolean,boolean>>|No
|
49
53
|
| <<plugins-{type}s-{plugin}-strip_leading_underscore>> |<<boolean,boolean>>|No
|
50
54
|
|=======================================================================
|
@@ -62,6 +66,22 @@ input plugins.
|
|
62
66
|
|
63
67
|
The IP address or hostname to listen on.
|
64
68
|
|
69
|
+
[id="plugins-{type}s-{plugin}-use_udp"]
|
70
|
+
===== `use_udp`
|
71
|
+
|
72
|
+
* Value type is <<boolean,boolean>>
|
73
|
+
* Default value is `true`
|
74
|
+
|
75
|
+
Whether to listen for gelf messages sent over udp
|
76
|
+
|
77
|
+
[id="plugins-{type}s-{plugin}-use_tcp"]
|
78
|
+
===== `use_tcp`
|
79
|
+
|
80
|
+
* Value type is <<boolean,boolean>>
|
81
|
+
* Default value is `false`
|
82
|
+
|
83
|
+
Whether to listen for gelf messages sent over tcp
|
84
|
+
|
65
85
|
[id="plugins-{type}s-{plugin}-port"]
|
66
86
|
===== `port`
|
67
87
|
|
@@ -70,6 +90,22 @@ The IP address or hostname to listen on.
|
|
70
90
|
|
71
91
|
The port to listen on. Remember that ports less than 1024 (privileged
|
72
92
|
ports) may require root to use.
|
93
|
+
port_tcp and port_udp can be used to set a specific port for each protocol.
|
94
|
+
[id="plugins-{type}s-{plugin}-port_tcp"]
|
95
|
+
===== `port_tcp`
|
96
|
+
|
97
|
+
* Value type is <<number,number>>
|
98
|
+
* There is no default value for this setting.
|
99
|
+
|
100
|
+
Tcp port to listen on. Use port instead of this setting unless you need a different port for udp than tcp
|
101
|
+
|
102
|
+
[id="plugins-{type}s-{plugin}-port_udp"]
|
103
|
+
===== `port_udp`
|
104
|
+
|
105
|
+
* Value type is <<number,number>>
|
106
|
+
* There is no default value for this setting.
|
107
|
+
|
108
|
+
Udp port to listen on. Use port instead of this setting unless you need a different port for udp than tcp
|
73
109
|
|
74
110
|
[id="plugins-{type}s-{plugin}-remap"]
|
75
111
|
===== `remap`
|
@@ -102,4 +138,4 @@ e.g. `\_foo` becomes `foo`
|
|
102
138
|
|
103
139
|
|
104
140
|
[id="plugins-{type}s-{plugin}-common-options"]
|
105
|
-
include::{include_path}/{type}.asciidoc[]
|
141
|
+
include::{include_path}/{type}.asciidoc[]
|
data/lib/logstash/inputs/gelf.rb
CHANGED
@@ -6,6 +6,7 @@ require "logstash/timestamp"
|
|
6
6
|
require "stud/interval"
|
7
7
|
require "date"
|
8
8
|
require "socket"
|
9
|
+
require "json"
|
9
10
|
|
10
11
|
# This input will read GELF messages as events over the network,
|
11
12
|
# making it a good choice if you already use Graylog2 today.
|
@@ -29,9 +30,12 @@ class LogStash::Inputs::Gelf < LogStash::Inputs::Base
|
|
29
30
|
# The IP address or hostname to listen on.
|
30
31
|
config :host, :validate => :string, :default => "0.0.0.0"
|
31
32
|
|
32
|
-
# The
|
33
|
+
# The ports to listen on. Remember that ports less than 1024 (privileged
|
33
34
|
# ports) may require root to use.
|
35
|
+
# port_tcp and port_udp can be used to have a different port for udp than the tcp port.
|
34
36
|
config :port, :validate => :number, :default => 12201
|
37
|
+
config :port_tcp, :validate => :number
|
38
|
+
config :port_udp, :validate => :number
|
35
39
|
|
36
40
|
# Whether or not to remap the GELF message fields to Logstash event fields or
|
37
41
|
# leave them intact.
|
@@ -59,6 +63,10 @@ class LogStash::Inputs::Gelf < LogStash::Inputs::Base
|
|
59
63
|
PARSE_FAILURE_TAG = "_jsonparsefailure"
|
60
64
|
PARSE_FAILURE_LOG_MESSAGE = "JSON parse failure. Falling back to plain-text"
|
61
65
|
|
66
|
+
# Whether or not to use TCP or/and UDP
|
67
|
+
config :use_tcp, :validate => :boolean, :default => false
|
68
|
+
config :use_udp, :validate => :boolean, :default => true
|
69
|
+
|
62
70
|
public
|
63
71
|
def initialize(params)
|
64
72
|
super
|
@@ -68,13 +76,23 @@ class LogStash::Inputs::Gelf < LogStash::Inputs::Base
|
|
68
76
|
public
|
69
77
|
def register
|
70
78
|
require 'gelfd'
|
71
|
-
|
79
|
+
@port_tcp ||= @port
|
80
|
+
@port_udp ||= @port
|
81
|
+
end
|
72
82
|
|
73
83
|
public
|
74
84
|
def run(output_queue)
|
75
85
|
begin
|
76
|
-
|
77
|
-
|
86
|
+
if @use_tcp
|
87
|
+
tcp_thr = Thread.new(output_queue) do |output_queue|
|
88
|
+
tcp_listener(output_queue)
|
89
|
+
end
|
90
|
+
end
|
91
|
+
if @use_udp
|
92
|
+
udp_thr = Thread.new(output_queue) do |output_queue|
|
93
|
+
udp_listener(output_queue)
|
94
|
+
end
|
95
|
+
end
|
78
96
|
rescue => e
|
79
97
|
unless stop?
|
80
98
|
@logger.warn("gelf listener died", :exception => e, :backtrace => e.backtrace)
|
@@ -82,23 +100,113 @@ class LogStash::Inputs::Gelf < LogStash::Inputs::Base
|
|
82
100
|
retry unless stop?
|
83
101
|
end
|
84
102
|
end # begin
|
103
|
+
if @use_tcp
|
104
|
+
tcp_thr.join
|
105
|
+
end
|
106
|
+
if @use_udp
|
107
|
+
udp_thr.join
|
108
|
+
end
|
85
109
|
end # def run
|
86
110
|
|
87
111
|
public
|
88
112
|
def stop
|
89
|
-
|
90
|
-
|
113
|
+
begin
|
114
|
+
@udp.close if @use_udp
|
115
|
+
rescue IOError => e
|
116
|
+
@logger.warn("Caugh exception while closing udp socket", :exception => e.inspect)
|
117
|
+
end
|
118
|
+
begin
|
119
|
+
@tcp.close if @use_tcp
|
120
|
+
rescue IOError => e
|
121
|
+
@logger.warn("Caugh exception while closing tcp socket", :exception => e.inspect)
|
122
|
+
end
|
123
|
+
end
|
124
|
+
|
125
|
+
private
|
126
|
+
def tcp_listener(output_queue)
|
127
|
+
|
128
|
+
@logger.info("Starting gelf listener (tcp) ...", :address => "#{@host}:#{@port_tcp}")
|
129
|
+
|
130
|
+
if @tcp.nil?
|
131
|
+
@tcp = TCPServer.new(@host, @port_tcp)
|
132
|
+
end
|
133
|
+
|
134
|
+
while !@shutdown_requested
|
135
|
+
Thread.new(@tcp.accept) do |client|
|
136
|
+
@logger.debug? && @logger.debug("Gelf (tcp): Accepting connection from: #{client.peeraddr[2]}:#{client.peeraddr[1]}")
|
137
|
+
|
138
|
+
begin
|
139
|
+
while !client.nil? && !client.eof?
|
140
|
+
|
141
|
+
begin # Read from socket
|
142
|
+
data_in = client.gets("\u0000")
|
143
|
+
rescue => ex
|
144
|
+
@logger.warn("Gelf (tcp): failed gets from client socket:", :exception => ex, :backtrace => ex.backtrace)
|
145
|
+
end
|
146
|
+
|
147
|
+
if data_in.nil?
|
148
|
+
@logger.warn("Gelf (tcp): socket read succeeded, but data is nil. Skipping.")
|
149
|
+
next
|
150
|
+
end
|
151
|
+
|
152
|
+
# data received. Remove trailing \0
|
153
|
+
data_in[-1] == "\u0000" && data_in = data_in[0...-1]
|
154
|
+
begin # Parse JSON
|
155
|
+
jsonObj = JSON.parse(data_in)
|
156
|
+
rescue => ex
|
157
|
+
@logger.warn("Gelf (tcp): failed to parse a message. Skipping: " + data_in, :exception => ex, :backtrace => ex.backtrace)
|
158
|
+
next
|
159
|
+
end
|
160
|
+
|
161
|
+
begin # Create event
|
162
|
+
event = LogStash::Event.new(jsonObj)
|
163
|
+
event.set(SOURCE_HOST_FIELD, host.force_encoding("UTF-8"))
|
164
|
+
if event.get("timestamp").is_a?(Numeric)
|
165
|
+
event.set("timestamp", LogStash::Timestamp.at(event.get("timestamp")))
|
166
|
+
event.remove("timestamp")
|
167
|
+
end
|
168
|
+
remap_gelf(event) if @remap
|
169
|
+
strip_leading_underscore(event) if @strip_leading_underscore
|
170
|
+
decorate(event)
|
171
|
+
output_queue << event
|
172
|
+
rescue => ex
|
173
|
+
@logger.warn("Gelf (tcp): failed to create event from json object. Skipping: " + jsonObj.to_s, :exception => ex, :backtrace => ex.backtrace)
|
174
|
+
end
|
175
|
+
|
176
|
+
end # while client
|
177
|
+
@logger.debug? && @logger.debug("Gelf (tcp): Closing client connection")
|
178
|
+
client.close
|
179
|
+
client = nil
|
180
|
+
rescue => ex
|
181
|
+
@logger.warn("Gelf (tcp): client socket failed.", :exception => ex, :backtrace => ex.backtrace)
|
182
|
+
ensure
|
183
|
+
if !client.nil?
|
184
|
+
@logger.debug? && @logger.debug("Gelf (tcp): Ensuring client is closed")
|
185
|
+
client.close
|
186
|
+
client = nil
|
187
|
+
end
|
188
|
+
end # begin client
|
189
|
+
end # Thread.new
|
190
|
+
end # @shutdown_requested
|
191
|
+
|
91
192
|
end
|
92
193
|
|
93
194
|
private
|
94
195
|
def udp_listener(output_queue)
|
95
|
-
@logger.info("Starting gelf listener", :address => "#{@host}:#{@
|
196
|
+
@logger.info("Starting gelf listener (udp) ...", :address => "#{@host}:#{@port_udp}")
|
96
197
|
|
97
198
|
@udp = UDPSocket.new(Socket::AF_INET)
|
98
|
-
@udp.bind(@host, @
|
199
|
+
@udp.bind(@host, @port_udp)
|
99
200
|
|
100
|
-
while
|
101
|
-
|
201
|
+
while !@udp.closed?
|
202
|
+
begin
|
203
|
+
line, client = @udp.recvfrom(8192)
|
204
|
+
rescue => e
|
205
|
+
if !stop? # if we're shutting down there's no point in logging anything
|
206
|
+
@logger.error("Caught exception while reading from UDP socket", :exception => e.inspect)
|
207
|
+
end
|
208
|
+
next
|
209
|
+
end
|
102
210
|
|
103
211
|
begin
|
104
212
|
data = Gelfd::Parser.parse(line)
|
@@ -129,7 +237,7 @@ class LogStash::Inputs::Gelf < LogStash::Inputs::Base
|
|
129
237
|
event = parse(json_gelf)
|
130
238
|
return if event.nil?
|
131
239
|
|
132
|
-
event.set(SOURCE_HOST_FIELD, host)
|
240
|
+
event.set(SOURCE_HOST_FIELD, host.force_encoding("UTF-8"))
|
133
241
|
|
134
242
|
if (gelf_timestamp = event.get(TIMESTAMP_GELF_FIELD)).is_a?(Numeric)
|
135
243
|
event.timestamp = self.coerce_timestamp(gelf_timestamp)
|
data/logstash-input-gelf.gemspec
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-input-gelf'
|
4
|
-
s.version = '3.0
|
4
|
+
s.version = '3.1.0'
|
5
5
|
s.licenses = ['Apache License (2.0)']
|
6
6
|
s.summary = "Reads GELF-format messages from Graylog2 as events"
|
7
7
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
data/spec/inputs/gelf_spec.rb
CHANGED
@@ -16,36 +16,34 @@ describe LogStash::Inputs::Gelf do
|
|
16
16
|
before { producer.run }
|
17
17
|
after { producer.stop }
|
18
18
|
|
19
|
-
|
20
19
|
it_behaves_like "an interruptible input plugin"
|
21
20
|
end
|
22
21
|
|
23
|
-
|
24
|
-
port
|
25
|
-
host
|
26
|
-
chunksize
|
27
|
-
gelfclient
|
22
|
+
describe "chunked gelf messages" do
|
23
|
+
let(:port) { 12209 }
|
24
|
+
let(:host) { "127.0.0.1" }
|
25
|
+
let(:chunksize) { 1420 }
|
26
|
+
let(:gelfclient) { GELF::Notifier.new(host, port, chunksize) }
|
27
|
+
let(:large_random) { 2000.times.map{32 + rand(126 - 32)}.join("") }
|
28
28
|
|
29
|
-
|
30
|
-
|
31
|
-
gelf {
|
32
|
-
port => "#{port}"
|
33
|
-
host => "#{host}"
|
34
|
-
}
|
35
|
-
}
|
36
|
-
CONFIG
|
29
|
+
let(:config) { { "port" => port, "host" => host } }
|
30
|
+
let(:queue) { Queue.new }
|
37
31
|
|
38
|
-
|
32
|
+
subject { described_class.new(config) }
|
39
33
|
|
40
|
-
messages
|
34
|
+
let(:messages) { [
|
41
35
|
"hello",
|
42
36
|
"world",
|
43
37
|
large_random,
|
44
38
|
"we survived gelf!"
|
45
|
-
]
|
39
|
+
] }
|
46
40
|
|
47
|
-
|
48
|
-
|
41
|
+
before(:each) do
|
42
|
+
subject.register
|
43
|
+
Thread.new { subject.run(queue) }
|
44
|
+
end
|
45
|
+
|
46
|
+
it "processes them" do
|
49
47
|
while queue.size <= 0
|
50
48
|
gelfclient.notify!("short_message" => "prime")
|
51
49
|
sleep(0.1)
|
@@ -58,15 +56,15 @@ describe LogStash::Inputs::Gelf do
|
|
58
56
|
end
|
59
57
|
|
60
58
|
messages.each do |m|
|
61
|
-
|
59
|
+
gelfclient.notify!("short_message" => m)
|
62
60
|
end
|
63
61
|
|
64
|
-
messages.map{queue.pop}
|
65
|
-
end
|
62
|
+
events = messages.map{queue.pop}
|
66
63
|
|
67
|
-
|
68
|
-
|
69
|
-
|
64
|
+
events.each_with_index do |e, i|
|
65
|
+
expect(e.get("message")).to eq(messages[i])
|
66
|
+
expect(e.get("host")).to eq(Socket.gethostname)
|
67
|
+
end
|
70
68
|
end
|
71
69
|
end
|
72
70
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-input-gelf
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.0
|
4
|
+
version: 3.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2018-01-15 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -155,7 +155,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
155
155
|
version: '0'
|
156
156
|
requirements: []
|
157
157
|
rubyforge_project:
|
158
|
-
rubygems_version: 2.6.
|
158
|
+
rubygems_version: 2.6.13
|
159
159
|
signing_key:
|
160
160
|
specification_version: 4
|
161
161
|
summary: Reads GELF-format messages from Graylog2 as events
|