fluent-plugin-netflowipfix 1.1.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +6 -1
- data/fluent-plugin-netflowipfix.gemspec +1 -1
- data/lib/.gitignore +1 -0
- data/lib/fluent/.gitignore +1 -0
- data/lib/fluent/plugin/in_netflowipfix.rb +28 -63
- metadata +4 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: fe5554635cf4c0166f3b748a2df446cfe7761ba9
|
4
|
+
data.tar.gz: a681929a171a350c44dbab185157cd2fb7b77f17
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 5ad6ea83415dbbaefc176faa638ab5982adfd34560cff3ad37e73304bb6f6e858158229a963a68f449a00575d8334f59bb64ac3ea70554bc99dec0dbdd6d23b3
|
7
|
+
data.tar.gz: f6f380dbf85051517358e92ba9aa5c0dd8d46351d113cf370ceb0b02d9c6cc93d0a945766fa9ea509520809aebe0942c2a257f5a2cb0e9d658105a5b507ca00d
|
data/.gitignore
CHANGED
data/lib/.gitignore
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
.DS_Store
|
@@ -0,0 +1 @@
|
|
1
|
+
.DS_Store
|
@@ -30,15 +30,16 @@ module Fluent
|
|
30
30
|
include DetachMultiProcessMixin
|
31
31
|
|
32
32
|
class PortConnection
|
33
|
-
def initialize(bind, port, tag, cache_ttl, definitions, queuesleep)
|
33
|
+
def initialize(bind, port, tag, cache_ttl, definitions, queuesleep, log)
|
34
34
|
@bind = bind
|
35
35
|
@port = port
|
36
36
|
@tag = tag
|
37
37
|
@cache_ttl = cache_ttl
|
38
38
|
@definitions = definitions
|
39
39
|
@eventQueue = Queue.new
|
40
|
-
@udpQueue = Queue.new
|
40
|
+
# @udpQueue = Queue.new
|
41
41
|
@queuesleep = queuesleep
|
42
|
+
@log = log
|
42
43
|
end
|
43
44
|
|
44
45
|
def bind
|
@@ -52,8 +53,8 @@ class PortConnection
|
|
52
53
|
end
|
53
54
|
|
54
55
|
def start
|
55
|
-
@thread_udp = UdpListenerThread.new(@bind, @port, @udpQueue, @tag)
|
56
|
-
@thread_parser = ParserThread.new(@udpQueue, @queuesleep, @eventQueue, @cache_ttl, @definitions)
|
56
|
+
@thread_udp = UdpListenerThread.new(@bind, @port, @udpQueue, @tag, @log)
|
57
|
+
@thread_parser = ParserThread.new(@udpQueue, @queuesleep, @eventQueue, @cache_ttl, @definitions, @log)
|
57
58
|
@thread_udp.start
|
58
59
|
@thread_parser.start
|
59
60
|
end # def start
|
@@ -73,13 +74,6 @@ class PortConnection
|
|
73
74
|
end
|
74
75
|
|
75
76
|
|
76
|
-
# def udpqueue_pop
|
77
|
-
# @udpQueue.pop
|
78
|
-
# end
|
79
|
-
|
80
|
-
# def udpqueue_length
|
81
|
-
# @udpQueue.length
|
82
|
-
# end
|
83
77
|
end #class PortConnection
|
84
78
|
|
85
79
|
config_param :tag, :string
|
@@ -93,7 +87,7 @@ end #class PortConnection
|
|
93
87
|
@@connections ||= {}
|
94
88
|
if @@connections.nil?
|
95
89
|
end
|
96
|
-
@@connections[@port] = PortConnection.new(@bind, @port, @tag, @cache_ttl, @definitions, @queuesleep)
|
90
|
+
@@connections[@port] = PortConnection.new(@bind, @port, @tag, @cache_ttl, @definitions, @queuesleep, log)
|
97
91
|
log.debug "NetflowipfixInput::configure NB=#{@@connections.length}"
|
98
92
|
@total = 0
|
99
93
|
end
|
@@ -108,15 +102,8 @@ end #class PortConnection
|
|
108
102
|
$log.debug "start listening UDP on #{conn.bind}:#{conn.port}"
|
109
103
|
conn.start
|
110
104
|
end
|
111
|
-
end
|
112
|
-
|
105
|
+
end
|
113
106
|
|
114
|
-
# @eventQueue = Queue.new
|
115
|
-
# @udpQueue = Queue.new
|
116
|
-
# @thread_udp = UdpListenerThread.new(@bind, @port, @udpQueue)
|
117
|
-
# @thread_parser = ParserThread.new(@udpQueue, @queuesleep, @eventQueue, @cache_ttl, @definitions)
|
118
|
-
# @thread_udp.start
|
119
|
-
# @thread_parser.start
|
120
107
|
waitForEvents
|
121
108
|
end
|
122
109
|
|
@@ -125,8 +112,6 @@ end #class PortConnection
|
|
125
112
|
$log.debug "NetflowipfixInput::shutdown NB=#{@@connections.length}"
|
126
113
|
if @@connections.nil?
|
127
114
|
else
|
128
|
-
# $log.debug "listening UDP on #{@bind}:#{@port}"
|
129
|
-
# @connections[@port].stop
|
130
115
|
@@connections.each do | port, conn |
|
131
116
|
$log.debug "shutdown listening UDP on #{conn.bind}:#{conn.port}"
|
132
117
|
conn.stop
|
@@ -134,42 +119,29 @@ end #class PortConnection
|
|
134
119
|
@@connections = nil
|
135
120
|
end
|
136
121
|
|
137
|
-
# @thread_udp.close
|
138
|
-
# @thread_udp.join
|
139
|
-
# @thread_parser.close
|
140
|
-
# @thread_parser.join
|
141
122
|
end
|
142
123
|
|
143
124
|
|
144
125
|
def waitForEvents
|
145
|
-
# puts "Main::run begin #{@eventQueue.length}"
|
146
126
|
loop do
|
147
127
|
@@connections.each do | port, conn |
|
148
128
|
if (conn.event_queue_length > 0)
|
149
|
-
$log.
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
129
|
+
$log.trace "waitForEvents: #{conn.bind}:#{conn.port} queue has #{conn.event_queue_length} elements"
|
130
|
+
nbq = conn.event_queue_length
|
131
|
+
loop do
|
132
|
+
ar = conn.event_pop
|
133
|
+
time = ar[0]
|
134
|
+
record = ar[1]
|
135
|
+
router.emit(conn.tag, EventTime.new(time.to_i), record)
|
136
|
+
nbq = nbq - 1
|
137
|
+
break if nbq == 0
|
138
|
+
end
|
154
139
|
end
|
155
140
|
end
|
141
|
+
$log.trace "waitForEvents: sleep #{@queuesleep}"
|
156
142
|
sleep(@queuesleep)
|
157
143
|
|
158
|
-
# if @eventQueue.length > 0
|
159
|
-
|
160
|
-
|
161
|
-
# ar = @eventQueue.pop
|
162
|
-
# time = ar[0]
|
163
|
-
# record = ar[1]
|
164
|
-
# router.emit(@tag, EventTime.new(time.to_i), record)
|
165
|
-
|
166
|
-
|
167
|
-
# puts "Main::pop before #{@eventQueue.length} #{@tag} #{time}" # #{record.to_s}"
|
168
|
-
# puts "Main::pop after #{@eventQueue.length}"
|
169
|
-
# else
|
170
|
-
# end
|
171
144
|
end
|
172
|
-
# puts "Main::run end #{@eventQueue.length}"
|
173
145
|
|
174
146
|
end
|
175
147
|
|
@@ -178,18 +150,19 @@ end #class PortConnection
|
|
178
150
|
|
179
151
|
class UdpListenerThread
|
180
152
|
|
181
|
-
def initialize(bind, port, udpQueue, tag)
|
153
|
+
def initialize(bind, port, udpQueue, tag, log)
|
182
154
|
@port = port
|
183
155
|
@udpQueue = udpQueue
|
184
156
|
@udp_socket = UDPSocket.new
|
185
157
|
@udp_socket.bind(bind, port)
|
186
158
|
@total = 0
|
187
159
|
@tag = tag
|
160
|
+
@log = log
|
188
161
|
end
|
189
162
|
|
190
163
|
def start
|
191
164
|
@thread = Thread.new(&method(:run))
|
192
|
-
|
165
|
+
@log.trace "UdpListenerThread::start"
|
193
166
|
end
|
194
167
|
|
195
168
|
def close
|
@@ -205,8 +178,7 @@ class UdpListenerThread
|
|
205
178
|
loop do
|
206
179
|
msg, sender = @udp_socket.recvfrom(4096)
|
207
180
|
@total = @total + msg.length
|
208
|
-
|
209
|
-
# log.debug "Received #{msg.length} bytes for #{@total} total"
|
181
|
+
@log.trace "UdpListenerThread::recvfrom #{msg.length} bytes for #{@total} total on UDP/#{@port}"
|
210
182
|
record = {}
|
211
183
|
record["message"] = msg
|
212
184
|
record["length"] = msg.length
|
@@ -215,17 +187,17 @@ class UdpListenerThread
|
|
215
187
|
record["port"] = @port
|
216
188
|
# time = EventTime.new()
|
217
189
|
time = Time.now.getutc
|
218
|
-
# router.emit(@tag, EventTime.new(), record)
|
219
190
|
@udpQueue << [time, record]
|
220
191
|
end
|
221
192
|
end
|
222
193
|
end # class UdpListenerThread
|
223
194
|
|
224
195
|
class ParserThread
|
225
|
-
def initialize(udpQueue, queuesleep, eventQueue, cache_ttl, definitions)
|
196
|
+
def initialize(udpQueue, queuesleep, eventQueue, cache_ttl, definitions, log)
|
226
197
|
@udpQueue = udpQueue
|
227
198
|
@queuesleep = queuesleep
|
228
199
|
@eventQueue = eventQueue
|
200
|
+
@log = log
|
229
201
|
|
230
202
|
@parser_v5 = NetflowipfixInput::ParserNetflowv5.new
|
231
203
|
@parser_v9 = NetflowipfixInput::ParserNetflowv9.new
|
@@ -236,7 +208,7 @@ class ParserThread
|
|
236
208
|
end
|
237
209
|
def start
|
238
210
|
@thread = Thread.new(&method(:run))
|
239
|
-
|
211
|
+
@log.debug "ParserThread::start"
|
240
212
|
end
|
241
213
|
|
242
214
|
def close
|
@@ -247,16 +219,12 @@ class ParserThread
|
|
247
219
|
end
|
248
220
|
|
249
221
|
def run
|
250
|
-
# puts "ParserThread::run start #{@udpQueue.length}"
|
251
|
-
|
252
222
|
loop do
|
253
223
|
if @udpQueue.length == 0
|
254
|
-
# puts "ParserThread::run sleep #{@queuesleep}"
|
255
224
|
sleep(@queuesleep)
|
256
225
|
|
257
226
|
else
|
258
227
|
block = method(:emit)
|
259
|
-
#block = nil
|
260
228
|
ar = @udpQueue.pop
|
261
229
|
time = ar[0]
|
262
230
|
msg = ar[1]
|
@@ -264,8 +232,7 @@ class ParserThread
|
|
264
232
|
host = msg["sender"]
|
265
233
|
|
266
234
|
version,_ = payload[0,2].unpack('n')
|
267
|
-
|
268
|
-
|
235
|
+
@log.trace "ParserThread::pop #{@udpQueue.length} v#{version}"
|
269
236
|
|
270
237
|
case version
|
271
238
|
when 5
|
@@ -278,11 +245,9 @@ class ParserThread
|
|
278
245
|
packet = NetflowipfixInput::Netflow10Packet.read(payload)
|
279
246
|
@parser_v10.handle_v10(host, packet, block)
|
280
247
|
else
|
281
|
-
|
248
|
+
$log.warn "Unsupported Netflow version v#{version}: #{version.class}"
|
282
249
|
end # case
|
283
250
|
|
284
|
-
# parent_call(time, record)
|
285
|
-
# @eventQueue << [time, record]
|
286
251
|
end
|
287
252
|
end # loop do
|
288
253
|
end # def run
|
@@ -291,7 +256,7 @@ class ParserThread
|
|
291
256
|
event["host"] = host
|
292
257
|
end
|
293
258
|
@eventQueue << [time, event]
|
294
|
-
|
259
|
+
@log.trace "ParserThread::emit #{@eventQueue.length}"
|
295
260
|
end # def emit
|
296
261
|
|
297
262
|
end # class ParserThread
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-netflowipfix
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Yves Desharnaus
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2019-01-
|
11
|
+
date: 2019-01-08 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake
|
@@ -87,7 +87,9 @@ files:
|
|
87
87
|
- Rakefile
|
88
88
|
- fluent-plugin-netflowipfix.gemspec
|
89
89
|
- lib/.DS_Store
|
90
|
+
- lib/.gitignore
|
90
91
|
- lib/fluent/.DS_Store
|
92
|
+
- lib/fluent/.gitignore
|
91
93
|
- lib/fluent/plugin/in_netflowipfix.rb
|
92
94
|
- lib/fluent/plugin/ipfix_fields.yaml
|
93
95
|
- lib/fluent/plugin/netflow_fields.yaml
|