fluent-plugin-netflowipfix 1.1.0 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: '069c7af312866b4319eaa05bbf3a85b21dbe7284'
4
- data.tar.gz: 347842300a58938ced6d12a552df6df416a74765
3
+ metadata.gz: fe5554635cf4c0166f3b748a2df446cfe7761ba9
4
+ data.tar.gz: a681929a171a350c44dbab185157cd2fb7b77f17
5
5
  SHA512:
6
- metadata.gz: 5319acf004e86e279528485faa5ad760acaeb4b2a970aed0551a399ff34ef5b9a9692d3efd4c88072b65a76692ad96f46b19d1e84a8ede30cdce611d261cd16d
7
- data.tar.gz: df25aa1db4ba2990a1428d99f9072c1c8f7ddd21fa9838d314d9acb218bfcd04808c9d129cc3be140508b8cf63f21550f1bffb41c6d49626f84ad5c711f4c622
6
+ metadata.gz: 5ad6ea83415dbbaefc176faa638ab5982adfd34560cff3ad37e73304bb6f6e858158229a963a68f449a00575d8334f59bb64ac3ea70554bc99dec0dbdd6d23b3
7
+ data.tar.gz: f6f380dbf85051517358e92ba9aa5c0dd8d46351d113cf370ceb0b02d9c6cc93d0a945766fa9ea509520809aebe0942c2a257f5a2cb0e9d658105a5b507ca00d
data/.gitignore CHANGED
@@ -1 +1,6 @@
1
- .DS_Store
1
+ **/.DS_Store
2
+ *.gem
3
+ lib/.DS_Store
4
+ lib/fluent/.DS_Store
5
+ lib/.DS_Store
6
+ lib/fluent/.DS_Store
@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
3
3
 
4
4
  Gem::Specification.new do |spec|
5
5
  spec.name = "fluent-plugin-netflowipfix"
6
- spec.version = "1.1.0"
6
+ spec.version = "1.2.0"
7
7
  spec.authors = ["Yves Desharnaus"]
8
8
  spec.email = ["yvesbd@gmail.com"]
9
9
 
data/lib/.gitignore ADDED
@@ -0,0 +1 @@
1
+ .DS_Store
@@ -0,0 +1 @@
1
+ .DS_Store
@@ -30,15 +30,16 @@ module Fluent
30
30
  include DetachMultiProcessMixin
31
31
 
32
32
  class PortConnection
33
- def initialize(bind, port, tag, cache_ttl, definitions, queuesleep)
33
+ def initialize(bind, port, tag, cache_ttl, definitions, queuesleep, log)
34
34
  @bind = bind
35
35
  @port = port
36
36
  @tag = tag
37
37
  @cache_ttl = cache_ttl
38
38
  @definitions = definitions
39
39
  @eventQueue = Queue.new
40
- @udpQueue = Queue.new
40
+ # @udpQueue = Queue.new
41
41
  @queuesleep = queuesleep
42
+ @log = log
42
43
  end
43
44
 
44
45
  def bind
@@ -52,8 +53,8 @@ class PortConnection
52
53
  end
53
54
 
54
55
  def start
55
- @thread_udp = UdpListenerThread.new(@bind, @port, @udpQueue, @tag)
56
- @thread_parser = ParserThread.new(@udpQueue, @queuesleep, @eventQueue, @cache_ttl, @definitions)
56
+ @thread_udp = UdpListenerThread.new(@bind, @port, @udpQueue, @tag, @log)
57
+ @thread_parser = ParserThread.new(@udpQueue, @queuesleep, @eventQueue, @cache_ttl, @definitions, @log)
57
58
  @thread_udp.start
58
59
  @thread_parser.start
59
60
  end # def start
@@ -73,13 +74,6 @@ class PortConnection
73
74
  end
74
75
 
75
76
 
76
- # def udpqueue_pop
77
- # @udpQueue.pop
78
- # end
79
-
80
- # def udpqueue_length
81
- # @udpQueue.length
82
- # end
83
77
  end #class PortConnection
84
78
 
85
79
  config_param :tag, :string
@@ -93,7 +87,7 @@ end #class PortConnection
93
87
  @@connections ||= {}
94
88
  if @@connections.nil?
95
89
  end
96
- @@connections[@port] = PortConnection.new(@bind, @port, @tag, @cache_ttl, @definitions, @queuesleep)
90
+ @@connections[@port] = PortConnection.new(@bind, @port, @tag, @cache_ttl, @definitions, @queuesleep, log)
97
91
  log.debug "NetflowipfixInput::configure NB=#{@@connections.length}"
98
92
  @total = 0
99
93
  end
@@ -108,15 +102,8 @@ end #class PortConnection
108
102
  $log.debug "start listening UDP on #{conn.bind}:#{conn.port}"
109
103
  conn.start
110
104
  end
111
- end
112
-
105
+ end
113
106
 
114
- # @eventQueue = Queue.new
115
- # @udpQueue = Queue.new
116
- # @thread_udp = UdpListenerThread.new(@bind, @port, @udpQueue)
117
- # @thread_parser = ParserThread.new(@udpQueue, @queuesleep, @eventQueue, @cache_ttl, @definitions)
118
- # @thread_udp.start
119
- # @thread_parser.start
120
107
  waitForEvents
121
108
  end
122
109
 
@@ -125,8 +112,6 @@ end #class PortConnection
125
112
  $log.debug "NetflowipfixInput::shutdown NB=#{@@connections.length}"
126
113
  if @@connections.nil?
127
114
  else
128
- # $log.debug "listening UDP on #{@bind}:#{@port}"
129
- # @connections[@port].stop
130
115
  @@connections.each do | port, conn |
131
116
  $log.debug "shutdown listening UDP on #{conn.bind}:#{conn.port}"
132
117
  conn.stop
@@ -134,42 +119,29 @@ end #class PortConnection
134
119
  @@connections = nil
135
120
  end
136
121
 
137
- # @thread_udp.close
138
- # @thread_udp.join
139
- # @thread_parser.close
140
- # @thread_parser.join
141
122
  end
142
123
 
143
124
 
144
125
  def waitForEvents
145
- # puts "Main::run begin #{@eventQueue.length}"
146
126
  loop do
147
127
  @@connections.each do | port, conn |
148
128
  if (conn.event_queue_length > 0)
149
- $log.debug "waitForEvents: #{conn.bind}:#{conn.port}"
150
- ar = conn.event_pop
151
- time = ar[0]
152
- record = ar[1]
153
- router.emit(conn.tag, EventTime.new(time.to_i), record)
129
+ $log.trace "waitForEvents: #{conn.bind}:#{conn.port} queue has #{conn.event_queue_length} elements"
130
+ nbq = conn.event_queue_length
131
+ loop do
132
+ ar = conn.event_pop
133
+ time = ar[0]
134
+ record = ar[1]
135
+ router.emit(conn.tag, EventTime.new(time.to_i), record)
136
+ nbq = nbq - 1
137
+ break if nbq == 0
138
+ end
154
139
  end
155
140
  end
141
+ $log.trace "waitForEvents: sleep #{@queuesleep}"
156
142
  sleep(@queuesleep)
157
143
 
158
- # if @eventQueue.length > 0
159
-
160
-
161
- # ar = @eventQueue.pop
162
- # time = ar[0]
163
- # record = ar[1]
164
- # router.emit(@tag, EventTime.new(time.to_i), record)
165
-
166
-
167
- # puts "Main::pop before #{@eventQueue.length} #{@tag} #{time}" # #{record.to_s}"
168
- # puts "Main::pop after #{@eventQueue.length}"
169
- # else
170
- # end
171
144
  end
172
- # puts "Main::run end #{@eventQueue.length}"
173
145
 
174
146
  end
175
147
 
@@ -178,18 +150,19 @@ end #class PortConnection
178
150
 
179
151
  class UdpListenerThread
180
152
 
181
- def initialize(bind, port, udpQueue, tag)
153
+ def initialize(bind, port, udpQueue, tag, log)
182
154
  @port = port
183
155
  @udpQueue = udpQueue
184
156
  @udp_socket = UDPSocket.new
185
157
  @udp_socket.bind(bind, port)
186
158
  @total = 0
187
159
  @tag = tag
160
+ @log = log
188
161
  end
189
162
 
190
163
  def start
191
164
  @thread = Thread.new(&method(:run))
192
- puts "UdpListenerThread::start"
165
+ @log.trace "UdpListenerThread::start"
193
166
  end
194
167
 
195
168
  def close
@@ -205,8 +178,7 @@ class UdpListenerThread
205
178
  loop do
206
179
  msg, sender = @udp_socket.recvfrom(4096)
207
180
  @total = @total + msg.length
208
- # puts "UdpListenerThread::recvfrom #{msg.length} bytes for #{@total} total on UDP/#{@port}"
209
- # log.debug "Received #{msg.length} bytes for #{@total} total"
181
+ @log.trace "UdpListenerThread::recvfrom #{msg.length} bytes for #{@total} total on UDP/#{@port}"
210
182
  record = {}
211
183
  record["message"] = msg
212
184
  record["length"] = msg.length
@@ -215,17 +187,17 @@ class UdpListenerThread
215
187
  record["port"] = @port
216
188
  # time = EventTime.new()
217
189
  time = Time.now.getutc
218
- # router.emit(@tag, EventTime.new(), record)
219
190
  @udpQueue << [time, record]
220
191
  end
221
192
  end
222
193
  end # class UdpListenerThread
223
194
 
224
195
  class ParserThread
225
- def initialize(udpQueue, queuesleep, eventQueue, cache_ttl, definitions)
196
+ def initialize(udpQueue, queuesleep, eventQueue, cache_ttl, definitions, log)
226
197
  @udpQueue = udpQueue
227
198
  @queuesleep = queuesleep
228
199
  @eventQueue = eventQueue
200
+ @log = log
229
201
 
230
202
  @parser_v5 = NetflowipfixInput::ParserNetflowv5.new
231
203
  @parser_v9 = NetflowipfixInput::ParserNetflowv9.new
@@ -236,7 +208,7 @@ class ParserThread
236
208
  end
237
209
  def start
238
210
  @thread = Thread.new(&method(:run))
239
- puts "ParserThread::start"
211
+ @log.debug "ParserThread::start"
240
212
  end
241
213
 
242
214
  def close
@@ -247,16 +219,12 @@ class ParserThread
247
219
  end
248
220
 
249
221
  def run
250
- # puts "ParserThread::run start #{@udpQueue.length}"
251
-
252
222
  loop do
253
223
  if @udpQueue.length == 0
254
- # puts "ParserThread::run sleep #{@queuesleep}"
255
224
  sleep(@queuesleep)
256
225
 
257
226
  else
258
227
  block = method(:emit)
259
- #block = nil
260
228
  ar = @udpQueue.pop
261
229
  time = ar[0]
262
230
  msg = ar[1]
@@ -264,8 +232,7 @@ class ParserThread
264
232
  host = msg["sender"]
265
233
 
266
234
  version,_ = payload[0,2].unpack('n')
267
- # puts "ParserThread::pop #{@udpQueue.length} v#{version}"
268
-
235
+ @log.trace "ParserThread::pop #{@udpQueue.length} v#{version}"
269
236
 
270
237
  case version
271
238
  when 5
@@ -278,11 +245,9 @@ class ParserThread
278
245
  packet = NetflowipfixInput::Netflow10Packet.read(payload)
279
246
  @parser_v10.handle_v10(host, packet, block)
280
247
  else
281
- # $log.warn "Unsupported Netflow version v#{version}: #{version.class}"
248
+ $log.warn "Unsupported Netflow version v#{version}: #{version.class}"
282
249
  end # case
283
250
 
284
- # parent_call(time, record)
285
- # @eventQueue << [time, record]
286
251
  end
287
252
  end # loop do
288
253
  end # def run
@@ -291,7 +256,7 @@ class ParserThread
291
256
  event["host"] = host
292
257
  end
293
258
  @eventQueue << [time, event]
294
- # puts "ParserThread::emit #{@eventQueue.length}"
259
+ @log.trace "ParserThread::emit #{@eventQueue.length}"
295
260
  end # def emit
296
261
 
297
262
  end # class ParserThread
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-netflowipfix
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.1.0
4
+ version: 1.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Yves Desharnaus
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2019-01-03 00:00:00.000000000 Z
11
+ date: 2019-01-08 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: rake
@@ -87,7 +87,9 @@ files:
87
87
  - Rakefile
88
88
  - fluent-plugin-netflowipfix.gemspec
89
89
  - lib/.DS_Store
90
+ - lib/.gitignore
90
91
  - lib/fluent/.DS_Store
92
+ - lib/fluent/.gitignore
91
93
  - lib/fluent/plugin/in_netflowipfix.rb
92
94
  - lib/fluent/plugin/ipfix_fields.yaml
93
95
  - lib/fluent/plugin/netflow_fields.yaml